From ff54ab2852e13e25ecdeaaab8f66639b795b5c33 Mon Sep 17 00:00:00 2001 From: Vithorio Polten Date: Tue, 14 May 2024 15:14:40 -0300 Subject: [PATCH] fix(macos): fix broken streaming on MacOS (#2485) --- src/platform/macos/av_img_t.h | 48 ++++++++++++++++--------- src/platform/macos/display.mm | 44 +++++++++++++++-------- src/platform/macos/nv12_zero_device.cpp | 3 ++ src/platform/macos/nv12_zero_device.h | 1 - src/video.cpp | 9 +++-- 5 files changed, 71 insertions(+), 34 deletions(-) diff --git a/src/platform/macos/av_img_t.h b/src/platform/macos/av_img_t.h index fb0d7ecd0..427968006 100644 --- a/src/platform/macos/av_img_t.h +++ b/src/platform/macos/av_img_t.h @@ -11,42 +11,56 @@ namespace platf { struct av_sample_buf_t { + CMSampleBufferRef buf; + explicit av_sample_buf_t(CMSampleBufferRef buf): buf((CMSampleBufferRef) CFRetain(buf)) {} ~av_sample_buf_t() { - CFRelease(buf); + if (buf != nullptr) { + CFRelease(buf); + } } - - CMSampleBufferRef buf; }; struct av_pixel_buf_t { - explicit av_pixel_buf_t(CVPixelBufferRef buf): - buf((CVPixelBufferRef) CFRetain(buf)), - locked(false) {} + CVPixelBufferRef buf; + + // Constructor + explicit av_pixel_buf_t(CMSampleBufferRef sb): + buf( + CMSampleBufferGetImageBuffer(sb)) { + CVPixelBufferLockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); + } [[nodiscard]] uint8_t * - lock() const { - if (!locked) { - CVPixelBufferLockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); - } - return (uint8_t *) CVPixelBufferGetBaseAddress(buf); + data() const { + return static_cast(CVPixelBufferGetBaseAddress(buf)); } + // Destructor ~av_pixel_buf_t() { - if (locked) { + if (buf != nullptr) { CVPixelBufferUnlockBaseAddress(buf, kCVPixelBufferLock_ReadOnly); } - CFRelease(buf); } - - CVPixelBufferRef buf; - bool locked; }; - struct av_img_t: public img_t { + struct av_img_t: img_t { std::shared_ptr sample_buffer; std::shared_ptr pixel_buffer; }; + + struct temp_retain_av_img_t { + std::shared_ptr sample_buffer; + std::shared_ptr pixel_buffer; + uint8_t *data; + + temp_retain_av_img_t( + std::shared_ptr sb, + std::shared_ptr pb, + uint8_t *dt): + sample_buffer(std::move(sb)), + pixel_buffer(std::move(pb)), data(dt) {} + }; } // namespace platf diff --git a/src/platform/macos/display.mm b/src/platform/macos/display.mm index 47b83435e..dc7c7b685 100644 --- a/src/platform/macos/display.mm +++ b/src/platform/macos/display.mm @@ -31,6 +31,9 @@ namespace platf { capture_e capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override { auto signal = [av_capture capture:^(CMSampleBufferRef sampleBuffer) { + auto new_sample_buffer = std::make_shared(sampleBuffer); + auto new_pixel_buffer = std::make_shared(new_sample_buffer->buf); + std::shared_ptr img_out; if (!pull_free_image_cb(img_out)) { // got interrupt signal @@ -39,17 +42,22 @@ namespace platf { } auto av_img = std::static_pointer_cast(img_out); - CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + auto old_data_retainer = std::make_shared( + av_img->sample_buffer, + av_img->pixel_buffer, + img_out->data); - av_img->sample_buffer = std::make_shared(sampleBuffer); - av_img->pixel_buffer = std::make_shared(pixelBuffer); - img_out->data = av_img->pixel_buffer->lock(); + av_img->sample_buffer = new_sample_buffer; + av_img->pixel_buffer = new_pixel_buffer; + img_out->data = new_pixel_buffer->data(); - img_out->width = (int) CVPixelBufferGetWidth(pixelBuffer); - img_out->height = (int) CVPixelBufferGetHeight(pixelBuffer); - img_out->row_pitch = (int) CVPixelBufferGetBytesPerRow(pixelBuffer); + img_out->width = (int) CVPixelBufferGetWidth(new_pixel_buffer->buf); + img_out->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); + img_out->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); img_out->pixel_pitch = img_out->row_pitch / img_out->width; + old_data_retainer = nullptr; + if (!push_captured_image_cb(std::move(img_out), true)) { // got interrupt signal // returning false here stops capture backend @@ -93,19 +101,27 @@ namespace platf { int dummy_img(img_t *img) override { auto signal = [av_capture capture:^(CMSampleBufferRef sampleBuffer) { + auto new_sample_buffer = std::make_shared(sampleBuffer); + auto new_pixel_buffer = std::make_shared(new_sample_buffer->buf); + auto av_img = (av_img_t *) img; - CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + auto old_data_retainer = std::make_shared( + av_img->sample_buffer, + av_img->pixel_buffer, + img->data); - av_img->sample_buffer = std::make_shared(sampleBuffer); - av_img->pixel_buffer = std::make_shared(pixelBuffer); - img->data = av_img->pixel_buffer->lock(); + av_img->sample_buffer = new_sample_buffer; + av_img->pixel_buffer = new_pixel_buffer; + img->data = new_pixel_buffer->data(); - img->width = (int) CVPixelBufferGetWidth(pixelBuffer); - img->height = (int) CVPixelBufferGetHeight(pixelBuffer); - img->row_pitch = (int) CVPixelBufferGetBytesPerRow(pixelBuffer); + img->width = (int) CVPixelBufferGetWidth(new_pixel_buffer->buf); + img->height = (int) CVPixelBufferGetHeight(new_pixel_buffer->buf); + img->row_pitch = (int) CVPixelBufferGetBytesPerRow(new_pixel_buffer->buf); img->pixel_pitch = img->row_pitch / img->width; + old_data_retainer = nullptr; + // returning false here stops capture backend return false; }]; diff --git a/src/platform/macos/nv12_zero_device.cpp b/src/platform/macos/nv12_zero_device.cpp index c217c6376..ab0c478eb 100644 --- a/src/platform/macos/nv12_zero_device.cpp +++ b/src/platform/macos/nv12_zero_device.cpp @@ -4,6 +4,7 @@ */ #include +#include "src/platform/macos/av_img_t.h" #include "src/platform/macos/nv12_zero_device.h" #include "src/video.h" @@ -24,6 +25,8 @@ namespace platf { CVPixelBufferRelease((CVPixelBufferRef) data); } + util::safe_ptr av_frame; + int nv12_zero_device::convert(platf::img_t &img) { auto *av_img = (av_img_t *) &img; diff --git a/src/platform/macos/nv12_zero_device.h b/src/platform/macos/nv12_zero_device.h index b83210cb9..e0f3230f9 100644 --- a/src/platform/macos/nv12_zero_device.h +++ b/src/platform/macos/nv12_zero_device.h @@ -5,7 +5,6 @@ #pragma once #include "src/platform/common.h" -#include "src/platform/macos/av_img_t.h" struct AVFrame; diff --git a/src/video.cpp b/src/video.cpp index 12f8cb417..394e7d46f 100644 --- a/src/video.cpp +++ b/src/video.cpp @@ -909,7 +909,9 @@ namespace video { }, {}, // SDR-specific options {}, // HDR-specific options - {}, // Fallback options + { + { "flags"s, "-low_delay" }, + }, // Fallback options std::nullopt, "h264_videotoolbox"s, }, @@ -1451,7 +1453,10 @@ namespace video { } } - ctx->flags |= (AV_CODEC_FLAG_CLOSED_GOP | AV_CODEC_FLAG_LOW_DELAY); + // We forcefully reset the flags to avoid clash on reuse of AVCodecContext + ctx->flags = 0; + ctx->flags |= AV_CODEC_FLAG_CLOSED_GOP | AV_CODEC_FLAG_LOW_DELAY; + ctx->flags2 |= AV_CODEC_FLAG2_FAST; auto avcodec_colorspace = avcodec_colorspace_from_sunshine_colorspace(colorspace);