[PR] avdevice/avfoundation: wait for frame consumption to avoid dropping A/V frames (PR #23531)

PR #23531 opened by Jun Zhao (mypopydev)
URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23531
Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23531.patch

The capture callback dropped the previous frame instead of waiting for it
to be consumed, so frames were lost when avf_read_packet() fell behind;
39fbd06314 (return EAGAIN instead of waiting) made it easy to hit.

Add back a condition variable: the capture callback blocks until
avf_read_packet() takes the current frame, and the reader waits on it when
no frame is ready. An is_stopping flag set in destroy_context() wakes both
sides so teardown cannot deadlock. observed_quit is now set under the lock
and broadcast as well, so a blocked reader wakes and returns EOF when a
transport-control device stops delivering frames.

Based on a patch by Zhongxin Zhuang <[email protected]>; here
unlock_frames() only broadcasts and unlocks, so a read no longer releases
the other stream's still-unconsumed frame.

Verified by capturing camera+mic for 8s and comparing delivered packet
counts before/after this change:

  ffmpeg -f avfoundation -pixel_format nv12 -framerate 30 -i "0:0" -t 8 \
         -vf "scale=2560:1440,hqdn3d" -c:v libx264 -preset medium \
         -c:a aac out.mp4
  ffprobe -count_packets -show_entries stream=nb_read_packets out.mp4

  slow consumer (ideal: video ~240, audio ~375)
                     video   audio
    before (EAGAIN)   174     163   (audio ~57% dropped)
    after  (condvar)  234     376   (no drops)

  fast consumer (-preset ultrafast, no filter)
    before            240     328
    after             238     376

Signed-off-by: Jun Zhao <[email protected]>

# Summary of changes

Briefly describe what this PR does and why.

<!--
If this PR requires new FATE test samples, attach them to the PR and
list their target paths below (relative to the fate-suite root).

Attached filenames must match the sample's filename:

```fate-samples
# e.g. vorbis/new-sample.ogg
```
-->



>From 388eaee4fde6cc5ee8ee297588a3cd907fe0bfcf Mon Sep 17 00:00:00 2001
From: Jun Zhao <[email protected]>
Date: Thu, 18 Jun 2026 23:14:21 +0800
Subject: [PATCH] avdevice/avfoundation: wait for frame consumption to avoid
 dropping A/V frames

The capture callback dropped the previous frame instead of waiting for it
to be consumed, so frames were lost when avf_read_packet() fell behind;
39fbd06314 (return EAGAIN instead of waiting) made it easy to hit.

Add back a condition variable: the capture callback blocks until
avf_read_packet() takes the current frame, and the reader waits on it when
no frame is ready. An is_stopping flag set in destroy_context() wakes both
sides so teardown cannot deadlock. observed_quit is now set under the lock
and broadcast as well, so a blocked reader wakes and returns EOF when a
transport-control device stops delivering frames.

Based on a patch by Zhongxin Zhuang <[email protected]>; here
unlock_frames() only broadcasts and unlocks, so a read no longer releases
the other stream's still-unconsumed frame.

Verified by capturing camera+mic for 8s and comparing delivered packet
counts before/after this change:

  ffmpeg -f avfoundation -pixel_format nv12 -framerate 30 -i "0:0" -t 8 \
         -vf "scale=2560:1440,hqdn3d" -c:v libx264 -preset medium \
         -c:a aac out.mp4
  ffprobe -count_packets -show_entries stream=nb_read_packets out.mp4

  slow consumer (ideal: video ~240, audio ~375)
                     video   audio
    before (EAGAIN)   174     163   (audio ~57% dropped)
    after  (condvar)  234     376   (no drops)

  fast consumer (-preset ultrafast, no filter)
    before            240     328
    after             238     376

Signed-off-by: Jun Zhao <[email protected]>
---
 libavdevice/avfoundation.m | 56 ++++++++++++++++++++++++++++++++------
 1 file changed, 47 insertions(+), 9 deletions(-)

diff --git a/libavdevice/avfoundation.m b/libavdevice/avfoundation.m
index ebec1ac4f2..6f52989fc9 100644
--- a/libavdevice/avfoundation.m
+++ b/libavdevice/avfoundation.m
@@ -89,6 +89,8 @@ typedef struct
     int             frames_captured;
     int             audio_frames_captured;
     pthread_mutex_t frame_lock;
+    pthread_cond_t  frame_wait_cond;
+    int             is_stopping;
     id              avf_delegate;
     id              avf_audio_delegate;
 
@@ -147,6 +149,7 @@ static void lock_frames(AVFContext* ctx)
 
 static void unlock_frames(AVFContext* ctx)
 {
+    pthread_cond_broadcast(&ctx->frame_wait_cond);
     pthread_mutex_unlock(&ctx->frame_lock);
 }
 
@@ -210,7 +213,12 @@ static void unlock_frames(AVFContext* ctx)
 
         if (mode != _context->observed_mode) {
             if (mode == AVCaptureDeviceTransportControlsNotPlayingMode) {
+                // Set under the lock and broadcast so a reader blocked in
+                // avf_read_packet() wakes up and returns EOF instead of
+                // hanging once the device stops delivering frames.
+                lock_frames(_context);
                 _context->observed_quit = 1;
+                unlock_frames(_context);
             }
             _context->observed_mode = mode;
         }
@@ -229,8 +237,13 @@ static void unlock_frames(AVFContext* ctx)
 {
     lock_frames(_context);
 
-    if (_context->current_frame != nil) {
-        CFRelease(_context->current_frame);
+    while ((_context->current_frame != nil) && !_context->is_stopping) {
+        pthread_cond_wait(&_context->frame_wait_cond, &_context->frame_lock);
+    }
+
+    if (_context->is_stopping) {
+        unlock_frames(_context);
+        return;
     }
 
     _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);
@@ -273,8 +286,13 @@ static void unlock_frames(AVFContext* ctx)
 {
     lock_frames(_context);
 
-    if (_context->current_audio_frame != nil) {
-        CFRelease(_context->current_audio_frame);
+    while ((_context->current_audio_frame != nil) && !_context->is_stopping) {
+        pthread_cond_wait(&_context->frame_wait_cond, &_context->frame_lock);
+    }
+
+    if (_context->is_stopping) {
+        unlock_frames(_context);
+        return;
     }
 
     _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);
@@ -288,6 +306,12 @@ static void unlock_frames(AVFContext* ctx)
 
 static void destroy_context(AVFContext* ctx)
 {
+    // Wake any capture callback blocked waiting for the consumer and make it
+    // bail out, so stopRunning() can drain the session without a deadlock.
+    lock_frames(ctx);
+    ctx->is_stopping = 1;
+    unlock_frames(ctx);
+
     [ctx->capture_session stopRunning];
 
     [ctx->capture_session release];
@@ -305,10 +329,17 @@ static void destroy_context(AVFContext* ctx)
     av_freep(&ctx->url);
     av_freep(&ctx->audio_buffer);
 
+    pthread_cond_destroy(&ctx->frame_wait_cond);
     pthread_mutex_destroy(&ctx->frame_lock);
 
     if (ctx->current_frame) {
         CFRelease(ctx->current_frame);
+        ctx->current_frame = nil;
+    }
+
+    if (ctx->current_audio_frame) {
+        CFRelease(ctx->current_audio_frame);
+        ctx->current_audio_frame = nil;
     }
 }
 
@@ -836,6 +867,8 @@ static int avf_read_header(AVFormatContext *s)
     ctx->num_video_devices = [devices count] + [devices_muxed count];
 
     pthread_mutex_init(&ctx->frame_lock, NULL);
+    pthread_cond_init(&ctx->frame_wait_cond, NULL);
+    ctx->is_stopping = 0;
 
 #if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
     CGGetActiveDisplayList(0, NULL, &num_screens);
@@ -1120,10 +1153,10 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
 {
     AVFContext* ctx = (AVFContext*)s->priv_data;
 
+    lock_frames(ctx);
     do {
         CVImageBufferRef image_buffer;
         CMBlockBufferRef block_buffer;
-        lock_frames(ctx);
 
         if (ctx->current_frame != nil) {
             int status;
@@ -1253,16 +1286,21 @@ static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
             ctx->current_audio_frame = nil;
         } else {
             pkt->data = NULL;
-            unlock_frames(ctx);
             if (ctx->observed_quit) {
+                unlock_frames(ctx);
                 return AVERROR_EOF;
-            } else {
-                return AVERROR(EAGAIN);
             }
+            // No frame available yet: wait until a capture callback delivers
+            // one (or until the device is being torn down).
+            pthread_cond_wait(&ctx->frame_wait_cond, &ctx->frame_lock);
         }
+    } while (!pkt->data && !ctx->is_stopping);
 
+    if (ctx->is_stopping) {
         unlock_frames(ctx);
-    } while (!pkt->data);
+        return AVERROR_EOF;
+    }
+    unlock_frames(ctx);
 
     return 0;
 }
-- 
2.52.0

_______________________________________________
ffmpeg-devel mailing list -- [email protected]
To unsubscribe send an email to [email protected]