Re: lavcac3enc broken?

Reimar Döffinger <[email protected]> Thu, 30 Mar 2023 19:48:05 +0200
Newsgroups gmane.comp.video.mplayer.devel
Message-ID <[email protected]> 
> On 29 Mar 2023, at 21:20, eatdirt <[email protected]> wrote:
> 
> Woo! Thanks for looking at it.
> 
> Ok, it segfaults in av_helpers now

SIGFPE, division by 0 actually, not segfault.
Obviously my testing was testing even less than I hoped.
Maybe this one on top will help?
I'll probably have to see if I can find a system where I can actually test this.

_______________________________________________
MPlayer-dev-eng mailing list
[email protected]
https://lists.mplayerhq.hu/mailman/listinfo/mplayer-dev-eng
helpers_ch_layout.diff (application/octet-stream, 2.2 KB) 
Index: av_helpers.c
===================================================================
--- av_helpers.c	(revision 38410)
+++ av_helpers.c	(working copy)
@@ -116,38 +116,39 @@
 {
     void *orig_src = src;
     int bps = av_get_bytes_per_sample(ctx->sample_fmt);
-    int planar = ctx->channels > 1 && av_sample_fmt_is_planar(ctx->sample_fmt);
+    int channels = ctx->channels > 0 ? ctx->channels : ctx->ch_layout.nb_channels;
+    int planar = channels > 1 && av_sample_fmt_is_planar(ctx->sample_fmt);
     int isac3 = ctx->codec->id == AV_CODEC_ID_AC3;
     int n;
     int got;
     AVPacket pkt;
     AVFrame *frame = av_frame_alloc();
-    if ((ctx->channels == 6 || ctx->channels == 5) &&
+    if ((channels == 6 || channels == 5) &&
         (isac3 || !strcmp(ctx->codec->name,"libfaac"))) {
         reorder_channel_nch(src, AF_CHANNEL_LAYOUT_MPLAYER_DEFAULT,
                             isac3 ? AF_CHANNEL_LAYOUT_LAVC_DEFAULT : AF_CHANNEL_LAYOUT_AAC_DEFAULT,
-                            ctx->channels,
+                            channels,
                             src_len / bps, bps);
     }
-    frame->nb_samples = src_len / ctx->channels / bps;
+    frame->nb_samples = src_len / channels / bps;
     if (planar) {
         // TODO: this is horribly inefficient.
         int ch;
         src = av_mallocz(src_len);
-        for (ch = 0; ch < ctx->channels; ch++) {
+        for (ch = 0; ch < channels; ch++) {
             uint8_t *tmps = (uint8_t *)orig_src + ch*bps;
-            uint8_t *tmpd = (uint8_t *)src + ch*src_len/ctx->channels;
+            uint8_t *tmpd = (uint8_t *)src + ch*src_len/channels;
             int s;
             for (s = 0; s < frame->nb_samples; s++) {
                 memcpy(tmpd, tmps, bps);
-                tmps += ctx->channels * bps;
+                tmps += channels * bps;
                 tmpd += bps;
             }
         }
     }
     frame->format = ctx->sample_fmt;
-    frame->channels = ctx->channels;
-    n = avcodec_fill_audio_frame(frame, ctx->channels, ctx->sample_fmt, src, src_len, 1);
+    frame->channels = channels;
+    n = avcodec_fill_audio_frame(frame, channels, ctx->sample_fmt, src, src_len, 1);
     if (n < 0) return 0;
     n = avcodec_send_frame(ctx, frame);
     av_init_packet(&pkt);