[PR] swresample/rematrix: improve height channel downmixing (PR #24029)
AYOUB NABIL BOUBAGRAT via ffmpeg-devel <[email protected]>
| Newsgroups | gmane.comp.video.ffmpeg.devel |
|---|---|
| Message-ID | <178601719460.59.9785298226718700684@29965ddac10e> |
PR #24029 opened by AYOUB NABIL BOUBAGRAT (ayoubnabil) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24029 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24029.patch the default matrix used 0.707 when folding top-front channels into matching ear-level front channels if the input already contained front channels. ITU-R BS.2127-1 specifies unity gain for this mapping. top-back channels were left unaccounted and could be dropped. when the output retains top-front channels, fold top-back into them at 0.707 following IAMF 1.1.0. otherwise map them to rear, side, front, or mono outputs. add direct tests for the resulting coefficients. follow-up to #24024. >From cd292e4e96163dd7d396b43ef0601b63b63e55e6 Mon Sep 17 00:00:00 2001 From: Ayoub Nabil Boubagrat <[email protected]> Date: Thu, 6 Aug 2026 13:46:44 +0200 Subject: [PATCH 1/2] swresample/rematrix: use unity gain for top-front downmix ITU-R BS.2127-1 maps U+030 and U-030 to the matching ear-level front channels at unity gain. the current matrix uses 0.707 when the input also contains front channels. use 1.0 instead. Signed-off-by: Ayoub Nabil Boubagrat <[email protected]> --- libswresample/rematrix.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/libswresample/rematrix.c b/libswresample/rematrix.c index 6f4c16ee45..77ec2250a6 100644 --- a/libswresample/rematrix.c +++ b/libswresample/rematrix.c @@ -287,13 +287,9 @@ static void build_matrix(const AVChannelLayout *in_ch_layout, const AVChannelLay if (av_channel_layout_index_from_channel(in_ch_layout, AV_CHAN_TOP_FRONT_CENTER) >= 0) matrix[TOP_FRONT_CENTER][TOP_FRONT_CENTER] = center_mix_level * sqrt(2); } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_FRONT_LEFT) >= 0) { - if (av_channel_layout_index_from_channel(in_ch_layout, AV_CHAN_FRONT_LEFT) >= 0) { - matrix[FRONT_LEFT ][TOP_FRONT_LEFT ] += M_SQRT1_2; - matrix[FRONT_RIGHT][TOP_FRONT_RIGHT] += M_SQRT1_2; - } else { - matrix[FRONT_LEFT ][TOP_FRONT_LEFT ] += 1.0; - matrix[FRONT_RIGHT][TOP_FRONT_RIGHT] += 1.0; - } + /* U+030 -> M+030 in ITU-R BS.2127-1, Table 16. */ + matrix[FRONT_LEFT ][TOP_FRONT_LEFT ] += 1.0; + matrix[FRONT_RIGHT][TOP_FRONT_RIGHT] += 1.0; } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_FRONT_CENTER) >= 0) { matrix[FRONT_CENTER][TOP_FRONT_LEFT ] += M_SQRT1_2; matrix[FRONT_CENTER][TOP_FRONT_RIGHT] += M_SQRT1_2; -- 2.52.0 >From 8a02a7396e53b68f16d452bbe2680d4fd67d6b8b Mon Sep 17 00:00:00 2001 From: Ayoub Nabil Boubagrat <[email protected]> Date: Thu, 6 Aug 2026 13:46:58 +0200 Subject: [PATCH 2/2] swresample/rematrix: handle top-back channels top-back channels are currently left unaccounted and can be dropped during downmixing. when the output retains top-front channels, follow IAMF 1.1.0 and fold top-back into top-front at 0.707. prefer this path over ear-level rear channels to preserve the height layer in x.1.4 to x.1.2 downmixes. when no matching height output remains, map top-back to back or side channels, then fall back to front or mono outputs. handle top-back center separately and add direct tests for every matrix path. Signed-off-by: Ayoub Nabil Boubagrat <[email protected]> --- libswresample/Makefile | 3 +- libswresample/rematrix.c | 46 ++++++++++ libswresample/tests/.gitignore | 1 + libswresample/tests/rematrix.c | 152 +++++++++++++++++++++++++++++++++ tests/fate/libswresample.mak | 6 ++ tests/ref/fate/swr-rematrix | 0 6 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 libswresample/tests/rematrix.c create mode 100644 tests/ref/fate/swr-rematrix diff --git a/libswresample/Makefile b/libswresample/Makefile index 12fbfc35c1..8149de069f 100644 --- a/libswresample/Makefile +++ b/libswresample/Makefile @@ -24,5 +24,6 @@ SHLIBOBJS += log2_tab.o # Windows resource file SHLIBOBJS-$(HAVE_GNU_WINDRES) += swresampleres.o -TESTPROGS = swresample \ +TESTPROGS = rematrix \ + swresample \ swresample_resample_realloc \ diff --git a/libswresample/rematrix.c b/libswresample/rematrix.c index 77ec2250a6..723e70d78b 100644 --- a/libswresample/rematrix.c +++ b/libswresample/rematrix.c @@ -127,6 +127,8 @@ static int sane_layout(AVChannelLayout *ch_layout) { return 0; if(!even(av_channel_layout_subset(ch_layout, (AV_CH_TOP_FRONT_LEFT | AV_CH_TOP_FRONT_RIGHT)))) return 0; + if(!even(av_channel_layout_subset(ch_layout, (AV_CH_TOP_BACK_LEFT | AV_CH_TOP_BACK_RIGHT)))) + return 0; return 1; } @@ -297,6 +299,50 @@ static void build_matrix(const AVChannelLayout *in_ch_layout, const AVChannelLay av_assert0(0); } + if (unaccounted & AV_CH_TOP_BACK_LEFT) { + if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_TOP_BACK_CENTER) >= 0) { + matrix[TOP_BACK_CENTER][TOP_BACK_LEFT ] += M_SQRT1_2; + matrix[TOP_BACK_CENTER][TOP_BACK_RIGHT] += M_SQRT1_2; + } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_TOP_FRONT_LEFT) >= 0) { + /* IAMF v1.1.0, Section 7.3.2.1.1. */ + matrix[TOP_FRONT_LEFT ][TOP_BACK_LEFT ] += M_SQRT1_2; + matrix[TOP_FRONT_RIGHT][TOP_BACK_RIGHT] += M_SQRT1_2; + } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_BACK_LEFT) >= 0) { + matrix[BACK_LEFT ][TOP_BACK_LEFT ] += 1.0; + matrix[BACK_RIGHT][TOP_BACK_RIGHT] += 1.0; + } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_SIDE_LEFT) >= 0) { + matrix[SIDE_LEFT ][TOP_BACK_LEFT ] += 1.0; + matrix[SIDE_RIGHT][TOP_BACK_RIGHT] += 1.0; + } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_FRONT_LEFT) >= 0) { + matrix[FRONT_LEFT ][TOP_BACK_LEFT ] += M_SQRT1_2; + matrix[FRONT_RIGHT][TOP_BACK_RIGHT] += M_SQRT1_2; + } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_FRONT_CENTER) >= 0) { + matrix[FRONT_CENTER][TOP_BACK_LEFT ] += M_SQRT1_2; + matrix[FRONT_CENTER][TOP_BACK_RIGHT] += M_SQRT1_2; + } else + av_assert0(0); + } + + /* BS.2127-1 maps U+180 to rear outputs before front outputs. */ + if (unaccounted & AV_CH_TOP_BACK_CENTER) { + if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_TOP_BACK_LEFT) >= 0) { + matrix[TOP_BACK_LEFT ][TOP_BACK_CENTER] += M_SQRT1_2; + matrix[TOP_BACK_RIGHT][TOP_BACK_CENTER] += M_SQRT1_2; + } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_BACK_LEFT) >= 0) { + matrix[BACK_LEFT ][TOP_BACK_CENTER] += M_SQRT1_2; + matrix[BACK_RIGHT][TOP_BACK_CENTER] += M_SQRT1_2; + } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_SIDE_LEFT) >= 0) { + matrix[SIDE_LEFT ][TOP_BACK_CENTER] += M_SQRT1_2; + matrix[SIDE_RIGHT][TOP_BACK_CENTER] += M_SQRT1_2; + } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_FRONT_LEFT) >= 0) { + matrix[FRONT_LEFT ][TOP_BACK_CENTER] += 0.5; + matrix[FRONT_RIGHT][TOP_BACK_CENTER] += 0.5; + } else if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_FRONT_CENTER) >= 0) + matrix[FRONT_CENTER][TOP_BACK_CENTER] += 0.5; + else + av_assert0(0); + } + /* mix LFE into front left/right or center */ if (unaccounted & AV_CH_LOW_FREQUENCY) { if (av_channel_layout_index_from_channel(out_ch_layout, AV_CHAN_FRONT_CENTER) >= 0) { diff --git a/libswresample/tests/.gitignore b/libswresample/tests/.gitignore index 2dc986bd0e..e156fc35de 100644 --- a/libswresample/tests/.gitignore +++ b/libswresample/tests/.gitignore @@ -1 +1,2 @@ +/rematrix /swresample diff --git a/libswresample/tests/rematrix.c b/libswresample/tests/rematrix.c new file mode 100644 index 0000000000..9ce720d39d --- /dev/null +++ b/libswresample/tests/rematrix.c @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2026 Ayoub Nabil Boubagrat + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include <limits.h> +#include <stdio.h> + +#include "libavutil/channel_layout.h" +#include "libavutil/mathematics.h" +#include "libswresample/swresample.h" + +/* swr_build_matrix2() accesses an internal SWR_CH_MAX by SWR_CH_MAX matrix. */ +#define MATRIX_STRIDE 64 + +static int check_coefficient(const AVChannelLayout *in_layout, + const AVChannelLayout *out_layout, + enum AVChannel in_channel, + enum AVChannel out_channel, double expected) +{ + double matrix[MATRIX_STRIDE * MATRIX_STRIDE] = { 0 }; + char in_name[16], out_name[16]; + int in, out, ret; + + av_channel_name(in_name, sizeof(in_name), in_channel); + av_channel_name(out_name, sizeof(out_name), out_channel); + + if (in_layout->nb_channels > MATRIX_STRIDE || + out_layout->nb_channels > MATRIX_STRIDE) { + fprintf(stderr, "channel layout exceeds matrix capacity\n"); + return 1; + } + + in = av_channel_layout_index_from_channel(in_layout, in_channel); + out = av_channel_layout_index_from_channel(out_layout, out_channel); + if (in < 0) { + fprintf(stderr, "input channel %s is not in the input layout\n", in_name); + return 1; + } + if (out < 0) { + fprintf(stderr, "output channel %s is not in the output layout\n", out_name); + return 1; + } + + /* Disable normalization so the raw downmix gains can be checked. */ + ret = swr_build_matrix2(in_layout, out_layout, M_SQRT1_2, M_SQRT1_2, + 0.0, INT_MAX, 1.0, matrix, MATRIX_STRIDE, + AV_MATRIX_ENCODING_NONE, NULL); + if (ret < 0) { + fprintf(stderr, "swr_build_matrix2 failed with error %d\n", ret); + return 1; + } + + if (fabs(matrix[out * MATRIX_STRIDE + in] - expected) > 1e-12) { + fprintf(stderr, "%s -> %s: expected %.12f, got %.12f\n", + in_name, out_name, expected, + matrix[out * MATRIX_STRIDE + in]); + return 1; + } + + return 0; +} + +int main(void) +{ + const AVChannelLayout mono = AV_CHANNEL_LAYOUT_MONO; + const AVChannelLayout stereo = AV_CHANNEL_LAYOUT_STEREO; + const AVChannelLayout surround = AV_CHANNEL_LAYOUT_5POINT1; + const AVChannelLayout surround_back = AV_CHANNEL_LAYOUT_5POINT1_BACK; + const AVChannelLayout surround_2 = AV_CHANNEL_LAYOUT_5POINT1POINT2; + const AVChannelLayout surround_4 = AV_CHANNEL_LAYOUT_5POINT1POINT4_BACK; + const AVChannelLayout surround_tbc = AV_CHANNEL_LAYOUT_7POINT2POINT3; + int ret = 0; + + ret |= check_coefficient(&surround_2, &stereo, + AV_CHAN_TOP_FRONT_LEFT, AV_CHAN_FRONT_LEFT, 1.0); + ret |= check_coefficient(&surround_2, &stereo, + AV_CHAN_TOP_FRONT_RIGHT, AV_CHAN_FRONT_RIGHT, 1.0); + ret |= check_coefficient(&surround_4, &surround_2, + AV_CHAN_TOP_BACK_LEFT, AV_CHAN_TOP_FRONT_LEFT, + M_SQRT1_2); + ret |= check_coefficient(&surround_4, &surround_2, + AV_CHAN_TOP_BACK_RIGHT, AV_CHAN_TOP_FRONT_RIGHT, + M_SQRT1_2); + ret |= check_coefficient(&surround_4, &surround_tbc, + AV_CHAN_TOP_BACK_LEFT, AV_CHAN_TOP_BACK_CENTER, + M_SQRT1_2); + ret |= check_coefficient(&surround_4, &surround_tbc, + AV_CHAN_TOP_BACK_RIGHT, AV_CHAN_TOP_BACK_CENTER, + M_SQRT1_2); + ret |= check_coefficient(&surround_4, &surround, + AV_CHAN_TOP_BACK_LEFT, AV_CHAN_SIDE_LEFT, 1.0); + ret |= check_coefficient(&surround_4, &surround, + AV_CHAN_TOP_BACK_RIGHT, AV_CHAN_SIDE_RIGHT, 1.0); + ret |= check_coefficient(&surround_4, &surround_back, + AV_CHAN_TOP_BACK_LEFT, AV_CHAN_BACK_LEFT, 1.0); + ret |= check_coefficient(&surround_4, &surround_back, + AV_CHAN_TOP_BACK_RIGHT, AV_CHAN_BACK_RIGHT, 1.0); + ret |= check_coefficient(&surround_4, &stereo, + AV_CHAN_TOP_BACK_LEFT, AV_CHAN_FRONT_LEFT, + M_SQRT1_2); + ret |= check_coefficient(&surround_4, &stereo, + AV_CHAN_TOP_BACK_RIGHT, AV_CHAN_FRONT_RIGHT, + M_SQRT1_2); + ret |= check_coefficient(&surround_4, &mono, + AV_CHAN_TOP_BACK_LEFT, AV_CHAN_FRONT_CENTER, + M_SQRT1_2); + ret |= check_coefficient(&surround_4, &mono, + AV_CHAN_TOP_BACK_RIGHT, AV_CHAN_FRONT_CENTER, + M_SQRT1_2); + ret |= check_coefficient(&surround_tbc, &surround, + AV_CHAN_TOP_BACK_CENTER, AV_CHAN_SIDE_LEFT, + M_SQRT1_2); + ret |= check_coefficient(&surround_tbc, &surround, + AV_CHAN_TOP_BACK_CENTER, AV_CHAN_SIDE_RIGHT, + M_SQRT1_2); + ret |= check_coefficient(&surround_tbc, &surround_4, + AV_CHAN_TOP_BACK_CENTER, AV_CHAN_TOP_BACK_LEFT, + M_SQRT1_2); + ret |= check_coefficient(&surround_tbc, &surround_4, + AV_CHAN_TOP_BACK_CENTER, AV_CHAN_TOP_BACK_RIGHT, + M_SQRT1_2); + ret |= check_coefficient(&surround_tbc, &surround_back, + AV_CHAN_TOP_BACK_CENTER, AV_CHAN_BACK_LEFT, + M_SQRT1_2); + ret |= check_coefficient(&surround_tbc, &surround_back, + AV_CHAN_TOP_BACK_CENTER, AV_CHAN_BACK_RIGHT, + M_SQRT1_2); + ret |= check_coefficient(&surround_tbc, &stereo, + AV_CHAN_TOP_BACK_CENTER, AV_CHAN_FRONT_LEFT, 0.5); + ret |= check_coefficient(&surround_tbc, &stereo, + AV_CHAN_TOP_BACK_CENTER, AV_CHAN_FRONT_RIGHT, 0.5); + ret |= check_coefficient(&surround_tbc, &mono, + AV_CHAN_TOP_BACK_CENTER, AV_CHAN_FRONT_CENTER, 0.5); + + return ret; +} diff --git a/tests/fate/libswresample.mak b/tests/fate/libswresample.mak index 25073923b5..e7608e9d56 100644 --- a/tests/fate/libswresample.mak +++ b/tests/fate/libswresample.mak @@ -1106,6 +1106,12 @@ fate-swr-custom-rematrix: REF = 2a14a44deb4ae26e3b474ddbfbc048f8 FATE_SWR += $(FATE_SWR_CUSTOM_REMATRIX-yes) +FATE_SWR_REMATRIX-$(CONFIG_SWRESAMPLE) += fate-swr-rematrix +fate-swr-rematrix: libswresample/tests/rematrix$(EXESUF) +fate-swr-rematrix: CMD = run libswresample/tests/rematrix$(EXESUF) + +FATE_SWR += $(FATE_SWR_REMATRIX-yes) + FATE_SWR_REALLOC-$(CONFIG_SWRESAMPLE) += fate-swr-resample-realloc fate-swr-resample-realloc: libswresample/tests/swresample_resample_realloc$(EXESUF) fate-swr-resample-realloc: CMD = run libswresample/tests/swresample_resample_realloc$(EXESUF) diff --git a/tests/ref/fate/swr-rematrix b/tests/ref/fate/swr-rematrix new file mode 100644 index 0000000000..e69de29bb2 -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]