From bedfb6eca402037f5cbb115fa767d106b8c14f1c Mon Sep 17 00:00:00 2001
From: Lynne <dev@lynne.ee>
Date: Sat, 8 Feb 2025 04:35:31 +0100
Subject: [PATCH] aacenc_tns: clamp filter direction energy measurement

The issue is that:

float en[2];
...
tns->n_filt[w] = is8 ? 1 : order != TNS_MAX_ORDER ? 2 : 3;
for (g = 0; g < tns->n_filt[w]; g++) {
    tns->direction[w][g] = slant != 2 ? slant : en[g] < en[!g];

When using the AAC Main profile, n_filt = 3, and slant is by
default 2 (normal long frames), g can go above 1.

en is the evolution of energy in the frequency domain for every
band at the given window. E.g. whether the energy is concentrated
at the top of each band, or the bottom.

For 2-pole filters, its straightforward.
For 3-pole filters, we need more than 2 measurements.

This commit properly implements support for 3-pole filters, by measuring
the band energy across three areas.

Do note that even xHE-AAC caps n_filt to 2, and only AAC Main allows
n_filt == 3.

Fixes https://trac.ffmpeg.org/ticket/11418

CVE: CVE-2025-1594

Upstream-Status: Backport [https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/bedfb6eca402037f5cbb115fa767d106b8c14f1c]

Signed-off-by: Archana Polampalli <archana.polampalli@windriver.com>
---
 libavcodec/aacenc_tns.c | 33 ++++++++++++++++++++++++---------
 1 file changed, 24 insertions(+), 9 deletions(-)

diff --git a/libavcodec/aacenc_tns.c b/libavcodec/aacenc_tns.c
index 8dc6dfc..9ea3506 100644
--- a/libavcodec/aacenc_tns.c
+++ b/libavcodec/aacenc_tns.c
@@ -172,6 +172,7 @@ void ff_aac_search_for_tns(AACEncContext *s, SingleChannelElement *sce)
                       sce->ics.window_sequence[0] == LONG_START_SEQUENCE ? 0 : 2;
     const int sfb_len = sfb_end - sfb_start;
     const int coef_len = sce->ics.swb_offset[sfb_end] - sce->ics.swb_offset[sfb_start];
+    const int n_filt = is8 ? 1 : order != TNS_MAX_ORDER ? 2 : 3;
 
     if (coef_len <= 0 || sfb_len <= 0) {
         sce->tns.present = 0;
@@ -179,16 +180,30 @@ void ff_aac_search_for_tns(AACEncContext *s, SingleChannelElement *sce)
     }
 
     for (w = 0; w < sce->ics.num_windows; w++) {
-        float en[2] = {0.0f, 0.0f};
+	float en[4] = {0.0f, 0.0f, 0.0f, 0.0f};
         int oc_start = 0, os_start = 0;
         int coef_start = sce->ics.swb_offset[sfb_start];
 
-        for (g = sfb_start; g < sce->ics.num_swb && g <= sfb_end; g++) {
-            FFPsyBand *band = &s->psy.ch[s->cur_channel].psy_bands[w*16+g];
-            if (g > sfb_start + (sfb_len/2))
-                en[1] += band->energy;
-            else
-                en[0] += band->energy;
+	if (n_filt == 2) {
+            for (g = sfb_start; g < sce->ics.num_swb && g <= sfb_end; g++) {
+                FFPsyBand *band = &s->psy.ch[s->cur_channel].psy_bands[w*16+g];
+                    if (g > sfb_start + (sfb_len/2))
+                        en[1] += band->energy; /* End */
+                    else
+                        en[0] += band->energy; /* Start */
+            }
+            en[2] = en[0];
+        } else {
+            for (g = sfb_start; g < sce->ics.num_swb && g <= sfb_end; g++) {
+                FFPsyBand *band = &s->psy.ch[s->cur_channel].psy_bands[w*16+g];
+                    if (g > sfb_start + (sfb_len/2) + (sfb_len/4))
+                        en[2] += band->energy; /* End */
+                    else if (g > sfb_start + (sfb_len/2) - (sfb_len/4))
+                        en[1] += band->energy; /* Middle */
+                    else
+                        en[0] += band->energy; /* Start */
+            }
+            en[3] = en[0];
         }
 
         /* LPC */
@@ -198,9 +213,9 @@ void ff_aac_search_for_tns(AACEncContext *s, SingleChannelElement *sce)
         if (!order || !isfinite(gain) || gain < TNS_GAIN_THRESHOLD_LOW || gain > TNS_GAIN_THRESHOLD_HIGH)
             continue;
 
-        tns->n_filt[w] = is8 ? 1 : order != TNS_MAX_ORDER ? 2 : 3;
+	tns->n_filt[w] = n_filt;
         for (g = 0; g < tns->n_filt[w]; g++) {
-            tns->direction[w][g] = slant != 2 ? slant : en[g] < en[!g];
+	    tns->direction[w][g] = slant != 2 ? slant : en[g] < en[g + 1];
             tns->order[w][g] = g < tns->n_filt[w] ? order/tns->n_filt[w] : order - oc_start;
             tns->length[w][g] = g < tns->n_filt[w] ? sfb_len/tns->n_filt[w] : sfb_len - os_start;
             quantize_coefs(&coefs[oc_start], tns->coef_idx[w][g], tns->coef[w][g],
-- 
2.40.0

