From 194bdf50697689768335096b6c01c5b010f023ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Volker=20R=C3=BCmelin?= Date: Sun, 8 Mar 2020 20:33:20 +0100 Subject: audio: fix saturation nonlinearity in clip_* functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current positive limit for the saturation nonlinearity is only correct if the type of the result has 8 bits or less. Signed-off-by: Volker Rümelin Message-id: 20200308193321.20668-5-vr_qemu@t-online.de Signed-off-by: Gerd Hoffmann --- audio/mixeng_template.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'audio') diff --git a/audio/mixeng_template.h b/audio/mixeng_template.h index fc8e1d4d9e..bc8509e423 100644 --- a/audio/mixeng_template.h +++ b/audio/mixeng_template.h @@ -83,10 +83,9 @@ static inline int64_t glue (conv_, ET) (IN_T v) static inline IN_T glue (clip_, ET) (int64_t v) { - if (v >= 0x7f000000) { + if (v >= 0x7fffffffLL) { return IN_MAX; - } - else if (v < -2147483648LL) { + } else if (v < -2147483648LL) { return IN_MIN; } -- cgit v1.2.3