From 40a355a42d4a9444dc753c04c6608dade2f06a23 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 19 Apr 2024 03:13:27 +0200 Subject: Adding upstream version 125.0.1. Signed-off-by: Daniel Baumann --- media/libopus/celt/celt_decoder.c | 360 ++++++++++++++++++++++++++++++-------- 1 file changed, 288 insertions(+), 72 deletions(-) (limited to 'media/libopus/celt/celt_decoder.c') diff --git a/media/libopus/celt/celt_decoder.c b/media/libopus/celt/celt_decoder.c index 883dae15d2..743c2031bc 100644 --- a/media/libopus/celt/celt_decoder.c +++ b/media/libopus/celt/celt_decoder.c @@ -51,6 +51,11 @@ #include "celt_lpc.h" #include "vq.h" +#ifdef ENABLE_DEEP_PLC +#include "lpcnet.h" +#include "lpcnet_private.h" +#endif + /* The maximum pitch lag to allow in the pitch-based PLC. It's possible to save CPU time in the PLC pitch search by making this smaller than MAX_PERIOD. The current value corresponds to a pitch of 66.67 Hz. */ @@ -59,9 +64,6 @@ pitch of 480 Hz. */ #define PLC_PITCH_LAG_MIN (100) -#if defined(SMALL_FOOTPRINT) && defined(FIXED_POINT) -#define NORM_ALIASING_HACK -#endif /**********************************************************************/ /* */ /* DECODER */ @@ -69,6 +71,9 @@ /**********************************************************************/ #define DECODE_BUFFER_SIZE 2048 +#define PLC_UPDATE_FRAMES 4 +#define PLC_UPDATE_SAMPLES (PLC_UPDATE_FRAMES*FRAME_SIZE) + /** Decoder state @brief Decoder state */ @@ -82,6 +87,7 @@ struct OpusCustomDecoder { int start, end; int signalling; int disable_inv; + int complexity; int arch; /* Everything beyond this point gets cleared on a reset */ @@ -98,11 +104,18 @@ struct OpusCustomDecoder { opus_val16 postfilter_gain_old; int postfilter_tapset; int postfilter_tapset_old; + int prefilter_and_fold; celt_sig preemph_memD[2]; +#ifdef ENABLE_DEEP_PLC + opus_int16 plc_pcm[PLC_UPDATE_SAMPLES]; + int plc_fill; + float plc_preemphasis_mem; +#endif + celt_sig _decode_mem[1]; /* Size = channels*(DECODE_BUFFER_SIZE+mode->overlap) */ - /* opus_val16 lpc[], Size = channels*LPC_ORDER */ + /* opus_val16 lpc[], Size = channels*CELT_LPC_ORDER */ /* opus_val16 oldEBands[], Size = 2*mode->nbEBands */ /* opus_val16 oldLogE[], Size = 2*mode->nbEBands */ /* opus_val16 oldLogE2[], Size = 2*mode->nbEBands */ @@ -157,7 +170,7 @@ OPUS_CUSTOM_NOSTATIC int opus_custom_decoder_get_size(const CELTMode *mode, int { int size = sizeof(struct CELTDecoder) + (channels*(DECODE_BUFFER_SIZE+mode->overlap)-1)*sizeof(celt_sig) - + channels*LPC_ORDER*sizeof(opus_val16) + + channels*CELT_LPC_ORDER*sizeof(opus_val16) + 4*2*mode->nbEBands*sizeof(opus_val16); return size; } @@ -499,7 +512,100 @@ static int celt_plc_pitch_search(celt_sig *decode_mem[2], int C, int arch) return pitch_index; } -static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) +static void prefilter_and_fold(CELTDecoder * OPUS_RESTRICT st, int N) +{ + int c; + int CC; + int i; + int overlap; + celt_sig *decode_mem[2]; + const OpusCustomMode *mode; + VARDECL(opus_val32, etmp); + mode = st->mode; + overlap = st->overlap; + CC = st->channels; + ALLOC(etmp, overlap, opus_val32); + c=0; do { + decode_mem[c] = st->_decode_mem + c*(DECODE_BUFFER_SIZE+overlap); + } while (++cpostfilter_period_old, st->postfilter_period, overlap, + -st->postfilter_gain_old, -st->postfilter_gain, + st->postfilter_tapset_old, st->postfilter_tapset, NULL, 0, st->arch); + + /* Simulate TDAC on the concealed audio so that it blends with the + MDCT of the next frame. */ + for (i=0;iwindow[i], etmp[overlap-1-i]) + + MULT16_32_Q15(mode->window[overlap-i-1], etmp[i]); + } + } while (++cfec_read_pos; + tmp_fec_skip = lpcnet->fec_skip; + for (i=0;ifec_read_pos = tmp_read_post; + lpcnet->fec_skip = tmp_fec_skip; +} +#endif + +static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM +#ifdef ENABLE_DEEP_PLC + ,LPCNetPLCState *lpcnet +#endif + ) { int c; int i; @@ -527,22 +633,22 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) out_syn[c] = decode_mem[c]+DECODE_BUFFER_SIZE-N; } while (++c_decode_mem+(DECODE_BUFFER_SIZE+overlap)*C); - oldBandE = lpc+C*LPC_ORDER; + oldBandE = lpc+C*CELT_LPC_ORDER; oldLogE = oldBandE + 2*nbEBands; oldLogE2 = oldLogE + 2*nbEBands; backgroundLogE = oldLogE2 + 2*nbEBands; loss_duration = st->loss_duration; start = st->start; +#ifdef ENABLE_DEEP_PLC + noise_based = start != 0 || (lpcnet->fec_fill_pos == 0 && (st->skip_plc || loss_duration >= 80)); +#else noise_based = loss_duration >= 40 || start != 0 || st->skip_plc; +#endif if (noise_based) { /* Noise-based PLC/CNG */ -#ifdef NORM_ALIASING_HACK - celt_norm *X; -#else VARDECL(celt_norm, X); -#endif opus_uint32 seed; int end; int effEnd; @@ -550,18 +656,16 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) end = st->end; effEnd = IMAX(start, IMIN(end, mode->effEBands)); -#ifdef NORM_ALIASING_HACK - /* This is an ugly hack that breaks aliasing rules and would be easily broken, - but it saves almost 4kB of stack. */ - X = (celt_norm*)(out_syn[C-1]+overlap/2); -#else ALLOC(X, C*N, celt_norm); /**< Interleaved normalised MDCTs */ -#endif c=0; do { OPUS_MOVE(decode_mem[c], decode_mem[c]+N, - DECODE_BUFFER_SIZE-N+(overlap>>1)); + DECODE_BUFFER_SIZE-N+overlap); } while (++cprefilter_and_fold) { + prefilter_and_fold(st, N); + } + /* Energy decay */ decay = loss_duration==0 ? QCONST16(1.5f, DB_SHIFT) : QCONST16(.5f, DB_SHIFT); c=0; do @@ -590,6 +694,9 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) st->rng = seed; celt_synthesis(mode, X, out_syn, oldBandE, start, effEnd, C, C, 0, LM, st->downsample, 0, st->arch); + st->prefilter_and_fold = 0; + /* Skip regular PLC until we get two consecutive packets. */ + st->skip_plc = 1; } else { int exc_length; /* Pitch-based PLC */ @@ -597,12 +704,14 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) opus_val16 *exc; opus_val16 fade = Q15ONE; int pitch_index; - VARDECL(opus_val32, etmp); VARDECL(opus_val16, _exc); VARDECL(opus_val16, fir_tmp); if (loss_duration == 0) { +#ifdef ENABLE_DEEP_PLC + if (lpcnet->loaded) update_plc_state(lpcnet, decode_mem, &st->plc_preemphasis_mem, C); +#endif st->last_pitch_index = pitch_index = celt_plc_pitch_search(decode_mem, C, st->arch); } else { pitch_index = st->last_pitch_index; @@ -613,10 +722,9 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) decaying signal, but we can't get more than MAX_PERIOD. */ exc_length = IMIN(2*pitch_index, MAX_PERIOD); - ALLOC(etmp, overlap, opus_val32); - ALLOC(_exc, MAX_PERIOD+LPC_ORDER, opus_val16); + ALLOC(_exc, MAX_PERIOD+CELT_LPC_ORDER, opus_val16); ALLOC(fir_tmp, exc_length, opus_val16); - exc = _exc+LPC_ORDER; + exc = _exc+CELT_LPC_ORDER; window = mode->window; c=0; do { opus_val16 decay; @@ -628,16 +736,16 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) int j; buf = decode_mem[c]; - for (i=0;iarch); + CELT_LPC_ORDER, MAX_PERIOD, st->arch); /* Add a noise floor of -40 dB. */ #ifdef FIXED_POINT ac[0] += SHR32(ac[0],13); @@ -645,7 +753,7 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) ac[0] *= 1.0001f; #endif /* Use lag windowing to stabilize the Levinson-Durbin recursion. */ - for (i=1;i<=LPC_ORDER;i++) + for (i=1;i<=CELT_LPC_ORDER;i++) { /*ac[i] *= exp(-.5*(2*M_PI*.002*i)*(2*M_PI*.002*i));*/ #ifdef FIXED_POINT @@ -654,7 +762,7 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) ac[i] -= ac[i]*(0.008f*0.008f)*i*i; #endif } - _celt_lpc(lpc+c*LPC_ORDER, ac, LPC_ORDER); + _celt_lpc(lpc+c*CELT_LPC_ORDER, ac, CELT_LPC_ORDER); #ifdef FIXED_POINT /* For fixed-point, apply bandwidth expansion until we can guarantee that no overflow can happen in the IIR filter. This means: @@ -662,13 +770,13 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) while (1) { opus_val16 tmp=Q15ONE; opus_val32 sum=QCONST16(1., SIG_SHIFT); - for (i=0;iarch); + celt_fir(exc+MAX_PERIOD-exc_length, lpc+c*CELT_LPC_ORDER, + fir_tmp, exc_length, CELT_LPC_ORDER, st->arch); OPUS_COPY(exc+MAX_PERIOD-exc_length, fir_tmp, exc_length); } @@ -737,15 +845,15 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) S1 += SHR32(MULT16_16(tmp, tmp), 10); } { - opus_val16 lpc_mem[LPC_ORDER]; + opus_val16 lpc_mem[CELT_LPC_ORDER]; /* Copy the last decoded samples (prior to the overlap region) to synthesis filter memory so we can have a continuous signal. */ - for (i=0;iarch); #ifdef FIXED_POINT for (i=0; i < extrapolation_len; i++) @@ -792,23 +900,65 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) } } - /* Apply the pre-filter to the MDCT overlap for the next frame because - the post-filter will be re-applied in the decoder after the MDCT - overlap. */ - comb_filter(etmp, buf+DECODE_BUFFER_SIZE, - st->postfilter_period, st->postfilter_period, overlap, - -st->postfilter_gain, -st->postfilter_gain, - st->postfilter_tapset, st->postfilter_tapset, NULL, 0, st->arch); - - /* Simulate TDAC on the concealed audio so that it blends with the - MDCT of the next frame. */ - for (i=0;iloaded && (st->complexity >= 5 || lpcnet->fec_fill_pos > 0)) { + float overlap_mem; + int samples_needed16k; + celt_sig *buf; + VARDECL(float, buf_copy); + buf = decode_mem[0]; + ALLOC(buf_copy, C*overlap, float); + c=0; do { + OPUS_COPY(buf_copy+c*overlap, &decode_mem[c][DECODE_BUFFER_SIZE-N], overlap); + } while (++cplc_fill = 0; + } + while (st->plc_fill < samples_needed16k) { + lpcnet_plc_conceal(lpcnet, &st->plc_pcm[st->plc_fill]); + st->plc_fill += FRAME_SIZE; + } + /* Resample to 48 kHz. */ + for (i=0;i<(N+overlap)/3;i++) { + int j; + float sum; + for (sum=0, j=0;j<17;j++) sum += 3*st->plc_pcm[i+j]*sinc_filter[3*j]; + buf[DECODE_BUFFER_SIZE-N+3*i] = sum; + for (sum=0, j=0;j<16;j++) sum += 3*st->plc_pcm[i+j+1]*sinc_filter[3*j+2]; + buf[DECODE_BUFFER_SIZE-N+3*i+1] = sum; + for (sum=0, j=0;j<16;j++) sum += 3*st->plc_pcm[i+j+1]*sinc_filter[3*j+1]; + buf[DECODE_BUFFER_SIZE-N+3*i+2] = sum; + } + OPUS_MOVE(st->plc_pcm, &st->plc_pcm[N/3], st->plc_fill-N/3); + st->plc_fill -= N/3; + for (i=0;iplc_preemphasis_mem; + st->plc_preemphasis_mem = tmp; + } + overlap_mem = st->plc_preemphasis_mem; + for (i=0;iprefilter_and_fold = 1; } /* Saturate to soemthing large to avoid wrap-around. */ @@ -817,18 +967,18 @@ static void celt_decode_lost(CELTDecoder * OPUS_RESTRICT st, int N, int LM) RESTORE_STACK; } -int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st, const unsigned char *data, - int len, opus_val16 * OPUS_RESTRICT pcm, int frame_size, ec_dec *dec, int accum) +int celt_decode_with_ec_dred(CELTDecoder * OPUS_RESTRICT st, const unsigned char *data, + int len, opus_val16 * OPUS_RESTRICT pcm, int frame_size, ec_dec *dec, int accum +#ifdef ENABLE_DEEP_PLC + ,LPCNetPLCState *lpcnet +#endif + ) { int c, i, N; int spread_decision; opus_int32 bits; ec_dec _dec; -#ifdef NORM_ALIASING_HACK - celt_norm *X; -#else VARDECL(celt_norm, X); -#endif VARDECL(int, fine_quant); VARDECL(int, pulses); VARDECL(int, cap); @@ -881,7 +1031,7 @@ int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st, const unsigned char *dat frame_size *= st->downsample; lpc = (opus_val16*)(st->_decode_mem+(DECODE_BUFFER_SIZE+overlap)*CC); - oldBandE = lpc+CC*LPC_ORDER; + oldBandE = lpc+CC*CELT_LPC_ORDER; oldLogE = oldBandE + 2*nbEBands; oldLogE2 = oldLogE + 2*nbEBands; backgroundLogE = oldLogE2 + 2*nbEBands; @@ -935,15 +1085,25 @@ int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st, const unsigned char *dat if (data == NULL || len<=1) { - celt_decode_lost(st, N, LM); + celt_decode_lost(st, N, LM +#ifdef ENABLE_DEEP_PLC + , lpcnet +#endif + ); deemphasis(out_syn, pcm, N, CC, st->downsample, mode->preemph, st->preemph_memD, accum); RESTORE_STACK; return frame_size/st->downsample; } +#ifdef ENABLE_DEEP_PLC + else { + /* FIXME: This is a bit of a hack just to make sure opus_decode_native() knows we're no longer in PLC. */ + if (lpcnet) lpcnet->blend = 0; + } +#endif /* Check if there are at least two packets received consecutively before * turning on the pitch-based PLC */ - st->skip_plc = st->loss_duration != 0; + if (st->loss_duration == 0) st->skip_plc = 0; if (dec == NULL) { @@ -1006,6 +1166,36 @@ int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st, const unsigned char *dat /* Decode the global flags (first symbols in the stream) */ intra_ener = tell+3<=total_bits ? ec_dec_bit_logp(dec, 3) : 0; + /* If recovering from packet loss, make sure we make the energy prediction safe to reduce the + risk of getting loud artifacts. */ + if (!intra_ener && st->loss_duration != 0) { + c=0; do + { + opus_val16 safety = 0; + int missing = IMIN(10, st->loss_duration>>LM); + if (LM==0) safety = QCONST16(1.5f,DB_SHIFT); + else if (LM==1) safety = QCONST16(.5f,DB_SHIFT); + for (i=start;iprefilter_and_fold) { + prefilter_and_fold(st, N); + } celt_synthesis(mode, X, out_syn, oldBandE, start, effEnd, C, CC, isTransient, LM, st->downsample, silence, st->arch); @@ -1173,6 +1359,7 @@ int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st, const unsigned char *dat deemphasis(out_syn, pcm, N, CC, st->downsample, mode->preemph, st->preemph_memD, accum); st->loss_duration = 0; + st->prefilter_and_fold = 0; RESTORE_STACK; if (ec_tell(dec) > 8*len) return OPUS_INTERNAL_ERROR; @@ -1181,6 +1368,15 @@ int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st, const unsigned char *dat return frame_size/st->downsample; } +int celt_decode_with_ec(CELTDecoder * OPUS_RESTRICT st, const unsigned char *data, + int len, opus_val16 * OPUS_RESTRICT pcm, int frame_size, ec_dec *dec, int accum) +{ + return celt_decode_with_ec_dred(st, data, len, pcm, frame_size, dec, accum +#ifdef ENABLE_DEEP_PLC + , NULL +#endif + ); +} #ifdef CUSTOM_MODES @@ -1254,6 +1450,26 @@ int opus_custom_decoder_ctl(CELTDecoder * OPUS_RESTRICT st, int request, ...) va_start(ap, request); switch (request) { + case OPUS_SET_COMPLEXITY_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>10) + { + goto bad_arg; + } + st->complexity = value; + } + break; + case OPUS_GET_COMPLEXITY_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->complexity; + } + break; case CELT_SET_START_BAND_REQUEST: { opus_int32 value = va_arg(ap, opus_int32); @@ -1300,7 +1516,7 @@ int opus_custom_decoder_ctl(CELTDecoder * OPUS_RESTRICT st, int request, ...) int i; opus_val16 *lpc, *oldBandE, *oldLogE, *oldLogE2; lpc = (opus_val16*)(st->_decode_mem+(DECODE_BUFFER_SIZE+st->overlap)*st->channels); - oldBandE = lpc+st->channels*LPC_ORDER; + oldBandE = lpc+st->channels*CELT_LPC_ORDER; oldLogE = oldBandE + 2*st->mode->nbEBands; oldLogE2 = oldLogE + 2*st->mode->nbEBands; OPUS_CLEAR((char*)&st->DECODER_RESET_START, -- cgit v1.2.3