diff options
Diffstat (limited to 'media/libopus/src/opus_decoder.c')
-rw-r--r-- | media/libopus/src/opus_decoder.c | 495 |
1 files changed, 481 insertions, 14 deletions
diff --git a/media/libopus/src/opus_decoder.c b/media/libopus/src/opus_decoder.c index 6520e748ea..b57c809434 100644 --- a/media/libopus/src/opus_decoder.c +++ b/media/libopus/src/opus_decoder.c @@ -52,6 +52,15 @@ #include "mathops.h" #include "cpu_support.h" +#ifdef ENABLE_DEEP_PLC +#include "dred_rdovae_dec_data.h" +#include "dred_rdovae_dec.h" +#endif + +#ifdef ENABLE_OSCE +#include "osce.h" +#endif + struct OpusDecoder { int celt_dec_offset; int silk_dec_offset; @@ -59,7 +68,11 @@ struct OpusDecoder { opus_int32 Fs; /** Sampling rate (at the API level) */ silk_DecControlStruct DecControl; int decode_gain; + int complexity; int arch; +#ifdef ENABLE_DEEP_PLC + LPCNetPLCState lpcnet; +#endif /* Everything beyond this point gets cleared on a reset */ #define OPUS_DECODER_RESET_START stream_channels @@ -135,6 +148,7 @@ int opus_decoder_init(OpusDecoder *st, opus_int32 Fs, int channels) silk_dec = (char*)st+st->silk_dec_offset; celt_dec = (CELTDecoder*)((char*)st+st->celt_dec_offset); st->stream_channels = st->channels = channels; + st->complexity = 0; st->Fs = Fs; st->DecControl.API_sampleRate = st->Fs; @@ -152,6 +166,9 @@ int opus_decoder_init(OpusDecoder *st, opus_int32 Fs, int channels) st->prev_mode = 0; st->frame_size = Fs/400; +#ifdef ENABLE_DEEP_PLC + lpcnet_plc_init( &st->lpcnet); +#endif st->arch = opus_select_arch(); return OPUS_OK; } @@ -370,7 +387,7 @@ static int opus_decode_frame(OpusDecoder *st, const unsigned char *data, pcm_ptr = pcm_silk; if (st->prev_mode==MODE_CELT_ONLY) - silk_InitDecoder( silk_dec ); + silk_ResetDecoder( silk_dec ); /* The SILK PLC cannot produce frames of less than 10 ms */ st->DecControl.payloadSize_ms = IMAX(10, 1000 * audiosize / st->Fs); @@ -394,14 +411,28 @@ static int opus_decode_frame(OpusDecoder *st, const unsigned char *data, st->DecControl.internalSampleRate = 16000; } } + st->DecControl.enable_deep_plc = st->complexity >= 5; +#ifdef ENABLE_OSCE + st->DecControl.osce_method = OSCE_METHOD_NONE; +#ifndef DISABLE_LACE + if (st->complexity >= 6) {st->DecControl.osce_method = OSCE_METHOD_LACE;} +#endif +#ifndef DISABLE_NOLACE + if (st->complexity >= 7) {st->DecControl.osce_method = OSCE_METHOD_NOLACE;} +#endif +#endif - lost_flag = data == NULL ? 1 : 2 * decode_fec; + lost_flag = data == NULL ? 1 : 2 * !!decode_fec; decoded_samples = 0; do { /* Call SILK decoder */ int first_frame = decoded_samples == 0; silk_ret = silk_Decode( silk_dec, &st->DecControl, - lost_flag, first_frame, &dec, pcm_ptr, &silk_frame_size, st->arch ); + lost_flag, first_frame, &dec, pcm_ptr, &silk_frame_size, +#ifdef ENABLE_DEEP_PLC + &st->lpcnet, +#endif + st->arch ); if( silk_ret ) { if (lost_flag) { /* PLC failure should not be fatal */ @@ -521,8 +552,12 @@ static int opus_decode_frame(OpusDecoder *st, const unsigned char *data, if (mode != st->prev_mode && st->prev_mode > 0 && !st->prev_redundancy) MUST_SUCCEED(celt_decoder_ctl(celt_dec, OPUS_RESET_STATE)); /* Decode CELT */ - celt_ret = celt_decode_with_ec(celt_dec, decode_fec ? NULL : data, - len, pcm, celt_frame_size, &dec, celt_accum); + celt_ret = celt_decode_with_ec_dred(celt_dec, decode_fec ? NULL : data, + len, pcm, celt_frame_size, &dec, celt_accum +#ifdef ENABLE_DEEP_PLC + , &st->lpcnet +#endif + ); } else { unsigned char silence[2] = {0xFF, 0xFF}; if (!celt_accum) @@ -634,7 +669,7 @@ static int opus_decode_frame(OpusDecoder *st, const unsigned char *data, int opus_decode_native(OpusDecoder *st, const unsigned char *data, opus_int32 len, opus_val16 *pcm, int frame_size, int decode_fec, - int self_delimited, opus_int32 *packet_offset, int soft_clip) + int self_delimited, opus_int32 *packet_offset, int soft_clip, const OpusDRED *dred, opus_int32 dred_offset) { int i, nb_samples; int count, offset; @@ -648,6 +683,35 @@ int opus_decode_native(OpusDecoder *st, const unsigned char *data, /* For FEC/PLC, frame_size has to be to have a multiple of 2.5 ms */ if ((decode_fec || len==0 || data==NULL) && frame_size%(st->Fs/400)!=0) return OPUS_BAD_ARG; +#ifdef ENABLE_DRED + if (dred != NULL && dred->process_stage == 2) { + int F10; + int features_per_frame; + int needed_feature_frames; + int init_frames; + lpcnet_plc_fec_clear(&st->lpcnet); + F10 = st->Fs/100; + /* if blend==0, the last PLC call was "update" and we need to feed two extra 10-ms frames. */ + init_frames = (st->lpcnet.blend == 0) ? 2 : 0; + features_per_frame = IMAX(1, frame_size/F10); + needed_feature_frames = init_frames + features_per_frame; + lpcnet_plc_fec_clear(&st->lpcnet); + for (i=0;i<needed_feature_frames;i++) { + int feature_offset; + /* We floor instead of rounding because 5-ms overlap compensates for the missing 0.5 rounding offset. */ + feature_offset = init_frames - i - 2 + (int)floor(((float)dred_offset + dred->dred_offset*F10/4)/F10); + if (feature_offset <= 4*dred->nb_latents-1 && feature_offset >= 0) { + lpcnet_plc_fec_add(&st->lpcnet, dred->fec_features+feature_offset*DRED_NUM_FEATURES); + } else { + if (feature_offset >= 0) lpcnet_plc_fec_add(&st->lpcnet, NULL); + } + + } + } +#else + (void)dred; + (void)dred_offset; +#endif if (len==0 || data==NULL) { int pcm_count=0; @@ -672,7 +736,7 @@ int opus_decode_native(OpusDecoder *st, const unsigned char *data, packet_stream_channels = opus_packet_get_nb_channels(data); count = opus_packet_parse_impl(data, len, self_delimited, &toc, NULL, - size, &offset, packet_offset); + size, &offset, packet_offset, NULL, NULL); if (count<0) return count; @@ -684,12 +748,12 @@ int opus_decode_native(OpusDecoder *st, const unsigned char *data, int ret; /* If no FEC can be present, run the PLC (recursive call) */ if (frame_size < packet_frame_size || packet_mode == MODE_CELT_ONLY || st->mode == MODE_CELT_ONLY) - return opus_decode_native(st, NULL, 0, pcm, frame_size, 0, 0, NULL, soft_clip); + return opus_decode_native(st, NULL, 0, pcm, frame_size, 0, 0, NULL, soft_clip, NULL, 0); /* Otherwise, run the PLC on everything except the size for which we might have FEC */ duration_copy = st->last_packet_duration; if (frame_size-packet_frame_size!=0) { - ret = opus_decode_native(st, NULL, 0, pcm, frame_size-packet_frame_size, 0, 0, NULL, soft_clip); + ret = opus_decode_native(st, NULL, 0, pcm, frame_size-packet_frame_size, 0, 0, NULL, soft_clip, NULL, 0); if (ret<0) { st->last_packet_duration = duration_copy; @@ -753,7 +817,7 @@ int opus_decode(OpusDecoder *st, const unsigned char *data, { if(frame_size<=0) return OPUS_BAD_ARG; - return opus_decode_native(st, data, len, pcm, frame_size, decode_fec, 0, NULL, 0); + return opus_decode_native(st, data, len, pcm, frame_size, decode_fec, 0, NULL, 0, NULL, 0); } #ifndef DISABLE_FLOAT_API @@ -781,7 +845,7 @@ int opus_decode_float(OpusDecoder *st, const unsigned char *data, celt_assert(st->channels == 1 || st->channels == 2); ALLOC(out, frame_size*st->channels, opus_int16); - ret = opus_decode_native(st, data, len, out, frame_size, decode_fec, 0, NULL, 0); + ret = opus_decode_native(st, data, len, out, frame_size, decode_fec, 0, NULL, 0, NULL, 0); if (ret > 0) { for (i=0;i<ret*st->channels;i++) @@ -819,7 +883,7 @@ int opus_decode(OpusDecoder *st, const unsigned char *data, celt_assert(st->channels == 1 || st->channels == 2); ALLOC(out, frame_size*st->channels, float); - ret = opus_decode_native(st, data, len, out, frame_size, decode_fec, 0, NULL, 1); + ret = opus_decode_native(st, data, len, out, frame_size, decode_fec, 0, NULL, 1, NULL, 0); if (ret > 0) { for (i=0;i<ret*st->channels;i++) @@ -834,7 +898,7 @@ int opus_decode_float(OpusDecoder *st, const unsigned char *data, { if(frame_size<=0) return OPUS_BAD_ARG; - return opus_decode_native(st, data, len, pcm, frame_size, decode_fec, 0, NULL, 0); + return opus_decode_native(st, data, len, pcm, frame_size, decode_fec, 0, NULL, 0, NULL, 0); } #endif @@ -864,6 +928,27 @@ int opus_decoder_ctl(OpusDecoder *st, int request, ...) *value = st->bandwidth; } break; + case OPUS_SET_COMPLEXITY_REQUEST: + { + opus_int32 value = va_arg(ap, opus_int32); + if(value<0 || value>10) + { + goto bad_arg; + } + st->complexity = value; + celt_decoder_ctl(celt_dec, OPUS_SET_COMPLEXITY(value)); + } + break; + case OPUS_GET_COMPLEXITY_REQUEST: + { + opus_int32 *value = va_arg(ap, opus_int32*); + if (!value) + { + goto bad_arg; + } + *value = st->complexity; + } + break; case OPUS_GET_FINAL_RANGE_REQUEST: { opus_uint32 *value = va_arg(ap, opus_uint32*); @@ -881,9 +966,12 @@ int opus_decoder_ctl(OpusDecoder *st, int request, ...) ((char*)&st->OPUS_DECODER_RESET_START - (char*)st)); celt_decoder_ctl(celt_dec, OPUS_RESET_STATE); - silk_InitDecoder( silk_dec ); + silk_ResetDecoder( silk_dec ); st->stream_channels = st->channels; st->frame_size = st->Fs/400; +#ifdef ENABLE_DEEP_PLC + lpcnet_plc_reset( &st->lpcnet ); +#endif } break; case OPUS_GET_SAMPLE_RATE_REQUEST: @@ -959,6 +1047,20 @@ int opus_decoder_ctl(OpusDecoder *st, int request, ...) ret = celt_decoder_ctl(celt_dec, OPUS_GET_PHASE_INVERSION_DISABLED(value)); } break; +#ifdef USE_WEIGHTS_FILE + case OPUS_SET_DNN_BLOB_REQUEST: + { + const unsigned char *data = va_arg(ap, const unsigned char *); + opus_int32 len = va_arg(ap, opus_int32); + if(len<0 || data == NULL) + { + goto bad_arg; + } + ret = lpcnet_plc_load_model(&st->lpcnet, data, len); + ret = silk_LoadOSCEModels(silk_dec, data, len) || ret; + } + break; +#endif default: /*fprintf(stderr, "unknown opus_decoder_ctl() request: %d", request);*/ ret = OPUS_UNIMPLEMENTED; @@ -1034,8 +1136,373 @@ int opus_packet_get_nb_samples(const unsigned char packet[], opus_int32 len, return samples; } +int opus_packet_has_lbrr(const unsigned char packet[], opus_int32 len) +{ + int ret; + const unsigned char *frames[48]; + opus_int16 size[48]; + int packet_mode, packet_frame_size, packet_stream_channels; + int nb_frames=1; + int lbrr; + + packet_mode = opus_packet_get_mode(packet); + if (packet_mode == MODE_CELT_ONLY) + return 0; + packet_frame_size = opus_packet_get_samples_per_frame(packet, 48000); + if (packet_frame_size > 960) + nb_frames = packet_frame_size/960; + packet_stream_channels = opus_packet_get_nb_channels(packet); + ret = opus_packet_parse(packet, len, NULL, frames, size, NULL); + if (ret <= 0) + return ret; + lbrr = (frames[0][0] >> (7-nb_frames)) & 0x1; + if (packet_stream_channels == 2) + lbrr = lbrr || ((frames[0][0] >> (6-2*nb_frames)) & 0x1); + return lbrr; +} + int opus_decoder_get_nb_samples(const OpusDecoder *dec, const unsigned char packet[], opus_int32 len) { return opus_packet_get_nb_samples(packet, len, dec->Fs); } + +struct OpusDREDDecoder { +#ifdef ENABLE_DRED + RDOVAEDec model; +#endif + int loaded; + int arch; + opus_uint32 magic; +}; + +#if defined(ENABLE_DRED) && (defined(ENABLE_HARDENING) || defined(ENABLE_ASSERTIONS)) +static void validate_dred_decoder(OpusDREDDecoder *st) +{ + celt_assert(st->magic == 0xD8EDDEC0); +#ifdef OPUS_ARCHMASK + celt_assert(st->arch >= 0); + celt_assert(st->arch <= OPUS_ARCHMASK); +#endif +} +#define VALIDATE_DRED_DECODER(st) validate_dred_decoder(st) +#else +#define VALIDATE_DRED_DECODER(st) +#endif + + +int opus_dred_decoder_get_size(void) +{ + return sizeof(OpusDREDDecoder); +} + +#ifdef ENABLE_DRED +int dred_decoder_load_model(OpusDREDDecoder *dec, const unsigned char *data, int len) +{ + WeightArray *list; + int ret; + parse_weights(&list, data, len); + ret = init_rdovaedec(&dec->model, list); + opus_free(list); + if (ret == 0) dec->loaded = 1; + return (ret == 0) ? OPUS_OK : OPUS_BAD_ARG; +} +#endif + +int opus_dred_decoder_init(OpusDREDDecoder *dec) +{ + int ret = 0; + dec->loaded = 0; +#if defined(ENABLE_DRED) && !defined(USE_WEIGHTS_FILE) + ret = init_rdovaedec(&dec->model, rdovaedec_arrays); + if (ret == 0) dec->loaded = 1; +#endif + dec->arch = opus_select_arch(); + /* To make sure nobody forgets to init, use a magic number. */ + dec->magic = 0xD8EDDEC0; + return (ret == 0) ? OPUS_OK : OPUS_UNIMPLEMENTED; +} + +OpusDREDDecoder *opus_dred_decoder_create(int *error) +{ + int ret; + OpusDREDDecoder *dec; + dec = (OpusDREDDecoder *)opus_alloc(opus_dred_decoder_get_size()); + if (dec == NULL) + { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + ret = opus_dred_decoder_init(dec); + if (error) + *error = ret; + if (ret != OPUS_OK) + { + opus_free(dec); + dec = NULL; + } + return dec; +} + +void opus_dred_decoder_destroy(OpusDREDDecoder *dec) +{ + if (dec) dec->magic = 0xDE57801D; + opus_free(dec); +} + +int opus_dred_decoder_ctl(OpusDREDDecoder *dred_dec, int request, ...) +{ +#ifdef ENABLE_DRED + int ret = OPUS_OK; + va_list ap; + + va_start(ap, request); + (void)dred_dec; + switch (request) + { +# ifdef USE_WEIGHTS_FILE + case OPUS_SET_DNN_BLOB_REQUEST: + { + const unsigned char *data = va_arg(ap, const unsigned char *); + opus_int32 len = va_arg(ap, opus_int32); + if(len<0 || data == NULL) + { + goto bad_arg; + } + return dred_decoder_load_model(dred_dec, data, len); + } + break; +# endif + default: + /*fprintf(stderr, "unknown opus_decoder_ctl() request: %d", request);*/ + ret = OPUS_UNIMPLEMENTED; + break; + } + va_end(ap); + return ret; +# ifdef USE_WEIGHTS_FILE +bad_arg: + va_end(ap); + return OPUS_BAD_ARG; +# endif +#else + (void)dred_dec; + (void)request; + return OPUS_UNIMPLEMENTED; +#endif +} + +#ifdef ENABLE_DRED +static int dred_find_payload(const unsigned char *data, opus_int32 len, const unsigned char **payload, int *dred_frame_offset) +{ + const unsigned char *data0; + int len0; + int frame = 0; + int ret; + const unsigned char *frames[48]; + opus_int16 size[48]; + int frame_size; + + *payload = NULL; + /* Get the padding section of the packet. */ + ret = opus_packet_parse_impl(data, len, 0, NULL, frames, size, NULL, NULL, &data0, &len0); + if (ret < 0) + return ret; + frame_size = opus_packet_get_samples_per_frame(data, 48000); + data = data0; + len = len0; + /* Scan extensions in order until we find the earliest frame with DRED data. */ + while (len > 0) + { + opus_int32 header_size; + int id, L; + len0 = len; + data0 = data; + id = *data0 >> 1; + L = *data0 & 0x1; + len = skip_extension(&data, len, &header_size); + if (len < 0) + break; + if (id == 1) + { + if (L==0) + { + frame++; + } else { + frame += data0[1]; + } + } else if (id == DRED_EXTENSION_ID) + { + const unsigned char *curr_payload; + opus_int32 curr_payload_len; + curr_payload = data0+header_size; + curr_payload_len = (data-data0)-header_size; + /* DRED position in the packet, in units of 2.5 ms like for the signaled DRED offset. */ + *dred_frame_offset = frame*frame_size/120; +#ifdef DRED_EXPERIMENTAL_VERSION + /* Check that temporary extension type and version match. + This check will be removed once extension is finalized. */ + if (curr_payload_len > DRED_EXPERIMENTAL_BYTES && curr_payload[0] == 'D' && curr_payload[1] == DRED_EXPERIMENTAL_VERSION) { + *payload = curr_payload+2; + return curr_payload_len-2; + } +#else + if (curr_payload_len > 0) { + *payload = curr_payload; + return curr_payload_len; + } +#endif + } + } + return 0; +} +#endif + +int opus_dred_get_size(void) +{ +#ifdef ENABLE_DRED + return sizeof(OpusDRED); +#else + return 0; +#endif +} + +OpusDRED *opus_dred_alloc(int *error) +{ +#ifdef ENABLE_DRED + OpusDRED *dec; + dec = (OpusDRED *)opus_alloc(opus_dred_get_size()); + if (dec == NULL) + { + if (error) + *error = OPUS_ALLOC_FAIL; + return NULL; + } + return dec; +#else + if (error) + *error = OPUS_UNIMPLEMENTED; + return NULL; +#endif +} + +void opus_dred_free(OpusDRED *dec) +{ +#ifdef ENABLE_DRED + opus_free(dec); +#else + (void)dec; +#endif +} + +int opus_dred_parse(OpusDREDDecoder *dred_dec, OpusDRED *dred, const unsigned char *data, opus_int32 len, opus_int32 max_dred_samples, opus_int32 sampling_rate, int *dred_end, int defer_processing) +{ +#ifdef ENABLE_DRED + const unsigned char *payload; + opus_int32 payload_len; + int dred_frame_offset=0; + VALIDATE_DRED_DECODER(dred_dec); + if (!dred_dec->loaded) return OPUS_UNIMPLEMENTED; + dred->process_stage = -1; + payload_len = dred_find_payload(data, len, &payload, &dred_frame_offset); + if (payload_len < 0) + return payload_len; + if (payload != NULL) + { + int offset; + int min_feature_frames; + offset = 100*max_dred_samples/sampling_rate; + min_feature_frames = IMIN(2 + offset, 2*DRED_NUM_REDUNDANCY_FRAMES); + dred_ec_decode(dred, payload, payload_len, min_feature_frames, dred_frame_offset); + if (!defer_processing) + opus_dred_process(dred_dec, dred, dred); + if (dred_end) *dred_end = IMAX(0, -dred->dred_offset*sampling_rate/400); + return IMAX(0, dred->nb_latents*sampling_rate/25 - dred->dred_offset* sampling_rate/400); + } + if (dred_end) *dred_end = 0; + return 0; +#else + (void)dred_dec; + (void)dred; + (void)data; + (void)len; + (void)max_dred_samples; + (void)sampling_rate; + (void)defer_processing; + (void)dred_end; + return OPUS_UNIMPLEMENTED; +#endif +} + +int opus_dred_process(OpusDREDDecoder *dred_dec, const OpusDRED *src, OpusDRED *dst) +{ +#ifdef ENABLE_DRED + if (dred_dec == NULL || src == NULL || dst == NULL || (src->process_stage != 1 && src->process_stage != 2)) + return OPUS_BAD_ARG; + VALIDATE_DRED_DECODER(dred_dec); + if (!dred_dec->loaded) return OPUS_UNIMPLEMENTED; + if (src != dst) + OPUS_COPY(dst, src, 1); + if (dst->process_stage == 2) + return OPUS_OK; + DRED_rdovae_decode_all(&dred_dec->model, dst->fec_features, dst->state, dst->latents, dst->nb_latents, dred_dec->arch); + dst->process_stage = 2; + return OPUS_OK; +#else + (void)dred_dec; + (void)src; + (void)dst; + return OPUS_UNIMPLEMENTED; +#endif +} + +int opus_decoder_dred_decode(OpusDecoder *st, const OpusDRED *dred, opus_int32 dred_offset, opus_int16 *pcm, opus_int32 frame_size) +{ +#ifdef ENABLE_DRED + VARDECL(float, out); + int ret, i; + ALLOC_STACK; + + if(frame_size<=0) + { + RESTORE_STACK; + return OPUS_BAD_ARG; + } + + celt_assert(st->channels == 1 || st->channels == 2); + ALLOC(out, frame_size*st->channels, float); + + ret = opus_decode_native(st, NULL, 0, out, frame_size, 0, 0, NULL, 1, dred, dred_offset); + if (ret > 0) + { + for (i=0;i<ret*st->channels;i++) + pcm[i] = FLOAT2INT16(out[i]); + } + RESTORE_STACK; + return ret; +#else + (void)st; + (void)dred; + (void)dred_offset; + (void)pcm; + (void)frame_size; + return OPUS_UNIMPLEMENTED; +#endif +} + +int opus_decoder_dred_decode_float(OpusDecoder *st, const OpusDRED *dred, opus_int32 dred_offset, float *pcm, opus_int32 frame_size) +{ +#ifdef ENABLE_DRED + if(frame_size<=0) + return OPUS_BAD_ARG; + return opus_decode_native(st, NULL, 0, pcm, frame_size, 0, 0, NULL, 0, dred, dred_offset); +#else + (void)st; + (void)dred; + (void)dred_offset; + (void)pcm; + (void)frame_size; + return OPUS_UNIMPLEMENTED; +#endif +} |