eden-emu/opus
15
0
Fork 0
mirror of https://github.com/xiph/opus.git synced 2025-06-05 23:10:54 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

cleanup: get rid of non-causal PLC and DC handling

Browse source
This commit is contained in:
Jean-Marc Valin 2023-06-21 17:48:21 -04:00
parent 247e6a587c
commit 07bb3f01b4
No known key found for this signature in database
GPG key ID: 531A52533318F00A
4 changed files with 4 additions and 217 deletions
Download patch file Download diff file Expand all files Collapse all files

2
dnn/include/lpcnet.h
View file

@ -189,10 +189,8 @@ LPCNET_EXPORT void lpcnet_synthesize(LPCNetState *st, const float *features, sho
#define LPCNET_PLC_CAUSAL 0 #define LPCNET_PLC_CAUSAL 0
#define LPCNET_PLC_NONCAUSAL 1
#define LPCNET_PLC_CODEC 2 #define LPCNET_PLC_CODEC 2
#define LPCNET_PLC_DC_FILTER 4
LPCNET_EXPORT int lpcnet_plc_get_size(void); LPCNET_EXPORT int lpcnet_plc_get_size(void);

9
dnn/lpcnet_demo.c
View file

@ -90,9 +90,7 @@ void usage(void) {
fprintf(stderr, " lpcnet_demo -addlpc <features_without_lpc.f32> <features_with_lpc.lpc>\n\n"); fprintf(stderr, " lpcnet_demo -addlpc <features_without_lpc.f32> <features_with_lpc.lpc>\n\n");
fprintf(stderr, " plc_options:\n"); fprintf(stderr, " plc_options:\n");
fprintf(stderr, " causal: normal (causal) PLC\n"); fprintf(stderr, " causal: normal (causal) PLC\n");
fprintf(stderr, " causal_dc: normal (causal) PLC with DC offset compensation\n"); fprintf(stderr, " codec: normal (causal) PLC without cross-fade (will glitch)\n");
fprintf(stderr, " noncausal: non-causal PLC\n");
fprintf(stderr, " noncausal_dc: non-causal PLC with DC offset compensation\n");
exit(1); exit(1);
} }
@ -134,9 +132,7 @@ int main(int argc, char **argv) {
} }
if (mode == MODE_PLC) { if (mode == MODE_PLC) {
if (strcmp(plc_options, "causal")==0) plc_flags = LPCNET_PLC_CAUSAL; if (strcmp(plc_options, "causal")==0) plc_flags = LPCNET_PLC_CAUSAL;
else if (strcmp(plc_options, "causal_dc")==0) plc_flags = LPCNET_PLC_CAUSAL | LPCNET_PLC_DC_FILTER; else if (strcmp(plc_options, "codec")==0) plc_flags = LPCNET_PLC_CODEC;
else if (strcmp(plc_options, "noncausal")==0) plc_flags = LPCNET_PLC_NONCAUSAL;
else if (strcmp(plc_options, "noncausal_dc")==0) plc_flags = LPCNET_PLC_NONCAUSAL | LPCNET_PLC_DC_FILTER;
else usage(); else usage();
} }
if (argc != 4) usage(); if (argc != 4) usage();
@ -191,7 +187,6 @@ int main(int argc, char **argv) {
int loss=0; int loss=0;
int skip=0, extra=0; int skip=0, extra=0;
LPCNetPLCState *net; LPCNetPLCState *net;
if ((plc_flags&0x3) == LPCNET_PLC_NONCAUSAL) skip=extra=80;
net = lpcnet_plc_create(plc_flags); net = lpcnet_plc_create(plc_flags);
#ifdef USE_WEIGHTS_FILE #ifdef USE_WEIGHTS_FILE
lpcnet_plc_load_model(net, data, len); lpcnet_plc_load_model(net, data, len);

202
dnn/lpcnet_plc.c
View file

@ -54,8 +54,6 @@ LPCNET_EXPORT void lpcnet_plc_reset(LPCNetPLCState *st) {
st->skip_analysis = 0; st->skip_analysis = 0;
st->blend = 0; st->blend = 0;
st->loss_count = 0; st->loss_count = 0;
st->dc_mem = 0;
st->queued_update = 0;
} }
LPCNET_EXPORT int lpcnet_plc_init(LPCNetPLCState *st, int options) { LPCNET_EXPORT int lpcnet_plc_init(LPCNetPLCState *st, int options) {
@ -64,17 +62,11 @@ LPCNET_EXPORT int lpcnet_plc_init(LPCNetPLCState *st, int options) {
lpcnet_encoder_init(&st->enc); lpcnet_encoder_init(&st->enc);
if ((options&0x3) == LPCNET_PLC_CAUSAL) { if ((options&0x3) == LPCNET_PLC_CAUSAL) {
st->enable_blending = 1; st->enable_blending = 1;
st->non_causal = 0;
} else if ((options&0x3) == LPCNET_PLC_NONCAUSAL) {
st->enable_blending = 1;
st->non_causal = 1;
} else if ((options&0x3) == LPCNET_PLC_CODEC) { } else if ((options&0x3) == LPCNET_PLC_CODEC) {
st->enable_blending = 0; st->enable_blending = 0;
st->non_causal = 0;
} else { } else {
return -1; return -1;
} }
st->remove_dc = !!(options&LPCNET_PLC_DC_FILTER);
#ifndef USE_WEIGHTS_FILE #ifndef USE_WEIGHTS_FILE
ret = init_plc_model(&st->model, lpcnet_plc_arrays); ret = init_plc_model(&st->model, lpcnet_plc_arrays);
#else #else
@ -180,28 +172,15 @@ void clear_state(LPCNetPLCState *st) {
RNN_CLEAR(st->lpcnet.nnet.gru_b_state, GRU_B_STATE_SIZE); RNN_CLEAR(st->lpcnet.nnet.gru_b_state, GRU_B_STATE_SIZE);
} }
#define DC_CONST 0.003
/* In this causal version of the code, the DNN model implemented by compute_plc_pred() /* In this causal version of the code, the DNN model implemented by compute_plc_pred()
needs to generate two feature vectors to conceal the first lost packet.*/ needs to generate two feature vectors to conceal the first lost packet.*/
static int lpcnet_plc_update_causal(LPCNetPLCState *st, short *pcm) { int lpcnet_plc_update(LPCNetPLCState *st, short *pcm) {
int i; int i;
float x[FRAME_SIZE]; float x[FRAME_SIZE];
short output[FRAME_SIZE]; short output[FRAME_SIZE];
float plc_features[2*NB_BANDS+NB_FEATURES+1]; float plc_features[2*NB_BANDS+NB_FEATURES+1];
short lp[FRAME_SIZE]={0};
int delta = 0; int delta = 0;
if (st->remove_dc) {
st->dc_mem += st->syn_dc;
delta = st->syn_dc;
st->syn_dc = 0;
for (i=0;i<FRAME_SIZE;i++) {
lp[i] = (int)floor(.5 + st->dc_mem);
st->dc_mem += DC_CONST*(pcm[i] - st->dc_mem);
pcm[i] -= lp[i];
}
}
for (i=0;i<FRAME_SIZE;i++) x[i] = pcm[i]; for (i=0;i<FRAME_SIZE;i++) x[i] = pcm[i];
burg_cepstral_analysis(plc_features, x); burg_cepstral_analysis(plc_features, x);
st->enc.pcount = 0; st->enc.pcount = 0;
@ -280,17 +259,12 @@ static int lpcnet_plc_update_causal(LPCNetPLCState *st, short *pcm) {
RNN_MOVE(st->pcm, &st->pcm[FRAME_SIZE], PLC_BUF_SIZE); RNN_MOVE(st->pcm, &st->pcm[FRAME_SIZE], PLC_BUF_SIZE);
} }
st->loss_count = 0; st->loss_count = 0;
if (st->remove_dc) {
for (i=0;i<FRAME_SIZE;i++) {
pcm[i] += lp[i];
}
}
st->blend = 0; st->blend = 0;
return 0; return 0;
} }
static const float att_table[10] = {0, 0, -.2, -.2, -.4, -.4, -.8, -.8, -1.6, -1.6}; static const float att_table[10] = {0, 0, -.2, -.2, -.4, -.4, -.8, -.8, -1.6, -1.6};
static int lpcnet_plc_conceal_causal(LPCNetPLCState *st, short *pcm) { int lpcnet_plc_conceal(LPCNetPLCState *st, short *pcm) {
int i; int i;
short output[FRAME_SIZE]; short output[FRAME_SIZE];
run_frame_network_flush(&st->lpcnet); run_frame_network_flush(&st->lpcnet);
@ -327,177 +301,5 @@ static int lpcnet_plc_conceal_causal(LPCNetPLCState *st, short *pcm) {
process_single_frame(&st->enc, NULL); process_single_frame(&st->enc, NULL);
} }
st->blend = 1; st->blend = 1;
if (st->remove_dc) {
for (i=0;i<FRAME_SIZE;i++) {
st->syn_dc += DC_CONST*(pcm[i] - st->syn_dc);
pcm[i] += (int)floor(.5 + st->dc_mem);
}
}
return 0; return 0;
} }
/* In this non-causal version of the code, the DNN model implemented by compute_plc_pred()
is always called once per frame. We process audio up to the current position minus TRAINING_OFFSET. */
void process_queued_update(LPCNetPLCState *st) {
if (st->queued_update) {
lpcnet_synthesize_impl(&st->lpcnet, st->features, st->queued_samples, FRAME_SIZE, FRAME_SIZE);
st->queued_update=0;
}
}
static int lpcnet_plc_update_non_causal(LPCNetPLCState *st, short *pcm) {
int i;
float x[FRAME_SIZE];
short pcm_save[FRAME_SIZE];
float plc_features[2*NB_BANDS+NB_FEATURES+1];
short lp[FRAME_SIZE]={0};
double mem_bak=0;
int delta = st->syn_dc;
if (FEATURES_DELAY != 0) {
fprintf(stderr, "Non-causal PLC cannot work with non-zero FEATURES_DELAY\n");
fprintf(stderr, "Recompile with a no-lookahead model (see README.md)\n");
exit(1);
}
process_queued_update(st);
if (st->remove_dc) {
st->dc_mem += st->syn_dc;
st->syn_dc = 0;
mem_bak = st->dc_mem;
for (i=0;i<FRAME_SIZE;i++) {
lp[i] = (int)floor(.5 + st->dc_mem);
st->dc_mem += DC_CONST*(pcm[i] - st->dc_mem);
pcm[i] -= lp[i];
}
}
RNN_COPY(pcm_save, pcm, FRAME_SIZE);
for (i=0;i<FRAME_SIZE;i++) x[i] = pcm[i];
burg_cepstral_analysis(plc_features, x);
st->enc.pcount = 0;
if (st->loss_count > 0) {
LPCNetState copy;
/* Handle blending. */
float zeros[2*NB_BANDS+NB_FEATURES+1] = {0};
RNN_COPY(zeros, plc_features, 2*NB_BANDS);
zeros[2*NB_BANDS+NB_FEATURES] = 1;
compute_plc_pred(st, st->features, zeros);
copy = st->lpcnet;
lpcnet_synthesize_impl(&st->lpcnet, st->features, &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, 0);
/* Undo initial DC offset removal so that we can take into account the last 5ms of synthesis. */
if (st->remove_dc) {
for (i=0;i<FRAME_SIZE;i++) pcm[i] += lp[i];
st->dc_mem = mem_bak;
for (i=0;i<TRAINING_OFFSET;i++) st->syn_dc += DC_CONST*(st->pcm[FRAME_SIZE-TRAINING_OFFSET+i] - st->syn_dc);
st->dc_mem += st->syn_dc;
delta += st->syn_dc;
st->syn_dc = 0;
for (i=0;i<FRAME_SIZE;i++) {
lp[i] = (int)floor(.5 + st->dc_mem);
st->dc_mem += DC_CONST*(pcm[i] - st->dc_mem);
pcm[i] -= lp[i];
}
RNN_COPY(pcm_save, pcm, FRAME_SIZE);
}
{
short rev[FRAME_SIZE];
for (i=0;i<FRAME_SIZE;i++) rev[i] = pcm[FRAME_SIZE-i-1];
clear_state(st);
lpcnet_synthesize_impl(&st->lpcnet, st->features, rev, FRAME_SIZE, FRAME_SIZE);
lpcnet_synthesize_tail_impl(&st->lpcnet, rev, TRAINING_OFFSET, 0);
for (i=0;i<TRAINING_OFFSET;i++) {
float w;
w = .5 - .5*cos(M_PI*i/(TRAINING_OFFSET));
st->pcm[FRAME_SIZE-1-i] = (int)floor(.5 + w*st->pcm[FRAME_SIZE-1-i] + (1-w)*(rev[i]+delta));
}
}
st->lpcnet = copy;
#if 1
st->queued_update = 1;
RNN_COPY(&st->queued_samples[0], &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET);
RNN_COPY(&st->queued_samples[TRAINING_OFFSET], pcm, FRAME_SIZE-TRAINING_OFFSET);
#else
lpcnet_synthesize_impl(&st->lpcnet, st->features, &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, TRAINING_OFFSET);
lpcnet_synthesize_tail_impl(&st->lpcnet, pcm, FRAME_SIZE-TRAINING_OFFSET, FRAME_SIZE-TRAINING_OFFSET);
#endif
for (i=0;i<FRAME_SIZE;i++) x[i] = st->pcm[i];
preemphasis(x, &st->enc.mem_preemph, x, PREEMPHASIS, FRAME_SIZE);
compute_frame_features(&st->enc, x);
process_single_frame(&st->enc, NULL);
}
for (i=0;i<FRAME_SIZE;i++) x[i] = pcm[i];
preemphasis(x, &st->enc.mem_preemph, x, PREEMPHASIS, FRAME_SIZE);
compute_frame_features(&st->enc, x);
process_single_frame(&st->enc, NULL);
if (st->loss_count == 0) {
RNN_COPY(&plc_features[2*NB_BANDS], st->enc.features[0], NB_FEATURES);
plc_features[2*NB_BANDS+NB_FEATURES] = 1;
compute_plc_pred(st, st->features, plc_features);
lpcnet_synthesize_impl(&st->lpcnet, st->enc.features[0], &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, TRAINING_OFFSET);
lpcnet_synthesize_tail_impl(&st->lpcnet, pcm, FRAME_SIZE-TRAINING_OFFSET, FRAME_SIZE-TRAINING_OFFSET);
}
RNN_COPY(&pcm[FRAME_SIZE-TRAINING_OFFSET], pcm, TRAINING_OFFSET);
RNN_COPY(pcm, &st->pcm[TRAINING_OFFSET], FRAME_SIZE-TRAINING_OFFSET);
RNN_COPY(st->pcm, pcm_save, FRAME_SIZE);
st->loss_count = 0;
if (st->remove_dc) {
for (i=0;i<TRAINING_OFFSET;i++) pcm[i] += st->dc_buf[i];
for (;i<FRAME_SIZE;i++) pcm[i] += lp[i-TRAINING_OFFSET];
for (i=0;i<TRAINING_OFFSET;i++) st->dc_buf[i] = lp[FRAME_SIZE-TRAINING_OFFSET+i];
}
return 0;
}
static int lpcnet_plc_conceal_non_causal(LPCNetPLCState *st, short *pcm) {
int i;
float x[FRAME_SIZE];
float zeros[2*NB_BANDS+NB_FEATURES+1] = {0};
process_queued_update(st);
st->enc.pcount = 0;
compute_plc_pred(st, st->features, zeros);
if (st->loss_count >= 10) st->features[0] = MAX16(-10, st->features[0]+att_table[9] - 2*(st->loss_count-9));
else st->features[0] = MAX16(-10, st->features[0]+att_table[st->loss_count]);
if (st->loss_count == 0) {
RNN_COPY(pcm, &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET);
lpcnet_synthesize_impl(&st->lpcnet, st->features, &st->pcm[FRAME_SIZE-TRAINING_OFFSET], TRAINING_OFFSET, TRAINING_OFFSET);
lpcnet_synthesize_tail_impl(&st->lpcnet, &pcm[TRAINING_OFFSET], FRAME_SIZE-TRAINING_OFFSET, 0);
} else {
lpcnet_synthesize_impl(&st->lpcnet, st->features, pcm, TRAINING_OFFSET, 0);
lpcnet_synthesize_tail_impl(&st->lpcnet, &pcm[TRAINING_OFFSET], FRAME_SIZE-TRAINING_OFFSET, 0);
RNN_COPY(&st->pcm[FRAME_SIZE-TRAINING_OFFSET], pcm, TRAINING_OFFSET);
for (i=0;i<FRAME_SIZE;i++) x[i] = st->pcm[i];
preemphasis(x, &st->enc.mem_preemph, x, PREEMPHASIS, FRAME_SIZE);
compute_frame_features(&st->enc, x);
process_single_frame(&st->enc, NULL);
}
RNN_COPY(st->pcm, &pcm[TRAINING_OFFSET], FRAME_SIZE-TRAINING_OFFSET);
if (st->remove_dc) {
int dc = (int)floor(.5 + st->dc_mem);
if (st->loss_count == 0) {
for (i=TRAINING_OFFSET;i<FRAME_SIZE;i++) st->syn_dc += DC_CONST*(pcm[i] - st->syn_dc);
} else {
for (i=0;i<FRAME_SIZE;i++) st->syn_dc += DC_CONST*(pcm[i] - st->syn_dc);
}
for (i=0;i<TRAINING_OFFSET;i++) pcm[i] += st->dc_buf[i];
for (;i<FRAME_SIZE;i++) pcm[i] += dc;
for (i=0;i<TRAINING_OFFSET;i++) st->dc_buf[i] = dc;
}
st->loss_count++;
return 0;
}
LPCNET_EXPORT int lpcnet_plc_update(LPCNetPLCState *st, short *pcm) {
if (st->non_causal) return lpcnet_plc_update_non_causal(st, pcm);
else return lpcnet_plc_update_causal(st, pcm);
}
LPCNET_EXPORT int lpcnet_plc_conceal(LPCNetPLCState *st, short *pcm) {
if (st->non_causal) return lpcnet_plc_conceal_non_causal(st, pcm);
else return lpcnet_plc_conceal_causal(st, pcm);
}

8
dnn/lpcnet_private.h
View file

@ -80,8 +80,6 @@ struct LPCNetPLCState {
LPCNetState lpcnet; LPCNetState lpcnet;
LPCNetEncState enc; LPCNetEncState enc;
int enable_blending; int enable_blending;
int non_causal;
int remove_dc;
#define LPCNET_PLC_RESET_START fec #define LPCNET_PLC_RESET_START fec
float fec[PLC_MAX_FEC][NB_FEATURES]; float fec[PLC_MAX_FEC][NB_FEATURES];
@ -97,12 +95,6 @@ struct LPCNetPLCState {
int loss_count; int loss_count;
PLCNetState plc_net; PLCNetState plc_net;
PLCNetState plc_copy[FEATURES_DELAY+1]; PLCNetState plc_copy[FEATURES_DELAY+1];
double dc_mem;
double syn_dc;
short dc_buf[TRAINING_OFFSET];
int queued_update;
short queued_samples[FRAME_SIZE];
}; };
void preemphasis(float *y, float *mem, const float *x, float coef, int N); void preemphasis(float *y, float *mem, const float *x, float coef, int N);