diff --git a/dnn/dump_data.c b/dnn/dump_data.c index f5544db3..cc272993 100644 --- a/dnn/dump_data.c +++ b/dnn/dump_data.c @@ -138,9 +138,18 @@ int main(int argc, char **argv) { int encode = 0; int decode = 0; int quantize = 0; + int burg = 0; srand(getpid()); st = lpcnet_encoder_create(); argv0=argv[0]; + if (argc == 5 && strcmp(argv[1], "-btrain")==0) { + burg = 1; + training = 1; + } + if (argc == 4 && strcmp(argv[1], "-btest")==0) { + burg = 1; + training = 0; + } if (argc == 5 && strcmp(argv[1], "-train")==0) training = 1; if (argc == 5 && strcmp(argv[1], "-qtrain")==0) { training = 1; @@ -236,7 +245,8 @@ int main(int argc, char **argv) { if (count*FRAME_SIZE_5MS>=10000000 && one_pass_completed) break; if (training && ++gain_change_count > 2821) { float tmp, tmp2; - speech_gain = pow(10., (-20+(rand()%40))/20.); + speech_gain = pow(10., (-30+(rand()%40))/20.); + if (rand()&1) speech_gain = -speech_gain; if (rand()%20==0) speech_gain *= .01; if (rand()%100==0) speech_gain = 0; gain_change_count = 0; @@ -247,13 +257,18 @@ int main(int argc, char **argv) { } biquad(x, mem_hp_x, x, b_hp, a_hp, FRAME_SIZE); biquad(x, mem_resp_x, x, b_sig, a_sig, FRAME_SIZE); - preemphasis(x, &mem_preemph, x, PREEMPHASIS, FRAME_SIZE); for (i=0;ienc.pcount = 0; if (st->skip_analysis) { @@ -105,8 +107,8 @@ LPCNET_EXPORT int lpcnet_plc_update(LPCNetPLCState *st, short *pcm) { process_single_frame(&st->enc, NULL); #if PLC_DNN_PRED if (st->skip_analysis <= 1) { - RNN_COPY(plc_features, st->enc.features[0], NB_FEATURES); - plc_features[NB_FEATURES] = 1; + RNN_COPY(&plc_features[2*NB_BANDS], st->enc.features[0], NB_FEATURES); + plc_features[2*NB_BANDS+NB_FEATURES] = 1; compute_plc_pred(&st->plc_net, st->features, plc_features); } #else @@ -142,7 +144,7 @@ LPCNET_EXPORT int lpcnet_plc_conceal(LPCNetPLCState *st, short *pcm) { int i; #endif short output[FRAME_SIZE]; - float zeros[NB_FEATURES+1] = {0}; + float zeros[2*NB_BANDS+NB_FEATURES+1] = {0}; st->enc.pcount = 0; /* If we concealed the previous frame, finish synthesizing the rest of the samples. */ /* FIXME: Copy/predict features. */ diff --git a/dnn/lpcnet_private.h b/dnn/lpcnet_private.h index e1b4c99a..36234426 100644 --- a/dnn/lpcnet_private.h +++ b/dnn/lpcnet_private.h @@ -64,6 +64,7 @@ struct LPCNetEncState{ float features[4][NB_TOTAL_FEATURES]; float sig_mem[LPC_ORDER]; int exc_mem; + float burg_cepstrum[2*NB_BANDS]; }; #define PLC_BUF_SIZE (FEATURES_DELAY*FRAME_SIZE + TRAINING_OFFSET) diff --git a/dnn/training_tf2/lpcnet_plc.py b/dnn/training_tf2/lpcnet_plc.py index 04be0e0a..9acea419 100644 --- a/dnn/training_tf2/lpcnet_plc.py +++ b/dnn/training_tf2/lpcnet_plc.py @@ -62,8 +62,8 @@ class WeightClip(Constraint): constraint = WeightClip(0.992) -def new_lpcnet_plc_model(rnn_units=256, nb_used_features=20, batch_size=128, training=False, adaptation=False, quantize=False, cond_size=128): - feat = Input(shape=(None, nb_used_features), batch_size=batch_size) +def new_lpcnet_plc_model(rnn_units=256, nb_used_features=20, nb_burg_features=36, batch_size=128, training=False, adaptation=False, quantize=False, cond_size=128): + feat = Input(shape=(None, nb_used_features+nb_burg_features), batch_size=batch_size) lost = Input(shape=(None, 1), batch_size=batch_size) fdense1 = Dense(cond_size, activation='tanh', name='plc_dense1') @@ -96,5 +96,6 @@ def new_lpcnet_plc_model(rnn_units=256, nb_used_features=20, batch_size=128, tra model.rnn_units = rnn_units model.cond_size = cond_size model.nb_used_features = nb_used_features + model.nb_burg_features = nb_burg_features return model diff --git a/dnn/training_tf2/plc_loader.py b/dnn/training_tf2/plc_loader.py index 004fd70d..13414fd8 100644 --- a/dnn/training_tf2/plc_loader.py +++ b/dnn/training_tf2/plc_loader.py @@ -29,12 +29,13 @@ import numpy as np from tensorflow.keras.utils import Sequence class PLCLoader(Sequence): - def __init__(self, features, lost, batch_size): + def __init__(self, features, lost, nb_burg_features, batch_size): self.batch_size = batch_size self.nb_batches = features.shape[0]//self.batch_size self.features = features[:self.nb_batches*self.batch_size, :, :] self.lost = lost.astype('float') self.lost = self.lost[:(len(self.lost)//features.shape[1]-1)*features.shape[1]] + self.nb_burg_features = nb_burg_features self.on_epoch_end() def on_epoch_end(self): @@ -51,7 +52,7 @@ class PLCLoader(Sequence): lost = np.reshape(lost, (features.shape[0], features.shape[1], 1)) lost_mask = np.tile(lost, (1,1,features.shape[2])) - out_features = np.concatenate([features, 1.-lost], axis=-1) + out_features = np.concatenate([features[:,:,self.nb_burg_features:], 1.-lost], axis=-1) inputs = [features*lost_mask, lost] outputs = [out_features] return (inputs, outputs) diff --git a/dnn/training_tf2/train_plc.py b/dnn/training_tf2/train_plc.py index 0ccca6e1..c7a280ef 100644 --- a/dnn/training_tf2/train_plc.py +++ b/dnn/training_tf2/train_plc.py @@ -140,8 +140,9 @@ with strategy.scope(): lpc_order = 16 feature_file = args.features -nb_features = model.nb_used_features + lpc_order +nb_features = model.nb_used_features + lpc_order + model.nb_burg_features nb_used_features = model.nb_used_features +nb_burg_features = model.nb_burg_features sequence_size = args.seq_length # u for unquantised, load 16 bit PCM samples and convert to mu-law @@ -153,7 +154,7 @@ features = features[:nb_sequences*sequence_size*nb_features] features = np.reshape(features, (nb_sequences, sequence_size, nb_features)) -features = features[:, :, :nb_used_features] +features = features[:, :, :nb_used_features+model.nb_burg_features] lost = np.memmap(args.lost_file, dtype='int8', mode='r') @@ -169,7 +170,7 @@ if quantize or retrain: model.save_weights('{}_{}_initial.h5'.format(args.output, args.gru_size)) -loader = PLCLoader(features, lost, batch_size) +loader = PLCLoader(features, lost, nb_burg_features, batch_size) callbacks = [checkpoint] if args.logdir is not None: