Computing signals in C

dnn/dump_data.c

@@ -58,6 +58,9 @@ typedef struct {
   float pitch_buf[PITCH_BUF_SIZE];
   float last_gain;
   int last_period;
+  float lpc[LPC_ORDER];
+  float sig_mem[LPC_ORDER];
+  int exc_mem;
 } DenoiseState;
 
 static int rnnoise_get_size() {
@@ -108,7 +111,6 @@ static void compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_c
   int i;
   float E = 0;
   float Ly[NB_BANDS];
-  float lpc[LPC_ORDER];
   float p[WINDOW_SIZE];
   float pitch_buf[PITCH_BUF_SIZE];
   int pitch_index;
@@ -150,7 +152,7 @@ static void compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_c
   }
   dct(features, Ly);
   features[0] -= 4;
-  g = lpc_from_cepstrum(lpc, features);
+  g = lpc_from_cepstrum(st->lpc, features);
 #if 0
   for (i=0;i<NB_BANDS;i++) printf("%f ", Ly[i]);
   printf("\n");
@@ -158,7 +160,7 @@ static void compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_c
   features[2*NB_BANDS] = .01*(pitch_index-200);
   features[2*NB_BANDS+1] = gain;
   features[2*NB_BANDS+2] = log10(g);
-  for (i=0;i<LPC_ORDER;i++) features[2*NB_BANDS+3+i] = lpc[i];
+  for (i=0;i<LPC_ORDER;i++) features[2*NB_BANDS+3+i] = st->lpc[i];
 #if 0
   for (i=0;i<NB_FEATURES;i++) printf("%f ", features[i]);
   printf("\n");
@@ -198,6 +200,36 @@ static void rand_resp(float *a, float *b) {
   b[1] = .75*uni_rand();
 }
 
+void write_audio(DenoiseState *st, const short *pcm, float noise_std, FILE *file) {
+  int i;
+  unsigned char data[4*FRAME_SIZE];
+  for (i=0;i<FRAME_SIZE;i++) {
+    int noise;
+    float p=0;
+    float e;
+    int j;
+    for (j=0;j<LPC_ORDER;j++) p -= st->lpc[j]*st->sig_mem[j];
+    e = lin2ulaw(pcm[i] - p);
+    /* Signal. */
+    data[4*i] = lin2ulaw(st->sig_mem[0]);
+    /* Prediction. */
+    data[4*i+1] = lin2ulaw(p);
+    /* Excitation in. */
+    data[4*i+2] = st->exc_mem;
+    /* Excitation out. */
+    data[4*i+3] = e;
+    /* Simulate error on excitation. */
+    noise = (int)floor(.5 + noise_std*.707*(log((float)rand()/RAND_MAX)-log((float)rand()/RAND_MAX)));
+    e += noise;
+    e = IMIN(255, IMAX(0, e));
+    
+    RNN_MOVE(&st->sig_mem[1], &st->sig_mem[0], LPC_ORDER-1);
+    st->sig_mem[0] = p + ulaw2lin(e);
+    st->exc_mem = e;
+  }
+  fwrite(data, 4*FRAME_SIZE, 1, file);
+}
+
 int main(int argc, char **argv) {
   int i;
   int count=0;
@@ -221,6 +253,7 @@ int main(int argc, char **argv) {
   float old_speech_gain = 1;
   int one_pass_completed = 0;
   DenoiseState *st;
+  float noise_std=0;
   int training = -1;
   st = rnnoise_create();
   if (argc == 5 && strcmp(argv[1], "-train")==0) training = 1;
@@ -287,6 +320,7 @@ int main(int argc, char **argv) {
       if (rand()%100==0) speech_gain = 0;
       gain_change_count = 0;
       rand_resp(a_sig, b_sig);
+      noise_std = 3*(float)rand()/RAND_MAX;
     }
     biquad(x, mem_hp_x, x, b_hp, a_hp, FRAME_SIZE);
     biquad(x, mem_resp_x, x, b_sig, a_sig, FRAME_SIZE);
@@ -302,7 +336,8 @@ int main(int argc, char **argv) {
     fwrite(features, sizeof(float), NB_FEATURES, ffeat);
     /* PCM is delayed by 1/2 frame to make the features centered on the frames. */
     for (i=0;i<FRAME_SIZE-TRAINING_OFFSET;i++) pcm[i+TRAINING_OFFSET] = float2short(x[i]);
-    if (fpcm) fwrite(pcm, sizeof(short), FRAME_SIZE, fpcm);
+    if (fpcm) write_audio(st, pcm, noise_std, fpcm);
+    //if (fpcm) fwrite(pcm, sizeof(short), FRAME_SIZE, fpcm);
     for (i=0;i<TRAINING_OFFSET;i++) pcm[i] = float2short(x[i+FRAME_SIZE-TRAINING_OFFSET]);
     old_speech_gain = speech_gain;
     count++;

dnn/train_lpcnet.py

@@ -66,85 +66,39 @@ pcm_chunk_size = frame_size*feature_chunk_size
 
 # u for unquantised, load 16 bit PCM samples and convert to mu-law
 
-udata = np.fromfile(pcm_file, dtype='int16')
+data = np.fromfile(pcm_file, dtype='uint8')
-data = lin2ulaw(udata)
+nb_frames = len(data)//(4*pcm_chunk_size)
-nb_frames = len(data)//pcm_chunk_size
 
 features = np.fromfile(feature_file, dtype='float32')
 
 # limit to discrete number of frames
-data = data[:nb_frames*pcm_chunk_size]
+data = data[:nb_frames*4*pcm_chunk_size]
-udata = udata[:nb_frames*pcm_chunk_size]
 features = features[:nb_frames*feature_chunk_size*nb_features]
 
-# Noise injection: the idea is that the real system is going to be
-# predicting samples based on previously predicted samples rather than
-# from the original. Since the previously predicted samples aren't
-# expected to be so good, I add noise to the training data.  Exactly
-# how the noise is added makes a huge difference
-
-in_data = np.concatenate([data[0:1], data[:-1]]);
-noise = np.concatenate([np.zeros((len(data)*1//5)), np.random.randint(-3, 3, len(data)*1//5), np.random.randint(-2, 2, len(data)*1//5), np.random.randint(-1, 1, len(data)*2//5)])
-#noise = np.round(np.concatenate([np.zeros((len(data)*1//5)), np.random.laplace(0, 1.2, len(data)*1//5), np.random.laplace(0, .77, len(data)*1//5), np.random.laplace(0, .33, len(data)*1//5), np.random.randint(-1, 1, len(data)*1//5)]))
-del data
-in_data = in_data + noise
-del noise
-in_data = np.clip(in_data, 0, 255)
-
 features = np.reshape(features, (nb_frames*feature_chunk_size, nb_features))
 
-# Note: the LPC predictor output is now calculated by the loop below, this code was
+sig = np.reshape(data[0::4], (nb_frames, pcm_chunk_size, 1))
-# for an ealier version that implemented the prediction filter in C
+pred = np.reshape(data[1::4], (nb_frames, pcm_chunk_size, 1))
-
+in_exc = np.reshape(data[2::4], (nb_frames, pcm_chunk_size, 1))
-upred = np.zeros((nb_frames*pcm_chunk_size,), dtype='float32')
+out_exc = np.reshape(data[3::4], (nb_frames, pcm_chunk_size, 1))
-
+del data
-# Use 16th order LPC to generate LPC prediction output upred[] and (in
-# mu-law form) pred[]
-
-pred_in = ulaw2lin(in_data)
-for i in range(2, nb_frames*feature_chunk_size):
-    upred[i*frame_size:(i+1)*frame_size] = 0
-    for k in range(16):
-        upred[i*frame_size:(i+1)*frame_size] = upred[i*frame_size:(i+1)*frame_size] - \
-            pred_in[i*frame_size-k:(i+1)*frame_size-k]*features[i, nb_features-16+k]
-del pred_in
-
-pred = lin2ulaw(upred)
-
-in_data = np.reshape(in_data, (nb_frames, pcm_chunk_size, 1))
-in_data = in_data.astype('uint8')
-
-# LPC residual, which is the difference between the input speech and
-# the predictor output, with a slight time shift this is also the
-# ideal excitation in_exc
-
-out_data = lin2ulaw(udata-upred)
-del upred
-del udata
-in_exc = np.concatenate([out_data[0:1], out_data[:-1]]);
-
-out_data = np.reshape(out_data, (nb_frames, pcm_chunk_size, 1))
-out_data = out_data.astype('uint8')
-
-in_exc = np.reshape(in_exc, (nb_frames, pcm_chunk_size, 1))
-in_exc = in_exc.astype('uint8')
 
+print("ulaw std = ", np.std(out_exc))
 
 features = np.reshape(features, (nb_frames, feature_chunk_size, nb_features))
 features = features[:, :, :nb_used_features]
 features[:,:,18:36] = 0
-pred = np.reshape(pred, (nb_frames, pcm_chunk_size, 1))
-pred = pred.astype('uint8')
 
 periods = (.1 + 50*features[:,:,36:37]+100).astype('int16')
 
-in_data = np.concatenate([in_data, pred], axis=-1)
+in_data = np.concatenate([sig, pred], axis=-1)
 
+del sig
 del pred
 
 # dump models to disk as we go
-checkpoint = ModelCheckpoint('lpcnet15_384_10_G16_{epoch:02d}.h5')
+checkpoint = ModelCheckpoint('lpcnet18_384_10_G16_{epoch:02d}.h5')
 
-#model.load_weights('lpcnet9b_384_10_G16_01.h5')
+model.load_weights('lpcnet9b_384_10_G16_01.h5')
 model.compile(optimizer=Adam(0.001, amsgrad=True, decay=5e-5), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])
-model.fit([in_data, in_exc, features, periods], out_data, batch_size=batch_size, epochs=nb_epochs, validation_split=0.0, callbacks=[checkpoint, lpcnet.Sparsify(2000, 40000, 400, (0.1, 0.1, 0.1))])
+model.fit([in_data, in_exc, features, periods], out_exc, batch_size=batch_size, epochs=nb_epochs, validation_split=0.0, callbacks=[checkpoint, lpcnet.Sparsify(2000, 40000, 400, (0.1, 0.1, 0.1))])