diff --git a/dnn/lpcnet.py b/dnn/lpcnet.py index a275ce99..fcdf2425 100644 --- a/dnn/lpcnet.py +++ b/dnn/lpcnet.py @@ -46,6 +46,7 @@ def new_wavernn_model(): exc = Input(shape=(None, 1)) pitch = Input(shape=(None, 1)) feat = Input(shape=(None, nb_used_features)) + pitch = Input(shape=(None, 1)) dec_feat = Input(shape=(None, 32)) dec_state = Input(shape=(rnn_units,)) @@ -67,7 +68,10 @@ def new_wavernn_model(): embed2 = Embedding(256, embed_size, embeddings_initializer=PCMInit()) cexc = Reshape((-1, embed_size))(embed2(exc)) - cfeat = fconv2(fconv1(feat)) + pembed = Embedding(256, 64) + cat_feat = Concatenate()([feat, Reshape((-1, 64))(pembed(pitch))]) + + cfeat = fconv2(fconv1(cat_feat)) rep = Lambda(lambda x: K.repeat_elements(x, 160, 1)) @@ -77,8 +81,8 @@ def new_wavernn_model(): gru_out, state = rnn(rnn_in) ulaw_prob = md(gru_out) - model = Model([pcm, exc, feat], ulaw_prob) - encoder = Model(feat, cfeat) + model = Model([pcm, exc, feat, pitch], ulaw_prob) + encoder = Model([feat, pitch], cfeat) dec_rnn_in = Concatenate()([cpcm, cexc, dec_feat]) dec_gru_out, state = rnn(dec_rnn_in, initial_state=dec_state) diff --git a/dnn/test_wavenet_audio.py b/dnn/test_wavenet_audio.py index 01f63f91..3175ef3a 100755 --- a/dnn/test_wavenet_audio.py +++ b/dnn/test_wavenet_audio.py @@ -60,7 +60,7 @@ out_data = (out_data.astype('int16')+128).astype('uint8') features = np.reshape(features, (nb_frames, feature_chunk_size, nb_features)) features = features[:, :, :] - +periods = (50*features[:,:,36:37]+100).astype('int16') in_data = np.reshape(in_data, (nb_frames*pcm_chunk_size, 1)) out_data = np.reshape(data, (nb_frames*pcm_chunk_size, 1)) diff --git a/dnn/train_wavenet_audio.py b/dnn/train_wavenet_audio.py index 8856417b..9a304708 100755 --- a/dnn/train_wavenet_audio.py +++ b/dnn/train_wavenet_audio.py @@ -94,6 +94,7 @@ features = np.reshape(features, (nb_frames, feature_chunk_size, nb_features)) features = features[:, :, :nb_used_features] pred = np.reshape(pred, (nb_frames, pcm_chunk_size, 1)) pred = (pred.astype('int16')+128).astype('uint8') +periods = (50*features[:,:,36:37]+100).astype('int16') in_data = np.concatenate([in_data, pred], axis=-1) @@ -103,8 +104,8 @@ in_data = np.concatenate([in_data, pred], axis=-1) # f.create_dataset('data', data=in_data[:50000, :, :]) # f.create_dataset('feat', data=features[:50000, :, :]) -checkpoint = ModelCheckpoint('wavenet4d3_{epoch:02d}.h5') +checkpoint = ModelCheckpoint('wavenet4e_{epoch:02d}.h5') #model.load_weights('wavernn1c_01.h5') model.compile(optimizer=Adam(0.001, amsgrad=True, decay=2e-4), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy']) -model.fit([in_data, in_exc, features], out_data, batch_size=batch_size, epochs=30, validation_split=0.2, callbacks=[checkpoint]) +model.fit([in_data, in_exc, features, periods], out_data, batch_size=batch_size, epochs=30, validation_split=0.2, callbacks=[checkpoint])