eden-emu/opus
15
0
Fork 0
mirror of https://github.com/xiph/opus.git synced 2025-05-19 09:58:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

more pcm outputs

Browse source
This commit is contained in:
Jean-Marc Valin 2018-08-03 01:59:29 -04:00
parent 70789e6f43
commit 4fec1144f3
5 changed files with 57 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

49
dnn/denoise.c
View file

@ -38,6 +38,7 @@
#include "pitch.h" #include "pitch.h"
#include "arch.h" #include "arch.h"
#include "celt_lpc.h" #include "celt_lpc.h"
#include <assert.h>
#define PREEMPHASIS (0.85f) #define PREEMPHASIS (0.85f)
@ -64,7 +65,7 @@
#define CEPS_MEM 8 #define CEPS_MEM 8
#define NB_DELTA_CEPS 6 #define NB_DELTA_CEPS 6
#define NB_FEATURES (2*NB_BANDS+2+LPC_ORDER) #define NB_FEATURES (2*NB_BANDS+3+LPC_ORDER)
#ifndef TRAINING #ifndef TRAINING
@ -305,12 +306,20 @@ int lowpass = FREQ_SIZE;
int band_lp = NB_BANDS; int band_lp = NB_BANDS;
#endif #endif
static void frame_analysis(DenoiseState *st, signed char *iexc, float *lpc, kiss_fft_cpx *X, float *Ex, const float *in) { short float2short(float x)
{
int i;
i = (int)floor(.5+x);
return IMAX(-32767, IMIN(32767, i));
}
static float frame_analysis(DenoiseState *st, signed char *iexc, short *pred, short *pcm, float *lpc, kiss_fft_cpx *X, float *Ex, const float *in) {
int i; int i;
float x[WINDOW_SIZE]; float x[WINDOW_SIZE];
float x0[WINDOW_SIZE]; float x0[WINDOW_SIZE];
float ac[LPC_ORDER+1]; float ac[LPC_ORDER+1];
float rc[LPC_ORDER]; float rc[LPC_ORDER];
float g;
RNN_COPY(x, st->analysis_mem, FRAME_SIZE); RNN_COPY(x, st->analysis_mem, FRAME_SIZE);
for (i=0;i<FRAME_SIZE;i++) x[FRAME_SIZE + i] = in[i]; for (i=0;i<FRAME_SIZE;i++) x[FRAME_SIZE + i] = in[i];
RNN_COPY(st->analysis_mem, in, FRAME_SIZE); RNN_COPY(st->analysis_mem, in, FRAME_SIZE);
@ -325,7 +334,8 @@ static void frame_analysis(DenoiseState *st, signed char *iexc, float *lpc, kiss
/* Lag windowing. */ /* Lag windowing. */
for (i=1;i<LPC_ORDER+1;i++) ac[i] *= (1 - 6e-5*i*i); for (i=1;i<LPC_ORDER+1;i++) ac[i] *= (1 - 6e-5*i*i);
e = _celt_lpc(lpc, rc, ac, LPC_ORDER); e = _celt_lpc(lpc, rc, ac, LPC_ORDER);
g_1 = sqrt(FRAME_SIZE/(1e-10+e)); g = sqrt((1e-10+e)*(1./FRAME_SIZE));
g_1 = 1./g;
#if 0 #if 0
for(i=0;i<WINDOW_SIZE;i++) printf("%f ", x[i]); for(i=0;i<WINDOW_SIZE;i++) printf("%f ", x[i]);
printf("\n"); printf("\n");
@ -343,6 +353,8 @@ static void frame_analysis(DenoiseState *st, signed char *iexc, float *lpc, kiss
z = &x0[i]+FRAME_SIZE/2; z = &x0[i]+FRAME_SIZE/2;
tmp = z[0]; tmp = z[0];
for (j=0;j<LPC_ORDER;j++) tmp += lpc[j]*z[-1-j]; for (j=0;j<LPC_ORDER;j++) tmp += lpc[j]*z[-1-j];
pcm[i] = float2short(z[0]);
pred[i] = float2short(z[0] - tmp);
nexc = (int)floor(.5 + 16*g_1*tmp); nexc = (int)floor(.5 + 16*g_1*tmp);
nexc = IMAX(-128, IMIN(127, nexc)); nexc = IMAX(-128, IMIN(127, nexc));
iexc[i] = nexc; iexc[i] = nexc;
@ -357,9 +369,10 @@ static void frame_analysis(DenoiseState *st, signed char *iexc, float *lpc, kiss
X[i].r = X[i].i = 0; X[i].r = X[i].i = 0;
#endif #endif
compute_band_energy(Ex, X); compute_band_energy(Ex, X);
return g;
} }
static int compute_frame_features(DenoiseState *st, signed char *iexc, kiss_fft_cpx *X, kiss_fft_cpx *P, static int compute_frame_features(DenoiseState *st, signed char *iexc, short *pred, short *pcm, kiss_fft_cpx *X, kiss_fft_cpx *P,
float *Ex, float *Ep, float *Exp, float *features, const float *in) { float *Ex, float *Ep, float *Exp, float *features, const float *in) {
int i; int i;
float E = 0; float E = 0;
@ -371,7 +384,8 @@ static int compute_frame_features(DenoiseState *st, signed char *iexc, kiss_fft_
float gain; float gain;
float tmp[NB_BANDS]; float tmp[NB_BANDS];
float follow, logMax; float follow, logMax;
frame_analysis(st, iexc, lpc, X, Ex, in); float g;
g = frame_analysis(st, iexc, pred, pcm, lpc, X, Ex, in);
RNN_MOVE(st->pitch_buf, &st->pitch_buf[FRAME_SIZE], PITCH_BUF_SIZE-FRAME_SIZE); RNN_MOVE(st->pitch_buf, &st->pitch_buf[FRAME_SIZE], PITCH_BUF_SIZE-FRAME_SIZE);
RNN_COPY(&st->pitch_buf[PITCH_BUF_SIZE-FRAME_SIZE], in, FRAME_SIZE); RNN_COPY(&st->pitch_buf[PITCH_BUF_SIZE-FRAME_SIZE], in, FRAME_SIZE);
//pre[0] = &st->pitch_buf[0]; //pre[0] = &st->pitch_buf[0];
@ -419,7 +433,8 @@ static int compute_frame_features(DenoiseState *st, signed char *iexc, kiss_fft_
#endif #endif
features[2*NB_BANDS] = .01*(pitch_index-200); features[2*NB_BANDS] = .01*(pitch_index-200);
features[2*NB_BANDS+1] = gain; features[2*NB_BANDS+1] = gain;
for (i=0;i<LPC_ORDER;i++) features[2*NB_BANDS+2+i] = lpc[i]; features[2*NB_BANDS+2] = log10(g);
for (i=0;i<LPC_ORDER;i++) features[2*NB_BANDS+3+i] = lpc[i];
#if 0 #if 0
for (i=0;i<NB_FEATURES;i++) printf("%f ", features[i]); for (i=0;i<NB_FEATURES;i++) printf("%f ", features[i]);
printf("\n"); printf("\n");
@ -505,11 +520,11 @@ float rnnoise_process_frame(DenoiseState *st, float *out, const float *in) {
float g[NB_BANDS]; float g[NB_BANDS];
float gf[FREQ_SIZE]={1}; float gf[FREQ_SIZE]={1};
float vad_prob = 0; float vad_prob = 0;
int silence; int silence=0;
static const float a_hp[2] = {-1.99599, 0.99600}; static const float a_hp[2] = {-1.99599, 0.99600};
static const float b_hp[2] = {-2, 1}; static const float b_hp[2] = {-2, 1};
biquad(x, st->mem_hp_x, in, b_hp, a_hp, FRAME_SIZE); biquad(x, st->mem_hp_x, in, b_hp, a_hp, FRAME_SIZE);
silence = compute_frame_features(st, NULL, X, P, Ex, Ep, Exp, features, x); //silence = compute_frame_features(st, NULL, X, P, Ex, Ep, Exp, features, x);
if (!silence) { if (!silence) {
pitch_filter(X, P, Ex, Ep, Exp, g); pitch_filter(X, P, Ex, Ep, Exp, g);
@ -554,15 +569,23 @@ int main(int argc, char **argv) {
float x[FRAME_SIZE]; float x[FRAME_SIZE];
FILE *f1; FILE *f1;
FILE *fexc; FILE *fexc;
FILE *ffeat;
FILE *fpred;
FILE *fpcm;
signed char iexc[FRAME_SIZE]; signed char iexc[FRAME_SIZE];
short pred[FRAME_SIZE];
short pcm[FRAME_SIZE];
DenoiseState *st; DenoiseState *st;
st = rnnoise_create(); st = rnnoise_create();
if (argc!=3) { if (argc!=6) {
fprintf(stderr, "usage: %s <speech> <exc out>\n", argv[0]); fprintf(stderr, "usage: %s <speech> <exc out> <features out> <prediction out> <pcm out> \n", argv[0]);
return 1; return 1;
} }
f1 = fopen(argv[1], "r"); f1 = fopen(argv[1], "r");
fexc = fopen(argv[2], "w"); fexc = fopen(argv[2], "w");
ffeat = fopen(argv[3], "w");
fpred = fopen(argv[4], "w");
fpcm = fopen(argv[5], "w");
while (1) { while (1) {
kiss_fft_cpx X[FREQ_SIZE], P[WINDOW_SIZE]; kiss_fft_cpx X[FREQ_SIZE], P[WINDOW_SIZE];
float Ex[NB_BANDS], Ep[NB_BANDS]; float Ex[NB_BANDS], Ep[NB_BANDS];
@ -582,10 +605,12 @@ int main(int argc, char **argv) {
biquad(x, mem_hp_x, x, b_hp, a_hp, FRAME_SIZE); biquad(x, mem_hp_x, x, b_hp, a_hp, FRAME_SIZE);
preemphasis(x, &mem_preemph, x, PREEMPHASIS, FRAME_SIZE); preemphasis(x, &mem_preemph, x, PREEMPHASIS, FRAME_SIZE);
compute_frame_features(st, iexc, X, P, Ex, Ep, Exp, features, x); compute_frame_features(st, iexc, pred, pcm, X, P, Ex, Ep, Exp, features, x);
#if 1 #if 1
fwrite(features, sizeof(float), NB_FEATURES, stdout);
fwrite(iexc, sizeof(signed char), FRAME_SIZE, fexc); fwrite(iexc, sizeof(signed char), FRAME_SIZE, fexc);
fwrite(features, sizeof(float), NB_FEATURES, ffeat);
fwrite(pred, sizeof(short), FRAME_SIZE, fpred);
fwrite(pcm, sizeof(short), FRAME_SIZE, fpcm);
#endif #endif
count++; count++;
} }

1
dnn/test_lpcnet.py
View file

@ -73,6 +73,7 @@ for c in range(1, nb_frames):
#fexc[0, 0, 0] = in_data[f*frame_size + i, 0] #fexc[0, 0, 0] = in_data[f*frame_size + i, 0]
#print(cfeat.shape) #print(cfeat.shape)
p, state = dec.predict([fexc, cfeat[:, fr:fr+1, :], state]) p, state = dec.predict([fexc, cfeat[:, fr:fr+1, :], state])
#p = np.maximum(p-0.003, 0)
p = p/(1e-5 + np.sum(p)) p = p/(1e-5 + np.sum(p))
#print(np.sum(p)) #print(np.sum(p))
iexc[0, 0, 0] = np.argmax(np.random.multinomial(1, p[0,0,:], 1))-128 iexc[0, 0, 0] = np.argmax(np.random.multinomial(1, p[0,0,:], 1))-128

13
dnn/test_wavenet_audio.py
View file

@ -10,11 +10,11 @@ from ulaw import ulaw2lin, lin2ulaw
import keras.backend as K import keras.backend as K
import h5py import h5py
#import tensorflow as tf import tensorflow as tf
#from keras.backend.tensorflow_backend import set_session from keras.backend.tensorflow_backend import set_session
#config = tf.ConfigProto() config = tf.ConfigProto()
#config.gpu_options.per_process_gpu_memory_fraction = 0.44 config.gpu_options.per_process_gpu_memory_fraction = 0.2
#set_session(tf.Session(config=config)) set_session(tf.Session(config=config))
nb_epochs = 40 nb_epochs = 40
batch_size = 64 batch_size = 64
@ -66,7 +66,7 @@ in_data = np.reshape(in_data, (nb_frames*pcm_chunk_size, 1))
out_data = np.reshape(data, (nb_frames*pcm_chunk_size, 1)) out_data = np.reshape(data, (nb_frames*pcm_chunk_size, 1))
model.load_weights('wavenet3e_30.h5') model.load_weights('wavenet3g_30.h5')
order = 16 order = 16
@ -92,6 +92,7 @@ for c in range(1, nb_frames):
#fexc[0, 0, 0] = in_data[f*frame_size + i, 0] #fexc[0, 0, 0] = in_data[f*frame_size + i, 0]
#print(cfeat.shape) #print(cfeat.shape)
p, state = dec.predict([fexc, cfeat[:, fr:fr+1, :], state]) p, state = dec.predict([fexc, cfeat[:, fr:fr+1, :], state])
#p = np.maximum(p-0.003, 0)
p = p/(1e-5 + np.sum(p)) p = p/(1e-5 + np.sum(p))
#print(np.sum(p)) #print(np.sum(p))
iexc[0, 0, 0] = np.argmax(np.random.multinomial(1, p[0,0,:], 1))-128 iexc[0, 0, 0] = np.argmax(np.random.multinomial(1, p[0,0,:], 1))-128

10
dnn/train_wavenet.py
View file

@ -9,11 +9,11 @@ from ulaw import ulaw2lin, lin2ulaw
import keras.backend as K import keras.backend as K
import h5py import h5py
#import tensorflow as tf import tensorflow as tf
#from keras.backend.tensorflow_backend import set_session from keras.backend.tensorflow_backend import set_session
#config = tf.ConfigProto() config = tf.ConfigProto()
#config.gpu_options.per_process_gpu_memory_fraction = 0.44 config.gpu_options.per_process_gpu_memory_fraction = 0.44
#set_session(tf.Session(config=config)) set_session(tf.Session(config=config))
nb_epochs = 40 nb_epochs = 40
batch_size = 64 batch_size = 64

13
dnn/train_wavenet_audio.py
View file

@ -10,11 +10,11 @@ from ulaw import ulaw2lin, lin2ulaw
import keras.backend as K import keras.backend as K
import h5py import h5py
#import tensorflow as tf import tensorflow as tf
#from keras.backend.tensorflow_backend import set_session from keras.backend.tensorflow_backend import set_session
#config = tf.ConfigProto() config = tf.ConfigProto()
#config.gpu_options.per_process_gpu_memory_fraction = 0.44 config.gpu_options.per_process_gpu_memory_fraction = 0.44
#set_session(tf.Session(config=config)) set_session(tf.Session(config=config))
nb_epochs = 40 nb_epochs = 40
batch_size = 64 batch_size = 64
@ -43,6 +43,7 @@ data = data[:nb_frames*pcm_chunk_size]
features = features[:nb_frames*feature_chunk_size*nb_features] features = features[:nb_frames*feature_chunk_size*nb_features]
in_data = np.concatenate([data[0:1], data[:-1]]); in_data = np.concatenate([data[0:1], data[:-1]]);
in_data = in_data + np.random.randint(-1, 1, len(data))
features = np.reshape(features, (nb_frames*feature_chunk_size, nb_features)) features = np.reshape(features, (nb_frames*feature_chunk_size, nb_features))
pitch = 1.*data pitch = 1.*data
@ -67,7 +68,7 @@ features = features[:, :, :nb_used_features]
# f.create_dataset('data', data=in_data[:50000, :, :]) # f.create_dataset('data', data=in_data[:50000, :, :])
# f.create_dataset('feat', data=features[:50000, :, :]) # f.create_dataset('feat', data=features[:50000, :, :])
checkpoint = ModelCheckpoint('wavenet3e_{epoch:02d}.h5') checkpoint = ModelCheckpoint('wavenet3g_{epoch:02d}.h5')
#model.load_weights('wavernn1c_01.h5') #model.load_weights('wavernn1c_01.h5')
model.compile(optimizer=Adam(0.001, amsgrad=True, decay=2e-4), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy']) model.compile(optimizer=Adam(0.001, amsgrad=True, decay=2e-4), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])