From 57f5681987bf1efd2718c712eceebf4b8e45054f Mon Sep 17 00:00:00 2001
From: Jean-Marc Valin <jmvalin@amazon.com>
Date: Mon, 24 Jan 2022 16:22:29 -0500
Subject: [PATCH] Add swish activation support

---
 dnn/nnet.c | 13 +++++++++++++
 dnn/nnet.h |  1 +
 2 files changed, 14 insertions(+)

diff --git a/dnn/nnet.c b/dnn/nnet.c
index 31472033..9bbd14bf 100644
--- a/dnn/nnet.c
+++ b/dnn/nnet.c
@@ -46,6 +46,17 @@
 
 #define SOFTMAX_HACK
 
+#define MAX_ACTIVATIONS (4096)
+
+static OPUS_INLINE void vec_swish(float *y, const float *x, int N)
+{
+   int i;
+   float tmp[MAX_ACTIVATIONS];
+   celt_assert(N <= MAX_ACTIVATIONS);
+   vec_sigmoid(tmp, x, N);
+   for (i=0;i<N;i++)
+      y[i] = x[i]*tmp[i];
+}
 
 static OPUS_INLINE float relu(float x)
 {
@@ -75,6 +86,8 @@ void compute_activation(float *output, const float *input, int N, int activation
       vec_sigmoid(output, input, N);
    } else if (activation == ACTIVATION_TANH) {
       vec_tanh(output, input, N);
+   } else if (activation == ACTIVATION_SWISH) {
+      vec_swish(output, input, N);
    } else if (activation == ACTIVATION_RELU) {
       for (i=0;i<N;i++)
          output[i] = relu(input[i]);
diff --git a/dnn/nnet.h b/dnn/nnet.h
index 456c3266..a5d50e6c 100644
--- a/dnn/nnet.h
+++ b/dnn/nnet.h
@@ -36,6 +36,7 @@
 #define ACTIVATION_TANH    2
 #define ACTIVATION_RELU    3
 #define ACTIVATION_SOFTMAX 4
+#define ACTIVATION_SWISH   5
 
 typedef struct {
   const float *bias;