The encoder would crash in the PVQ search if fed NaNs via the float interface. This patch protects against it in two sufficient ways: Making the PVQ search robust against NaNs and by squashing NaNs to zero on input.

Thanks to David Richards for reporting this failure mode.

libcelt/celt.c

@@ -1086,6 +1086,8 @@ int celt_encode_with_ec_float(CELTEncoder * restrict st, const celt_sig * pcm, i
 
             x = SCALEIN(*pcmp);
 #ifndef FIXED_POINT
+            if (!(x==x))
+               x = 0;
             if (st->clip)
                x = MAX32(-65536.f, MIN32(65536.f,x));
 #endif

libcelt/vq.c

@@ -223,7 +223,9 @@ unsigned alg_quant(celt_norm *X, int N, int K, int spread, int B,
 #ifdef FIXED_POINT
       if (sum <= K)
 #else
-      if (sum <= EPSILON)
+      /* Prevents infinities and NaNs from causing too many pulses
+         to be allocated. 64 is an approximation of infinity here. */
+      if (!(sum > EPSILON && sum < 64))
 #endif
       {
          X[0] = QCONST16(1.f,14);