Preparing for test vectors

doc/draft-ietf-codec-opus.xml

@@ -6528,11 +6528,51 @@ the latter shall take precedence.
 <t>
 Compliance with this specification means that a decoder's output MUST be
  within the thresholds specified by the opus_compare.c tool (included
- with the code) when compared to the reference implementation. Either the floating-point
+ with the code) when compared to the reference implementation for each of the 
+ test vectors provided (see <xref target="test-vectors"></xref>). Either the floating-point
  implementation or the fixed-point implementation can be used as a reference and being
- within the threshold for one of the two is sufficient.
+ within the threshold for one of the two is sufficient. In addition, a compilant
+ decoder implementation MUST have the same final range decoder state as that of the
+ reference decoder. 
 </t>
 
+<section title="Testing">
+<t>
+Using the reference code provided in <xref target="ref-implementation"></xref>,
+a mono test vector can be decoded with
+<list>
+<t>opus_demo -d 48000 1 test_mono.bit test_mono.out</t>
+</list>
+
+If the range decoder state is incorrect for one of the frames, the decoder will exit with
+"Error: Range coder state mismatch between encoder and decoder". If the decoder succeeds, then
+the output can be compared with the "reference" output with
+<list>
+<t>opus_compare test_mono.float test_mono.out</t>
+</list>
+or
+<list>
+<t>opus_compare test_mono.fixed test_mono.out</t>
+</list>
+
+For a stereo test vector, the command line for decoding is
+<list>
+<t>opus_demo -d 48000 2 test_stereo.bin test_stereo.out</t>
+</list>
+
+and the output can be compared with the reference output with
+<list>
+<t>opus_compare -s test_stereo.float test_stereo.out</t>
+</list>
+or
+<list>
+<t>opus_compare -s test_stereo.fixed test_stereo.out</t>
+</list>
+</t>
+
+</section>
+
+<section title="Opus Custom">
 <t>
 To complement the Opus specification, the "Opus Custom" codec is defined to
 handle special sample rates and frame rates that are not supported by the
@@ -6543,6 +6583,7 @@ be compatible with the "main" Opus codec. In Opus Custom operation,
 only the CELT layer is available, which is available using the celt_* function
 calls in celt.h.
 </t>
+</section>
 
 </section>
 
@@ -6891,6 +6932,11 @@ Development snapshots are provided at
 </t>
 </section>
 
+<section anchor="test-vectors" title="Test vectors">
+<t>
+</t>
+</section>
+
 </section>
  
 <section anchor="self-delimiting-framing" title="Self-Delimiting Framing">

tests/run_vectors.sh

@@ -0,0 +1,89 @@
+#!/bin/sh
+
+CMD_PATH=$1
+VECTOR_PATH=$2
+
+OPUS_DEMO=$CMD_PATH/opus_demo
+OPUS_COMPARE=$CMD_PATH/opus_compare
+
+if [ -d $VECTOR_PATH ]; then
+    echo Test vectors found in $VECTOR_PATH
+else
+    echo No test vectors found
+    #Don't make the test fail here because the test vectors will be 
+    #distributed separateyl
+    exit 0
+fi
+
+if [ -x $OPUS_DEMO ]; then
+    echo Decoding with $OPUS_DEMO
+else
+    echo ERROR: Decoder not found: $OPUS_DEMO
+    exit 1
+fi
+
+echo "=============="
+echo Testing mono
+echo "=============="
+echo
+
+for file in test1_mono
+do
+    if [ -e $VECTOR_PATH/$file.bit ]; then
+        echo Testing $file
+    else 
+        echo Bitstream file not found: $file
+    fi
+    if $OPUS_DEMO -d 48000 1 $VECTOR_PATH/$file.bit tmp.out > /dev/null 2>&1; then
+        echo successfully decoded
+    else
+        echo ERROR: decoding failed
+        exit 1
+    fi
+    $OPUS_COMPARE $VECTOR_PATH/$file.float tmp.out > /dev/null 2>&1
+    float_ret=$?
+    $OPUS_COMPARE $VECTOR_PATH/$file.fixed tmp.out > /dev/null 2>&1
+    fixed_ret=$?
+    if [ "$float_ret" -eq "0" -o "$fixed_ret" -eq "0" ]; then
+        echo output matches reference
+    else
+        echo ERROR: output does not match reference
+        exit 1
+    fi
+    echo
+done
+
+echo "=============="
+echo Testing stereo
+echo "=============="
+echo
+
+for file in test1_stereo
+do
+    if [ -e $VECTOR_PATH/$file.bit ]; then
+        echo Testing $file
+    else 
+        echo Bitstream file not found: $file
+    fi
+    if $OPUS_DEMO -d 48000 2 $VECTOR_PATH/$file.bit tmp.out > /dev/null 2>&1; then
+        echo successfully decoded
+    else
+        echo ERROR: decoding failed
+        exit 1
+    fi
+    $OPUS_COMPARE -s $VECTOR_PATH/$file.float tmp.out > /dev/null 2>&1
+    float_ret=$?
+    $OPUS_COMPARE -s $VECTOR_PATH/$file.fixed tmp.out > /dev/null 2>&1
+    fixed_ret=$?
+    if [ "$float_ret" -eq "0" -o "$fixed_ret" -eq "0" ]; then
+        echo output matches reference
+    else
+        echo ERROR: output does not match reference
+        exit 1
+    fi
+    echo
+done
+
+
+
+echo All tests have passed successfully