multistream: improve arg check

Avoid undefined behavior (signed arithmetic overflow) or
implementation-defined behavior (malloc(0)) on out-of-range arguments,
e.g. opus_multistream_encoder_create(48000, 2, 2147483647, 1, ...)
or opus_multistream_surround_encoder_create(48000, 3, 0, ...).

Signed-off-by: Jean-Marc Valin <jmvalin@jmvalin.ca>
This commit is contained in:
Mark Harris 2014-11-27 08:48:09 -08:00 committed by Jean-Marc Valin
parent d10755e95c
commit 25b27a9c16
2 changed files with 13 additions and 5 deletions

View file

@ -75,7 +75,7 @@ int opus_multistream_decoder_init(
char *ptr; char *ptr;
if ((channels>255) || (channels<1) || (coupled_streams>streams) || if ((channels>255) || (channels<1) || (coupled_streams>streams) ||
(coupled_streams+streams>255) || (streams<1) || (coupled_streams<0)) (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams))
return OPUS_BAD_ARG; return OPUS_BAD_ARG;
st->layout.nb_channels = channels; st->layout.nb_channels = channels;
@ -119,7 +119,7 @@ OpusMSDecoder *opus_multistream_decoder_create(
int ret; int ret;
OpusMSDecoder *st; OpusMSDecoder *st;
if ((channels>255) || (channels<1) || (coupled_streams>streams) || if ((channels>255) || (channels<1) || (coupled_streams>streams) ||
(coupled_streams+streams>255) || (streams<1) || (coupled_streams<0)) (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams))
{ {
if (error) if (error)
*error = OPUS_BAD_ARG; *error = OPUS_BAD_ARG;

View file

@ -408,7 +408,7 @@ static int opus_multistream_encoder_init_impl(
char *ptr; char *ptr;
if ((channels>255) || (channels<1) || (coupled_streams>streams) || if ((channels>255) || (channels<1) || (coupled_streams>streams) ||
(coupled_streams+streams>255) || (streams<1) || (coupled_streams<0)) (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams))
return OPUS_BAD_ARG; return OPUS_BAD_ARG;
st->layout.nb_channels = channels; st->layout.nb_channels = channels;
@ -530,7 +530,7 @@ OpusMSEncoder *opus_multistream_encoder_create(
int ret; int ret;
OpusMSEncoder *st; OpusMSEncoder *st;
if ((channels>255) || (channels<1) || (coupled_streams>streams) || if ((channels>255) || (channels<1) || (coupled_streams>streams) ||
(coupled_streams+streams>255) || (streams<1) || (coupled_streams<0)) (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams))
{ {
if (error) if (error)
*error = OPUS_BAD_ARG; *error = OPUS_BAD_ARG;
@ -566,6 +566,7 @@ OpusMSEncoder *opus_multistream_surround_encoder_create(
) )
{ {
int ret; int ret;
opus_int32 size;
OpusMSEncoder *st; OpusMSEncoder *st;
if ((channels>255) || (channels<1)) if ((channels>255) || (channels<1))
{ {
@ -573,7 +574,14 @@ OpusMSEncoder *opus_multistream_surround_encoder_create(
*error = OPUS_BAD_ARG; *error = OPUS_BAD_ARG;
return NULL; return NULL;
} }
st = (OpusMSEncoder *)opus_alloc(opus_multistream_surround_encoder_get_size(channels, mapping_family)); size = opus_multistream_surround_encoder_get_size(channels, mapping_family);
if (!size)
{
if (error)
*error = OPUS_UNIMPLEMENTED;
return NULL;
}
st = (OpusMSEncoder *)opus_alloc(size);
if (st==NULL) if (st==NULL)
{ {
if (error) if (error)