Description: fix denial of service and possible code execution via malformed file containing QDM2 stream Origin: upstream, http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=491eaf35ae1f9b619441314bec33766e31580184 Origin: upstream, http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=291d74a46d32183653db07818c7b3407fd50a288 Origin: upstream, http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=14db3af4f26dad8e6ddf2147e96ccc710952ad4d Origin: backport, http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=895d258e9ba065d035dd30dbc622423031f0185c Index: ffmpeg-0.6/libavcodec/qdm2.c =================================================================== --- ffmpeg-0.6.orig/libavcodec/qdm2.c 2011-12-21 10:46:44.980455995 -0500 +++ ffmpeg-0.6/libavcodec/qdm2.c 2011-12-21 12:12:04.972616378 -0500 @@ -75,6 +75,7 @@ #define SAMPLES_NEEDED_2(why) \ av_log (NULL,AV_LOG_INFO,"This file triggers some missing code. Please contact the developers.\nPosition: %s\n",why); +#define QDM2_MAX_FRAME_SIZE 512 typedef int8_t sb_int8_array[2][30][64]; @@ -167,7 +168,7 @@ /// I/O data const uint8_t *compressed_data; int compressed_size; - float output_buffer[1024]; + float output_buffer[QDM2_MAX_FRAME_SIZE * MPA_MAX_CHANNELS * 2]; /// Synthesis filter DECLARE_ALIGNED(16, MPA_INT, synth_buf)[MPA_MAX_CHANNELS][512*2]; @@ -1329,7 +1330,7 @@ local_int_10 = 1 << (q->group_order - duration - 1); offset = 1; - while (1) { + while (get_bits_left(gb)>0) { if (q->superblocktype_2_3) { while ((n = qdm2_get_vlc(gb, &vlc_tab_fft_tone_offset[local_int_8], 1, 2)) < 2) { offset = 1; @@ -1355,6 +1356,8 @@ return; local_int_14 = (offset >> local_int_8); + if (local_int_14 >= FF_ARRAY_ELEMS(fft_level_index_table)) + return; if (q->nb_channels > 1) { channel = get_bits1(gb); @@ -1799,6 +1802,8 @@ avctx->channels = s->nb_channels = s->channels = AV_RB32(extradata); extradata += 4; + if (s->channels > MPA_MAX_CHANNELS) + return AVERROR_INVALIDDATA; avctx->sample_rate = AV_RB32(extradata); extradata += 4; @@ -1820,6 +1825,8 @@ // something like max decodable tones s->group_order = av_log2(s->group_size) + 1; s->frame_size = s->group_size / 16; // 16 iterations per super block + if (s->frame_size > QDM2_MAX_FRAME_SIZE) + return AVERROR_INVALIDDATA; s->sub_sampling = s->fft_order - 7; s->frequency_range = 255 / (1 << (2 - s->sub_sampling)); @@ -1888,6 +1895,9 @@ int ch, i; const int frame_size = (q->frame_size * q->channels); + if((unsigned)frame_size > FF_ARRAY_ELEMS(q->output_buffer)/2) + return; + /* select input buffer */ q->compressed_data = in; q->compressed_size = q->checksum_size;