Description: fix denial of service and possible code execution via
 malformed file containing QDM2 stream
Origin: upstream, http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=491eaf35ae1f9b619441314bec33766e31580184
Origin: upstream, http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=291d74a46d32183653db07818c7b3407fd50a288
Origin: upstream, http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=14db3af4f26dad8e6ddf2147e96ccc710952ad4d
Origin: backport, http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=895d258e9ba065d035dd30dbc622423031f0185c

Index: ffmpeg-0.6/libavcodec/qdm2.c
===================================================================
--- ffmpeg-0.6.orig/libavcodec/qdm2.c	2011-12-21 10:46:44.980455995 -0500
+++ ffmpeg-0.6/libavcodec/qdm2.c	2011-12-21 12:12:04.972616378 -0500
@@ -75,6 +75,7 @@
 #define SAMPLES_NEEDED_2(why) \
      av_log (NULL,AV_LOG_INFO,"This file triggers some missing code. Please contact the developers.\nPosition: %s\n",why);
 
+#define QDM2_MAX_FRAME_SIZE 512
 
 typedef int8_t sb_int8_array[2][30][64];
 
@@ -167,7 +168,7 @@
     /// I/O data
     const uint8_t *compressed_data;
     int compressed_size;
-    float output_buffer[1024];
+    float output_buffer[QDM2_MAX_FRAME_SIZE * MPA_MAX_CHANNELS * 2];
 
     /// Synthesis filter
     DECLARE_ALIGNED(16, MPA_INT, synth_buf)[MPA_MAX_CHANNELS][512*2];
@@ -1329,7 +1330,7 @@
     local_int_10 = 1 << (q->group_order - duration - 1);
     offset = 1;
 
-    while (1) {
+    while (get_bits_left(gb)>0) {
         if (q->superblocktype_2_3) {
             while ((n = qdm2_get_vlc(gb, &vlc_tab_fft_tone_offset[local_int_8], 1, 2)) < 2) {
                 offset = 1;
@@ -1355,6 +1356,8 @@
             return;
 
         local_int_14 = (offset >> local_int_8);
+        if (local_int_14 >= FF_ARRAY_ELEMS(fft_level_index_table))
+            return;
 
         if (q->nb_channels > 1) {
             channel = get_bits1(gb);
@@ -1799,6 +1802,8 @@
 
     avctx->channels = s->nb_channels = s->channels = AV_RB32(extradata);
     extradata += 4;
+    if (s->channels > MPA_MAX_CHANNELS)
+        return AVERROR_INVALIDDATA;
 
     avctx->sample_rate = AV_RB32(extradata);
     extradata += 4;
@@ -1820,6 +1825,8 @@
     // something like max decodable tones
     s->group_order = av_log2(s->group_size) + 1;
     s->frame_size = s->group_size / 16; // 16 iterations per super block
+    if (s->frame_size > QDM2_MAX_FRAME_SIZE)
+        return AVERROR_INVALIDDATA;
 
     s->sub_sampling = s->fft_order - 7;
     s->frequency_range = 255 / (1 << (2 - s->sub_sampling));
@@ -1888,6 +1895,9 @@
     int ch, i;
     const int frame_size = (q->frame_size * q->channels);
 
+    if((unsigned)frame_size > FF_ARRAY_ELEMS(q->output_buffer)/2)
+        return;
+
     /* select input buffer */
     q->compressed_data = in;
     q->compressed_size = q->checksum_size;