commit 55cd1dd7c8d0a3de907d22e0f12718733f4e41d9 Author: Jerome Jiang <jianj@google.com> Date: Thu Oct 26 15:24:17 2017 -0700 DO NOT MERGE | libvpx: Fix OOB caused by odd frame width. Keep behaviors unchanged without external allocation. Bug: b/64710201 Test: poc provided in the bug. Change-Id: I319a47b64c7cfa7bb47ad01c702be6f2acffe3a4 (cherry picked from commit 51721c34847e6b4f935d5ecb1b44931c7716fd59) (cherry picked from commit 28a641201287106fbb73dfbad35dae2756cde265) Index: libvpx-1.3.0/vpx/src/vpx_image.c =================================================================== --- libvpx-1.3.0.orig/vpx/src/vpx_image.c +++ libvpx-1.3.0/vpx/src/vpx_image.c @@ -10,6 +10,7 @@ #include <stdlib.h> +#include <stdint.h> #include <string.h> #include "vpx/vpx_image.h" #include "vpx/vpx_integer.h" @@ -124,11 +125,10 @@ static vpx_image_t *img_alloc_helper(vpx break; } - /* Calculate storage sizes given the chroma subsampling */ - align = (1 << xcs) - 1; - w = (d_w + align) & ~align; - align = (1 << ycs) - 1; - h = (d_h + align) & ~align; + /* Calculate storage sizes. If the buffer was allocated externally, the width + * and height shouldn't be adjusted. */ + w = d_w; + h = d_h; s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8; s = (s + stride_align - 1) & ~(stride_align - 1); @@ -146,8 +145,18 @@ static vpx_image_t *img_alloc_helper(vpx img->img_data = img_data; if (!img_data) { - const uint64_t alloc_size = (fmt & VPX_IMG_FMT_PLANAR) ? - (uint64_t)h * s * bps / 8 : (uint64_t)h * s; + uint64_t alloc_size; + /* Calculate storage sizes given the chroma subsampling */ + align = xcs ? (1 << xcs) - 1 : 1; + w = (d_w + align - 1) & ~(align - 1); + align = ycs ? (1 << ycs) - 1 : 1; + h = (d_h + align - 1) & ~(align - 1); + + s = (fmt & VPX_IMG_FMT_PLANAR) ? w : bps * w / 8; + s = (s + stride_align - 1) & ~(stride_align - 1); + + alloc_size = (fmt & VPX_IMG_FMT_PLANAR) ? (uint64_t)h * s * bps / 8 + : (uint64_t)h * s; if (alloc_size != (size_t)alloc_size) goto fail;