From a486d800b60d0af4cc0836bf7ed8f21e12974129 Mon Sep 17 00:00:00 2001
From: James Zern <jzern@google.com>
Date: Wed, 22 Feb 2023 22:15:47 -0800
Subject: [PATCH] EncodeAlphaInternal: clear result->bw on error

This avoids a double free should the function fail prior to
VP8BitWriterInit() and a previous trial result's buffer carried over.
Previously in ApplyFiltersAndEncode() trial.bw (with a previous
iteration's buffer) would be freed, followed by best.bw pointing to the
same buffer.

Since:
187d379d add a fallback to ALPHA_NO_COMPRESSION

In addition, check the return value of VP8BitWriterInit() in this
function.

Bug: webp:603
Change-Id: Ic258381ee26c8c16bc211d157c8153831c8c6910
---
 src/enc/alpha_enc.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/enc/alpha_enc.c b/src/enc/alpha_enc.c
index f7c02690e3..7d205586fe 100644
--- a/src/enc/alpha_enc.c
+++ b/src/enc/alpha_enc.c
@@ -13,6 +13,7 @@
 
 #include <assert.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include "src/enc/vp8i_enc.h"
 #include "src/dsp/dsp.h"
@@ -148,6 +149,7 @@ static int EncodeAlphaInternal(const uint8_t* const data, int width, int height,
       }
     } else {
       VP8LBitWriterWipeOut(&tmp_bw);
+      memset(&result->bw, 0, sizeof(result->bw));
       return 0;
     }
   }
@@ -162,7 +164,7 @@ static int EncodeAlphaInternal(const uint8_t* const data, int width, int height,
   header = method | (filter << 2);
   if (reduce_levels) header |= ALPHA_PREPROCESSED_LEVELS << 4;
 
-  VP8BitWriterInit(&result->bw, ALPHA_HEADER_LEN + output_size);
+  if (!VP8BitWriterInit(&result->bw, ALPHA_HEADER_LEN + output_size)) ok = 0;
   ok = ok && VP8BitWriterAppend(&result->bw, &header, ALPHA_HEADER_LEN);
   ok = ok && VP8BitWriterAppend(&result->bw, output, output_size);