CVE-2023-44488: Improper Handling of Exceptional Conditions (CWE-755)
value: VP9 in libvpx before 1.13.1 mishandles widths, leading to a crash related to encoding.
CWE description: The product does not handle or incorrectly handles an exceptional condition.
CVSS v3 base score: 7.5
Published: 2023-09-30T20:15Z
Repository: https://github.com/webmproject/libvpx
Language: C
Files changed: test/resize_test.cc, vp9/common/vp9_alloccommon.c, vp9/encoder/vp9_encoder.c
Commit date: 2023-09-27 11:21:11 -0400
Commit message: Fix bug with smaller width bigger size

Fixed previous patch that clusterfuzz failed on.

Bug: webm:1642
Change-Id: If0e08e72abd2e042efe4dcfac21e4cc51afdfdb9

Diff: diff --git a/test/resize_test.cc b/test/resize_test.cc
index fd1c2a92de..20ad2229b4 100644
--- a/test/resize_test.cc
+++ b/test/resize_test.cc
@@ -100,15 +100,12 @@ void ScaleForFrameNumber(unsigned int frame, unsigned int initial_w,
 
   if (smaller_width_larger_size_) {
     if (frame < 30) {
       return;
     }
-    if (frame < 100) {
-      *w = initial_w * 7 / 10;
-      *h = initial_h * 16 / 10;
-      return;
-    }
+    *w = initial_w * 7 / 10;
+    *h = initial_h * 16 / 10;
     return;
   }
   if (frame < 10) {
     return;
   }
@@ -557,13 +554,11 @@ TEST_P(ResizeRealtimeTest, TestExternalResizeWorks) {
         << "Frame " << frame << " had unexpected height";
     EXPECT_EQ(static_cast<unsigned int>(0), GetMismatchFrames());
   }
 }
 
-// TODO(https://crbug.com/webm/1642): This causes a segfault in
-// init_encode_frame_mb_context().
-TEST_P(ResizeRealtimeTest, DISABLED_TestExternalResizeSmallerWidthBiggerSize) {
+TEST_P(ResizeRealtimeTest, TestExternalResizeSmallerWidthBiggerSize) {
   ResizingVideoSource video;
   video.flag_codec_ = true;
   video.smaller_width_larger_size_ = true;
   DefaultConfig();
   // Disable internal resize for this test.
diff --git a/vp9/common/vp9_alloccommon.c b/vp9/common/vp9_alloccommon.c
index e53883f621..9e73e40ea0 100644
--- a/vp9/common/vp9_alloccommon.c
+++ b/vp9/common/vp9_alloccommon.c
@@ -133,17 +133,10 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
   new_mi_size = cm->mi_stride * calc_mi_size(cm->mi_rows);
   if (cm->mi_alloc_size < new_mi_size) {
     cm->free_mi(cm);
     if (cm->alloc_mi(cm, new_mi_size)) goto fail;
   }
-
-  if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
-    // Create the segmentation map structure and set to 0.
-    free_seg_map(cm);
-    if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
-  }
-
   if (cm->above_context_alloc_cols < cm->mi_cols) {
     vpx_free(cm->above_context);
     cm->above_context = (ENTROPY_CONTEXT *)vpx_calloc(
         2 * mi_cols_aligned_to_sb(cm->mi_cols) * MAX_MB_PLANE,
         sizeof(*cm->above_context));
@@ -154,10 +147,16 @@ int vp9_alloc_context_buffers(VP9_COMMON *cm, int width, int height) {
         mi_cols_aligned_to_sb(cm->mi_cols), sizeof(*cm->above_seg_context));
     if (!cm->above_seg_context) goto fail;
     cm->above_context_alloc_cols = cm->mi_cols;
   }
 
+  if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
+    // Create the segmentation map structure and set to 0.
+    free_seg_map(cm);
+    if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
+  }
+
   if (vp9_alloc_loop_filter(cm)) goto fail;
 
   return 0;
 
 fail:
diff --git a/vp9/encoder/vp9_encoder.c b/vp9/encoder/vp9_encoder.c
index 69a4e3c314..e3ba294c32 100644
--- a/vp9/encoder/vp9_encoder.c
+++ b/vp9/encoder/vp9_encoder.c
@@ -2045,10 +2045,21 @@ static void alloc_copy_partition_data(VP9_COMP *cpi) {
         (uint8_t *)vpx_calloc((cm->mi_stride >> 3) * ((cm->mi_rows >> 3) + 1),
                               sizeof(*cpi->copied_frame_cnt)));
   }
 }
 
+static void free_copy_partition_data(VP9_COMP *cpi) {
+  vpx_free(cpi->prev_partition);
+  cpi->prev_partition = NULL;
+  vpx_free(cpi->prev_segment_id);
+  cpi->prev_segment_id = NULL;
+  vpx_free(cpi->prev_variance_low);
+  cpi->prev_variance_low = NULL;
+  vpx_free(cpi->copied_frame_cnt);
+  cpi->copied_frame_cnt = NULL;
+}
+
 void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
   VP9_COMMON *const cm = &cpi->common;
   RATE_CONTROL *const rc = &cpi->rc;
   int last_w = cpi->oxcf.width;
   int last_h = cpi->oxcf.height;
@@ -2124,10 +2135,12 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
     int new_mi_size = 0;
     vp9_set_mb_mi(cm, cm->width, cm->height);
     new_mi_size = cm->mi_stride * calc_mi_size(cm->mi_rows);
     if (cm->mi_alloc_size < new_mi_size) {
       vp9_free_context_buffers(cm);
+      vp9_free_pc_tree(&cpi->td);
+      vpx_free(cpi->mbmi_ext_base);
       alloc_compressor_data(cpi);
       realloc_segmentation_maps(cpi);
       cpi->initial_width = cpi->initial_height = 0;
       cpi->external_resize = 0;
     } else if (cm->mi_alloc_size == new_mi_size &&
@@ -2142,12 +2155,22 @@ void vp9_change_config(struct VP9_COMP *cpi, const VP9EncoderConfig *oxcf) {
   if (cm->current_video_frame == 0 || last_w != cpi->oxcf.width ||
       last_h != cpi->oxcf.height)
     update_frame_size(cpi);
 
   if (last_w != cpi->oxcf.width || last_h != cpi->oxcf.height) {
-    memset(cpi->consec_zero_mv, 0,
-           cm->mi_rows * cm->mi_cols * sizeof(*cpi->consec_zero_mv));
+    vpx_free(cpi->consec_zero_mv);
+    CHECK_MEM_ERROR(
+        &cm->error, cpi->consec_zero_mv,
+        vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(*cpi->consec_zero_mv)));
+
+    vpx_free(cpi->skin_map);
+    CHECK_MEM_ERROR(
+        &cm->error, cpi->skin_map,
+        vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(cpi->skin_map[0])));
+
+    free_copy_partition_data(cpi);
+    alloc_copy_partition_data(cpi);
     if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ)
       vp9_cyclic_refresh_reset_resize(cpi);
     rc->rc_1_frame = 0;
     rc->rc_2_frame = 0;
   }

Vulnerable code:     if (frame < 100) {
      *w = initial_w * 7 / 10;
      *h = initial_h * 16 / 10;
      return;
    }
// TODO(https://crbug.com/webm/1642): This causes a segfault in
// init_encode_frame_mb_context().
TEST_P(ResizeRealtimeTest, DISABLED_TestExternalResizeSmallerWidthBiggerSize) {

  if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
    // Create the segmentation map structure and set to 0.
    free_seg_map(cm);
    if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
  }

    memset(cpi->consec_zero_mv, 0,
           cm->mi_rows * cm->mi_cols * sizeof(*cpi->consec_zero_mv));
Fixed code:     *w = initial_w * 7 / 10;
    *h = initial_h * 16 / 10;
TEST_P(ResizeRealtimeTest, TestExternalResizeSmallerWidthBiggerSize) {
  if (cm->seg_map_alloc_size < cm->mi_rows * cm->mi_cols) {
    // Create the segmentation map structure and set to 0.
    free_seg_map(cm);
    if (alloc_seg_map(cm, cm->mi_rows * cm->mi_cols)) goto fail;
  }

static void free_copy_partition_data(VP9_COMP *cpi) {
  vpx_free(cpi->prev_partition);
  cpi->prev_partition = NULL;
  vpx_free(cpi->prev_segment_id);
  cpi->prev_segment_id = NULL;
  vpx_free(cpi->prev_variance_low);
  cpi->prev_variance_low = NULL;
  vpx_free(cpi->copied_frame_cnt);
  cpi->copied_frame_cnt = NULL;
}

      vp9_free_pc_tree(&cpi->td);
      vpx_free(cpi->mbmi_ext_base);
    vpx_free(cpi->consec_zero_mv);
    CHECK_MEM_ERROR(
        &cm->error, cpi->consec_zero_mv,
        vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(*cpi->consec_zero_mv)));

    vpx_free(cpi->skin_map);
    CHECK_MEM_ERROR(
        &cm->error, cpi->skin_map,
        vpx_calloc(cm->mi_rows * cm->mi_cols, sizeof(cpi->skin_map[0])));

    free_copy_partition_data(cpi);
    alloc_copy_partition_data(cpi);