CVE-2023-5217: Out-of-bounds Write (CWE-787)
value: Heap buffer overflow in vp8 encoding in libvpx in Google Chrome prior to 117.0.5938.132 and libvpx 1.13.1 allowed a remote attacker to potentially exploit heap corruption via a crafted HTML page. (Chromium security severity: High)
CWE description: The product writes data past the end, or before the beginning, of the intended buffer.
CVSS v3 base score: 8.8
Published: 2023-09-28T16:15Z
Repository: https://github.com/webmproject/libvpx
Language: C++
Files changed: test/encode_api_test.cc
Commit date: 2023-09-25 19:28:08 -0700
Commit message: encode_api_test: add ConfigResizeChangeThreadCount

Update thread counts and resolution to ensure allocations are updated
correctly. VP8 is disabled to avoid a crash.

Bug: chromium:1486441
Change-Id: Ie89776d9818d27dc351eff298a44c699e850761b

Diff: diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
index c8bd7daa4a..a8a4df2ddf 100644
--- a/test/encode_api_test.cc
+++ b/test/encode_api_test.cc
@@ -306,11 +306,10 @@ TEST(EncodeAPI, SetRoi) {
   }
 }
 
 void InitCodec(const vpx_codec_iface_t &iface, int width, int height,
                vpx_codec_ctx_t *enc, vpx_codec_enc_cfg_t *cfg) {
-  ASSERT_EQ(vpx_codec_enc_config_default(&iface, cfg, 0), VPX_CODEC_OK);
   cfg->g_w = width;
   cfg->g_h = height;
   cfg->g_lag_in_frames = 0;
   cfg->g_pass = VPX_RC_ONE_PASS;
   ASSERT_EQ(vpx_codec_enc_init(enc, &iface, cfg, 0), VPX_CODEC_OK);
@@ -344,10 +343,11 @@ TEST(EncodeAPI, ConfigChangeThreadCount) {
       struct Encoder {
         ~Encoder() { EXPECT_EQ(vpx_codec_destroy(&ctx), VPX_CODEC_OK); }
         vpx_codec_ctx_t ctx = {};
       } enc;
 
+      ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK);
       EXPECT_NO_FATAL_FAILURE(
           InitCodec(*iface, kWidth, kHeight, &enc.ctx, &cfg));
       if (IsVP9(iface)) {
         EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_TILE_COLUMNS, 6),
                   VPX_CODEC_OK);
@@ -362,10 +362,58 @@ TEST(EncodeAPI, ConfigChangeThreadCount) {
       }
     }
   }
 }
 
+TEST(EncodeAPI, ConfigResizeChangeThreadCount) {
+  constexpr int kInitWidth = 1024;
+  constexpr int kInitHeight = 1024;
+
+  for (const auto *iface : kCodecIfaces) {
+    SCOPED_TRACE(vpx_codec_iface_name(iface));
+    if (!IsVP9(iface)) {
+      GTEST_SKIP() << "TODO(https://crbug.com/1486441) remove this condition "
+                      "after VP8 is fixed.";
+    }
+    for (int i = 0; i < (IsVP9(iface) ? 2 : 1); ++i) {
+      vpx_codec_enc_cfg_t cfg = {};
+      struct Encoder {
+        ~Encoder() { EXPECT_EQ(vpx_codec_destroy(&ctx), VPX_CODEC_OK); }
+        vpx_codec_ctx_t ctx = {};
+      } enc;
+
+      ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK);
+      // Start in threaded mode to ensure resolution and thread related
+      // allocations are updated correctly across changes in resolution and
+      // thread counts. See https://crbug.com/1486441.
+      cfg.g_threads = 4;
+      EXPECT_NO_FATAL_FAILURE(
+          InitCodec(*iface, kInitWidth, kInitHeight, &enc.ctx, &cfg));
+      if (IsVP9(iface)) {
+        EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_TILE_COLUMNS, 6),
+                  VPX_CODEC_OK);
+        EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_ROW_MT, i),
+                  VPX_CODEC_OK);
+      }
+
+      cfg.g_w = 1000;
+      cfg.g_h = 608;
+      EXPECT_EQ(vpx_codec_enc_config_set(&enc.ctx, &cfg), VPX_CODEC_OK)
+          << vpx_codec_error_detail(&enc.ctx);
+
+      cfg.g_w = 16;
+      cfg.g_h = 720;
+
+      for (const auto threads : { 1, 4, 8, 6, 2, 1 }) {
+        cfg.g_threads = threads;
+        EXPECT_NO_FATAL_FAILURE(EncodeWithConfig(cfg, &enc.ctx))
+            << "iteration: " << i << " threads: " << threads;
+      }
+    }
+  }
+}
+
 #if CONFIG_VP9_ENCODER
 class EncodeApiGetTplStatsTest
     : public ::libvpx_test::EncoderTest,
       public ::testing::TestWithParam<const libvpx_test::CodecFactory *> {
  public:

Vulnerable code:   ASSERT_EQ(vpx_codec_enc_config_default(&iface, cfg, 0), VPX_CODEC_OK);
Fixed code:       ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK);
TEST(EncodeAPI, ConfigResizeChangeThreadCount) {
  constexpr int kInitWidth = 1024;
  constexpr int kInitHeight = 1024;

  for (const auto *iface : kCodecIfaces) {
    SCOPED_TRACE(vpx_codec_iface_name(iface));
    if (!IsVP9(iface)) {
      GTEST_SKIP() << "TODO(https://crbug.com/1486441) remove this condition "
                      "after VP8 is fixed.";
    }
    for (int i = 0; i < (IsVP9(iface) ? 2 : 1); ++i) {
      vpx_codec_enc_cfg_t cfg = {};
      struct Encoder {
        ~Encoder() { EXPECT_EQ(vpx_codec_destroy(&ctx), VPX_CODEC_OK); }
        vpx_codec_ctx_t ctx = {};
      } enc;

      ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK);
      // Start in threaded mode to ensure resolution and thread related
      // allocations are updated correctly across changes in resolution and
      // thread counts. See https://crbug.com/1486441.
      cfg.g_threads = 4;
      EXPECT_NO_FATAL_FAILURE(
          InitCodec(*iface, kInitWidth, kInitHeight, &enc.ctx, &cfg));
      if (IsVP9(iface)) {
        EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_TILE_COLUMNS, 6),
                  VPX_CODEC_OK);
        EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_ROW_MT, i),
                  VPX_CODEC_OK);
      }

      cfg.g_w = 1000;
      cfg.g_h = 608;
      EXPECT_EQ(vpx_codec_enc_config_set(&enc.ctx, &cfg), VPX_CODEC_OK)
          << vpx_codec_error_detail(&enc.ctx);

      cfg.g_w = 16;
      cfg.g_h = 720;

      for (const auto threads : { 1, 4, 8, 6, 2, 1 }) {
        cfg.g_threads = threads;
        EXPECT_NO_FATAL_FAILURE(EncodeWithConfig(cfg, &enc.ctx))
            << "iteration: " << i << " threads: " << threads;
      }
    }
  }
}