Commit message: None
CVE: CVE-2016-2450
CWE: CWE-264
Vulnerability description: codecs/on2/enc/SoftVPXEncoder.cpp in libstagefright in mediaserver in Android 4.x before 4.4.4, 5.0.x before 5.0.2, 5.1.x before 5.1.1, and 6.x before 2016-05-01 does not validate OMX buffer sizes, which allows attackers to gain privileges via a crafted application, as demonstrated by obtaining Signature or SignatureOrSystem access, aka internal bug 27569635.

Project: Android

static int GetCPUCoreCount() {
 int cpuCoreCount = 1;
#if defined(_SC_NPROCESSORS_ONLN)
    cpuCoreCount = sysconf(_SC_NPROCESSORS_ONLN);
#else
    cpuCoreCount = sysconf(_SC_NPROC_ONLN);
#endif
    CHECK_GE(cpuCoreCount, 1);
 return cpuCoreCount;
}

Project: Android

static void InitOMXParams(T *params) {
    params->nSize = sizeof(T);
    params->nVersion.s.nVersionMajor = 1;
    params->nVersion.s.nVersionMinor = 1;
    params->nVersion.s.nRevision = 2;
    params->nVersion.s.nStep = 0;
}

Project: Android

SoftVPXEncoder::SoftVPXEncoder(const char *name,
 const OMX_CALLBACKTYPE *callbacks,
                               OMX_PTR appData,
                               OMX_COMPONENTTYPE **component)
 : SoftVideoEncoderOMXComponent(
            name, "video_encoder.vp8", OMX_VIDEO_CodingVP8,
            kProfileLevels, NELEM(kProfileLevels),
 176 /* width */, 144 /* height */,
            callbacks, appData, component),
      mCodecContext(NULL),
      mCodecConfiguration(NULL),
      mCodecInterface(NULL),
      mBitrateUpdated(false),
      mBitrateControlMode(VPX_VBR), // variable bitrate
      mDCTPartitions(0),
      mErrorResilience(OMX_FALSE),
      mLevel(OMX_VIDEO_VP8Level_Version0),
      mKeyFrameInterval(0),
      mMinQuantizer(0),
      mMaxQuantizer(0),
      mTemporalLayers(0),
      mTemporalPatternType(OMX_VIDEO_VPXTemporalLayerPatternNone),
      mTemporalPatternLength(0),
      mTemporalPatternIdx(0),
      mLastTimestamp(0x7FFFFFFFFFFFFFFFLL),
      mConversionBuffer(NULL),
      mKeyFrameRequested(false) {
    memset(mTemporalLayerBitrateRatio, 0, sizeof(mTemporalLayerBitrateRatio));
    mTemporalLayerBitrateRatio[0] = 100;

 const size_t kMinOutputBufferSize = 1024 * 1024; // arbitrary

    initPorts(
            kNumBuffers, kNumBuffers, kMinOutputBufferSize,
            MEDIA_MIMETYPE_VIDEO_VP8, 2 /* minCompressionRatio */);
}

Project: Android

android::SoftOMXComponent *createSoftOMXComponent(
 const char *name, const OMX_CALLBACKTYPE *callbacks,
        OMX_PTR appData, OMX_COMPONENTTYPE **component) {
 return new android::SoftVPXEncoder(name, callbacks, appData, component);
}

Project: Android

vpx_enc_frame_flags_t SoftVPXEncoder::getEncodeFlags() {
 vpx_enc_frame_flags_t flags = 0;
 int patternIdx = mTemporalPatternIdx % mTemporalPatternLength;
    mTemporalPatternIdx++;
 switch (mTemporalPattern[patternIdx]) {
 case kTemporalUpdateLast:
            flags |= VP8_EFLAG_NO_UPD_GF;
            flags |= VP8_EFLAG_NO_UPD_ARF;
            flags |= VP8_EFLAG_NO_REF_GF;
            flags |= VP8_EFLAG_NO_REF_ARF;
 break;
 case kTemporalUpdateGoldenWithoutDependency:
            flags |= VP8_EFLAG_NO_REF_GF;
 case kTemporalUpdateGolden:
            flags |= VP8_EFLAG_NO_REF_ARF;
            flags |= VP8_EFLAG_NO_UPD_ARF;
            flags |= VP8_EFLAG_NO_UPD_LAST;
 break;
 case kTemporalUpdateAltrefWithoutDependency:
            flags |= VP8_EFLAG_NO_REF_ARF;
            flags |= VP8_EFLAG_NO_REF_GF;
 case kTemporalUpdateAltref:
            flags |= VP8_EFLAG_NO_UPD_GF;
            flags |= VP8_EFLAG_NO_UPD_LAST;
 break;
 case kTemporalUpdateNoneNoRefAltref:
            flags |= VP8_EFLAG_NO_REF_ARF;
 case kTemporalUpdateNone:
            flags |= VP8_EFLAG_NO_UPD_GF;
            flags |= VP8_EFLAG_NO_UPD_ARF;
            flags |= VP8_EFLAG_NO_UPD_LAST;
            flags |= VP8_EFLAG_NO_UPD_ENTROPY;
 break;
 case kTemporalUpdateNoneNoRefGoldenRefAltRef:
            flags |= VP8_EFLAG_NO_REF_GF;
            flags |= VP8_EFLAG_NO_UPD_GF;
            flags |= VP8_EFLAG_NO_UPD_ARF;
            flags |= VP8_EFLAG_NO_UPD_LAST;
            flags |= VP8_EFLAG_NO_UPD_ENTROPY;
 break;
 case kTemporalUpdateGoldenWithoutDependencyRefAltRef:
            flags |= VP8_EFLAG_NO_REF_GF;
            flags |= VP8_EFLAG_NO_UPD_ARF;
            flags |= VP8_EFLAG_NO_UPD_LAST;
 break;
 case kTemporalUpdateLastRefAltRef:
            flags |= VP8_EFLAG_NO_UPD_GF;
            flags |= VP8_EFLAG_NO_UPD_ARF;
            flags |= VP8_EFLAG_NO_REF_GF;
 break;
 case kTemporalUpdateGoldenRefAltRef:
            flags |= VP8_EFLAG_NO_UPD_ARF;
            flags |= VP8_EFLAG_NO_UPD_LAST;
 break;
 case kTemporalUpdateLastAndGoldenRefAltRef:
            flags |= VP8_EFLAG_NO_UPD_ARF;
            flags |= VP8_EFLAG_NO_REF_GF;
 break;
 case kTemporalUpdateLastRefAll:
            flags |= VP8_EFLAG_NO_UPD_ARF;
            flags |= VP8_EFLAG_NO_UPD_GF;
 break;
 }
 return flags;
}

Project: Android

status_t SoftVPXEncoder::initEncoder() {
 vpx_codec_err_t codec_return;

    mCodecContext = new vpx_codec_ctx_t;
    mCodecConfiguration = new vpx_codec_enc_cfg_t;
    mCodecInterface = vpx_codec_vp8_cx();

 if (mCodecInterface == NULL) {
 return UNKNOWN_ERROR;
 }
    ALOGD("VP8: initEncoder. BRMode: %u. TSLayers: %zu. KF: %u. QP: %u - %u",
 (uint32_t)mBitrateControlMode, mTemporalLayers, mKeyFrameInterval,
          mMinQuantizer, mMaxQuantizer);
    codec_return = vpx_codec_enc_config_default(mCodecInterface,
                                                mCodecConfiguration,
 0); // Codec specific flags

 if (codec_return != VPX_CODEC_OK) {
        ALOGE("Error populating default configuration for vpx encoder.");
 return UNKNOWN_ERROR;
 }

    mCodecConfiguration->g_w = mWidth;
    mCodecConfiguration->g_h = mHeight;
    mCodecConfiguration->g_threads = GetCPUCoreCount();
    mCodecConfiguration->g_error_resilient = mErrorResilience;

 switch (mLevel) {
 case OMX_VIDEO_VP8Level_Version0:
            mCodecConfiguration->g_profile = 0;
 break;

 case OMX_VIDEO_VP8Level_Version1:
            mCodecConfiguration->g_profile = 1;
 break;

 case OMX_VIDEO_VP8Level_Version2:
            mCodecConfiguration->g_profile = 2;
 break;

 case OMX_VIDEO_VP8Level_Version3:
            mCodecConfiguration->g_profile = 3;
 break;

 default:
            mCodecConfiguration->g_profile = 0;
 }

    mCodecConfiguration->g_timebase.num = 1;
    mCodecConfiguration->g_timebase.den = 1000000;
    mCodecConfiguration->rc_target_bitrate = (mBitrate + 500) / 1000;
    mCodecConfiguration->rc_end_usage = mBitrateControlMode;
    mCodecConfiguration->rc_dropframe_thresh = 0;
 if (mBitrateControlMode == VPX_CBR) {
        mCodecConfiguration->rc_resize_allowed = 0;
        mCodecConfiguration->g_pass = VPX_RC_ONE_PASS;
        mCodecConfiguration->rc_undershoot_pct = 100;
        mCodecConfiguration->rc_overshoot_pct = 15;
        mCodecConfiguration->rc_buf_initial_sz = 500;
        mCodecConfiguration->rc_buf_optimal_sz = 600;
        mCodecConfiguration->rc_buf_sz = 1000;
        mCodecConfiguration->g_error_resilient = 1;
        mCodecConfiguration->g_lag_in_frames = 0;
        mCodecConfiguration->kf_max_dist = 3000;
        mCodecConfiguration->kf_mode = VPX_KF_AUTO;
 }

 switch (mTemporalLayers) {
 case 0:
 {
            mTemporalPatternLength = 0;
 break;
 }
 case 1:
 {
            mCodecConfiguration->ts_number_layers = 1;
            mCodecConfiguration->ts_rate_decimator[0] = 1;
            mCodecConfiguration->ts_periodicity = 1;
            mCodecConfiguration->ts_layer_id[0] = 0;
            mTemporalPattern[0] = kTemporalUpdateLastRefAll;
            mTemporalPatternLength = 1;
 break;
 }
 case 2:
 {
            mCodecConfiguration->ts_number_layers = 2;
            mCodecConfiguration->ts_rate_decimator[0] = 2;
            mCodecConfiguration->ts_rate_decimator[1] = 1;
            mCodecConfiguration->ts_periodicity = 2;
            mCodecConfiguration->ts_layer_id[0] = 0;
            mCodecConfiguration->ts_layer_id[1] = 1;
            mTemporalPattern[0] = kTemporalUpdateLastAndGoldenRefAltRef;
            mTemporalPattern[1] = kTemporalUpdateGoldenWithoutDependencyRefAltRef;
            mTemporalPattern[2] = kTemporalUpdateLastRefAltRef;
            mTemporalPattern[3] = kTemporalUpdateGoldenRefAltRef;
            mTemporalPattern[4] = kTemporalUpdateLastRefAltRef;
            mTemporalPattern[5] = kTemporalUpdateGoldenRefAltRef;
            mTemporalPattern[6] = kTemporalUpdateLastRefAltRef;
            mTemporalPattern[7] = kTemporalUpdateNone;
            mTemporalPatternLength = 8;
 break;
 }
 case 3:
 {
            mCodecConfiguration->ts_number_layers = 3;
            mCodecConfiguration->ts_rate_decimator[0] = 4;
            mCodecConfiguration->ts_rate_decimator[1] = 2;
            mCodecConfiguration->ts_rate_decimator[2] = 1;
            mCodecConfiguration->ts_periodicity = 4;
            mCodecConfiguration->ts_layer_id[0] = 0;
            mCodecConfiguration->ts_layer_id[1] = 2;
            mCodecConfiguration->ts_layer_id[2] = 1;
            mCodecConfiguration->ts_layer_id[3] = 2;
            mTemporalPattern[0] = kTemporalUpdateLastAndGoldenRefAltRef;
            mTemporalPattern[1] = kTemporalUpdateNoneNoRefGoldenRefAltRef;
            mTemporalPattern[2] = kTemporalUpdateGoldenWithoutDependencyRefAltRef;
            mTemporalPattern[3] = kTemporalUpdateNone;
            mTemporalPattern[4] = kTemporalUpdateLastRefAltRef;
            mTemporalPattern[5] = kTemporalUpdateNone;
            mTemporalPattern[6] = kTemporalUpdateGoldenRefAltRef;
            mTemporalPattern[7] = kTemporalUpdateNone;
            mTemporalPatternLength = 8;
 break;
 }
 default:
 {
            ALOGE("Wrong number of temporal layers %zu", mTemporalLayers);
 return UNKNOWN_ERROR;
 }
 }

 for (size_t i = 0; i < mCodecConfiguration->ts_number_layers; i++) {
        mCodecConfiguration->ts_target_bitrate[i] =
            mCodecConfiguration->rc_target_bitrate *
            mTemporalLayerBitrateRatio[i] / 100;
 }
 if (mKeyFrameInterval > 0) {
        mCodecConfiguration->kf_max_dist = mKeyFrameInterval;
        mCodecConfiguration->kf_min_dist = mKeyFrameInterval;
        mCodecConfiguration->kf_mode = VPX_KF_AUTO;
 }
 if (mMinQuantizer > 0) {
        mCodecConfiguration->rc_min_quantizer = mMinQuantizer;
 }
 if (mMaxQuantizer > 0) {
        mCodecConfiguration->rc_max_quantizer = mMaxQuantizer;
 }

    codec_return = vpx_codec_enc_init(mCodecContext,
                                      mCodecInterface,
                                      mCodecConfiguration,
 0); // flags

 if (codec_return != VPX_CODEC_OK) {
        ALOGE("Error initializing vpx encoder");
 return UNKNOWN_ERROR;
 }

    codec_return = vpx_codec_control(mCodecContext,
                                     VP8E_SET_TOKEN_PARTITIONS,
                                     mDCTPartitions);
 if (codec_return != VPX_CODEC_OK) {
        ALOGE("Error setting dct partitions for vpx encoder.");
 return UNKNOWN_ERROR;
 }

 if (mBitrateControlMode == VPX_CBR) {
        codec_return = vpx_codec_control(mCodecContext,
                                         VP8E_SET_STATIC_THRESHOLD,
 1);
 if (codec_return == VPX_CODEC_OK) {
 uint32_t rc_max_intra_target =
                mCodecConfiguration->rc_buf_optimal_sz * (mFramerate >> 17) / 10;
 if (rc_max_intra_target < 300) {
                rc_max_intra_target = 300;
 }
            codec_return = vpx_codec_control(mCodecContext,
                                             VP8E_SET_MAX_INTRA_BITRATE_PCT,
                                             rc_max_intra_target);
 }
 if (codec_return == VPX_CODEC_OK) {
            codec_return = vpx_codec_control(mCodecContext,
                                             VP8E_SET_CPUUSED,
 -8);
 }
 if (codec_return != VPX_CODEC_OK) {
            ALOGE("Error setting cbr parameters for vpx encoder.");
 return UNKNOWN_ERROR;
 }
 }

 if (mColorFormat != OMX_COLOR_FormatYUV420Planar || mInputDataIsMeta) {
        free(mConversionBuffer);
        mConversionBuffer = NULL;
 if (((uint64_t)mWidth * mHeight) > ((uint64_t)INT32_MAX / 3)) {
            ALOGE("b/25812794, Buffer size is too big.");
 return UNKNOWN_ERROR;
 }
        mConversionBuffer = (uint8_t *)malloc(mWidth * mHeight * 3 / 2);
 if (mConversionBuffer == NULL) {
            ALOGE("Allocating conversion buffer failed.");
 return UNKNOWN_ERROR;
 }
 }
 return OK;
}

Project: Android

OMX_ERRORTYPE SoftVPXEncoder::internalSetAndroidVp8Params(
 const OMX_VIDEO_PARAM_ANDROID_VP8ENCODERTYPE* vp8AndroidParams) {
 if (vp8AndroidParams->nPortIndex != kOutputPortIndex) {
 return OMX_ErrorUnsupportedIndex;
 }
 if (vp8AndroidParams->eTemporalPattern != OMX_VIDEO_VPXTemporalLayerPatternNone &&
        vp8AndroidParams->eTemporalPattern != OMX_VIDEO_VPXTemporalLayerPatternWebRTC) {
 return OMX_ErrorBadParameter;
 }
 if (vp8AndroidParams->nTemporalLayerCount > OMX_VIDEO_ANDROID_MAXVP8TEMPORALLAYERS) {
 return OMX_ErrorBadParameter;
 }
 if (vp8AndroidParams->nMinQuantizer > vp8AndroidParams->nMaxQuantizer) {
 return OMX_ErrorBadParameter;
 }

    mTemporalPatternType = vp8AndroidParams->eTemporalPattern;
 if (vp8AndroidParams->eTemporalPattern == OMX_VIDEO_VPXTemporalLayerPatternWebRTC) {
        mTemporalLayers = vp8AndroidParams->nTemporalLayerCount;
 } else if (vp8AndroidParams->eTemporalPattern == OMX_VIDEO_VPXTemporalLayerPatternNone) {
        mTemporalLayers = 0;
 }
 if (mTemporalLayers > 1) {
 for (size_t i = 0; i < mTemporalLayers - 1; i++) {
 if (vp8AndroidParams->nTemporalLayerBitrateRatio[i + 1] <=
                    vp8AndroidParams->nTemporalLayerBitrateRatio[i]) {
                ALOGE("Wrong bitrate ratio - should be in increasing order.");
 return OMX_ErrorBadParameter;
 }
 }
 }
    mKeyFrameInterval = vp8AndroidParams->nKeyFrameInterval;
    mMinQuantizer = vp8AndroidParams->nMinQuantizer;
    mMaxQuantizer = vp8AndroidParams->nMaxQuantizer;
    memcpy(mTemporalLayerBitrateRatio, vp8AndroidParams->nTemporalLayerBitrateRatio,
 sizeof(mTemporalLayerBitrateRatio));
    ALOGD("VP8: internalSetAndroidVp8Params. BRMode: %u. TS: %zu. KF: %u."
 " QP: %u - %u BR0: %u. BR1: %u. BR2: %u",
 (uint32_t)mBitrateControlMode, mTemporalLayers, mKeyFrameInterval,
          mMinQuantizer, mMaxQuantizer, mTemporalLayerBitrateRatio[0],
          mTemporalLayerBitrateRatio[1], mTemporalLayerBitrateRatio[2]);
 return OMX_ErrorNone;
}

Project: Android

OMX_ERRORTYPE SoftVPXEncoder::internalSetVp8Params(
 const OMX_VIDEO_PARAM_VP8TYPE* vp8Params) {
 if (vp8Params->nPortIndex != kOutputPortIndex) {
 return OMX_ErrorUnsupportedIndex;
 }

 if (vp8Params->eProfile != OMX_VIDEO_VP8ProfileMain) {
 return OMX_ErrorBadParameter;
 }

 if (vp8Params->eLevel == OMX_VIDEO_VP8Level_Version0 ||
        vp8Params->eLevel == OMX_VIDEO_VP8Level_Version1 ||
        vp8Params->eLevel == OMX_VIDEO_VP8Level_Version2 ||
        vp8Params->eLevel == OMX_VIDEO_VP8Level_Version3) {
        mLevel = vp8Params->eLevel;
 } else {
 return OMX_ErrorBadParameter;
 }

 if (vp8Params->nDCTPartitions <= kMaxDCTPartitions) {
        mDCTPartitions = vp8Params->nDCTPartitions;
 } else {
 return OMX_ErrorBadParameter;
 }

    mErrorResilience = vp8Params->bErrorResilientMode;
 return OMX_ErrorNone;
}

Project: Android

status_t SoftVPXEncoder::releaseEncoder() {
 if (mCodecContext != NULL) {
        vpx_codec_destroy(mCodecContext);
 delete mCodecContext;
        mCodecContext = NULL;
 }

 if (mCodecConfiguration != NULL) {
 delete mCodecConfiguration;
        mCodecConfiguration = NULL;
 }

 if (mConversionBuffer != NULL) {
        free(mConversionBuffer);
        mConversionBuffer = NULL;
 }

    mCodecInterface = NULL;

 return OK;
}

Project: Android

SoftVPXEncoder::~SoftVPXEncoder() {
    releaseEncoder();
}