Commit message: None
CVE: CVE-2016-3872
CWE: CWE-119
Vulnerability description: Buffer overflow in codecs/on2/dec/SoftVPX.cpp in libstagefright in mediaserver in Android 4.x before 4.4.4, 5.0.x before 5.0.2, 5.1.x before 5.1.1, 6.x before 2016-09-01, and 7.0 before 2016-09-01 allows attackers to gain privileges via a crafted application, aka internal bug 29421675.

Project: Android

static int GetCPUCoreCount() {
 int cpuCoreCount = 1;
#if defined(_SC_NPROCESSORS_ONLN)
    cpuCoreCount = sysconf(_SC_NPROCESSORS_ONLN);
#else
    cpuCoreCount = sysconf(_SC_NPROC_ONLN);
#endif
    CHECK(cpuCoreCount >= 1);
    ALOGV("Number of CPU cores: %d", cpuCoreCount);
 return cpuCoreCount;
}

Project: Android

SoftVPX::SoftVPX(
 const char *name,
 const char *componentRole,
        OMX_VIDEO_CODINGTYPE codingType,
 const OMX_CALLBACKTYPE *callbacks,
        OMX_PTR appData,
        OMX_COMPONENTTYPE **component)
 : SoftVideoDecoderOMXComponent(
            name, componentRole, codingType,
            codingType == OMX_VIDEO_CodingVP8 ? NULL : kVP9ProfileLevels,
            codingType == OMX_VIDEO_CodingVP8 ? 0 : NELEM(kVP9ProfileLevels),
 320 /* width */, 240 /* height */, callbacks, appData, component),
      mMode(codingType == OMX_VIDEO_CodingVP8 ? MODE_VP8 : MODE_VP9),
      mEOSStatus(INPUT_DATA_AVAILABLE),
      mCtx(NULL),
      mFrameParallelMode(false),
      mTimeStampIdx(0),
      mImg(NULL) {
 const size_t kMinCompressionRatio = mMode == MODE_VP8 ? 2 : 4;
 const char *mime = mMode == MODE_VP8 ? MEDIA_MIMETYPE_VIDEO_VP8 : MEDIA_MIMETYPE_VIDEO_VP9;
 const size_t kMaxOutputBufferSize = 2048 * 2048 * 3 / 2;
    initPorts(
            kNumBuffers, kMaxOutputBufferSize / kMinCompressionRatio /* inputBufferSize */,
            kNumBuffers, mime, kMinCompressionRatio);
    CHECK_EQ(initDecoder(), (status_t)OK);
}

Project: Android

android::SoftOMXComponent *createSoftOMXComponent(
 const char *name, const OMX_CALLBACKTYPE *callbacks,
        OMX_PTR appData, OMX_COMPONENTTYPE **component) {
 if (!strcmp(name, "OMX.google.vp8.decoder")) {
 return new android::SoftVPX(
                name, "video_decoder.vp8", OMX_VIDEO_CodingVP8,
                callbacks, appData, component);
 } else if (!strcmp(name, "OMX.google.vp9.decoder")) {
 return new android::SoftVPX(
                name, "video_decoder.vp9", OMX_VIDEO_CodingVP9,
                callbacks, appData, component);
 } else {
        CHECK(!"Unknown component");
 }
 return NULL;
}

Project: Android

status_t SoftVPX::destroyDecoder() {
    vpx_codec_destroy((vpx_codec_ctx_t *)mCtx);
 delete (vpx_codec_ctx_t *)mCtx;
    mCtx = NULL;
 return OK;
}

Project: Android

status_t SoftVPX::initDecoder() {
    mCtx = new vpx_codec_ctx_t;
 vpx_codec_err_t vpx_err;
 vpx_codec_dec_cfg_t cfg;
 vpx_codec_flags_t flags;
    memset(&cfg, 0, sizeof(vpx_codec_dec_cfg_t));
    memset(&flags, 0, sizeof(vpx_codec_flags_t));
    cfg.threads = GetCPUCoreCount();

 if (mFrameParallelMode) {
        flags |= VPX_CODEC_USE_FRAME_THREADING;
 }

 if ((vpx_err = vpx_codec_dec_init(
 (vpx_codec_ctx_t *)mCtx,
                 mMode == MODE_VP8 ? &vpx_codec_vp8_dx_algo : &vpx_codec_vp9_dx_algo,
 &cfg, flags))) {
        ALOGE("on2 decoder failed to initialize. (%d)", vpx_err);
 return UNKNOWN_ERROR;
 }

 return OK;
}

Project: Android

void SoftVPX::onPortFlushCompleted(OMX_U32 portIndex) {
 if (portIndex == kInputPortIndex) {
 bool portWillReset = false;
 if (!outputBuffers(
 true /* flushDecoder */, false /* display */, false /* eos */, &portWillReset)) {
            ALOGE("Failed to flush decoder.");
            notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
 return;
 }
        mEOSStatus = INPUT_DATA_AVAILABLE;
 }
}

Project: Android

void SoftVPX::onQueueFilled(OMX_U32 /* portIndex */) {
 if (mOutputPortSettingsChange != NONE || mEOSStatus == OUTPUT_FRAMES_FLUSHED) {
 return;
 }

 List<BufferInfo *> &inQueue = getPortQueue(0);
 List<BufferInfo *> &outQueue = getPortQueue(1);
 bool EOSseen = false;
 bool portWillReset = false;

 while ((mEOSStatus == INPUT_EOS_SEEN || !inQueue.empty())
 && !outQueue.empty()) {
 if (mEOSStatus == INPUT_EOS_SEEN || mImg != NULL) {
 if (!outputBuffers(
                     mEOSStatus == INPUT_EOS_SEEN, true /* display */,
                     mEOSStatus == INPUT_EOS_SEEN, &portWillReset)) {
                ALOGE("on2 decoder failed to output frame.");
                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
 return;
 }
 if (portWillReset || mEOSStatus == OUTPUT_FRAMES_FLUSHED ||
                    mEOSStatus == INPUT_EOS_SEEN) {
 return;
 }
 continue;
 }

 BufferInfo *inInfo = *inQueue.begin();
        OMX_BUFFERHEADERTYPE *inHeader = inInfo->mHeader;

 if (inHeader->nFlags & OMX_BUFFERFLAG_CODECCONFIG) {
 if (mMode == MODE_VP9) {
                inQueue.erase(inQueue.begin());
                inInfo->mOwnedByUs = false;
                notifyEmptyBufferDone(inHeader);
 continue;
 } else {
                ALOGW("WARNING: Got CSD buffer for VP8.");
 }
 }

        mTimeStamps[mTimeStampIdx] = inHeader->nTimeStamp;

 if (inHeader->nFlags & OMX_BUFFERFLAG_EOS) {
            mEOSStatus = INPUT_EOS_SEEN;
 EOSseen = true;
 }

 if (inHeader->nFilledLen > 0) {
 vpx_codec_err_t err = vpx_codec_decode(
 (vpx_codec_ctx_t *)mCtx, inHeader->pBuffer + inHeader->nOffset,
                    inHeader->nFilledLen, &mTimeStamps[mTimeStampIdx], 0);
 if (err == VPX_CODEC_OK) {
                inInfo->mOwnedByUs = false;
                inQueue.erase(inQueue.begin());
                inInfo = NULL;
                notifyEmptyBufferDone(inHeader);
                inHeader = NULL;
 } else {
                ALOGE("on2 decoder failed to decode frame. err: %d", err);
                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
 return;
 }
 }

        mTimeStampIdx = (mTimeStampIdx + 1) % kNumBuffers;

 if (!outputBuffers(
 EOSseen /* flushDecoder */, true /* display */, EOSseen, &portWillReset)) {
            ALOGE("on2 decoder failed to output frame.");
            notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
 return;
 }
 if (portWillReset) {
 return;
 }
 }
}

Project: Android

void SoftVPX::onReset() {
 bool portWillReset = false;
 if (!outputBuffers(
 true /* flushDecoder */, false /* display */, false /* eos */, &portWillReset)) {
        ALOGW("Failed to flush decoder. Try to hard reset decoder");
        destroyDecoder();
        initDecoder();
 }
    mEOSStatus = INPUT_DATA_AVAILABLE;
}

Project: Android

SoftVPX::~SoftVPX() {
    destroyDecoder();
}