Commit message: None
CVE: CVE-2016-3899
CWE: CWE-284
Vulnerability description: OMXCodec.cpp in libstagefright in mediaserver in Android 4.x before 4.4.4, 5.0.x before 5.0.2, 5.1.x before 5.1.1, 6.x before 2016-09-01, and 7.0 before 2016-09-01 does not validate a certain pointer, which allows remote attackers to cause a denial of service (device hang or reboot) via a crafted media file, aka internal bug 29421811.

Project: Android

status_t OMXCodec::allocateOutputBuffersFromNativeWindow() {
    OMX_PARAM_PORTDEFINITIONTYPE def;
 InitOMXParams(&def);
    def.nPortIndex = kPortIndexOutput;

 status_t err = mOMX->getParameter(
            mNode, OMX_IndexParamPortDefinition, &def, sizeof(def));
 if (err != OK) {
        CODEC_LOGE("getParameter failed: %d", err);
 return err;
 }

    sp<MetaData> meta = mSource->getFormat();

 int32_t rotationDegrees;
 if (!meta->findInt32(kKeyRotation, &rotationDegrees)) {
        rotationDegrees = 0;
 }

    OMX_U32 usage = 0;
    err = mOMX->getGraphicBufferUsage(mNode, kPortIndexOutput, &usage);
 if (err != 0) {
        ALOGW("querying usage flags from OMX IL component failed: %d", err);
        usage = 0;
 }

 if (mFlags & kEnableGrallocUsageProtected) {
        usage |= GRALLOC_USAGE_PROTECTED;
 }

    err = setNativeWindowSizeFormatAndUsage(
            mNativeWindow.get(),
            def.format.video.nFrameWidth,
            def.format.video.nFrameHeight,
            def.format.video.eColorFormat,
            rotationDegrees,
            usage | GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_EXTERNAL_DISP);
 if (err != 0) {
 return err;
 }

 int minUndequeuedBufs = 0;
    err = mNativeWindow->query(mNativeWindow.get(),
            NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS, &minUndequeuedBufs);
 if (err != 0) {
        ALOGE("NATIVE_WINDOW_MIN_UNDEQUEUED_BUFFERS query failed: %s (%d)",
                strerror(-err), -err);
 return err;
 }

    CODEC_LOGI("OMX-buffers: min=%u actual=%u undeq=%d+1",
            def.nBufferCountMin, def.nBufferCountActual, minUndequeuedBufs);

 for (OMX_U32 extraBuffers = 2 + 1; /* condition inside loop */; extraBuffers--) {
        OMX_U32 newBufferCount =
            def.nBufferCountMin + minUndequeuedBufs + extraBuffers;
        def.nBufferCountActual = newBufferCount;
        err = mOMX->setParameter(
                mNode, OMX_IndexParamPortDefinition, &def, sizeof(def));

 if (err == OK) {
            minUndequeuedBufs += extraBuffers;
 break;
 }

        CODEC_LOGW("setting nBufferCountActual to %u failed: %d",
                newBufferCount, err);
 /* exit condition */
 if (extraBuffers == 0) {
 return err;
 }
 }
    CODEC_LOGI("OMX-buffers: min=%u actual=%u undeq=%d+1",
            def.nBufferCountMin, def.nBufferCountActual, minUndequeuedBufs);

    err = native_window_set_buffer_count(
            mNativeWindow.get(), def.nBufferCountActual);
 if (err != 0) {
        ALOGE("native_window_set_buffer_count failed: %s (%d)", strerror(-err),
 -err);
 return err;
 }

    CODEC_LOGV("allocating %u buffers from a native window of size %u on "
 "output port", def.nBufferCountActual, def.nBufferSize);

 for (OMX_U32 i = 0; i < def.nBufferCountActual; i++) {
 ANativeWindowBuffer* buf;
        err = native_window_dequeue_buffer_and_wait(mNativeWindow.get(), &buf);
 if (err != 0) {
            ALOGE("dequeueBuffer failed: %s (%d)", strerror(-err), -err);
 break;
 }

        sp<GraphicBuffer> graphicBuffer(new GraphicBuffer(buf, false));
 BufferInfo info;
        info.mData = NULL;
        info.mSize = def.nBufferSize;
        info.mStatus = OWNED_BY_US;
        info.mMem = NULL;
        info.mMediaBuffer = new MediaBuffer(graphicBuffer);
        info.mMediaBuffer->setObserver(this);
        mPortBuffers[kPortIndexOutput].push(info);

        IOMX::buffer_id bufferId;
        err = mOMX->useGraphicBuffer(mNode, kPortIndexOutput, graphicBuffer,
 &bufferId);
 if (err != 0) {
            CODEC_LOGE("registering GraphicBuffer with OMX IL component "
 "failed: %d", err);
 break;
 }

        mPortBuffers[kPortIndexOutput].editItemAt(i).mBuffer = bufferId;

        CODEC_LOGV("registered graphic buffer with ID %u (pointer = %p)",
                bufferId, graphicBuffer.get());
 }

    OMX_U32 cancelStart;
    OMX_U32 cancelEnd;
 if (err != 0) {
        cancelStart = 0;
        cancelEnd = mPortBuffers[kPortIndexOutput].size();
 } else {
        cancelStart = def.nBufferCountActual - minUndequeuedBufs;
        cancelEnd = def.nBufferCountActual;
 }

 for (OMX_U32 i = cancelStart; i < cancelEnd; i++) {
 BufferInfo *info = &mPortBuffers[kPortIndexOutput].editItemAt(i);
        cancelBufferToNativeWindow(info);
 }

 return err;
}

Project: Android

void OMXCodec::fillOutputBuffer(IOMX::buffer_id buffer) {
 Vector<BufferInfo> *buffers = &mPortBuffers[kPortIndexOutput];
 for (size_t i = 0; i < buffers->size(); ++i) {
 if ((*buffers)[i].mBuffer == buffer) {
            fillOutputBuffer(&buffers->editItemAt(i));
 return;
 }
 }

    CHECK(!"should not be here.");
}

Project: Android

bool OMXCodec::findCodecQuirks(const char *componentName, uint32_t *quirks) {
 const sp<IMediaCodecList> list = MediaCodecList::getInstance();
 if (list == NULL) {
 return false;
 }

 ssize_t index = list->findCodecByName(componentName);

 if (index < 0) {
 return false;
 }

 const sp<MediaCodecInfo> info = list->getCodecInfo(index);
    CHECK(info != NULL);
 *quirks = getComponentQuirks(info);

 return true;
}

Project: Android

void OMXCodec::findMatchingCodecs(
 const char *mime,
 bool createEncoder, const char *matchComponentName,
 uint32_t flags,
 Vector<CodecNameAndQuirks> *matchingCodecs) {
    matchingCodecs->clear();

 const sp<IMediaCodecList> list = MediaCodecList::getInstance();
 if (list == NULL) {
 return;
 }

 size_t index = 0;
 for (;;) {
 ssize_t matchIndex =
 list->findCodecByType(mime, createEncoder, index);

 if (matchIndex < 0) {
 break;
 }

        index = matchIndex + 1;

 const sp<MediaCodecInfo> info = list->getCodecInfo(matchIndex);
        CHECK(info != NULL);
 const char *componentName = info->getCodecName();

 if (matchComponentName && strcmp(componentName, matchComponentName)) {
 continue;
 }

 if (((flags & kSoftwareCodecsOnly) && IsSoftwareCodec(componentName)) ||
 ((flags & kHardwareCodecsOnly) && !IsSoftwareCodec(componentName)) ||
 (!(flags & (kSoftwareCodecsOnly | kHardwareCodecsOnly)))) {

 ssize_t index = matchingCodecs->add();
 CodecNameAndQuirks *entry = &matchingCodecs->editItemAt(index);
            entry->mName = String8(componentName);
            entry->mQuirks = getComponentQuirks(info);

            ALOGV("matching '%s' quirks 0x%08x",
                  entry->mName.string(), entry->mQuirks);
 }
 }

 if (flags & kPreferSoftwareCodecs) {
        matchingCodecs->sort(CompareSoftwareCodecsFirst);
 }
}

Project: Android

status_t getOMXChannelMapping(size_t numChannels, OMX_AUDIO_CHANNELTYPE map[]) {
 switch (numChannels) {
 case 1:
 map[0] = OMX_AUDIO_ChannelCF;
 break;
 case 2:
 map[0] = OMX_AUDIO_ChannelLF;
 map[1] = OMX_AUDIO_ChannelRF;
 break;
 case 3:
 map[0] = OMX_AUDIO_ChannelLF;
 map[1] = OMX_AUDIO_ChannelRF;
 map[2] = OMX_AUDIO_ChannelCF;
 break;
 case 4:
 map[0] = OMX_AUDIO_ChannelLF;
 map[1] = OMX_AUDIO_ChannelRF;
 map[2] = OMX_AUDIO_ChannelLR;
 map[3] = OMX_AUDIO_ChannelRR;
 break;
 case 5:
 map[0] = OMX_AUDIO_ChannelLF;
 map[1] = OMX_AUDIO_ChannelRF;
 map[2] = OMX_AUDIO_ChannelCF;
 map[3] = OMX_AUDIO_ChannelLR;
 map[4] = OMX_AUDIO_ChannelRR;
 break;
 case 6:
 map[0] = OMX_AUDIO_ChannelLF;
 map[1] = OMX_AUDIO_ChannelRF;
 map[2] = OMX_AUDIO_ChannelCF;
 map[3] = OMX_AUDIO_ChannelLFE;
 map[4] = OMX_AUDIO_ChannelLR;
 map[5] = OMX_AUDIO_ChannelRR;
 break;
 case 7:
 map[0] = OMX_AUDIO_ChannelLF;
 map[1] = OMX_AUDIO_ChannelRF;
 map[2] = OMX_AUDIO_ChannelCF;
 map[3] = OMX_AUDIO_ChannelLFE;
 map[4] = OMX_AUDIO_ChannelLR;
 map[5] = OMX_AUDIO_ChannelRR;
 map[6] = OMX_AUDIO_ChannelCS;
 break;
 case 8:
 map[0] = OMX_AUDIO_ChannelLF;
 map[1] = OMX_AUDIO_ChannelRF;
 map[2] = OMX_AUDIO_ChannelCF;
 map[3] = OMX_AUDIO_ChannelLFE;
 map[4] = OMX_AUDIO_ChannelLR;
 map[5] = OMX_AUDIO_ChannelRR;
 map[6] = OMX_AUDIO_ChannelLS;
 map[7] = OMX_AUDIO_ChannelRS;
 break;
 default:
 return -EINVAL;
 }

 return OK;
}

Project: Android

status_t OMXCodec::pause() {
 Mutex::Autolock autoLock(mLock);

    mPaused = true;

 return OK;
}

Project: Android

void OMXCodec::setAMRFormat(bool isWAMR, int32_t bitRate) {
    OMX_U32 portIndex = mIsEncoder ? kPortIndexOutput : kPortIndexInput;

    OMX_AUDIO_PARAM_AMRTYPE def;
 InitOMXParams(&def);
    def.nPortIndex = portIndex;

 status_t err =
        mOMX->getParameter(mNode, OMX_IndexParamAudioAmr, &def, sizeof(def));

    CHECK_EQ(err, (status_t)OK);

    def.eAMRFrameFormat = OMX_AUDIO_AMRFrameFormatFSF;

    def.eAMRBandMode = pickModeFromBitRate(isWAMR, bitRate);
    err = mOMX->setParameter(mNode, OMX_IndexParamAudioAmr, &def, sizeof(def));
    CHECK_EQ(err, (status_t)OK);


 if (mIsEncoder) {
        sp<MetaData> format = mSource->getFormat();
 int32_t sampleRate;
 int32_t numChannels;
        CHECK(format->findInt32(kKeySampleRate, &sampleRate));
        CHECK(format->findInt32(kKeyChannelCount, &numChannels));

        setRawAudioFormat(kPortIndexInput, sampleRate, numChannels);
 }
}

Project: Android

static OMX_U32 setPFramesSpacing(int32_t iFramesInterval, int32_t frameRate) {
 if (iFramesInterval < 0) {
 return 0xFFFFFFFF;
 } else if (iFramesInterval == 0) {
 return 0;
 }
    OMX_U32 ret = frameRate * iFramesInterval - 1;
 return ret;
}

Project: Android

status_t OMXCodec::setupH263EncoderParameters(const sp<MetaData>& meta) {
 int32_t iFramesInterval, frameRate, bitRate;
 bool success = meta->findInt32(kKeyBitRate, &bitRate);
    success = success && meta->findInt32(kKeyFrameRate, &frameRate);
    success = success && meta->findInt32(kKeyIFramesInterval, &iFramesInterval);
    CHECK(success);
    OMX_VIDEO_PARAM_H263TYPE h263type;
 InitOMXParams(&h263type);
    h263type.nPortIndex = kPortIndexOutput;

 status_t err = mOMX->getParameter(
            mNode, OMX_IndexParamVideoH263, &h263type, sizeof(h263type));
    CHECK_EQ(err, (status_t)OK);

    h263type.nAllowedPictureTypes =
        OMX_VIDEO_PictureTypeI | OMX_VIDEO_PictureTypeP;

    h263type.nPFrames = setPFramesSpacing(iFramesInterval, frameRate);
 if (h263type.nPFrames == 0) {
        h263type.nAllowedPictureTypes = OMX_VIDEO_PictureTypeI;
 }
    h263type.nBFrames = 0;

 CodecProfileLevel defaultProfileLevel, profileLevel;
    defaultProfileLevel.mProfile = h263type.eProfile;
    defaultProfileLevel.mLevel = h263type.eLevel;
    err = getVideoProfileLevel(meta, defaultProfileLevel, profileLevel);
 if (err != OK) return err;
    h263type.eProfile = static_cast<OMX_VIDEO_H263PROFILETYPE>(profileLevel.mProfile);
    h263type.eLevel = static_cast<OMX_VIDEO_H263LEVELTYPE>(profileLevel.mLevel);

    h263type.bPLUSPTYPEAllowed = OMX_FALSE;
    h263type.bForceRoundingTypeToZero = OMX_FALSE;
    h263type.nPictureHeaderRepetition = 0;
    h263type.nGOBHeaderInterval = 0;

    err = mOMX->setParameter(
            mNode, OMX_IndexParamVideoH263, &h263type, sizeof(h263type));
    CHECK_EQ(err, (status_t)OK);

    CHECK_EQ(setupBitRate(bitRate), (status_t)OK);
    CHECK_EQ(setupErrorCorrectionParameters(), (status_t)OK);

 return OK;
}

Project: Android

status_t OMXCodec::stopOmxComponent_l() {
    CODEC_LOGV("stopOmxComponent_l mState=%d", mState);

 while (isIntermediateState(mState)) {
        mAsyncCompletion.wait(mLock);
 }

 bool isError = false;
 switch (mState) {
 case LOADED:
 break;

 case ERROR:
 {
 if (mPortStatus[kPortIndexOutput] == ENABLING) {
                freeBuffersOnPort(kPortIndexInput, true /* onlyThoseWeOwn */);
                freeBuffersOnPort(kPortIndexOutput, true /* onlyThoseWeOwn */);
                setState(LOADED);
 break;
 } else {
                OMX_STATETYPE state = OMX_StateInvalid;
 status_t err = mOMX->getState(mNode, &state);
                CHECK_EQ(err, (status_t)OK);

 if (state != OMX_StateExecuting) {
 break;
 }
 }

            isError = true;
 }

 case EXECUTING:
 {
            setState(EXECUTING_TO_IDLE);

 if (mQuirks & kRequiresFlushBeforeShutdown) {
                CODEC_LOGV("This component requires a flush before transitioning "
 "from EXECUTING to IDLE...");

 bool emulateInputFlushCompletion =
 !flushPortAsync(kPortIndexInput);

 bool emulateOutputFlushCompletion =
 !flushPortAsync(kPortIndexOutput);

 if (emulateInputFlushCompletion) {
                    onCmdComplete(OMX_CommandFlush, kPortIndexInput);
 }

 if (emulateOutputFlushCompletion) {
                    onCmdComplete(OMX_CommandFlush, kPortIndexOutput);
 }
 } else {
                mPortStatus[kPortIndexInput] = SHUTTING_DOWN;
                mPortStatus[kPortIndexOutput] = SHUTTING_DOWN;

 status_t err =
                    mOMX->sendCommand(mNode, OMX_CommandStateSet, OMX_StateIdle);
                CHECK_EQ(err, (status_t)OK);
 }

 while (mState != LOADED && mState != ERROR) {
                mAsyncCompletion.wait(mLock);
 }

 if (isError) {
                setState(ERROR);
 }

 break;
 }

 default:
 {
            CHECK(!"should not be here.");
 break;
 }
 }

 if (mLeftOverBuffer) {
        mLeftOverBuffer->release();
        mLeftOverBuffer = NULL;
 }

 return OK;
}

Project: Android

OMXCodec::~OMXCodec() {
    mSource.clear();

    CHECK(mState == LOADED || mState == ERROR || mState == LOADED_TO_IDLE);

 status_t err = mOMX->freeNode(mNode);
    CHECK_EQ(err, (status_t)OK);

    mNode = 0;
    setState(DEAD);

    clearCodecSpecificData();

    free(mComponentName);
    mComponentName = NULL;

    free(mMIME);
    mMIME = NULL;
}