Commit message: dv: Fix null pointer dereference due to ach=0

dv: Fix null pointer dereference due to ach=0

Fixes part2 of CVE-2011-3929

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Reviewed-by: Roman Shaposhnik <roman@shaposhnik.org>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
Signed-off-by: Alex Converse <alex.converse@gmail.com>
CVE: CVE-2011-3929
CWE: CWE-119
Vulnerability description: The avpriv_dv_produce_packet function in libavcodec in FFmpeg 0.7.x before 0.7.12 and 0.8.x before 0.8.11 and in Libav 0.5.x before 0.5.9, 0.6.x before 0.6.6, 0.7.x before 0.7.5, and 0.8.x before 0.8.1 allows remote attackers to cause a denial of service (NULL pointer dereference and application crash) and possibly execute arbitrary code via a crafted DV file.

Project: libav
File: dv.c

DVDemuxContext* avpriv_dv_init_demux(AVFormatContext *s)
{
    DVDemuxContext *c;

    c = av_mallocz(sizeof(DVDemuxContext));
    if (!c)
        return NULL;

    c->vst = avformat_new_stream(s, NULL);
    if (!c->vst) {
        av_free(c);
        return NULL;
    }

    c->sys  = NULL;
    c->fctx = s;
    memset(c->ast, 0, sizeof(c->ast));
    c->ach    = 0;
    c->frames = 0;
    c->abytes = 0;

    c->vst->codec->codec_type = AVMEDIA_TYPE_VIDEO;
    c->vst->codec->codec_id   = CODEC_ID_DVVIDEO;
    c->vst->codec->bit_rate   = 25000000;
    c->vst->start_time        = 0;

    return c;
}

Project: libav
File: dv.c

static int dv_extract_audio_info(DVDemuxContext* c, uint8_t* frame)
{
    const uint8_t* as_pack;
    int freq, stype, smpls, quant, i, ach;

    as_pack = dv_extract_pack(frame, dv_audio_source);
    if (!as_pack || !c->sys) {    /* No audio ? */
        c->ach = 0;
        return 0;
    }

    smpls =  as_pack[1] & 0x3f;       /* samples in this frame - min. samples */
    freq  = (as_pack[4] >> 3) & 0x07; /* 0 - 48kHz, 1 - 44,1kHz, 2 - 32kHz */
    stype = (as_pack[3] & 0x1f);      /* 0 - 2CH, 2 - 4CH, 3 - 8CH */
    quant =  as_pack[4] & 0x07;       /* 0 - 16bit linear, 1 - 12bit nonlinear */

    if (stype > 3) {
        av_log(c->fctx, AV_LOG_ERROR, "stype %d is invalid\n", stype);
        c->ach = 0;
        return 0;
    }

    /* note: ach counts PAIRS of channels (i.e. stereo channels) */
    ach = ((int[4]){  1,  0,  2,  4})[stype];
    if (ach == 1 && quant && freq == 2)
        ach = 2;

    /* Dynamic handling of the audio streams in DV */
    for (i = 0; i < ach; i++) {
       if (!c->ast[i]) {
           c->ast[i] = avformat_new_stream(c->fctx, NULL);
           if (!c->ast[i])
               break;
           avpriv_set_pts_info(c->ast[i], 64, 1, 30000);
           c->ast[i]->codec->codec_type = AVMEDIA_TYPE_AUDIO;
           c->ast[i]->codec->codec_id   = CODEC_ID_PCM_S16LE;

           av_init_packet(&c->audio_pkt[i]);
           c->audio_pkt[i].size         = 0;
           c->audio_pkt[i].data         = c->audio_buf[i];
           c->audio_pkt[i].stream_index = c->ast[i]->index;
           c->audio_pkt[i].flags       |= AV_PKT_FLAG_KEY;
       }
       c->ast[i]->codec->sample_rate = dv_audio_frequency[freq];
       c->ast[i]->codec->channels    = 2;
       c->ast[i]->codec->bit_rate    = 2 * dv_audio_frequency[freq] * 16;
       c->ast[i]->start_time         = 0;
    }
    c->ach = i;

    return (c->sys->audio_min_samples[freq] + smpls) * 4; /* 2ch, 2bytes */;
}

Project: libav
File: dv.c

static int dv_extract_video_info(DVDemuxContext *c, uint8_t* frame)
{
    const uint8_t* vsc_pack;
    AVCodecContext* avctx;
    int apt, is16_9;
    int size = 0;

    if (c->sys) {
        avctx = c->vst->codec;

        avpriv_set_pts_info(c->vst, 64, c->sys->time_base.num,
                        c->sys->time_base.den);
        avctx->time_base= c->sys->time_base;
        if (!avctx->width){
            avctx->width = c->sys->width;
            avctx->height = c->sys->height;
        }
        avctx->pix_fmt = c->sys->pix_fmt;

        /* finding out SAR is a little bit messy */
        vsc_pack = dv_extract_pack(frame, dv_video_control);
        apt      = frame[4] & 0x07;
        is16_9   = (vsc_pack && ((vsc_pack[2] & 0x07) == 0x02 ||
                                (!apt && (vsc_pack[2] & 0x07) == 0x07)));
        c->vst->sample_aspect_ratio = c->sys->sar[is16_9];
        avctx->bit_rate = av_rescale_q(c->sys->frame_size, (AVRational){8,1},
                                       c->sys->time_base);
        size = c->sys->frame_size;
    }
    return size;
}