Commit message: avutil/mem: Fix flipped condition

Fixes return code and later null pointer dereference

Found-by: Laurent Butti <laurentb@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
CVE: CVE-2013-4265
CWE: CWE-476
Vulnerability description: The av_reallocp_array function in libavutil/mem.c in FFmpeg before 2.0.1 has an unspecified impact and remote vectors related to a "wrong return code" and a resultant NULL pointer dereference.

Project: FFmpeg
File: mem.c

void *av_calloc(size_t nmemb, size_t size)
{
    if (size <= 0 || nmemb >= INT_MAX / size)
        return NULL;
    return av_mallocz(nmemb * size);
}

Project: FFmpeg
File: mem.c

void *av_dynarray2_add(void **tab_ptr, int *nb_ptr, size_t elem_size,
                       const uint8_t *elem_data)
{
    int nb = *nb_ptr, nb_alloc;
    uint8_t *tab = *tab_ptr, *tab_elem_data;

    if ((nb & (nb - 1)) == 0) {
        if (nb == 0) {
            nb_alloc = 1;
        } else {
            if (nb > INT_MAX / (2 * elem_size))
                goto fail;
            nb_alloc = nb * 2;
        }
        tab = av_realloc(tab, nb_alloc * elem_size);
        if (!tab)
            goto fail;
        *tab_ptr = tab;
    }
    *nb_ptr = nb + 1;
    tab_elem_data = tab + nb*elem_size;
    if (elem_data)
        memcpy(tab_elem_data, elem_data, elem_size);
    else if (CONFIG_MEMORY_POISONING)
        memset(tab_elem_data, FF_MEMORY_POISON, elem_size);
    return tab_elem_data;

fail:
    av_freep(tab_ptr);
    *nb_ptr = 0;
    return NULL;
}

Project: FFmpeg
File: mem.c

void av_free(void *ptr)
{
#if CONFIG_MEMALIGN_HACK
    if (ptr) {
        int v= ((char *)ptr)[-1];
        av_assert0(v>0 && v<=ALIGN);
        free((char *)ptr - v);
    }
#elif HAVE_ALIGNED_MALLOC
    _aligned_free(ptr);
#else
    free(ptr);
#endif
}

Project: FFmpeg
File: mem.c

void av_freep(void *arg)
{
    void **ptr = (void **)arg;
    av_free(*ptr);
    *ptr = NULL;
}

Project: FFmpeg
File: mem.c

void *av_mallocz(size_t size)
{
    void *ptr = av_malloc(size);
    if (ptr)
        memset(ptr, 0, size);
    return ptr;
}

Project: FFmpeg
File: mem.c

void av_max_alloc(size_t max){
    max_alloc_size = max;
}

Project: FFmpeg
File: mem.c

void av_memcpy_backptr(uint8_t *dst, int back, int cnt)
{
    const uint8_t *src = &dst[-back];
    if (!back)
        return;

    if (back == 1) {
        memset(dst, *src, cnt);
    } else if (back == 2) {
        fill16(dst, cnt);
    } else if (back == 3) {
        fill24(dst, cnt);
    } else if (back == 4) {
        fill32(dst, cnt);
    } else {
        if (cnt >= 16) {
            int blocklen = back;
            while (cnt > blocklen) {
                memcpy(dst, src, blocklen);
                dst       += blocklen;
                cnt       -= blocklen;
                blocklen <<= 1;
            }
            memcpy(dst, src, cnt);
            return;
        }
        if (cnt >= 8) {
            AV_COPY32U(dst,     src);
            AV_COPY32U(dst + 4, src + 4);
            src += 8;
            dst += 8;
            cnt -= 8;
        }
        if (cnt >= 4) {
            AV_COPY32U(dst, src);
            src += 4;
            dst += 4;
            cnt -= 4;
        }
        if (cnt >= 2) {
            AV_COPY16U(dst, src);
            src += 2;
            dst += 2;
            cnt -= 2;
        }
        if (cnt)
            *dst = *src;
    }
}

Project: FFmpeg
File: mem.c

void *av_realloc_array(void *ptr, size_t nmemb, size_t size)
{
    if (size <= 0 || nmemb >= INT_MAX / size)
        return NULL;
    return av_realloc(ptr, nmemb * size);
}

Project: FFmpeg
File: mem.c

void *av_realloc_f(void *ptr, size_t nelem, size_t elsize)
{
    size_t size;
    void *r;

    if (av_size_mult(elsize, nelem, &size)) {
        av_free(ptr);
        return NULL;
    }
    r = av_realloc(ptr, size);
    if (!r && size)
        av_free(ptr);
    return r;
}

Project: FFmpeg
File: mem.c

char *av_strdup(const char *s)
{
    char *ptr = NULL;
    if (s) {
        int len = strlen(s) + 1;
        ptr = av_malloc(len);
        if (ptr)
            memcpy(ptr, s, len);
    }
    return ptr;
}

Project: FFmpeg
File: mem.c

static void fill16(uint8_t *dst, int len)
{
    uint32_t v = AV_RN16(dst - 2);

    v |= v << 16;

    while (len >= 4) {
        AV_WN32(dst, v);
        dst += 4;
        len -= 4;
    }

    while (len--) {
        *dst = dst[-2];
        dst++;
    }
}

Project: FFmpeg
File: mem.c

static void fill32(uint8_t *dst, int len)
{
    uint32_t v = AV_RN32(dst - 4);

    while (len >= 4) {
        AV_WN32(dst, v);
        dst += 4;
        len -= 4;
    }

    while (len--) {
        *dst = dst[-4];
        dst++;
    }
}