Commit message: Fix BN_hex2bn/BN_dec2bn NULL ptr/heap corruption

In the BN_hex2bn function the number of hex digits is calculated using
an int value |i|. Later |bn_expand| is called with a value of |i * 4|.
For large values of |i| this can result in |bn_expand| not allocating any
memory because |i * 4| is negative. This leaves ret->d as NULL leading
to a subsequent NULL ptr deref. For very large values of |i|, the
calculation |i * 4| could be a positive value smaller than |i|. In this
case memory is allocated to ret->d, but it is insufficiently sized
leading to heap corruption. A similar issue exists in BN_dec2bn.

This could have security consequences if BN_hex2bn/BN_dec2bn is ever
called by user applications with very large untrusted hex/dec data. This is
anticipated to be a rare occurrence.

All OpenSSL internal usage of this function uses data that is not expected
to be untrusted, e.g. config file data or application command line
arguments. If user developed applications generate config file data based
on untrusted data then it is possible that this could also lead to security
consequences. This is also anticipated to be a rare.

Issue reported by Guido Vranken.

CVE-2016-0797

Reviewed-by: Andy Polyakov <appro@openssl.org>
CVE: CVE-2016-0797
CWE: NVD-CWE-Other
Vulnerability description: Multiple integer overflows in OpenSSL 1.0.1 before 1.0.1s and 1.0.2 before 1.0.2g allow remote attackers to cause a denial of service (heap memory corruption or NULL pointer dereference) or possibly have unspecified other impact via a long digit string that is mishandled by the (1) BN_dec2bn or (2) BN_hex2bn function, related to crypto/bn/bn.h and crypto/bn/bn_print.c.

Project: openssl

int BN_print_fp(FILE *fp, const BIGNUM *a)
{
    BIO *b;
    int ret;

    if ((b = BIO_new(BIO_s_file())) == NULL)
        return (0);
    BIO_set_fp(b, fp, BIO_NOCLOSE);
    ret = BN_print(b, a);
    BIO_free(b);
    return (ret);
}

Project: openssl

static ossl_inline BIGNUM *bn_expand(BIGNUM *a, int bits)
{
    if (bits > (INT_MAX - BN_BITS2 + 1))
        return NULL;

    if(((bits+BN_BITS2-1)/BN_BITS2) <= (a)->dmax)
        return a;

    return bn_expand2((a),(bits+BN_BITS2-1)/BN_BITS2);
}

Project: openssl

int BN_asc2bn(BIGNUM **bn, const char *a)
{
    const char *p = a;
    if (*p == '-')
        p++;

    if (p[0] == '0' && (p[1] == 'X' || p[1] == 'x')) {
        if (!BN_hex2bn(bn, p + 2))
            return 0;
    } else {
        if (!BN_dec2bn(bn, p))
            return 0;
    }
    if (*a == '-')
        (*bn)->neg = 1;
    return 1;
}

Project: openssl

char *BN_options(void)
{
    static int init = 0;
    static char data[16];

    if (!init) {
        init++;
#ifdef BN_LLONG
        BIO_snprintf(data, sizeof data, "bn(%d,%d)",
                     (int)sizeof(BN_ULLONG) * 8, (int)sizeof(BN_ULONG) * 8);
#else
        BIO_snprintf(data, sizeof data, "bn(%d,%d)",
                     (int)sizeof(BN_ULONG) * 8, (int)sizeof(BN_ULONG) * 8);
#endif
    }
    return (data);
}

Project: openssl

int BN_hex2bn(BIGNUM **bn, const char *a)
{
    BIGNUM *ret = NULL;
    BN_ULONG l = 0;
    int neg = 0, h, m, i, j, k, c;
    int num;

    if ((a == NULL) || (*a == '\0'))
        return (0);

    if (*a == '-') {
        neg = 1;
        a++;
    }

    for (i = 0; i <= (INT_MAX/4) && isxdigit((unsigned char)a[i]); i++)
        continue;

    if (i > INT_MAX/4)
        goto err;

    num = i + neg;
    if (bn == NULL)
        return (num);

    /* a is the start of the hex digits, and it is 'i' long */
    if (*bn == NULL) {
        if ((ret = BN_new()) == NULL)
            return (0);
    } else {
        ret = *bn;
        BN_zero(ret);
    }

    /* i is the number of hex digits */
    if (bn_expand(ret, i * 4) == NULL)
        goto err;

    j = i;                      /* least significant 'hex' */
    m = 0;
    h = 0;
    while (j > 0) {
        m = ((BN_BYTES * 2) <= j) ? (BN_BYTES * 2) : j;
        l = 0;
        for (;;) {
            c = a[j - m];
            if ((c >= '0') && (c <= '9'))
                k = c - '0';
            else if ((c >= 'a') && (c <= 'f'))
                k = c - 'a' + 10;
            else if ((c >= 'A') && (c <= 'F'))
                k = c - 'A' + 10;
            else
                k = 0;          /* paranoia */
            l = (l << 4) | k;

            if (--m <= 0) {
                ret->d[h++] = l;
                break;
            }
        }
        j -= (BN_BYTES * 2);
    }
    ret->top = h;
    bn_correct_top(ret);
    ret->neg = neg;

    *bn = ret;
    bn_check_top(ret);
    return (num);
 err:
    if (*bn == NULL)
        BN_free(ret);
    return (0);
}

Project: openssl

int BN_print(BIO *bp, const BIGNUM *a)
{
    int i, j, v, z = 0;
    int ret = 0;

    if ((a->neg) && (BIO_write(bp, "-", 1) != 1))
        goto end;
    if (BN_is_zero(a) && (BIO_write(bp, "0", 1) != 1))
        goto end;
    for (i = a->top - 1; i >= 0; i--) {
        for (j = BN_BITS2 - 4; j >= 0; j -= 4) {
            /* strip leading zeros */
            v = ((int)(a->d[i] >> (long)j)) & 0x0f;
            if (z || (v != 0)) {
                if (BIO_write(bp, &(Hex[v]), 1) != 1)
                    goto end;
                z = 1;
            }
        }
    }
    ret = 1;
 end:
    return (ret);
}

Project: openssl

int BN_dec2bn(BIGNUM **bn, const char *a)
{
    BIGNUM *ret = NULL;
    BN_ULONG l = 0;
    int neg = 0, i, j;
    int num;

    if ((a == NULL) || (*a == '\0'))
        return (0);
    if (*a == '-') {
        neg = 1;
        a++;
    }

    for (i = 0; i <= (INT_MAX/4) && isdigit((unsigned char)a[i]); i++)
        continue;

    if (i > INT_MAX/4)
        goto err;

    num = i + neg;
    if (bn == NULL)
        return (num);

    /*
     * a is the start of the digits, and it is 'i' long. We chop it into
     * BN_DEC_NUM digits at a time
     */
    if (*bn == NULL) {
        if ((ret = BN_new()) == NULL)
            return (0);
    } else {
        ret = *bn;
        BN_zero(ret);
    }

    /* i is the number of digits, a bit of an over expand */
    if (bn_expand(ret, i * 4) == NULL)
        goto err;

    j = BN_DEC_NUM - (i % BN_DEC_NUM);
    if (j == BN_DEC_NUM)
        j = 0;
    l = 0;
    while (*a) {
        l *= 10;
        l += *a - '0';
        a++;
        if (++j == BN_DEC_NUM) {
            BN_mul_word(ret, BN_DEC_CONV);
            BN_add_word(ret, l);
            l = 0;
            j = 0;
        }
    }
    ret->neg = neg;

    bn_correct_top(ret);
    *bn = ret;
    bn_check_top(ret);
    return (num);
 err:
    if (*bn == NULL)
        BN_free(ret);
    return (0);
}