Commit message: Avoid overflow in EVP_EncodeUpdate

An overflow can occur in the EVP_EncodeUpdate function which is used for
Base64 encoding of binary data. If an attacker is able to supply very large
amounts of input data then a length check can overflow resulting in a heap
corruption. Due to the very large amounts of data involved this will most
likely result in a crash.

Internally to OpenSSL the EVP_EncodeUpdate function is primarly used by the
PEM_write_bio* family of functions. These are mainly used within the
OpenSSL command line applications, so any application which processes
data from an untrusted source and outputs it as a PEM file should be
considered vulnerable to this issue.

User applications that call these APIs directly with large amounts of
untrusted data may also be vulnerable.

Issue reported by Guido Vranken.

CVE-2016-2105

Reviewed-by: Richard Levitte <levitte@openssl.org>
CVE: CVE-2016-2105
CWE: CWE-189
Vulnerability description: Integer overflow in the EVP_EncodeUpdate function in crypto/evp/encode.c in OpenSSL before 1.0.1t and 1.0.2 before 1.0.2h allows remote attackers to cause a denial of service (heap memory corruption) via a large amount of binary data.

Project: openssl

int EVP_DecodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl)
{
    int i;

    *outl = 0;
    if (ctx->num != 0) {
        i = EVP_DecodeBlock(out, ctx->enc_data, ctx->num);
        if (i < 0)
            return (-1);
        ctx->num = 0;
        *outl = i;
        return (1);
    } else
        return (1);
}

Project: openssl

void EVP_DecodeInit(EVP_ENCODE_CTX *ctx)
{
    /* Only ctx->num is used during decoding. */
    ctx->num = 0;
    ctx->length = 0;
    ctx->line_num = 0;
    ctx->expect_nl = 0;
}

Project: openssl

int EVP_DecodeUpdate(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl,
                     const unsigned char *in, int inl)
{
    int seof = 0, eof = 0, rv = -1, ret = 0, i, v, tmp, n, decoded_len;
    unsigned char *d;

    n = ctx->num;
    d = ctx->enc_data;

    if (n > 0 && d[n - 1] == '=') {
        eof++;
        if (n > 1 && d[n - 2] == '=')
            eof++;
    }

     /* Legacy behaviour: an empty input chunk signals end of input. */
    if (inl == 0) {
        rv = 0;
        goto end;
    }

    for (i = 0; i < inl; i++) {
        tmp = *(in++);
        v = conv_ascii2bin(tmp);
        if (v == B64_ERROR) {
            rv = -1;
            goto end;
        }

        if (tmp == '=') {
            eof++;
        } else if (eof > 0 && B64_BASE64(v)) {
            /* More data after padding. */
            rv = -1;
            goto end;
        }

        if (eof > 2) {
            rv = -1;
            goto end;
        }

        if (v == B64_EOF) {
            seof = 1;
            goto tail;
        }

        /* Only save valid base64 characters. */
        if (B64_BASE64(v)) {
            if (n >= 64) {
                /*
                 * We increment n once per loop, and empty the buffer as soon as
                 * we reach 64 characters, so this can only happen if someone's
                 * manually messed with the ctx. Refuse to write any more data.
                 */
                rv = -1;
                goto end;
            }
            OPENSSL_assert(n < (int)sizeof(ctx->enc_data));
            d[n++] = tmp;
        }

        if (n == 64) {
            decoded_len = EVP_DecodeBlock(out, d, n);
            n = 0;
            if (decoded_len < 0 || eof > decoded_len) {
                rv = -1;
                goto end;
            }
            ret += decoded_len - eof;
            out += decoded_len - eof;
        }
    }

    /*
     * Legacy behaviour: if the current line is a full base64-block (i.e., has
     * 0 mod 4 base64 characters), it is processed immediately. We keep this
     * behaviour as applications may not be calling EVP_DecodeFinal properly.
     */
tail:
    if (n > 0) {
        if ((n & 3) == 0) {
            decoded_len = EVP_DecodeBlock(out, d, n);
            n = 0;
            if (decoded_len < 0 || eof > decoded_len) {
                rv = -1;
                goto end;
            }
            ret += (decoded_len - eof);
        } else if (seof) {
            /* EOF in the middle of a base64 block. */
            rv = -1;
            goto end;
        }
    }

    rv = seof || (n == 0 && eof) ? 0 : 1;
end:
    /* Legacy behaviour. This should probably rather be zeroed on error. */
    *outl = ret;
    ctx->num = n;
    return (rv);
}

Project: openssl

int EVP_DecodeValid(unsigned char *buf, int len)
{
    int i, num = 0, bad = 0;

    if (len == 0)
        return (-1);
    while (conv_ascii2bin(*buf) == B64_WS) {
        buf++;
        len--;
        if (len == 0)
            return (-1);
    }

    for (i = len; i >= 4; i -= 4) {
        if ((conv_ascii2bin(buf[0]) >= 0x40) ||
            (conv_ascii2bin(buf[1]) >= 0x40) ||
            (conv_ascii2bin(buf[2]) >= 0x40) ||
            (conv_ascii2bin(buf[3]) >= 0x40))
            return (-1);
        buf += 4;
        num += 1 + (buf[2] != '=') + (buf[3] != '=');
    }
    if ((i == 1) && (conv_ascii2bin(buf[0]) == B64_EOLN))
        return (num);
    if ((i == 2) && (conv_ascii2bin(buf[0]) == B64_EOLN) &&
        (conv_ascii2bin(buf[0]) == B64_EOLN))
        return (num);
    return (1);
}

Project: openssl

int EVP_EncodeBlock(unsigned char *t, const unsigned char *f, int dlen)
{
    int i, ret = 0;
    unsigned long l;

    for (i = dlen; i > 0; i -= 3) {
        if (i >= 3) {
            l = (((unsigned long)f[0]) << 16L) |
                (((unsigned long)f[1]) << 8L) | f[2];
            *(t++) = conv_bin2ascii(l >> 18L);
            *(t++) = conv_bin2ascii(l >> 12L);
            *(t++) = conv_bin2ascii(l >> 6L);
            *(t++) = conv_bin2ascii(l);
        } else {
            l = ((unsigned long)f[0]) << 16L;
            if (i == 2)
                l |= ((unsigned long)f[1] << 8L);

            *(t++) = conv_bin2ascii(l >> 18L);
            *(t++) = conv_bin2ascii(l >> 12L);
            *(t++) = (i == 1) ? '=' : conv_bin2ascii(l >> 6L);
            *(t++) = '=';
        }
        ret += 4;
        f += 3;
    }

    *t = '\0';
    return (ret);
}

Project: openssl

void EVP_EncodeFinal(EVP_ENCODE_CTX *ctx, unsigned char *out, int *outl)
{
    unsigned int ret = 0;

    if (ctx->num != 0) {
        ret = EVP_EncodeBlock(out, ctx->enc_data, ctx->num);
        out[ret++] = '\n';
        out[ret] = '\0';
        ctx->num = 0;
    }
    *outl = ret;
}

Project: openssl

void EVP_EncodeInit(EVP_ENCODE_CTX *ctx)
{
    ctx->length = 48;
    ctx->num = 0;
    ctx->line_num = 0;
}

Project: openssl

static unsigned char conv_ascii2bin(unsigned char a)
{
    if (a & 0x80)
        return B64_ERROR;
    return data_ascii2bin[a];
}

Project: openssl

static unsigned char conv_ascii2bin(unsigned char a)
{
    a = os_toascii[a];
    if (a & 0x80)
        return B64_ERROR;
    return data_ascii2bin[a];
}