Commit message: Fix blur coefficient calculation buffer overflow

Found by fuzzer test case id:000082,sig:11,src:002579,op:havoc,rep:8.
Correctness should be checked, but this fixes the overflow for good.
CVE: CVE-2016-7970
CWE: CWE-119
Vulnerability description: Buffer overflow in the calc_coeff function in libass/ass_blur.c in libass before 0.13.4 allows remote attackers to cause a denial of service via unspecified vectors.

Project: libass
File: ass_blur.c

void ass_blur1234_horz_c(int16_t *dst, const int16_t *src,
                         uintptr_t src_width, uintptr_t src_height,
                         const int16_t *param)
{
    uintptr_t dst_width = src_width + 8;
    uintptr_t size = ((src_width + STRIPE_MASK) & ~STRIPE_MASK) * src_height;
    uintptr_t step = STRIPE_WIDTH * src_height;

    uintptr_t offs = 0;
    int16_t buf[2 * STRIPE_WIDTH];
    int16_t *ptr = buf + STRIPE_WIDTH;
    for (uintptr_t x = 0; x < dst_width; x += STRIPE_WIDTH) {
        for (uintptr_t y = 0; y < src_height; ++y) {
            copy_line(ptr - 1 * STRIPE_WIDTH, src, offs - 1 * step, size);
            copy_line(ptr - 0 * STRIPE_WIDTH, src, offs - 0 * step, size);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = blur_func(ptr[k - 8], ptr[k - 7], ptr[k - 6], ptr[k - 5], ptr[k - 4],
                                   ptr[k - 3], ptr[k - 2], ptr[k - 1], ptr[k - 0], param);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
    }
}

Project: libass
File: ass_blur.c

void ass_blur1234_vert_c(int16_t *dst, const int16_t *src,
                         uintptr_t src_width, uintptr_t src_height,
                         const int16_t *param)
{
    uintptr_t dst_height = src_height + 8;
    uintptr_t step = STRIPE_WIDTH * src_height;

    for (uintptr_t x = 0; x < src_width; x += STRIPE_WIDTH) {
        uintptr_t offs = 0;
        for (uintptr_t y = 0; y < dst_height; ++y) {
            const int16_t *p4 = get_line(src, offs - 8 * STRIPE_WIDTH, step);
            const int16_t *p3 = get_line(src, offs - 7 * STRIPE_WIDTH, step);
            const int16_t *p2 = get_line(src, offs - 6 * STRIPE_WIDTH, step);
            const int16_t *p1 = get_line(src, offs - 5 * STRIPE_WIDTH, step);
            const int16_t *z0 = get_line(src, offs - 4 * STRIPE_WIDTH, step);
            const int16_t *n1 = get_line(src, offs - 3 * STRIPE_WIDTH, step);
            const int16_t *n2 = get_line(src, offs - 2 * STRIPE_WIDTH, step);
            const int16_t *n3 = get_line(src, offs - 1 * STRIPE_WIDTH, step);
            const int16_t *n4 = get_line(src, offs - 0 * STRIPE_WIDTH, step);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = blur_func(p4[k], p3[k], p2[k], p1[k], z0[k],
                                   n1[k], n2[k], n3[k], n4[k], param);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
        src += step;
    }
}

Project: libass
File: ass_blur.c

void ass_blur1235_horz_c(int16_t *dst, const int16_t *src,
                         uintptr_t src_width, uintptr_t src_height,
                         const int16_t *param)
{
    uintptr_t dst_width = src_width + 10;
    uintptr_t size = ((src_width + STRIPE_MASK) & ~STRIPE_MASK) * src_height;
    uintptr_t step = STRIPE_WIDTH * src_height;

    uintptr_t offs = 0;
#if STRIPE_WIDTH < 10
    int16_t buf[3 * STRIPE_WIDTH];
    int16_t *ptr = buf + 2 * STRIPE_WIDTH;
#else
    int16_t buf[2 * STRIPE_WIDTH];
    int16_t *ptr = buf + STRIPE_WIDTH;
#endif
    for (uintptr_t x = 0; x < dst_width; x += STRIPE_WIDTH) {
        for (uintptr_t y = 0; y < src_height; ++y) {
#if STRIPE_WIDTH < 10
            copy_line(ptr - 2 * STRIPE_WIDTH, src, offs - 2 * step, size);
#endif
            copy_line(ptr - 1 * STRIPE_WIDTH, src, offs - 1 * step, size);
            copy_line(ptr - 0 * STRIPE_WIDTH, src, offs - 0 * step, size);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = blur_func(ptr[k - 10], ptr[k - 8], ptr[k - 7], ptr[k - 6], ptr[k - 5],
                                   ptr[k -  4], ptr[k - 3], ptr[k - 2], ptr[k - 0], param);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
    }
}

Project: libass
File: ass_blur.c

void ass_blur1246_horz_c(int16_t *dst, const int16_t *src,
                         uintptr_t src_width, uintptr_t src_height,
                         const int16_t *param)
{
    uintptr_t dst_width = src_width + 12;
    uintptr_t size = ((src_width + STRIPE_MASK) & ~STRIPE_MASK) * src_height;
    uintptr_t step = STRIPE_WIDTH * src_height;

    uintptr_t offs = 0;
#if STRIPE_WIDTH < 12
    int16_t buf[3 * STRIPE_WIDTH];
    int16_t *ptr = buf + 2 * STRIPE_WIDTH;
#else
    int16_t buf[2 * STRIPE_WIDTH];
    int16_t *ptr = buf + STRIPE_WIDTH;
#endif
    for (uintptr_t x = 0; x < dst_width; x += STRIPE_WIDTH) {
        for (uintptr_t y = 0; y < src_height; ++y) {
#if STRIPE_WIDTH < 12
            copy_line(ptr - 2 * STRIPE_WIDTH, src, offs - 2 * step, size);
#endif
            copy_line(ptr - 1 * STRIPE_WIDTH, src, offs - 1 * step, size);
            copy_line(ptr - 0 * STRIPE_WIDTH, src, offs - 0 * step, size);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = blur_func(ptr[k - 12], ptr[k - 10], ptr[k - 8], ptr[k - 7], ptr[k - 6],
                                   ptr[k -  5], ptr[k -  4], ptr[k - 2], ptr[k - 0], param);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
    }
}

Project: libass
File: ass_blur.c

void ass_blur1246_vert_c(int16_t *dst, const int16_t *src,
                         uintptr_t src_width, uintptr_t src_height,
                         const int16_t *param)
{
    uintptr_t dst_height = src_height + 12;
    uintptr_t step = STRIPE_WIDTH * src_height;

    for (uintptr_t x = 0; x < src_width; x += STRIPE_WIDTH) {
        uintptr_t offs = 0;
        for (uintptr_t y = 0; y < dst_height; ++y) {
            const int16_t *p4 = get_line(src, offs - 12 * STRIPE_WIDTH, step);
            const int16_t *p3 = get_line(src, offs - 10 * STRIPE_WIDTH, step);
            const int16_t *p2 = get_line(src, offs -  8 * STRIPE_WIDTH, step);
            const int16_t *p1 = get_line(src, offs -  7 * STRIPE_WIDTH, step);
            const int16_t *z0 = get_line(src, offs -  6 * STRIPE_WIDTH, step);
            const int16_t *n1 = get_line(src, offs -  5 * STRIPE_WIDTH, step);
            const int16_t *n2 = get_line(src, offs -  4 * STRIPE_WIDTH, step);
            const int16_t *n3 = get_line(src, offs -  2 * STRIPE_WIDTH, step);
            const int16_t *n4 = get_line(src, offs -  0 * STRIPE_WIDTH, step);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = blur_func(p4[k], p3[k], p2[k], p1[k], z0[k],
                                   n1[k], n2[k], n3[k], n4[k], param);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
        src += step;
    }
}

Project: libass
File: ass_blur.c

void ass_expand_horz_c(int16_t *dst, const int16_t *src,
                       uintptr_t src_width, uintptr_t src_height)
{
    uintptr_t dst_width = 2 * src_width + 4;
    uintptr_t size = ((src_width + STRIPE_MASK) & ~STRIPE_MASK) * src_height;
    uintptr_t step = STRIPE_WIDTH * src_height;

    uintptr_t offs = 0;
    int16_t buf[2 * STRIPE_WIDTH];
    int16_t *ptr = buf + STRIPE_WIDTH;
    for (uintptr_t x = STRIPE_WIDTH; x < dst_width; x += 2 * STRIPE_WIDTH) {
        for (uintptr_t y = 0; y < src_height; ++y) {
            copy_line(ptr - 1 * STRIPE_WIDTH, src, offs - 1 * step, size);
            copy_line(ptr - 0 * STRIPE_WIDTH, src, offs - 0 * step, size);
            for (int k = 0; k < STRIPE_WIDTH / 2; ++k)
                expand_func(&dst[2 * k], &dst[2 * k + 1],
                            ptr[k - 2], ptr[k - 1], ptr[k]);
            int16_t *next = dst + step - STRIPE_WIDTH;
            for (int k = STRIPE_WIDTH / 2; k < STRIPE_WIDTH; ++k)
                expand_func(&next[2 * k], &next[2 * k + 1],
                            ptr[k - 2], ptr[k - 1], ptr[k]);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
        dst += step;
    }
    if ((dst_width - 1) & STRIPE_WIDTH)
        return;

    for (uintptr_t y = 0; y < src_height; ++y) {
        copy_line(ptr - 1 * STRIPE_WIDTH, src, offs - 1 * step, size);
        copy_line(ptr - 0 * STRIPE_WIDTH, src, offs - 0 * step, size);
        for (int k = 0; k < STRIPE_WIDTH / 2; ++k)
            expand_func(&dst[2 * k], &dst[2 * k + 1],
                        ptr[k - 2], ptr[k - 1], ptr[k]);
        dst += STRIPE_WIDTH;
        offs += STRIPE_WIDTH;
    }
}

Project: libass
File: ass_blur.c

void ass_expand_vert_c(int16_t *dst, const int16_t *src,
                       uintptr_t src_width, uintptr_t src_height)
{
    uintptr_t dst_height = 2 * src_height + 4;
    uintptr_t step = STRIPE_WIDTH * src_height;

    for (uintptr_t x = 0; x < src_width; x += STRIPE_WIDTH) {
        uintptr_t offs = 0;
        for (uintptr_t y = 0; y < dst_height; y += 2) {
            const int16_t *p1 = get_line(src, offs - 2 * STRIPE_WIDTH, step);
            const int16_t *z0 = get_line(src, offs - 1 * STRIPE_WIDTH, step);
            const int16_t *n1 = get_line(src, offs - 0 * STRIPE_WIDTH, step);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                expand_func(&dst[k], &dst[k + STRIPE_WIDTH],
                            p1[k], z0[k], n1[k]);
            dst += 2 * STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
        src += step;
    }
}

Project: libass
File: ass_blur.c

bool ass_gaussian_blur(const BitmapEngine *engine, Bitmap *bm, double r2)
{
    BlurMethod blur;
    find_best_method(&blur, r2);

    int w = bm->w, h = bm->h;
    int offset = ((2 * (blur.prefilter + blur.filter) + 17) << blur.level) - 5;
    int end_w = ((w + offset) & ~((1 << blur.level) - 1)) - 4;
    int end_h = ((h + offset) & ~((1 << blur.level) - 1)) - 4;

    const int stripe_width = 1 << (engine->align_order - 1);
    int size = end_h * ((end_w + stripe_width - 1) & ~(stripe_width - 1));
    int16_t *tmp = ass_aligned_alloc(2 * stripe_width, 4 * size, false);
    if (!tmp)
        return false;

    engine->stripe_unpack(tmp, bm->buffer, bm->stride, w, h);
    int16_t *buf[2] = {tmp, tmp + size};
    int index = 0;

    for (int i = 0; i < blur.level; ++i) {
        engine->shrink_vert(buf[index ^ 1], buf[index], w, h);
        h = (h + 5) >> 1;
        index ^= 1;
    }
    for (int i = 0; i < blur.level; ++i) {
        engine->shrink_horz(buf[index ^ 1], buf[index], w, h);
        w = (w + 5) >> 1;
        index ^= 1;
    }
    if (blur.prefilter) {
        engine->pre_blur_horz[blur.prefilter - 1](buf[index ^ 1], buf[index], w, h);
        w += 2 * blur.prefilter;
        index ^= 1;
    }
    engine->main_blur_horz[blur.filter](buf[index ^ 1], buf[index], w, h, blur.coeff);
    w += 2 * blur.filter + 8;
    index ^= 1;
    for (int i = 0; i < blur.level; ++i) {
        engine->expand_horz(buf[index ^ 1], buf[index], w, h);
        w = 2 * w + 4;
        index ^= 1;
    }
    if (blur.prefilter) {
        engine->pre_blur_vert[blur.prefilter - 1](buf[index ^ 1], buf[index], w, h);
        h += 2 * blur.prefilter;
        index ^= 1;
    }
    engine->main_blur_vert[blur.filter](buf[index ^ 1], buf[index], w, h, blur.coeff);
    h += 2 * blur.filter + 8;
    index ^= 1;
    for (int i = 0; i < blur.level; ++i) {
        engine->expand_vert(buf[index ^ 1], buf[index], w, h);
        h = 2 * h + 4;
        index ^= 1;
    }
    assert(w == end_w && h == end_h);

    if (!realloc_bitmap(engine, bm, w, h)) {
        ass_aligned_free(tmp);
        return false;
    }
    offset = ((blur.prefilter + blur.filter + 8) << blur.level) - 4;
    bm->left -= offset;
    bm->top  -= offset;

    engine->stripe_pack(bm->buffer, bm->stride, buf[index], w, h);
    ass_aligned_free(tmp);
    return true;
}

Project: libass
File: ass_blur.c

void ass_pre_blur1_vert_c(int16_t *dst, const int16_t *src,
                          uintptr_t src_width, uintptr_t src_height)
{
    uintptr_t dst_height = src_height + 2;
    uintptr_t step = STRIPE_WIDTH * src_height;

    for (uintptr_t x = 0; x < src_width; x += STRIPE_WIDTH) {
        uintptr_t offs = 0;
        for (uintptr_t y = 0; y < dst_height; ++y) {
            const int16_t *p1 = get_line(src, offs - 2 * STRIPE_WIDTH, step);
            const int16_t *z0 = get_line(src, offs - 1 * STRIPE_WIDTH, step);
            const int16_t *n1 = get_line(src, offs - 0 * STRIPE_WIDTH, step);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = pre_blur1_func(p1[k], z0[k], n1[k]);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
        src += step;
    }
}

Project: libass
File: ass_blur.c

void ass_pre_blur2_horz_c(int16_t *dst, const int16_t *src,
                          uintptr_t src_width, uintptr_t src_height)
{
    uintptr_t dst_width = src_width + 4;
    uintptr_t size = ((src_width + STRIPE_MASK) & ~STRIPE_MASK) * src_height;
    uintptr_t step = STRIPE_WIDTH * src_height;

    uintptr_t offs = 0;
    int16_t buf[2 * STRIPE_WIDTH];
    int16_t *ptr = buf + STRIPE_WIDTH;
    for (uintptr_t x = 0; x < dst_width; x += STRIPE_WIDTH) {
        for (uintptr_t y = 0; y < src_height; ++y) {
            copy_line(ptr - 1 * STRIPE_WIDTH, src, offs - 1 * step, size);
            copy_line(ptr - 0 * STRIPE_WIDTH, src, offs - 0 * step, size);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = pre_blur2_func(ptr[k - 4], ptr[k - 3], ptr[k - 2], ptr[k - 1], ptr[k]);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
    }
}

Project: libass
File: ass_blur.c

void ass_pre_blur2_vert_c(int16_t *dst, const int16_t *src,
                          uintptr_t src_width, uintptr_t src_height)
{
    uintptr_t dst_height = src_height + 4;
    uintptr_t step = STRIPE_WIDTH * src_height;

    for (uintptr_t x = 0; x < src_width; x += STRIPE_WIDTH) {
        uintptr_t offs = 0;
        for (uintptr_t y = 0; y < dst_height; ++y) {
            const int16_t *p2 = get_line(src, offs - 4 * STRIPE_WIDTH, step);
            const int16_t *p1 = get_line(src, offs - 3 * STRIPE_WIDTH, step);
            const int16_t *z0 = get_line(src, offs - 2 * STRIPE_WIDTH, step);
            const int16_t *n1 = get_line(src, offs - 1 * STRIPE_WIDTH, step);
            const int16_t *n2 = get_line(src, offs - 0 * STRIPE_WIDTH, step);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = pre_blur2_func(p2[k], p1[k], z0[k], n1[k], n2[k]);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
        src += step;
    }
}

Project: libass
File: ass_blur.c

void ass_pre_blur3_horz_c(int16_t *dst, const int16_t *src,
                          uintptr_t src_width, uintptr_t src_height)
{
    uintptr_t dst_width = src_width + 6;
    uintptr_t size = ((src_width + STRIPE_MASK) & ~STRIPE_MASK) * src_height;
    uintptr_t step = STRIPE_WIDTH * src_height;

    uintptr_t offs = 0;
    int16_t buf[2 * STRIPE_WIDTH];
    int16_t *ptr = buf + STRIPE_WIDTH;
    for (uintptr_t x = 0; x < dst_width; x += STRIPE_WIDTH) {
        for (uintptr_t y = 0; y < src_height; ++y) {
            copy_line(ptr - 1 * STRIPE_WIDTH, src, offs - 1 * step, size);
            copy_line(ptr - 0 * STRIPE_WIDTH, src, offs - 0 * step, size);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = pre_blur3_func(ptr[k - 6], ptr[k - 5], ptr[k - 4], ptr[k - 3],
                                        ptr[k - 2], ptr[k - 1], ptr[k]);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
    }
}

Project: libass
File: ass_blur.c

void ass_pre_blur3_vert_c(int16_t *dst, const int16_t *src,
                          uintptr_t src_width, uintptr_t src_height)
{
    uintptr_t dst_height = src_height + 6;
    uintptr_t step = STRIPE_WIDTH * src_height;

    for (uintptr_t x = 0; x < src_width; x += STRIPE_WIDTH) {
        uintptr_t offs = 0;
        for (uintptr_t y = 0; y < dst_height; ++y) {
            const int16_t *p3 = get_line(src, offs - 6 * STRIPE_WIDTH, step);
            const int16_t *p2 = get_line(src, offs - 5 * STRIPE_WIDTH, step);
            const int16_t *p1 = get_line(src, offs - 4 * STRIPE_WIDTH, step);
            const int16_t *z0 = get_line(src, offs - 3 * STRIPE_WIDTH, step);
            const int16_t *n1 = get_line(src, offs - 2 * STRIPE_WIDTH, step);
            const int16_t *n2 = get_line(src, offs - 1 * STRIPE_WIDTH, step);
            const int16_t *n3 = get_line(src, offs - 0 * STRIPE_WIDTH, step);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = pre_blur3_func(p3[k], p2[k], p1[k], z0[k], n1[k], n2[k], n3[k]);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
        src += step;
    }
}

Project: libass
File: ass_blur.c

void ass_shrink_horz_c(int16_t *dst, const int16_t *src,
                       uintptr_t src_width, uintptr_t src_height)
{
    uintptr_t dst_width = (src_width + 5) >> 1;
    uintptr_t size = ((src_width + STRIPE_MASK) & ~STRIPE_MASK) * src_height;
    uintptr_t step = STRIPE_WIDTH * src_height;

    uintptr_t offs = 0;
    int16_t buf[3 * STRIPE_WIDTH];
    int16_t *ptr = buf + STRIPE_WIDTH;
    for (uintptr_t x = 0; x < dst_width; x += STRIPE_WIDTH) {
        for (uintptr_t y = 0; y < src_height; ++y) {
            copy_line(ptr - 1 * STRIPE_WIDTH, src, offs - 1 * step, size);
            copy_line(ptr + 0 * STRIPE_WIDTH, src, offs + 0 * step, size);
            copy_line(ptr + 1 * STRIPE_WIDTH, src, offs + 1 * step, size);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = shrink_func(ptr[2 * k - 4], ptr[2 * k - 3],
                                     ptr[2 * k - 2], ptr[2 * k - 1],
                                     ptr[2 * k + 0], ptr[2 * k + 1]);
            dst += STRIPE_WIDTH;
            offs += STRIPE_WIDTH;
        }
        offs += step;
    }
}

Project: libass
File: ass_blur.c

void ass_shrink_vert_c(int16_t *dst, const int16_t *src,
                       uintptr_t src_width, uintptr_t src_height)
{
    uintptr_t dst_height = (src_height + 5) >> 1;
    uintptr_t step = STRIPE_WIDTH * src_height;

    for (uintptr_t x = 0; x < src_width; x += STRIPE_WIDTH) {
        uintptr_t offs = 0;
        for (uintptr_t y = 0; y < dst_height; ++y) {
            const int16_t *p1p = get_line(src, offs - 4 * STRIPE_WIDTH, step);
            const int16_t *p1n = get_line(src, offs - 3 * STRIPE_WIDTH, step);
            const int16_t *z0p = get_line(src, offs - 2 * STRIPE_WIDTH, step);
            const int16_t *z0n = get_line(src, offs - 1 * STRIPE_WIDTH, step);
            const int16_t *n1p = get_line(src, offs - 0 * STRIPE_WIDTH, step);
            const int16_t *n1n = get_line(src, offs + 1 * STRIPE_WIDTH, step);
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                dst[k] = shrink_func(p1p[k], p1n[k], z0p[k], z0n[k], n1p[k], n1n[k]);
            dst += STRIPE_WIDTH;
            offs += 2 * STRIPE_WIDTH;
        }
        src += step;
    }
}

Project: libass

void ass_stripe_unpack_c(int16_t *dst, const uint8_t *src, ptrdiff_t src_stride,
                         uintptr_t width, uintptr_t height)
{
    for (uintptr_t y = 0; y < height; ++y) {
        int16_t *ptr = dst;
        for (uintptr_t x = 0; x < width; x += STRIPE_WIDTH) {
            for (int k = 0; k < STRIPE_WIDTH; ++k)
                ptr[k] = (uint16_t)(((src[x + k] << 7) | (src[x + k] >> 1)) + 1) >> 1;
            ptr += STRIPE_WIDTH * height;
        }
        dst += STRIPE_WIDTH;
        src += src_stride;
    }
}

Project: libass
File: ass_blur.c

static inline int16_t blur_func(int16_t p4, int16_t p3, int16_t p2, int16_t p1, int16_t z0,
                                int16_t n1, int16_t n2, int16_t n3, int16_t n4, const int16_t c[])
{
    p1 -= z0;
    p2 -= z0;
    p3 -= z0;
    p4 -= z0;
    n1 -= z0;
    n2 -= z0;
    n3 -= z0;
    n4 -= z0;
    return (((p1 + n1) * c[0] +
             (p2 + n2) * c[1] +
             (p3 + n3) * c[2] +
             (p4 + n4) * c[3] +
             0x8000) >> 16) + z0;
}

Project: libass
File: ass_blur.c

static void calc_gauss(double *res, int n, double r2)
{
    double alpha = 0.5 / r2;
    double mul = exp(-alpha), mul2 = mul * mul;
    double cur = sqrt(alpha / M_PI);

    res[0] = cur;
    cur *= mul;
    res[1] = cur;
    for (int i = 2; i <= n; ++i) {
        mul *= mul2;
        cur *= mul;
        res[i] = cur;
    }
}

Project: libass

static void calc_matrix(double mat[4][4], const double *mat_freq, const int *index)
{
    for (int i = 0; i < 4; ++i) {
        mat[i][i] = mat_freq[2 * index[i]] + 3 * mat_freq[0] - 4 * mat_freq[index[i]];
        for (int j = i + 1; j < 4; ++j)
            mat[i][j] = mat[j][i] =
                mat_freq[index[i] + index[j]] + mat_freq[index[j] - index[i]] +
                2 * (mat_freq[0] - mat_freq[index[i]] - mat_freq[index[j]]);
    }

    for (int k = 0; k < 4; ++k) {
        int ip = k, jp = k;  // pivot
        double z = 1 / mat[ip][jp];
        mat[ip][jp] = 1;
        for (int i = 0; i < 4; ++i) {
            if (i == ip)
                continue;

            double mul = mat[i][jp] * z;
            mat[i][jp] = 0;
            for (int j = 0; j < 4; ++j)
                mat[i][j] -= mat[ip][j] * mul;
        }
        for (int j = 0; j < 4; ++j)
            mat[ip][j] *= z;
    }
}

Project: libass
File: ass_blur.c

static void coeff_blur121(double *coeff, int n)
{
    double prev = coeff[1];
    for (int i = 0; i <= n; ++i) {
        double res = (prev + 2 * coeff[i] + coeff[i + 1]) / 4;
        prev = coeff[i];
        coeff[i] = res;
    }
}

Project: libass
File: ass_blur.c

static void coeff_filter(double *coeff, int n, const double kernel[4])
{
    double prev1 = coeff[1], prev2 = coeff[2], prev3 = coeff[3];
    for (int i = 0; i <= n; ++i) {
        double res = coeff[i + 0]  * kernel[0] +
            (prev1 + coeff[i + 1]) * kernel[1] +
            (prev2 + coeff[i + 2]) * kernel[2] +
            (prev3 + coeff[i + 3]) * kernel[3];
        prev3 = prev2;
        prev2 = prev1;
        prev1 = coeff[i];
        coeff[i] = res;
    }
}

Project: libass
File: ass_blur.c

inline static void copy_line(int16_t *buf, const int16_t *ptr, uintptr_t offs, uintptr_t size)
{
    ptr = get_line(ptr, offs, size);
    for (int k = 0; k < STRIPE_WIDTH; ++k)
        buf[k] = ptr[k];
}

Project: libass
File: ass_blur.c

static void find_best_method(BlurMethod *blur, double r2)
{
    static const int index[][4] = {
        { 1, 2, 3, 4 },
        { 1, 2, 3, 5 },
        { 1, 2, 4, 6 },
    };

    double mu[5];
    if (r2 < 1.9) {
        blur->level = blur->prefilter = blur->filter = 0;

        if (r2 < 0.5) {
            mu[2] = 0.085 * r2 * r2 * r2;
            mu[1] = 0.5 * r2 - 4 * mu[2];
            mu[3] = mu[4] = 0;
        } else {
            calc_gauss(mu, 4, r2);
        }
    } else {
        double mul = 1;
        if (r2 < 6.693) {
            blur->level = 0;

            if (r2 < 2.8)
                blur->prefilter = 1;
            else if (r2 < 4.4)
                blur->prefilter = 2;
            else
                blur->prefilter = 3;

            blur->filter = blur->prefilter - 1;
        } else {
            frexp((r2 + 0.7) / 26.5, &blur->level);
            blur->level = (blur->level + 3) >> 1;
            mul = pow(0.25, blur->level);
            r2 *= mul;

            if (r2 < 3.15 - 1.5 * mul)
                blur->prefilter = 0;
            else if (r2 < 5.3 - 5.2 * mul)
                blur->prefilter = 1;
            else
                blur->prefilter = 2;

            blur->filter = blur->prefilter;
        }
        calc_coeff(mu + 1, index[blur->filter], blur->prefilter, r2, mul);
    }

    for (int i = 1; i <= 4; ++i)
        blur->coeff[i - 1] = (int)(0x10000 * mu[i] + 0.5);
}

Project: libass
File: ass_blur.c

static inline int16_t pre_blur1_func(int16_t p1, int16_t z0, int16_t n1)
{
    /*
    return (1 * p1 + 2 * z0 + 1 * n1 + 2) >> 2;
    */
    return (uint16_t)(((uint16_t)(p1 + n1) >> 1) + z0 + 1) >> 1;
}

Project: libass
File: ass_blur.c

static inline int16_t pre_blur2_func(int16_t p2, int16_t p1, int16_t z0,
                                     int16_t n1, int16_t n2)
{
    /*
    return (1 * p2 + 4 * p1 + 6 * z0 + 4 * n1 + 1 * n2 + 8) >> 4;
    */
    uint16_t r1 = ((uint16_t)(((uint16_t)(p2 + n2) >> 1) + z0) >> 1) + z0;
    uint16_t r2 = p1 + n1;
    uint16_t r = ((uint16_t)(r1 + r2) >> 1) | (0x8000 & r1 & r2);
    return (uint16_t)(r + 1) >> 1;
}

Project: libass
File: ass_blur.c

static inline int16_t pre_blur3_func(int16_t p3, int16_t p2, int16_t p1, int16_t z0,
                                     int16_t n1, int16_t n2, int16_t n3)
{
    /*
    return (1 * p3 + 6 * p2 + 15 * p1 + 20 * z0 + 15 * n1 + 6 * n2 + 1 * n3 + 32) >> 6;
    */
    return (20 * (uint16_t)z0 +
            15 * (uint16_t)(p1 + n1) +
             6 * (uint16_t)(p2 + n2) +
             1 * (uint16_t)(p3 + n3) + 32) >> 6;
}

Project: libass
File: ass_blur.c

static inline int16_t shrink_func(int16_t p1p, int16_t p1n,
                                  int16_t z0p, int16_t z0n,
                                  int16_t n1p, int16_t n1n)
{
    /*
    return (1 * p1p + 5 * p1n + 10 * z0p + 10 * z0n + 5 * n1p + 1 * n1n + 16) >> 5;
    */
    int32_t r = (p1p + p1n + n1p + n1n) >> 1;
    r = (r + z0p + z0n) >> 1;
    r = (r + p1n + n1p) >> 1;
    return (r + z0p + z0n + 2) >> 2;
}