Commit message: Avoid illegal memory access problem with 1 color paletted png(#73),
Thanks to HongxuChen.
CVE: CVE-2020-21048
CWE: CWE-200
Vulnerability description: An issue in the dither.c component of libsixel prior to v1.8.4 allows attackers to cause a denial of service (DOS) via a crafted PNG file.

Project: libsixel
File: dither.c

sixel_dither_set_pixelformat(
    sixel_dither_t /* in */ *dither,     /* dither context object */
    int            /* in */ pixelformat) /* one of enum pixelFormat */
{
    dither->pixelformat = pixelformat;
}

Project: libsixel
File: dither.c

sixel_dither_create(
    int     /* in */ ncolors)
{
    SIXELSTATUS status = SIXEL_FALSE;
    sixel_dither_t *dither = NULL;

    status = sixel_dither_new(&dither, ncolors, NULL);
    if (SIXEL_FAILED(status)) {
        goto end;
    }

end:
    return dither;
}

Project: libsixel
File: dither.c

sixel_dither_get_num_of_histgram_colors(
    sixel_dither_t /* in */ *dither)  /* dither context object */
{
    return sixel_dither_get_num_of_histogram_colors(dither);
}

Project: libsixel
File: dither.c

sixel_dither_apply_palette(
    sixel_dither_t  /* in */ *dither,
    unsigned char   /* in */ *pixels,
    int             /* in */ width,
    int             /* in */ height)
{
    SIXELSTATUS status = SIXEL_FALSE;
    size_t bufsize;
    unsigned char *dest = NULL;
    int ncolors;
    unsigned char *normalized_pixels = NULL;
    unsigned char *input_pixels;

    if (dither == NULL) {
        sixel_helper_set_additional_message(
            "sixel_dither_apply_palette: dither is null.");
        status = SIXEL_BAD_ARGUMENT;
        goto end;
    }

    sixel_dither_ref(dither);

    bufsize = (size_t)(width * height) * sizeof(unsigned char);
    dest = (unsigned char *)sixel_allocator_malloc(dither->allocator, bufsize);
    if (dest == NULL) {
        sixel_helper_set_additional_message(
            "sixel_dither_new: sixel_allocator_malloc() failed.");
        status = SIXEL_BAD_ALLOCATION;
        goto end;
    }

    /* if quality_mode is full, do not use palette caching */
    if (dither->quality_mode == SIXEL_QUALITY_FULL) {
        dither->optimized = 0;
    }

    if (dither->cachetable == NULL && dither->optimized) {
        if (dither->palette != pal_mono_dark && dither->palette != pal_mono_light) {
            dither->cachetable = (unsigned short *)sixel_allocator_calloc(dither->allocator,
                                                                          (size_t)(1 << 3 * 5),
                                                                          sizeof(unsigned short));
            if (dither->cachetable == NULL) {
                sixel_helper_set_additional_message(
                    "sixel_dither_new: sixel_allocator_calloc() failed.");
                status = SIXEL_BAD_ALLOCATION;
                goto end;
            }
        }
    }

    if (dither->pixelformat != SIXEL_PIXELFORMAT_RGB888) {
        /* normalize pixelformat */
        normalized_pixels
            = (unsigned char *)sixel_allocator_malloc(dither->allocator, (size_t)(width * height * 3));
        if (normalized_pixels == NULL) {
            sixel_helper_set_additional_message(
                "sixel_dither_new: sixel_allocator_malloc() failed.");
            status = SIXEL_BAD_ALLOCATION;
            goto end;
        }
        status = sixel_helper_normalize_pixelformat(normalized_pixels,
                                                    &dither->pixelformat,
                                                    pixels, dither->pixelformat,
                                                    width, height);
        if (SIXEL_FAILED(status)) {
            goto end;
        }
        input_pixels = normalized_pixels;
    } else {
        input_pixels = pixels;
    }

    status = sixel_quant_apply_palette(dest,
                                       input_pixels,
                                       width, height, 3,
                                       dither->palette,
                                       dither->ncolors,
                                       dither->method_for_diffuse,
                                       dither->optimized,
                                       dither->optimize_palette,
                                       dither->complexion,
                                       dither->cachetable,
                                       &ncolors,
                                       dither->allocator);
    if (SIXEL_FAILED(status)) {
        free(dest);
        dest = NULL;
        goto end;
    }

    dither->ncolors = ncolors;

end:
    free(normalized_pixels);
    sixel_dither_unref(dither);
    return dest;
}

Project: libsixel
File: dither.c

sixel_dither_set_quality_mode(
    sixel_dither_t  /* in */  *dither,
    int             /* in */  quality_mode)
{
    if (quality_mode == SIXEL_QUALITY_AUTO) {
        if (dither->ncolors <= 8) {
            quality_mode = SIXEL_QUALITY_HIGH;
        } else {
            quality_mode = SIXEL_QUALITY_LOW;
        }
    }
    dither->quality_mode = quality_mode;
}

Project: libsixel
File: dither.c

sixel_dither_set_complexion_score(
    sixel_dither_t /* in */ *dither,  /* dither context object */
    int            /* in */ score)    /* complexion score (>= 1) */
{
    dither->complexion = score;
}

Project: libsixel
File: dither.c

sixel_dither_set_diffusion_type(
    sixel_dither_t  /* in */ *dither,
    int             /* in */ method_for_diffuse)
{
    if (method_for_diffuse == SIXEL_DIFFUSE_AUTO) {
        if (dither->ncolors > 16) {
            method_for_diffuse = SIXEL_DIFFUSE_FS;
        } else {
            method_for_diffuse = SIXEL_DIFFUSE_ATKINSON;
        }
    }
    dither->method_for_diffuse = method_for_diffuse;
}

Project: libsixel
File: dither.c

sixel_dither_initialize(
    sixel_dither_t  /* in */ *dither,
    unsigned char   /* in */ *data,
    int             /* in */ width,
    int             /* in */ height,
    int             /* in */ pixelformat,
    int             /* in */ method_for_largest,
    int             /* in */ method_for_rep,
    int             /* in */ quality_mode)
{
    unsigned char *buf = NULL;
    unsigned char *normalized_pixels = NULL;
    unsigned char *input_pixels;
    SIXELSTATUS status = SIXEL_FALSE;

    if (dither == NULL) {
        sixel_helper_set_additional_message(
            "sixel_dither_new: dither is null.");
        status = SIXEL_BAD_ARGUMENT;
        goto end;
    }

    sixel_dither_ref(dither);

    sixel_dither_set_pixelformat(dither, pixelformat);

    switch (pixelformat) {
    case SIXEL_PIXELFORMAT_RGB888:
        input_pixels = data;
        break;
    default:
        /* normalize pixelformat */
        normalized_pixels
            = (unsigned char *)sixel_allocator_malloc(dither->allocator, (size_t)(width * height * 3));
        if (normalized_pixels == NULL) {
            sixel_helper_set_additional_message(
                "sixel_dither_initialize: sixel_allocator_malloc() failed.");
            status = SIXEL_BAD_ALLOCATION;
            goto end;
        }

        status = sixel_helper_normalize_pixelformat(
            normalized_pixels,
            &pixelformat,
            data,
            pixelformat,
            width,
            height);
        if (SIXEL_FAILED(status)) {
            goto end;
        }
        input_pixels = normalized_pixels;
        break;
    }

    sixel_dither_set_method_for_largest(dither, method_for_largest);
    sixel_dither_set_method_for_rep(dither, method_for_rep);
    sixel_dither_set_quality_mode(dither, quality_mode);

    status = sixel_quant_make_palette(&buf,
                                      input_pixels,
                                      (unsigned int)(width * height * 3),
                                      SIXEL_PIXELFORMAT_RGB888,
                                      (unsigned int)dither->reqcolors,
                                      (unsigned int *)&dither->ncolors,
                                      (unsigned int *)&dither->origcolors,
                                      dither->method_for_largest,
                                      dither->method_for_rep,
                                      dither->quality_mode,
                                      dither->allocator);
    if (SIXEL_FAILED(status)) {
        goto end;
    }
    memcpy(dither->palette, buf, (size_t)(dither->ncolors * 3));

    dither->optimized = 1;
    if (dither->origcolors <= dither->ncolors) {
        dither->method_for_diffuse = SIXEL_DIFFUSE_NONE;
    }

    sixel_quant_free_palette(buf, dither->allocator);
    status = SIXEL_OK;

end:
    free(normalized_pixels);
    sixel_dither_unref(dither);
    return status;
}

Project: libsixel
File: dither.c

test2(void)
{
    sixel_dither_t *dither = NULL;
    int colors;
    int nret = EXIT_FAILURE;

#if HAVE_DIAGNOSTIC_DEPRECATED_DECLARATIONS
#  pragma GCC diagnostic push
#  pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
    dither = sixel_dither_create(INT_MAX);
#if HAVE_DIAGNOSTIC_DEPRECATED_DECLARATIONS
#  pragma GCC diagnostic pop
#endif
    if (dither == NULL) {
        goto error;
    }
    sixel_dither_set_body_only(dither, 1);
    colors = sixel_dither_get_num_of_histogram_colors(dither);
    if (colors != -1) {
        goto error;
    }
    nret = EXIT_SUCCESS;

error:
    sixel_dither_unref(dither);
    return nret;
}

Project: libsixel
File: dither.c

sixel_dither_get_num_of_histogram_colors(
    sixel_dither_t /* in */ *dither)  /* dither context object */
{
    return dither->origcolors;
}

Project: libsixel
File: dither.c

sixel_dither_get_num_of_palette_colors(
    sixel_dither_t  /* in */ *dither)
{
    return dither->ncolors;
}

Project: libsixel
File: dither.c

sixel_dither_new(
    sixel_dither_t    /* out */ **ppdither, /* dither object to be created */
    int               /* in */  ncolors,    /* required colors */
    sixel_allocator_t /* in */  *allocator) /* allocator, null if you use
                                               default allocator */
{
    SIXELSTATUS status = SIXEL_FALSE;
    size_t headsize;
    size_t datasize;
    size_t wholesize;
    int quality_mode;

    if (ppdither == NULL) {
        sixel_helper_set_additional_message(
            "sixel_dither_new: ppdither is null.");
        status = SIXEL_BAD_ARGUMENT;
        goto end;
    }

    if (allocator == NULL) {
        status = sixel_allocator_new(&allocator, NULL, NULL, NULL, NULL);
        if (SIXEL_FAILED(status)) {
            *ppdither = NULL;
            goto end;
        }
    } else {
        sixel_allocator_ref(allocator);
    }

    if (ncolors < 0) {
        ncolors = 256;
        quality_mode = SIXEL_QUALITY_HIGHCOLOR;
    } else {
        if (ncolors > SIXEL_PALETTE_MAX) {
            status = SIXEL_BAD_INPUT;
            ncolors = 256;
        } else if (ncolors < 1) {
            status = SIXEL_BAD_INPUT;
            sixel_helper_set_additional_message(
                "sixel_dither_new: palette colors must be more than 0");
            goto end;
        }
        quality_mode = SIXEL_QUALITY_LOW;
    }
    headsize = sizeof(sixel_dither_t);
    datasize = (size_t)(ncolors * 3);
    wholesize = headsize + datasize;

    *ppdither = (sixel_dither_t *)sixel_allocator_malloc(allocator, wholesize);
    if (*ppdither == NULL) {
        sixel_allocator_unref(allocator);
        sixel_helper_set_additional_message(
            "sixel_dither_new: sixel_allocator_malloc() failed.");
        status = SIXEL_BAD_ALLOCATION;
        goto end;
    }

    (*ppdither)->ref = 1;
    (*ppdither)->palette = (unsigned char*)(*ppdither + 1);
    (*ppdither)->cachetable = NULL;
    (*ppdither)->reqcolors = ncolors;
    (*ppdither)->ncolors = ncolors;
    (*ppdither)->origcolors = (-1);
    (*ppdither)->keycolor = (-1);
    (*ppdither)->optimized = 0;
    (*ppdither)->optimize_palette = 0;
    (*ppdither)->complexion = 1;
    (*ppdither)->bodyonly = 0;
    (*ppdither)->method_for_largest = SIXEL_LARGE_NORM;
    (*ppdither)->method_for_rep = SIXEL_REP_CENTER_BOX;
    (*ppdither)->method_for_diffuse = SIXEL_DIFFUSE_FS;
    (*ppdither)->quality_mode = quality_mode;
    (*ppdither)->pixelformat = SIXEL_PIXELFORMAT_RGB888;
    (*ppdither)->allocator = allocator;

    status = SIXEL_OK;

end:
    return status;
}

Project: libsixel
File: dither.c

sixel_dither_set_palette(
    sixel_dither_t /* in */ *dither,   /* dither context object */
    unsigned char  /* in */ *palette)
{
    memcpy(dither->palette, palette, (size_t)(dither->ncolors * 3));
}

Project: libsixel
File: dither.c

sixel_dither_tests_main(void)
{
    int nret = EXIT_FAILURE;
    size_t i;
    typedef int (* testcase)(void);

    static testcase const testcases[] = {
        test1,
        test2,
    };

    for (i = 0; i < sizeof(testcases) / sizeof(testcase); ++i) {
        nret = testcases[i]();
        if (nret != EXIT_SUCCESS) {
            goto error;
        }
    }

    nret = EXIT_SUCCESS;

error:
    return nret;
}

Project: libsixel
File: dither.c

sixel_dither_ref(
    sixel_dither_t  /* in */ *dither)
{
    /* TODO: be thread safe */
    ++dither->ref;
}

Project: libsixel
File: dither.c

sixel_dither_set_transparent(
    sixel_dither_t /* in */ *dither,      /* dither context object */
    int            /* in */ transparent)  /* transparent color index */
{
    dither->keycolor = transparent;
}

Project: libsixel
File: dither.c

sixel_dither_set_method_for_largest(
    sixel_dither_t  /* in */ *dither,
    int             /* in */ method_for_largest)
{
    if (method_for_largest == SIXEL_LARGE_AUTO) {
        method_for_largest = SIXEL_LARGE_NORM;
    }
    dither->method_for_largest = method_for_largest;
}

Project: libsixel
File: dither.c

sixel_dither_unref(
    sixel_dither_t  /* in */ *dither)
{
    /* TODO: be thread safe */
    if (dither != NULL && --dither->ref == 0) {
        sixel_dither_destroy(dither);
    }
}

Project: libsixel
File: dither.c

sixel_dither_destroy(
    sixel_dither_t  /* in */ *dither)
{
    sixel_allocator_t *allocator;

    if (dither) {
        allocator = dither->allocator;
        sixel_allocator_free(allocator, dither->cachetable);
        dither->cachetable = NULL;
        sixel_allocator_free(allocator, dither);
        sixel_allocator_unref(allocator);
    }
}

Project: libsixel
File: dither.c

sixel_dither_set_method_for_rep(
    sixel_dither_t  /* in */ *dither,
    int             /* in */ method_for_rep)
{
    if (method_for_rep == SIXEL_REP_AUTO) {
        method_for_rep = SIXEL_REP_CENTER_BOX;
    }
    dither->method_for_rep = method_for_rep;
}

Project: libsixel
File: dither.c

sixel_dither_set_body_only(
    sixel_dither_t /* in */ *dither,     /* dither context object */
    int            /* in */ bodyonly)    /* 0: output palette section
                                            1: do not output palette section  */
{
    dither->bodyonly = bodyonly;
}

Project: libsixel
File: dither.c

sixel_dither_set_optimize_palette(
    sixel_dither_t /* in */ *dither,   /* dither context object */
    int            /* in */ do_opt)    /* 0: optimize palette size
                                          1: don't optimize palette size */
{
    dither->optimize_palette = do_opt;
}

Project: libsixel
File: dither.c

test1(void)
{
    sixel_dither_t *dither = NULL;
    int nret = EXIT_FAILURE;

#if HAVE_DIAGNOSTIC_DEPRECATED_DECLARATIONS
#  pragma GCC diagnostic push
#  pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
    dither = sixel_dither_create(0);
#if HAVE_DIAGNOSTIC_DEPRECATED_DECLARATIONS
#  pragma GCC diagnostic pop
#endif
    if (dither == NULL) {
        goto error;
    }
    sixel_dither_ref(dither);
    sixel_dither_unref(dither);
    nret = EXIT_SUCCESS;

error:
    sixel_dither_unref(dither);
    return nret;
}

Project: libsixel
File: dither.c

sixel_dither_get_palette(
    sixel_dither_t /* in */ *dither)  /* dither context object */
{
    return dither->palette;
}

Project: libsixel
File: dither.c

sixel_dither_get(
    int     /* in */ builtin_dither)
{
    SIXELSTATUS status = SIXEL_FALSE;
    unsigned char *palette;
    int ncolors;
    int keycolor;
    sixel_dither_t *dither = NULL;

    switch (builtin_dither) {
    case SIXEL_BUILTIN_MONO_DARK:
        ncolors = 2;
        palette = (unsigned char *)pal_mono_dark;
        keycolor = 0;
        break;
    case SIXEL_BUILTIN_MONO_LIGHT:
        ncolors = 2;
        palette = (unsigned char *)pal_mono_light;
        keycolor = 0;
        break;
    case SIXEL_BUILTIN_XTERM16:
        ncolors = 16;
        palette = (unsigned char *)pal_xterm256;
        keycolor = (-1);
        break;
    case SIXEL_BUILTIN_XTERM256:
        ncolors = 256;
        palette = (unsigned char *)pal_xterm256;
        keycolor = (-1);
        break;
    case SIXEL_BUILTIN_VT340_MONO:
        ncolors = 16;
        palette = (unsigned char *)pal_vt340_mono;
        keycolor = (-1);
        break;
    case SIXEL_BUILTIN_VT340_COLOR:
        ncolors = 16;
        palette = (unsigned char *)pal_vt340_color;
        keycolor = (-1);
        break;
    case SIXEL_BUILTIN_G1:
        ncolors = 2;
        palette = (unsigned char *)pal_gray_1bit;
        keycolor = (-1);
        break;
    case SIXEL_BUILTIN_G2:
        ncolors = 4;
        palette = (unsigned char *)pal_gray_2bit;
        keycolor = (-1);
        break;
    case SIXEL_BUILTIN_G4:
        ncolors = 16;
        palette = (unsigned char *)pal_gray_4bit;
        keycolor = (-1);
        break;
    case SIXEL_BUILTIN_G8:
        ncolors = 256;
        palette = (unsigned char *)pal_gray_8bit;
        keycolor = (-1);
        break;
    default:
        goto end;
    }

    status = sixel_dither_new(&dither, ncolors, NULL);
    if (SIXEL_FAILED(status)) {
        dither = NULL;
        goto end;
    }

    dither->palette = palette;
    dither->keycolor = keycolor;
    dither->optimized = 1;
    dither->optimize_palette = 0;

end:
    return dither;
}