Commit message: Fix bug 697459 Buffer overflow in fill_threshold_buffer

There was an overflow check for ht_buffer size, but none for the larger
threshold_buffer. Note that this file didn't fail on Windows because the
combination of the ht_buffer and the size of the (miscalculated due to
overflow) threshold_buffer would have exceeded the 2Gb limit.
CVE: CVE-2016-10317
CWE: CWE-119
Vulnerability description: The fill_threshhold_buffer function in base/gxht_thresh.c in Artifex Software, Inc. Ghostscript 9.20 allows remote attackers to cause a denial of service (heap-based buffer overflow and application crash) or possibly have unspecified other impact via a crafted PostScript document.

Project: ghostpdl
File: gxht_thresh.c

gx_ht_threshold_row_bit_sub(byte *contone,  byte *threshold_strip,  int contone_stride,
                  byte *halftone, int dithered_stride, int width,
                  int num_rows, int offset_bits)
{
#ifndef HAVE_SSE2
    int k, j;
    byte *contone_ptr;
    byte *thresh_ptr;
    byte *halftone_ptr;
    byte bit_init;

    /* For the moment just do a very slow compare until we get
       get this working.  This could use some serious optimization */
    width -= offset_bits;
    for (j = 0; j < num_rows; j++) {
        byte h;
        contone_ptr = contone;
        thresh_ptr = threshold_strip + contone_stride * j;
        halftone_ptr = halftone + dithered_stride * j;
        /* First get the left remainder portion.  Put into MSBs of first byte */
        bit_init = 0x80;
        h = 0;
        k = offset_bits;
        if (k > 0) {
            do {
                if (*contone_ptr++ > *thresh_ptr++) {
                    h |=  bit_init;
                }
                bit_init >>= 1;
                if (bit_init == 0) {
                    bit_init = 0x80;
                    *halftone_ptr++ = h;
                    h = 0;
                }
                k--;
            } while (k > 0);
            bit_init = 0x80;
            *halftone_ptr++ = h;
            h = 0;
            if (offset_bits < 8)
                *halftone_ptr++ = 0;
        }
        /* Now get the rest, which will be 16 bit aligned. */
        k = width;
        if (k > 0) {
            do {
                if (*contone_ptr++ > *thresh_ptr++) {
                    h |=  bit_init;
                }
                bit_init >>= 1;
                if (bit_init == 0) {
                    bit_init = 0x80;
                    *halftone_ptr++ = h;
                    h = 0;
                }
                k--;
            } while (k > 0);
            if (bit_init != 0x80) {
                *halftone_ptr++ = h;
            }
            if ((width & 15) < 8)
                *halftone_ptr++ = 0;
        }
    }
#else
    byte *contone_ptr;
    byte *thresh_ptr;
    byte *halftone_ptr;
    int num_tiles = (width - offset_bits + 15)>>4;
    int k, j;

    for (j = 0; j < num_rows; j++) {
        /* contone and thresh_ptr are 128 bit aligned.  We do need to do this in
           two steps to ensure that we pack the bits in an aligned fashion
           into halftone_ptr.  */
        contone_ptr = contone;
        thresh_ptr = threshold_strip + contone_stride * j;
        halftone_ptr = halftone + dithered_stride * j;
        if (offset_bits > 0) {
            /* Since we allowed for 16 bits in our left remainder
               we can go directly in to the destination.  threshold_16_SSE
               requires 128 bit alignment.  contone_ptr and thresh_ptr
               are set up so that after we move in by offset_bits elements
               then we are 128 bit aligned.  */
            threshold_16_SSE_unaligned(thresh_ptr, contone_ptr,
                                       halftone_ptr);
            halftone_ptr += 2;
            thresh_ptr += offset_bits;
            contone_ptr += offset_bits;
        }
        /* Now we should have 128 bit aligned with our input data. Iterate
           over sets of 16 going directly into our HT buffer.  Sources and
           halftone_ptr buffers should be padded to allow 15 bit overrun */
        for (k = 0; k < num_tiles; k++) {
            threshold_16_SSE(thresh_ptr, contone_ptr, halftone_ptr);
            thresh_ptr += 16;
            contone_ptr += 16;
            halftone_ptr += 2;
        }
    }
#endif
}

Project: ghostpdl
File: gxht_thresh.c

gx_ht_threshold_row_byte(byte *contone, byte *threshold_strip, int contone_stride,
                              byte *halftone, int dithered_stride, int width,
                              int num_rows)
{
    int k, j;
    byte *contone_ptr;
    byte *thresh_ptr;
    byte *halftone_ptr;

    /* For the moment just do a very slow compare until we get
       get this working */
    for (j = 0; j < num_rows; j++) {
        contone_ptr = contone;
        thresh_ptr = threshold_strip + contone_stride * j;
        halftone_ptr = halftone + dithered_stride * j;
        for (k = 0; k < width; k++) {
            if (contone_ptr[k] < thresh_ptr[k]) {
                halftone_ptr[k] = 0;
            } else {
                halftone_ptr[k] = 255;
            }
        }
    }
}

Project: ghostpdl
File: gxht_thresh.c

gx_ht_threshold_landscape(byte *contone_align, byte *thresh_align,
                    ht_landscape_info_t *ht_landscape, byte *halftone,
                    int data_length)
{
    __align16 byte contone[LAND_BITS];
    int position_start, position, curr_position;
    int *widths = &(ht_landscape->widths[0]);
    int local_widths[LAND_BITS];
    int num_contone = ht_landscape->num_contones;
    int k, j, w, contone_out_posit;
    byte *contone_ptr, *thresh_ptr, *halftone_ptr;
#ifdef PACIFY_VALGRIND
    int extra = 0;
#endif

    /* Work through chunks of 16.  */
    /* Data may have come in left to right or right to left. */
    if (ht_landscape->index > 0) {
        position = position_start = 0;
    } else {
        position = position_start = ht_landscape->curr_pos + 1;
    }
    thresh_ptr = thresh_align;
    halftone_ptr = halftone;
    /* Copy the widths to a local array, and truncate the last one (which may
     * be the first one!) if required. */
    k = 0;
    for (j = 0; j < num_contone; j++)
        k += (local_widths[j] = widths[position_start+j]);
    if (k > LAND_BITS) {
        if (ht_landscape->index > 0) {
            local_widths[num_contone-1] -= k-LAND_BITS;
        } else {
            local_widths[0] -= k-LAND_BITS;
        }
    }
#ifdef PACIFY_VALGRIND
    if (k < LAND_BITS) {
        extra = LAND_BITS - k;
    }
#endif
    for (k = data_length; k > 0; k--) { /* Loop on rows */
        contone_ptr = &(contone_align[position]); /* Point us to our row start */
        curr_position = 0; /* We use this in keeping track of widths */
        contone_out_posit = 0; /* Our index out */
        for (j = num_contone; j > 0; j--) {
            byte c = *contone_ptr;
            /* The microsoft compiler, cleverly spots that the following loop
             * can be replaced by a memset. Unfortunately, it can't spot that
             * the typical length values of the memset are so small that we'd
             * be better off doing it the slow way. We therefore introduce a
             * sneaky 'volatile' cast below that stops this optimisation. */
            w = local_widths[curr_position];
            do {
                ((volatile byte *)contone)[contone_out_posit] = c;
                contone_out_posit++;
            } while (--w);
#ifdef PACIFY_VALGRIND
            if (extra)
                memset(contone+contone_out_posit, 0, extra);
#endif
            curr_position++; /* Move us to the next position in our width array */
            contone_ptr++;   /* Move us to a new location in our contone buffer */
        }
        /* Now we have our left justified and expanded contone data for
           LAND_BITS/16 sets of 16 bits. Go ahead and threshold these. */
        contone_ptr = &contone[0];
#if LAND_BITS > 16
        j = LAND_BITS;
        do {
#endif
#ifdef HAVE_SSE2
            threshold_16_SSE(contone_ptr, thresh_ptr, halftone_ptr);
#else
            threshold_16_bit(contone_ptr, thresh_ptr, halftone_ptr);
#endif
            thresh_ptr += 16;
            position += 16;
            halftone_ptr += 2;
            contone_ptr += 16;
#if LAND_BITS > 16
            j -= 16;
        } while (j > 0);
#endif
    }
}

Project: ghostpdl
File: gxht_thresh.c

gx_ht_threshold_row_bit(byte *contone,  byte *threshold_strip,  int contone_stride,
                  byte *halftone, int dithered_stride, int width,
                  int num_rows, int offset_bits)
{
#ifndef HAVE_SSE2
    int k, j;
    byte *contone_ptr;
    byte *thresh_ptr;
    byte *halftone_ptr;
    byte bit_init;

    /* For the moment just do a very slow compare until we get
       get this working.  This could use some serious optimization */
    width -= offset_bits;
    for (j = 0; j < num_rows; j++) {
        byte h;
        contone_ptr = contone;
        thresh_ptr = threshold_strip + contone_stride * j;
        halftone_ptr = halftone + dithered_stride * j;
        /* First get the left remainder portion.  Put into MSBs of first byte */
        bit_init = 0x80;
        h = 0;
        k = offset_bits;
        if (k > 0) {
            do {
                if (*contone_ptr++ < *thresh_ptr++) {
                    h |=  bit_init;
                }
                bit_init >>= 1;
                if (bit_init == 0) {
                    bit_init = 0x80;
                    *halftone_ptr++ = h;
                    h = 0;
                }
                k--;
            } while (k > 0);
            bit_init = 0x80;
            *halftone_ptr++ = h;
            h = 0;
            if (offset_bits < 8)
                *halftone_ptr++ = 0;
        }
        /* Now get the rest, which will be 16 bit aligned. */
        k = width;
        if (k > 0) {
            do {
                if (*contone_ptr++ < *thresh_ptr++) {
                    h |=  bit_init;
                }
                bit_init >>= 1;
                if (bit_init == 0) {
                    bit_init = 0x80;
                    *halftone_ptr++ = h;
                    h = 0;
                }
                k--;
            } while (k > 0);
            if (bit_init != 0x80) {
                *halftone_ptr++ = h;
            }
            if ((width & 15) < 8)
                *halftone_ptr++ = 0;
        }
    }
#else
    byte *contone_ptr;
    byte *thresh_ptr;
    byte *halftone_ptr;
    int num_tiles = (width - offset_bits + 15)>>4;
    int k, j;

    for (j = 0; j < num_rows; j++) {
        /* contone and thresh_ptr are 128 bit aligned.  We do need to do this in
           two steps to ensure that we pack the bits in an aligned fashion
           into halftone_ptr.  */
        contone_ptr = contone;
        thresh_ptr = threshold_strip + contone_stride * j;
        halftone_ptr = halftone + dithered_stride * j;
        if (offset_bits > 0) {
            /* Since we allowed for 16 bits in our left remainder
               we can go directly in to the destination.  threshold_16_SSE
               requires 128 bit alignment.  contone_ptr and thresh_ptr
               are set up so that after we move in by offset_bits elements
               then we are 128 bit aligned.  */
            threshold_16_SSE_unaligned(contone_ptr, thresh_ptr,
                                       halftone_ptr);
            halftone_ptr += 2;
            thresh_ptr += offset_bits;
            contone_ptr += offset_bits;
        }
        /* Now we should have 128 bit aligned with our input data. Iterate
           over sets of 16 going directly into our HT buffer.  Sources and
           halftone_ptr buffers should be padded to allow 15 bit overrun */
        for (k = 0; k < num_tiles; k++) {
            threshold_16_SSE(contone_ptr, thresh_ptr, halftone_ptr);
            thresh_ptr += 16;
            contone_ptr += 16;
            halftone_ptr += 2;
        }
    }
#endif
}

Project: ghostpdl
File: gxht_thresh.c

int gxht_dda_length(gx_dda_fixed *dda, int src_size)
{
    gx_dda_fixed d = (*dda);
    dda_advance(d, src_size);
    return abs(fixed2int_var_rounded(dda_current(d)) - fixed2int_var_rounded(dda_current(*dda)));
}

Project: ghostpdl
File: gxht_thresh.c

reset_landscape_buffer(ht_landscape_info_t *ht_landscape, byte *contone_align,
                       int data_length, int num_used)
{
    int delta;
    int curr_x_pos = ht_landscape->xstart;

    if (ht_landscape->index < 0) {
        /* Moving right to left, move column to far right */
        delta = ht_landscape->count - num_used;
        memset(&(ht_landscape->widths[0]), 0, sizeof(int)*LAND_BITS);
        ht_landscape->widths[LAND_BITS-1] = delta;
        ht_landscape->curr_pos = LAND_BITS-2;
        ht_landscape->xstart = curr_x_pos - num_used;
    } else {
        /* Moving left to right, move column to far left */
        delta = ht_landscape->count - num_used;
        memset(&(ht_landscape->widths[0]), 0, sizeof(int)*LAND_BITS);
        ht_landscape->widths[0] = delta;
        ht_landscape->curr_pos = 1;
        ht_landscape->xstart = curr_x_pos + num_used;
    }
    ht_landscape->count = delta;
    ht_landscape->num_contones = 1;
}

Project: ghostpdl
File: gxht_thresh.c

fill_threshold_buffer(byte *dest_strip, byte *src_strip, int src_width,
                       int left_offset, int left_width, int num_tiles,
                       int right_width)
{
    byte *ptr_out_temp = dest_strip;
    int ii;

    /* Left part */
    memcpy(dest_strip, src_strip + left_offset, left_width);
    ptr_out_temp += left_width;
    /* Now the full parts */
    for (ii = 0; ii < num_tiles; ii++){
        memcpy(ptr_out_temp, src_strip, src_width);
        ptr_out_temp += src_width;
    }
    /* Now the remainder */
    memcpy(ptr_out_temp, src_strip, right_width);
#ifdef PACIFY_VALGRIND
    ptr_out_temp += right_width;
    ii = (dest_strip-ptr_out_temp) % (LAND_BITS-1);
    if (ii > 0)
        memset(ptr_out_temp, 0, ii);
#endif
}

Project: ghostpdl
File: gxht_thresh.c

move_landscape_buffer(ht_landscape_info_t *ht_landscape, byte *contone_align,
                       int data_length)
{
    int k;
    int position_curr, position_new;

    if (ht_landscape->index < 0) {
        /* Moving right to left, move column to far right */
        position_curr = ht_landscape->curr_pos + 1;
        position_new = LAND_BITS-1;
    } else {
        /* Moving left to right, move column to far left */
        position_curr = ht_landscape->curr_pos - 1;
        position_new = 0;
    }
    if (position_curr != position_new) {
        for (k = 0; k < data_length; k++) {
                contone_align[position_new] = contone_align[position_curr];
                position_curr += LAND_BITS;
                position_new += LAND_BITS;
        }
    }
}

Project: ghostpdl
File: gxht_thresh.c

gxht_thresh_image_init(gx_image_enum *penum)
{
    int code = 0;
    fixed ox;
    int temp;
    int dev_width, max_height;
    int spp_out;
    int k;
    gx_ht_order *d_order;
    gx_dda_fixed dda_ht;

    if (gx_device_must_halftone(penum->dev)) {
        if (penum->pgs != NULL && penum->pgs->dev_ht != NULL) {
            for (k = 0; k < penum->pgs->dev_ht->num_comp; k++) {
                d_order = &(penum->pgs->dev_ht->components[k].corder);
                code = gx_ht_construct_threshold(d_order, penum->dev,
                                                 penum->pgs, k);
                if (code < 0 ) {
                    return gs_rethrow(code, "threshold creation failed");
                }
            }
        } else {
            return -1;
        }
    }
    spp_out = penum->dev->color_info.num_components;
    /* Precompute values needed for rasterizing. */
    penum->dxx = float2fixed(penum->matrix.xx + fixed2float(fixed_epsilon) / 2);
    /* If the image is landscaped then we want to maintain a buffer
       that is sufficiently large so that we can hold a byte
       of halftoned data along the column.  This way we avoid doing
       multiple writes into the same position over and over.
       The size of the buffer we need depends upon the bitdepth of
       the output device, the number of device coloranants and the
       number of  colorants in the source space.  Note we will
       need to eventually  consider  multi-level halftone case
       here too.  For now, to make use of the SSE2 stuff, we would
       like to have a multiple of 16 bytes of data to process at a time.
       So we will collect the columns of data in a buffer that is LAND_BITS
       wide.  We will also keep track of the widths of each column.  When
       the total width count reaches LAND_BITS, we will create our
       threshold array and apply it.  We may have one column that is
       buffered between calls in this case.  Also if a call is made
       with h=0 we will flush the buffer as we are at the end of the
       data.  */
    if (penum->posture == image_landscape) {
        int col_length = fixed2int_var_rounded(any_abs(penum->x_extent.y));
        dda_ht = penum->dda.pixel0.y;
        if (penum->dxx > 0)
            dda_translate(dda_ht, -fixed_epsilon);      /* to match rounding in non-fast code */

        ox = dda_current(penum->dda.pixel0.x);
        temp = gxht_dda_length(&dda_ht, penum->rect.w);
        if (col_length < temp)
            col_length = temp;          /* choose max to make sure line_size is large enough */
        temp = (col_length + LAND_BITS)/LAND_BITS;      /* round up to allow for offset bits */
        /* bitmap_raster() expects the width in bits, hence "* 8" */
        penum->line_size = bitmap_raster((temp * LAND_BITS) * 8);  /* The stride */
        /* Now we need at most LAND_BITS of these */
        penum->line = gs_alloc_bytes(penum->memory,
                                     LAND_BITS * penum->line_size * spp_out + 16,
                                     "gxht_thresh");
        /* Same with this.  However, we only need one plane here */
        penum->thresh_buffer = gs_alloc_bytes(penum->memory,
                                           penum->line_size * LAND_BITS + 16,
                                           "gxht_thresh");
        /* That maps into (LAND_BITS/8) bytes of Halftone data */
        penum->ht_buffer =
                        gs_alloc_bytes(penum->memory,
                           penum->line_size * (LAND_BITS>>3) * spp_out,
                           "gxht_thresh");
        penum->ht_plane_height = penum->line_size;
        penum->ht_stride = penum->line_size;
        if (penum->line == NULL || penum->thresh_buffer == NULL
                    || penum->ht_buffer == NULL)
            return -1;
        penum->ht_landscape.count = 0;
        penum->ht_landscape.num_contones = 0;
        if (penum->y_extent.x < 0) {
            /* Going right to left */
            penum->ht_landscape.curr_pos = LAND_BITS-1;
            penum->ht_landscape.index = -1;
        } else {
            /* Going left to right */
            penum->ht_landscape.curr_pos = 0;
            penum->ht_landscape.index = 1;
        }
        if (penum->x_extent.y < 0) {
            penum->ht_landscape.flipy = true;
            penum->ht_landscape.y_pos =
                fixed2int_pixround_perfect(dda_current(penum->dda.pixel0.y) + penum->x_extent.y);
        } else {
            penum->ht_landscape.flipy = false;
            penum->ht_landscape.y_pos =
                fixed2int_pixround_perfect(dda_current(penum->dda.pixel0.y));
        }
        memset(&(penum->ht_landscape.widths[0]), 0, sizeof(int)*LAND_BITS);
        penum->ht_landscape.offset_set = false;
        penum->ht_offset_bits = 0; /* Will get set in call to render */
        if (code >= 0) {
#if defined(DEBUG) || defined(PACIFY_VALGRIND)
            memset(penum->line, 0, LAND_BITS * penum->line_size * spp_out + 16);
            memset(penum->ht_buffer, 0, penum->line_size * (LAND_BITS>>3) * spp_out);
            memset(penum->thresh_buffer, 0, LAND_BITS * penum->line_size + 16);
#endif
        }
    } else {
        /* In the portrait case we allocate a single line buffer
           in device width, a threshold buffer of the same size
           and possibly wider and the buffer for the halftoned
           bits. We have to do a bit of work to enable 16 byte
           boundary after an offset to ensure that we can make use
           of  the SSE2 operations for thresholding.  We do the
           allocations now to avoid doing them with every line */
        dda_ht = penum->dda.pixel0.x;
        if (penum->dxx > 0)
            dda_translate(dda_ht, -fixed_epsilon);      /* to match rounding in non-fast code */
        /* Initialize the ht_landscape stuff to zero */
        memset(&(penum->ht_landscape), 0, sizeof(ht_landscape_info_t));
        ox = dda_current(dda_ht);
        dev_width = gxht_dda_length(&dda_ht, penum->rect.w);
        /* Get the bit position so that we can do a copy_mono for
           the left remainder and then 16 bit aligned copies for the
           rest.  The right remainder will be OK as it will land in
           the MSBit positions. Note the #define chunk bits16 in
           gdevm1.c.  Allow also for a 15 sample over run.
        */
        penum->ht_offset_bits = (-fixed2int_var_rounded(ox)) & (bitmap_raster(1) - 1);
        if (penum->ht_offset_bits > 0) {
            penum->ht_stride = bitmap_raster((7 + (dev_width + 4)) + (ARCH_SIZEOF_LONG * 8));
        } else {
            penum->ht_stride = bitmap_raster((7 + (dev_width + 2)) + (ARCH_SIZEOF_LONG * 8));
        }
        /* We want to figure out the maximum height that we may
           have in taking a single source row and going to device
           space */
        max_height = (int) ceil(fixed2float(any_abs(penum->dst_height)) /
                                            (float) penum->Height);
        if (max_height <= 0)
            return -1;		/* shouldn't happen, but check so we don't div by zero */
        if (penum->ht_stride * spp_out > max_int / max_height)
            return -1;         /* overflow */

        penum->ht_buffer =
                        gs_alloc_bytes(penum->memory,
                           penum->ht_stride * max_height * spp_out,
                           "gxht_thresh");
        penum->ht_plane_height = penum->ht_stride * max_height;
        /* We want to have 128 bit alignement for our contone and
           threshold strips so that we can use SSE operations
           in the threshold operation.  Add in a minor buffer and offset
           to ensure this.  If gs_alloc_bytes provides at least 16
           bit alignment so we may need to move 14 bytes.  However, the
           HT process is split in two operations.  One that involves
           the HT of a left remainder and the rest which ensures that
           we pack in the HT data in the bits with no skew for a fast
           copy into the gdevm1 device (16 bit copies).  So, we
           need to account for those pixels which occur first and which
           are NOT aligned for the contone buffer.  After we offset
           by this remainder portion we should be 128 bit aligned.
           Also allow a 15 sample over run during the execution.  */
        temp = (int) ceil((float) ((dev_width + 15.0) + 15.0)/16.0);
        penum->line_size = bitmap_raster(temp * 16 * 8);  /* The stride */
        if (penum->line_size > max_int / max_height) {
            gs_free_object(penum->memory, penum->ht_buffer, "gxht_thresh");
            penum->ht_buffer = NULL;
            return -1;         /* thresh_buffer size overflow */
        }
        penum->line = gs_alloc_bytes(penum->memory, penum->line_size * spp_out,
                                     "gxht_thresh");
        penum->thresh_buffer = gs_alloc_bytes(penum->memory,
                                              penum->line_size * max_height,
                                              "gxht_thresh");
        if (penum->line == NULL || penum->thresh_buffer == NULL ||
            penum->ht_buffer == NULL) {
            return -1;
        } else {
#if defined(DEBUG) || defined(PACIFY_VALGRIND)
            memset(penum->line, 0, penum->line_size * spp_out);
            memset(penum->ht_buffer, 0, penum->ht_stride * max_height * spp_out);
            memset(penum->thresh_buffer, 0, penum->line_size * max_height);
#endif
        }
    }
    return code;
}

Project: ghostpdl
File: gxht_thresh.c

threshold_16_SSE(byte *contone_ptr, byte *thresh_ptr, byte *ht_data)
{
    __m128i input1;
    __m128i input2;
    register int result_int;
    const unsigned int mask1 = 0x80808080;
    __m128i sign_fix = _mm_set_epi32(mask1, mask1, mask1, mask1);

    /* Load */
    input1 = _mm_load_si128((const __m128i *)contone_ptr);
    input2 = _mm_load_si128((const __m128i *) thresh_ptr);
    /* Unsigned subtraction does Unsigned saturation so we
       have to use the signed operation */
    input1 = _mm_xor_si128(input1, sign_fix);
    input2 = _mm_xor_si128(input2, sign_fix);
    /* Subtract the two */
    input2 = _mm_subs_epi8(input1, input2);
    /* Grab the sign mask */
    result_int = _mm_movemask_epi8(input2);
    /* bit wise reversal on 16 bit word */
    ht_data[0] = bitreverse[(result_int & 0xff)];
    ht_data[1] = bitreverse[((result_int >> 8) & 0xff)];
}

Project: ghostpdl
File: gxht_thresh.c

gxht_thresh_planes(gx_image_enum *penum, fixed xrun,
                   int dest_width, int dest_height,
                   byte *thresh_align, gx_device * dev, int offset_contone[],
                   int contone_stride)
{
    int thresh_width, thresh_height, dx;
    int left_rem_end, left_width, vdi;
    int num_full_tiles, right_tile_width;
    int k, jj, dy, j;
    byte *thresh_tile;
    int position;
    bool replicate_tile;
    image_posture posture = penum->posture;
    const int y_pos = penum->yci;
    int width = 0; /* Init to silence compiler warnings */
    byte *ptr_out, *row_ptr, *ptr_out_temp;
    byte *threshold;
    int init_tile, in_row_offset, ii, num_tiles, tile_remainder;
    int offset_bits = penum->ht_offset_bits;
    byte *halftone;
    int dithered_stride = penum->ht_stride;
    bool is_planar_dev = dev->is_planar;
    gx_color_index dev_white = gx_device_white(dev);
    gx_color_index dev_black = gx_device_black(dev);
    int spp_out = dev->color_info.num_components;
    byte *contone_align = NULL; /* Init to silence compiler warnings */

    /* Go ahead and fill the threshold line buffer with tiled threshold values.
       First just grab the row or column that we are going to tile with and
       then do memcpy into the buffer */

    /* Figure out the tile steps.  Left offset, Number of tiles, Right offset. */
    switch (posture) {
        case image_portrait:
            vdi = penum->hci;
            /*  Iterate over the vdi and fill up our threshold buffer.  We
                 also need to loop across the planes of data */
            for (j = 0; j < spp_out; j++) {
                bool threshold_inverted = penum->pgs->dev_ht->components[j].corder.threshold_inverted;

                thresh_width = penum->pgs->dev_ht->components[j].corder.width;
                thresh_height = penum->pgs->dev_ht->components[j].corder.full_height;
                halftone = penum->ht_buffer + j * vdi * dithered_stride;
                /* Compute the tiling positions with dest_width */
                dx = (fixed2int_var_rounded(xrun) + penum->pgs->screen_phase[0].x) % thresh_width;
                /* Left remainder part */
                left_rem_end = min(dx + dest_width, thresh_width);
                /* The left width of our tile part */
                left_width = left_rem_end - dx;
                /* Now the middle part */
                num_full_tiles =
                    (int)fastfloor((dest_width - left_width)/ (float) thresh_width);
                /* Now the right part */
                right_tile_width = dest_width -  num_full_tiles * thresh_width -
                                   left_width;
                /* Get the proper threshold for the colorant count */
                threshold = penum->pgs->dev_ht->components[j].corder.threshold;
                /* Point to the proper contone data */
                contone_align = penum->line + contone_stride * j +
                                offset_contone[j];
                for (k = 0; k < vdi; k++) {
                    /* Get a pointer to our tile row */
                    dy = (penum->yci + k + penum->dev->band_offset_y) % thresh_height;
                    if (dy < 0)
                        dy += thresh_height;
                    thresh_tile = threshold + thresh_width * dy;
                    /* Fill the buffer, can be multiple rows.  Make sure
                       to update with stride */
                    position = contone_stride * k;
                    /* Tile into the 128 bit aligned threshold strip */
                    fill_threshold_buffer(&(thresh_align[position]),
                                           thresh_tile, thresh_width, dx, left_width,
                                           num_full_tiles, right_tile_width);
                }
                /* Apply the threshold operation */
                if (offset_bits > dest_width)
                    offset_bits = dest_width;

                if (threshold_inverted ||
                    (dev->color_info.polarity == GX_CINFO_POLARITY_SUBTRACTIVE && is_planar_dev)) {
                    gx_ht_threshold_row_bit_sub(contone_align, thresh_align, contone_stride,
                                      halftone, dithered_stride, dest_width, vdi,
                                      offset_bits);
                } else {
                    gx_ht_threshold_row_bit(contone_align, thresh_align, contone_stride,
                          halftone, dithered_stride, dest_width, vdi,
                          offset_bits);
                }
            }
            /* FIXME: An improvement here would be to generate the initial
             * offset_bits at the correct offset within the byte so that they
             * align with the remainder of the line. This would mean not
             * always packing them into the first offset_bits (in MSB order)
             * of our 16 bit word, but rather into the last offset_bits
             * (in MSB order) (except when the entire run is small!).
             *
             * This would enable us to do just one aligned copy_mono call for
             * the entire scanline. */
            /* Now do the copy mono or copy plane operation */
            /* First the left remainder bits */
            if (offset_bits > 0) {
                int x_pos = fixed2int_var_rounded(xrun);
                if (!is_planar_dev) {
                    (*dev_proc(dev, copy_mono)) (dev, penum->ht_buffer, 0, dithered_stride,
                                                 gx_no_bitmap_id, x_pos, y_pos,
                                                 offset_bits, vdi, dev_white,
                                                 dev_black);
                } else {
                    (*dev_proc(dev, copy_planes)) (dev, penum->ht_buffer, 0, dithered_stride,
                                                 gx_no_bitmap_id, x_pos, y_pos,
                                                 offset_bits, vdi, vdi);
                }
            }
            if ((dest_width - offset_bits) > 0 ) {
                /* Now the primary aligned bytes */
                int curr_width = dest_width - offset_bits;
                int x_pos = fixed2int_var_rounded(xrun) + offset_bits;
                /* FIXME: This assumes the allowed offset_bits will always be <= 16 */
                int xoffs = offset_bits > 0 ? 16 : 0;

                if (!is_planar_dev) {
                    (*dev_proc(dev, copy_mono)) (dev, penum->ht_buffer, xoffs, dithered_stride,
                                                 gx_no_bitmap_id, x_pos, y_pos,
                                                 curr_width, vdi, dev_white,
                                                 dev_black);
                } else {
                    (*dev_proc(dev, copy_planes)) (dev, penum->ht_buffer, xoffs, dithered_stride,
                                                 gx_no_bitmap_id, x_pos, y_pos,
                                                 curr_width, vdi, vdi);
                }
            }

            break;
        case image_landscape:
            /* Go ahead and paint the chunk if we have LAND_BITS values or a
             * partial to get us in sync with the 1 bit devices 16 bit
             * positions. */
            vdi = penum->wci;
                /* Now do the haftoning into our buffer.  We basically check
                   first if we have enough data or are all done */
            while ( (penum->ht_landscape.count >= LAND_BITS ||
                   ((penum->ht_landscape.count >= offset_bits) &&
                    penum->ht_landscape.offset_set))) {
                /* Go ahead and 2D tile in the threshold buffer at this time */
                /* Always work the tiling from the upper left corner of our
                   LAND_BITS columns */
                for (j = 0; j < spp_out; j++) {
                    halftone = penum->ht_buffer +
                                   j * penum->ht_plane_height * (LAND_BITS>>3);
                    thresh_width = penum->pgs->dev_ht->components[j].corder.width;
                    thresh_height =
                          penum->pgs->dev_ht->components[j].corder.full_height;
                    /* Get the proper threshold for the colorant count */
                    threshold = penum->pgs->dev_ht->components[j].corder.threshold;
                    /* Point to the proper contone data */
                    contone_align = penum->line + offset_contone[j] +
                                      LAND_BITS * j * contone_stride;
                    if (penum->ht_landscape.offset_set) {
                        width = offset_bits;
                    } else {
                        width = LAND_BITS;
                    }
                    if (penum->y_extent.x < 0) {
                        dx = penum->ht_landscape.xstart - width + 1;
                    } else {
                        dx = penum->ht_landscape.xstart;
                    }
                    dx = (dx + penum->pgs->screen_phase[0].x) % thresh_width;
                    dy = (penum->ht_landscape.y_pos +
                              penum->dev->band_offset_y) % thresh_height;
                    if (dy < 0)
                        dy += thresh_height;
                    /* Left remainder part */
                    left_rem_end = min(dx + LAND_BITS, thresh_width);
                    left_width = left_rem_end - dx;
                    /* Now the middle part */
                    num_full_tiles = (LAND_BITS - left_width) / thresh_width;
                    /* Now the right part */
                    right_tile_width =
                        LAND_BITS - num_full_tiles * thresh_width - left_width;
                    /* Now loop over the y stuff */
                    ptr_out = thresh_align;
                    /* Do this in three parts.  We do a top part, followed by
                       larger mem copies followed by a bottom partial. After
                       a slower initial fill we are able to do larger faster
                       expansions */
                    if (dest_height <= 2 * thresh_height) {
                        init_tile = dest_height;
                        replicate_tile = false;
                    } else {
                        init_tile = thresh_height;
                        replicate_tile = true;
                    }
                    for (jj = 0; jj < init_tile; jj++) {
                        in_row_offset = (jj + dy) % thresh_height;
                        row_ptr = threshold + in_row_offset * thresh_width;
                        ptr_out_temp = ptr_out;
                        /* Left part */
                        memcpy(ptr_out_temp, row_ptr + dx, left_width);
                        ptr_out_temp += left_width;
                        /* Now the full tiles */
                        for (ii = 0; ii < num_full_tiles; ii++) {
                            memcpy(ptr_out_temp, row_ptr, thresh_width);
                            ptr_out_temp += thresh_width;
                        }
                        /* Now the remainder */
                        memcpy(ptr_out_temp, row_ptr, right_tile_width);
                        ptr_out += LAND_BITS;
                    }
                    if (replicate_tile) {
                        /* Find out how many we need to copy */
                        num_tiles =
                            (int)fastfloor((float) (dest_height - thresh_height)/ (float) thresh_height);
                        tile_remainder = dest_height - (num_tiles + 1) * thresh_height;
                        for (jj = 0; jj < num_tiles; jj ++) {
                            memcpy(ptr_out, thresh_align, LAND_BITS * thresh_height);
                            ptr_out += LAND_BITS * thresh_height;
                        }
                        /* Now fill in the remainder */
                        memcpy(ptr_out, thresh_align, LAND_BITS * tile_remainder);
                    }
                    /* Apply the threshold operation */
                    if (dev->color_info.polarity == GX_CINFO_POLARITY_SUBTRACTIVE
                        && is_planar_dev) {
                        gx_ht_threshold_landscape_sub(contone_align, thresh_align,
                                            &(penum->ht_landscape), halftone, dest_height);
                    } else {
                        gx_ht_threshold_landscape(contone_align, thresh_align,
                                            &(penum->ht_landscape), halftone, dest_height);
                    }
                    /* We may have a line left over that has to be maintained
                       due to line replication in the resolution conversion. */
                    if (width != penum->ht_landscape.count) {
                        /* move the line do not reset the stuff */
                        move_landscape_buffer(&(penum->ht_landscape),
                                              contone_align, dest_height);
                    }
                }
                /* Perform the copy mono */
                if (penum->ht_landscape.index < 0) {
                    if (!is_planar_dev) {
                        (*dev_proc(dev, copy_mono))
                                       (dev, penum->ht_buffer, 0, LAND_BITS>>3,
                                        gx_no_bitmap_id,
                                        penum->ht_landscape.xstart - width + 1,
                                        penum->ht_landscape.y_pos,
                                        width, dest_height,
                                        dev_white, dev_black);
                    } else {
                        (*dev_proc(dev, copy_planes))
                                       (dev, penum->ht_buffer, 0, LAND_BITS>>3,
                                        gx_no_bitmap_id,
                                        penum->ht_landscape.xstart - width + 1,
                                        penum->ht_landscape.y_pos,
                                        width, dest_height,
                                        penum->ht_plane_height);
                    }
                } else {
                    if (!is_planar_dev) {
                        (*dev_proc(dev, copy_mono)) (dev, penum->ht_buffer, 0, LAND_BITS>>3,
                                                     gx_no_bitmap_id,
                                                     penum->ht_landscape.xstart,
                                                     penum->ht_landscape.y_pos,
                                                     width, dest_height,
                                                     dev_white, dev_black);
                    } else {
                        (*dev_proc(dev, copy_planes)) (dev, penum->ht_buffer, 0, LAND_BITS>>3,
                                                     gx_no_bitmap_id,
                                                     penum->ht_landscape.xstart,
                                                     penum->ht_landscape.y_pos,
                                                     width, dest_height,
                                                     penum->ht_plane_height);
                    }
                }
                penum->ht_landscape.offset_set = false;
                if (width != penum->ht_landscape.count) {
                    reset_landscape_buffer(&(penum->ht_landscape),
                                           contone_align, dest_height,
                                           width);
                } else {
                    /* Reset the whole buffer */
                    penum->ht_landscape.count = 0;
                    if (penum->ht_landscape.index < 0) {
                        /* Going right to left */
                        penum->ht_landscape.curr_pos = LAND_BITS-1;
                    } else {
                        /* Going left to right */
                        penum->ht_landscape.curr_pos = 0;
                    }
                    penum->ht_landscape.num_contones = 0;
                    memset(&(penum->ht_landscape.widths[0]), 0, sizeof(int)*LAND_BITS);
                }
            }
            break;
        default:
            return gs_rethrow(-1, "Invalid orientation for thresholding");
    }
    return 0;
}

Project: ghostpdl
File: gxht_thresh.c

gx_ht_threshold_landscape_sub(byte *contone_align, byte *thresh_align,
                    ht_landscape_info_t *ht_landscape, byte *halftone,
                    int data_length)
{
    __align16 byte contone[LAND_BITS];
    int position_start, position, curr_position;
    int *widths = &(ht_landscape->widths[0]);
    int local_widths[LAND_BITS];
    int num_contone = ht_landscape->num_contones;
    int k, j, w, contone_out_posit;
    byte *contone_ptr, *thresh_ptr, *halftone_ptr;
#ifdef PACIFY_VALGRIND
    int extra = 0;
#endif

    /* Work through chunks of 16.  */
    /* Data may have come in left to right or right to left. */
    if (ht_landscape->index > 0) {
        position = position_start = 0;
    } else {
        position = position_start = ht_landscape->curr_pos + 1;
    }
    thresh_ptr = thresh_align;
    halftone_ptr = halftone;
    /* Copy the widths to a local array, and truncate the last one (which may
     * be the first one!) if required. */
    k = 0;
    for (j = 0; j < num_contone; j++)
        k += (local_widths[j] = widths[position_start+j]);
    if (k > LAND_BITS) {
        if (ht_landscape->index > 0) {
            local_widths[num_contone-1] -= k-LAND_BITS;
        } else {
            local_widths[0] -= k-LAND_BITS;
        }
    }
#ifdef PACIFY_VALGRIND
    if (k < LAND_BITS) {
        extra = LAND_BITS - k;
    }
#endif
    for (k = data_length; k > 0; k--) { /* Loop on rows */
        contone_ptr = &(contone_align[position]); /* Point us to our row start */
        curr_position = 0; /* We use this in keeping track of widths */
        contone_out_posit = 0; /* Our index out */
        for (j = num_contone; j > 0; j--) {
            byte c = *contone_ptr;
            /* The microsoft compiler, cleverly spots that the following loop
             * can be replaced by a memset. Unfortunately, it can't spot that
             * the typical length values of the memset are so small that we'd
             * be better off doing it the slow way. We therefore introduce a
             * sneaky 'volatile' cast below that stops this optimisation. */
            w = local_widths[curr_position];
            do {
                ((volatile byte *)contone)[contone_out_posit] = c;
                contone_out_posit++;
            } while (--w);
#ifdef PACIFY_VALGRIND
            if (extra)
                memset(contone+contone_out_posit, 0, extra);
#endif
            curr_position++; /* Move us to the next position in our width array */
            contone_ptr++;   /* Move us to a new location in our contone buffer */
        }
        /* Now we have our left justified and expanded contone data for
           LAND_BITS/16 sets of 16 bits. Go ahead and threshold these. */
        contone_ptr = &contone[0];
#if LAND_BITS > 16
        j = LAND_BITS;
        do {
#endif
#ifdef HAVE_SSE2
            threshold_16_SSE(thresh_ptr, contone_ptr, halftone_ptr);
#else
            threshold_16_bit(thresh_ptr, contone_ptr, halftone_ptr);
#endif
            thresh_ptr += 16;
            position += 16;
            halftone_ptr += 2;
            contone_ptr += 16;
#if LAND_BITS > 16
            j -= 16;
        } while (j > 0);
#endif
    }
}

Project: ghostpdl
File: gxht_thresh.c

threshold_16_SSE_unaligned(byte *contone_ptr, byte *thresh_ptr, byte *ht_data)
{
    __m128i input1;
    __m128i input2;
    int result_int;
    byte *sse_data;
    const unsigned int mask1 = 0x80808080;
    __m128i sign_fix = _mm_set_epi32(mask1, mask1, mask1, mask1);

    sse_data = (byte*) &(result_int);
    /* Load */
    input1 = _mm_loadu_si128((const __m128i *)contone_ptr);
    input2 = _mm_loadu_si128((const __m128i *) thresh_ptr);
    /* Unsigned subtraction does Unsigned saturation so we
       have to use the signed operation */
    input1 = _mm_xor_si128(input1, sign_fix);
    input2 = _mm_xor_si128(input2, sign_fix);
    /* Subtract the two */
    input2 = _mm_subs_epi8(input1, input2);
    /* Grab the sign mask */
    result_int = _mm_movemask_epi8(input2);
    /* bit wise reversal on 16 bit word */
    ht_data[0] = bitreverse[sse_data[0]];
    ht_data[1] = bitreverse[sse_data[1]];
}

Project: ghostpdl
File: gxipixel.c

gx_image_enum_begin(gx_device * dev, const gs_gstate * pgs,
                    const gs_matrix *pmat, const gs_image_common_t * pic,
                const gx_drawing_color * pdcolor, const gx_clip_path * pcpath,
                gs_memory_t * mem, gx_image_enum *penum)
{
    const gs_pixel_image_t *pim = (const gs_pixel_image_t *)pic;
    gs_image_format_t format = pim->format;
    const int width = pim->Width;
    const int height = pim->Height;
    const int bps = pim->BitsPerComponent;
    bool masked = penum->masked;
    const float *decode = pim->Decode;
    gs_matrix_double mat;
    int index_bps;
    const gs_color_space *pcs = pim->ColorSpace;
    gs_logical_operation_t lop = (pgs ? pgs->log_op : lop_default);
    int code;
    int log2_xbytes = (bps <= 8 ? 0 : arch_log2_sizeof_frac);
    int spp, nplanes, spread;
    uint bsize;
    byte *buffer;
    fixed mtx, mty;
    gs_fixed_point row_extent, col_extent, x_extent, y_extent;
    bool device_color = true;
    gs_fixed_rect obox, cbox;
    bool gridfitimages = 0;
    bool in_pattern_accumulator = 0;
    int orthogonal;
    int force_interpolation = 0;

    penum->clues = NULL;
    penum->icc_setup.has_transfer = false;
    penum->icc_setup.is_lab = false;
    penum->icc_setup.must_halftone = false;
    penum->icc_setup.need_decode = false;
    penum->Width = width;
    penum->Height = height;

    if ((code = gx_image_compute_mat(pgs, pmat, &(pim->ImageMatrix), &mat)) < 0) {
        return code;
    }
    /* Grid fit: A common construction in postscript/PDF files is for images
     * to be constructed as a series of 'stacked' 1 pixel high images.
     * Furthermore, many of these are implemented as an imagemask plotted on
     * top of thin rectangles. The different fill rules for images and line
     * art produces problems; line art fills a pixel if any part of it is
     * touched - images only fill a pixel if the centre of the pixel is
     * covered. Bug 692666 is such a problem.
     *
     * As a workaround for this problem, the code below was introduced. The
     * concept is that orthogonal images can be 'grid fitted' (or 'stretch')
     * to entirely cover pixels that they touch. Initially I had this working
     * for all images regardless of type, but as testing has proceeded, this
     * showed more and more regressions, so I've cut the cases back in which
     * this code is used until it now only triggers on imagemasks that are
     * either 1 pixel high, or wide, and then not if we are rendering a
     * glyph (such as from a type3 font).
     */

    /* Ask the device if we are in a pattern accumulator */
    in_pattern_accumulator = (dev_proc(dev, dev_spec_op)(dev, gxdso_in_pattern_accumulator, NULL, 0));
    if (in_pattern_accumulator < 0)
        in_pattern_accumulator = 0;

    /* Figure out if we are orthogonal */
    if (mat.xy == 0 && mat.yx == 0)
        orthogonal = 1;
    else if (mat.xx == 0 && mat.yy == 0)
        orthogonal = 2;
    else
        orthogonal = 0;

    /* If we are in a pattern accumulator, we choose to always grid fit
     * orthogonal images. We do this by asking the device whether we
     * should grid fit. This allows us to avoid nasty blank lines around
     * the edges of cells.
     */
    gridfitimages = in_pattern_accumulator && orthogonal;

    if (pgs != NULL && pgs->show_gstate != NULL) {
        /* If we're a graphics state, and we're in a text object, then we
         * must be in a type3 font. Don't fiddle with it. */
    } else if (!gridfitimages &&
               (!penum->masked || penum->image_parent_type != 0)) {
        /* Other than for images we are specifically looking to grid fit (such as
         * ones in a pattern device), we only grid fit imagemasks */
    } else if (gridfitimages && (penum->masked && penum->image_parent_type == 0)) {
        /* We don't gridfit imagemasks in a pattern accumulator */
    } else if (pgs != NULL && pgs->fill_adjust.x == 0 && pgs->fill_adjust.y == 0) {
        /* If fill adjust is disabled, so is grid fitting */
    } else if (orthogonal == 1) {
        if (width == 1 || gridfitimages) {
            if (mat.xx > 0) {
                fixed ix0 = int2fixed(fixed2int(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.xx * width;
                fixed ix1 = int2fixed(fixed2int_ceiling(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.xx = (double)(fixed2float(ix1 - ix0)/width);
            } else if (mat.xx < 0) {
                fixed ix0 = int2fixed(fixed2int_ceiling(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.xx * width;
                fixed ix1 = int2fixed(fixed2int(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.xx = (double)(fixed2float(ix1 - ix0)/width);
            }
        }
        if (height == 1 || gridfitimages) {
            if (mat.yy > 0) {
                fixed iy0 = int2fixed(fixed2int(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.yy * height;
                fixed iy1 = int2fixed(fixed2int_ceiling(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.yy = (double)(fixed2float(iy1 - iy0)/height);
            } else if (mat.yy < 0) {
                fixed iy0 = int2fixed(fixed2int_ceiling(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.yy * height;
                fixed iy1 = int2fixed(fixed2int(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.yy = ((double)fixed2float(iy1 - iy0)/height);
            }
        }
    } else if (orthogonal == 2) {
        if (height == 1 || gridfitimages) {
            if (mat.yx > 0) {
                fixed ix0 = int2fixed(fixed2int(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.yx * height;
                fixed ix1 = int2fixed(fixed2int_ceiling(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.yx = (double)(fixed2float(ix1 - ix0)/height);
            } else if (mat.yx < 0) {
                fixed ix0 = int2fixed(fixed2int_ceiling(float2fixed(mat.tx)));
                double x1 = mat.tx + mat.yx * height;
                fixed ix1 = int2fixed(fixed2int(float2fixed(x1)));
                mat.tx = (double)fixed2float(ix0);
                mat.yx = (double)(fixed2float(ix1 - ix0)/height);
            }
        }
        if (width == 1 || gridfitimages) {
            if (mat.xy > 0) {
                fixed iy0 = int2fixed(fixed2int(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.xy * width;
                fixed iy1 = int2fixed(fixed2int_ceiling(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.xy = (double)(fixed2float(iy1 - iy0)/width);
            } else if (mat.xy < 0) {
                fixed iy0 = int2fixed(fixed2int_ceiling(float2fixed(mat.ty)));
                double y1 = mat.ty + mat.xy * width;
                fixed iy1 = int2fixed(fixed2int(float2fixed(y1)));
                mat.ty = (double)fixed2float(iy0);
                mat.xy = ((double)fixed2float(iy1 - iy0)/width);
            }
        }
    }

    /* When rendering to a pattern accumulator, if we are downscaling
     * then enable interpolation, as otherwise dropouts can cause
     * serious problems. */
    if (in_pattern_accumulator) {
        double ome = ((double)(fixed_1 - fixed_epsilon)) / (double)fixed_1; /* One Minus Epsilon */

        if (orthogonal == 1) {
            if ((mat.xx > -ome && mat.xx < ome) || (mat.yy > -ome && mat.yy < ome)) {
                force_interpolation = true;
            }
        } else if (orthogonal == 2) {
            if ((mat.xy > -ome && mat.xy < ome) || (mat.yx > -ome && mat.yx < ome)) {
                force_interpolation = true;
            }
        }
    }

    /* Can we restrict the amount of image we need? */
    while (pcpath) /* So we can break out of it */
    {
        gs_rect rect, rect_out;
        gs_matrix mi;
        const gs_matrix *m = pgs != NULL ? &ctm_only(pgs) : NULL;
        gs_fixed_rect obox;
        gs_int_rect irect;
        if (m == NULL || (code = gs_matrix_invert(m, &mi)) < 0 ||
            (code = gs_matrix_multiply(&mi, &pic->ImageMatrix, &mi)) < 0) {
            /* Give up trying to shrink the render box, but continue processing */
            break;
        }
        gx_cpath_outer_box(pcpath, &obox);
        rect.p.x = fixed2float(obox.p.x);
        rect.p.y = fixed2float(obox.p.y);
        rect.q.x = fixed2float(obox.q.x);
        rect.q.y = fixed2float(obox.q.y);
        code = gs_bbox_transform(&rect, &mi, &rect_out);
        if (code < 0) {
            /* Give up trying to shrink the render box, but continue processing */
            break;
        }
        irect.p.x = (int)(rect_out.p.x-1.0);
        irect.p.y = (int)(rect_out.p.y-1.0);
        irect.q.x = (int)(rect_out.q.x+1.0);
        irect.q.y = (int)(rect_out.q.y+1.0);
        /* Need to expand the region to allow for the fact that the mitchell
         * scaler reads multiple pixels in. This calculation can probably be
         * improved. */
        /* If mi.{xx,yy} > 1 then we are downscaling. During downscaling,
         * the support increases to ensure that we don't lose pixels contributions
         * entirely. */
        /* I do not understand the need for the +/- 1 fudge factors,
         * but they seem to be required. Increasing the render rectangle can
         * never be bad at least... RJW */
        {
            float support = any_abs(mi.xx);
            int isupport;
            if (any_abs(mi.yy) > support)
                support = any_abs(mi.yy);
            if (any_abs(mi.xy) > support)
                support = any_abs(mi.xy);
            if (any_abs(mi.yx) > support)
                support = any_abs(mi.yx);
            isupport = (int)(MAX_ISCALE_SUPPORT * (support+1)) + 1;
            irect.p.x -= isupport;
            irect.p.y -= isupport;
            irect.q.x += isupport;
            irect.q.y += isupport;
        }
        if (penum->rrect.x < irect.p.x) {
            penum->rrect.w -= irect.p.x - penum->rrect.x;
            if (penum->rrect.w < 0)
               penum->rrect.w = 0;
            penum->rrect.x = irect.p.x;
        }
        if (penum->rrect.x + penum->rrect.w > irect.q.x) {
            penum->rrect.w = irect.q.x - penum->rrect.x;
            if (penum->rrect.w < 0)
                penum->rrect.w = 0;
        }
        if (penum->rrect.y < irect.p.y) {
            penum->rrect.h -= irect.p.y - penum->rrect.y;
            if (penum->rrect.h < 0)
                penum->rrect.h = 0;
            penum->rrect.y = irect.p.y;
        }
        if (penum->rrect.y + penum->rrect.h > irect.q.y) {
            penum->rrect.h = irect.q.y - penum->rrect.y;
            if (penum->rrect.h < 0)
                penum->rrect.h = 0;
        }
        break; /* Out of the while */
    }
    /* Check for the intersection being null */
    if (penum->rrect.x + penum->rrect.w <= penum->rect.x  ||
        penum->rect.x  + penum->rect.w  <= penum->rrect.x ||
        penum->rrect.y + penum->rrect.h <= penum->rect.y  ||
        penum->rect.y  + penum->rect.h  <= penum->rrect.y)
    {
          /* Something may have gone wrong with the floating point above.
           * set the region to something sane. */
        penum->rrect.x = penum->rect.x;
        penum->rrect.y = penum->rect.y;
        penum->rrect.w = 0;
        penum->rrect.h = 0;
    }

    /*penum->matrix = mat;*/
    penum->matrix.xx = mat.xx;
    penum->matrix.xy = mat.xy;
    penum->matrix.yx = mat.yx;
    penum->matrix.yy = mat.yy;
    penum->matrix.tx = mat.tx;
    penum->matrix.ty = mat.ty;
    if_debug6m('b', mem, " [%g %g %g %g %g %g]\n",
              mat.xx, mat.xy, mat.yx, mat.yy, mat.tx, mat.ty);
    /* following works for 1, 2, 4, 8, 12, 16 */
    index_bps = (bps < 8 ? bps >> 1 : (bps >> 2) + 1);
    /*
     * Compute extents with distance transformation.
     */
    if (mat.tx > 0)
        mtx = float2fixed(mat.tx);
    else { /* Use positive values to ensure round down. */
        int f = (int)-mat.tx + 1;

        mtx = float2fixed(mat.tx + f) - int2fixed(f);
    }
    if (mat.ty > 0)
        mty = float2fixed(mat.ty);
    else {  /* Use positive values to ensure round down. */
        int f = (int)-mat.ty + 1;

        mty = float2fixed(mat.ty + f) - int2fixed(f);
    }

    row_extent.x = float2fixed_rounded_boxed(width * mat.xx);
    row_extent.y =
        (is_fzero(mat.xy) ? fixed_0 :
         float2fixed_rounded_boxed(width * mat.xy));
    col_extent.x =
        (is_fzero(mat.yx) ? fixed_0 :
         float2fixed_rounded_boxed(height * mat.yx));
    col_extent.y = float2fixed_rounded_boxed(height * mat.yy);
    gx_image_enum_common_init((gx_image_enum_common_t *)penum,
                              (const gs_data_image_t *)pim,
                              &image1_enum_procs, dev,
                              (masked ? 1 : (penum->alpha ? cs_num_components(pcs)+1 : cs_num_components(pcs))),
                              format);
    if (penum->rect.w == width && penum->rect.h == height) {
        x_extent = row_extent;
        y_extent = col_extent;
    } else {
        int rw = penum->rect.w, rh = penum->rect.h;

        x_extent.x = float2fixed_rounded_boxed(rw * mat.xx);
        x_extent.y =
            (is_fzero(mat.xy) ? fixed_0 :
             float2fixed_rounded_boxed(rw * mat.xy));
        y_extent.x =
            (is_fzero(mat.yx) ? fixed_0 :
             float2fixed_rounded_boxed(rh * mat.yx));
        y_extent.y = float2fixed_rounded_boxed(rh * mat.yy);
    }
    /* Set icolor0 and icolor1 to point to image clues locations if we have
       1spp or an imagemask, otherwise image clues is not used and
       we have these values point to other member variables */
    if (masked || cs_num_components(pcs) == 1) {
        /* Go ahead and allocate now if not already done.  For a mask
           we really should only do 2 values. For now, the goal is to
           eliminate the 256 bytes for the >8bpp image enumerator */
        penum->clues = (gx_image_clue*) gs_alloc_bytes(mem, sizeof(gx_image_clue)*256,
                             "gx_image_enum_begin");
        if (penum->clues == NULL)
            return_error(gs_error_VMerror);
        penum->icolor0 = &(penum->clues[0].dev_color);
        penum->icolor1 = &(penum->clues[255].dev_color);
    } else {
        penum->icolor0 = &(penum->icolor0_val);
        penum->icolor1 = &(penum->icolor1_val);
    }
    if (masked) {       /* This is imagemask. */
        if (bps != 1 || pcs != NULL || penum->alpha || decode[0] == decode[1]) {
            return_error(gs_error_rangecheck);
        }
        /* Initialize color entries 0 and 255. */
        set_nonclient_dev_color(penum->icolor0, gx_no_color_index);
        set_nonclient_dev_color(penum->icolor1, gx_no_color_index);
        *(penum->icolor1) = *pdcolor;
        memcpy(&penum->map[0].table.lookup4x1to32[0],
               (decode[0] < decode[1] ? lookup4x1to32_inverted :
                lookup4x1to32_identity),
               16 * 4);
        penum->map[0].decoding = sd_none;
        spp = 1;
        lop = rop3_know_S_0(lop);
    } else {                    /* This is image, not imagemask. */
        const gs_color_space_type *pcst = pcs->type;
        int b_w_color;

        spp = cs_num_components(pcs);
        if (spp < 0) {          /* Pattern not allowed */
            return_error(gs_error_rangecheck);
        }
        if (penum->alpha)
            ++spp;
        /* Use a less expensive format if possible. */
        switch (format) {
        case gs_image_format_bit_planar:
            if (bps > 1)
                break;
            format = gs_image_format_component_planar;
        case gs_image_format_component_planar:
            if (spp == 1)
                format = gs_image_format_chunky;
        default:                /* chunky */
            break;
        }

        if (pcs->cmm_icc_profile_data != NULL) {
            device_color = false;
        } else {
            device_color = (*pcst->concrete_space) (pcs, pgs) == pcs;
        }

        code = image_init_colors(penum, bps, spp, format, decode, pgs, dev,
                          pcs, &device_color);
        if (code < 0) 
            return gs_throw(code, "Image colors initialization failed");
        /* If we have a CIE based color space and the icc equivalent profile
           is not yet set, go ahead and handle that now.  It may already
           be done due to the above init_colors which may go through remap. */
        if (gs_color_space_is_PSCIE(pcs) && pcs->icc_equivalent == NULL) {
            code = gs_colorspace_set_icc_equivalent((gs_color_space *)pcs, &(penum->icc_setup.is_lab),
                                                pgs->memory);
            if (code < 0)
                return code;
            if (penum->icc_setup.is_lab) {
                /* Free what ever profile was created and use the icc manager's
                   cielab profile */
                gs_color_space *curr_pcs = (gs_color_space *)pcs;
                rc_decrement(curr_pcs->icc_equivalent,"gx_image_enum_begin");
                rc_decrement(curr_pcs->cmm_icc_profile_data,"gx_image_enum_begin");
                curr_pcs->cmm_icc_profile_data = pgs->icc_manager->lab_profile;
                rc_increment(curr_pcs->cmm_icc_profile_data);
            }
        }
        /* Try to transform non-default RasterOps to something */
        /* that we implement less expensively. */
        if (!pim->CombineWithColor)
            lop = rop3_know_T_0(lop) & ~lop_T_transparent;
        else if ((rop3_uses_T(lop) && color_draws_b_w(dev, pdcolor) == 0))
            lop = rop3_know_T_0(lop);

        if (lop != rop3_S &&    /* if best case, no more work needed */
            !rop3_uses_T(lop) && bps == 1 && spp == 1 &&
            (b_w_color =
             color_draws_b_w(dev, penum->icolor0)) >= 0 &&
            color_draws_b_w(dev, penum->icolor1) == (b_w_color ^ 1)
            ) {
            if (b_w_color) {    /* Swap the colors and invert the RasterOp source. */
                gx_device_color dcolor;

                dcolor = *(penum->icolor0);
                *(penum->icolor0) = *(penum->icolor1);
                *(penum->icolor1) = dcolor;
                lop = rop3_invert_S(lop);
            }
            /*
             * At this point, we know that the source pixels
             * correspond directly to the S input for the raster op,
             * i.e., icolor0 is black and icolor1 is white.
             */
            switch (lop) {
                case rop3_D & rop3_S:
                    /* Implement this as an inverted mask writing 0s. */
                    *(penum->icolor1) = *(penum->icolor0);
                    /* (falls through) */
                case rop3_D | rop3_not(rop3_S):
                    /* Implement this as an inverted mask writing 1s. */
                    memcpy(&penum->map[0].table.lookup4x1to32[0],
                           lookup4x1to32_inverted, 16 * 4);
                  rmask:        /* Fill in the remaining parameters for a mask. */
                    penum->masked = masked = true;
                    set_nonclient_dev_color(penum->icolor0, gx_no_color_index);
                    penum->map[0].decoding = sd_none;
                    lop = rop3_T;
                    break;
                case rop3_D & rop3_not(rop3_S):
                    /* Implement this as a mask writing 0s. */
                    *(penum->icolor1) = *(penum->icolor0);
                    /* (falls through) */
                case rop3_D | rop3_S:
                    /* Implement this as a mask writing 1s. */
                    memcpy(&penum->map[0].table.lookup4x1to32[0],
                           lookup4x1to32_identity, 16 * 4);
                    goto rmask;
                default:
                    ;
            }
        }
    }
    penum->device_color = device_color;
    /*
     * Adjust width upward for unpacking up to 7 trailing bits in
     * the row, plus 1 byte for end-of-run, plus up to 7 leading
     * bits for data_x offset within a packed byte.
     */
    bsize = ((bps > 8 ? width * 2 : width) + 15) * spp;
    buffer = gs_alloc_bytes(mem, bsize, "image buffer");
    if (buffer == 0) {
        return_error(gs_error_VMerror);
    }
    penum->bps = bps;
    penum->unpack_bps = bps;
    penum->log2_xbytes = log2_xbytes;
    penum->spp = spp;
    switch (format) {
    case gs_image_format_chunky:
        nplanes = 1;
        spread = 1 << log2_xbytes;
        break;
    case gs_image_format_component_planar:
        nplanes = spp;
        spread = spp << log2_xbytes;
        break;
    case gs_image_format_bit_planar:
        nplanes = spp * bps;
        spread = spp << log2_xbytes;
        break;
    default:
        /* No other cases are possible (checked by gx_image_enum_alloc). */
        return_error(gs_error_Fatal);
    }
    penum->num_planes = nplanes;
    penum->spread = spread;
    /*
     * If we're asked to interpolate in a partial image, we have to
     * assume that the client either really only is interested in
     * the given sub-image, or else is constructing output out of
     * overlapping pieces.
     */
    penum->interpolate = force_interpolation ? interp_force : pim->Interpolate ? interp_on : interp_off;
    penum->x_extent = x_extent;
    penum->y_extent = y_extent;
    penum->posture =
        ((x_extent.y | y_extent.x) == 0 ? image_portrait :
         (x_extent.x | y_extent.y) == 0 ? image_landscape :
         image_skewed);
    penum->pgs = pgs;
    penum->pcs = pcs;
    penum->memory = mem;
    penum->buffer = buffer;
    penum->buffer_size = bsize;
    penum->line = NULL;
    penum->icc_link = NULL;
    penum->color_cache = NULL;
    penum->ht_buffer = NULL;
    penum->thresh_buffer = NULL;
    penum->use_cie_range = false;
    penum->line_size = 0;
    penum->use_rop = lop != (masked ? rop3_T : rop3_S);
#ifdef DEBUG
    if (gs_debug_c('*')) {
        if (penum->use_rop)
            dmprintf1(mem, "[%03x]", lop);
        dmprintf5(mem, "%c%d%c%dx%d ",
                 (masked ? (color_is_pure(pdcolor) ? 'm' : 'h') : 'i'),
                 bps,
                 (penum->posture == image_portrait ? ' ' :
                  penum->posture == image_landscape ? 'L' : 'T'),
                 width, height);
    }
#endif
    penum->slow_loop = 0;
    if (pcpath == 0) {
        (*dev_proc(dev, get_clipping_box)) (dev, &obox);
        cbox = obox;
        penum->clip_image = 0;
    } else
        penum->clip_image =
            (gx_cpath_outer_box(pcpath, &obox) |        /* not || */
             gx_cpath_inner_box(pcpath, &cbox) ?
             0 : image_clip_region);
    penum->clip_outer = obox;
    penum->clip_inner = cbox;
    penum->log_op = rop3_T;     /* rop device takes care of this */
    penum->clip_dev = 0;        /* in case we bail out */
    penum->rop_dev = 0;         /* ditto */
    penum->scaler = 0;          /* ditto */
    /*
     * If all four extrema of the image fall within the clipping
     * rectangle, clipping is never required.  When making this check,
     * we must carefully take into account the fact that we only care
     * about pixel centers.
     */
    {
        fixed
            epx = min(row_extent.x, 0) + min(col_extent.x, 0),
            eqx = max(row_extent.x, 0) + max(col_extent.x, 0),
            epy = min(row_extent.y, 0) + min(col_extent.y, 0),
            eqy = max(row_extent.y, 0) + max(col_extent.y, 0);

        {
            int hwx, hwy;

            switch (penum->posture) {
                case image_portrait:
                    hwx = width, hwy = height;
                    break;
                case image_landscape:
                    hwx = height, hwy = width;
                    break;
                default:
                    hwx = hwy = 0;
            }
            /*
             * If the image is only 1 sample wide or high,
             * and is less than 1 device pixel wide or high,
             * move it slightly so that it covers pixel centers.
             * This is a hack to work around a bug in some old
             * versions of TeX/dvips, which use 1-bit-high images
             * to draw horizontal and vertical lines without
             * positioning them properly.
             */
            if (hwx == 1 && eqx - epx < fixed_1) {
                fixed diff =
                arith_rshift_1(row_extent.x + col_extent.x);

                mtx = (((mtx + diff) | fixed_half) & -fixed_half) - diff;
            }
            if (hwy == 1 && eqy - epy < fixed_1) {
                fixed diff =
                arith_rshift_1(row_extent.y + col_extent.y);

                mty = (((mty + diff) | fixed_half) & -fixed_half) - diff;
            }
        }
        if_debug5m('b', mem, "[b]Image: %sspp=%d, bps=%d, mt=(%g,%g)\n",
                   (masked? "masked, " : ""), spp, bps,
                   fixed2float(mtx), fixed2float(mty));
        if_debug9m('b', mem,
                   "[b]   cbox=(%g,%g),(%g,%g), obox=(%g,%g),(%g,%g), clip_image=0x%x\n",
                   fixed2float(cbox.p.x), fixed2float(cbox.p.y),
                   fixed2float(cbox.q.x), fixed2float(cbox.q.y),
                   fixed2float(obox.p.x), fixed2float(obox.p.y),
                   fixed2float(obox.q.x), fixed2float(obox.q.y),
                   penum->clip_image);
        /* These DDAs enumerate the starting position of each source pixel
         * row in device space. */
        dda_init(penum->dda.row.x, mtx, col_extent.x, height);
        dda_init(penum->dda.row.y, mty, col_extent.y, height);
        if (penum->posture == image_portrait) {
            penum->dst_width = row_extent.x;
            penum->dst_height = col_extent.y;
        } else {
            penum->dst_width = col_extent.x;
            penum->dst_height = row_extent.y;
        }
        /* For gs_image_class_0_interpolate. */
        penum->yi0 = fixed2int_pixround_perfect(dda_current(penum->dda.row.y)); /* For gs_image_class_0_interpolate. */
        if (penum->rect.y) {
            int y = penum->rect.y;

            while (y--) {
                dda_next(penum->dda.row.x);
                dda_next(penum->dda.row.y);
            }
        }
        penum->cur.x = penum->prev.x = dda_current(penum->dda.row.x);
        penum->cur.y = penum->prev.y = dda_current(penum->dda.row.y);
        /* These DDAs enumerate the starting positions of each row of our
         * source pixel data, in the subrectangle ('strip') that we are
         * actually rendering. */
        dda_init(penum->dda.strip.x, penum->cur.x, row_extent.x, width);
        dda_init(penum->dda.strip.y, penum->cur.y, row_extent.y, width);
        if (penum->rect.x) {
            dda_advance(penum->dda.strip.x, penum->rect.x);
            dda_advance(penum->dda.strip.y, penum->rect.x);
        }
        {
            fixed ox = dda_current(penum->dda.strip.x);
            fixed oy = dda_current(penum->dda.strip.y);

            if (!penum->clip_image)     /* i.e., not clip region */
                penum->clip_image =
                    (fixed_pixround(ox + epx) < fixed_pixround(cbox.p.x) ?
                     image_clip_xmin : 0) +
                    (fixed_pixround(ox + eqx) >= fixed_pixround(cbox.q.x) ?
                     image_clip_xmax : 0) +
                    (fixed_pixround(oy + epy) < fixed_pixround(cbox.p.y) ?
                     image_clip_ymin : 0) +
                    (fixed_pixround(oy + eqy) >= fixed_pixround(cbox.q.y) ?
                     image_clip_ymax : 0);
        }
    }
    penum->y = 0;
    penum->used.x = 0;
    penum->used.y = 0;
    {
        static sample_unpack_proc_t procs[2][6] = {
        {   sample_unpack_1, sample_unpack_2,
            sample_unpack_4, sample_unpack_8,
            sample_unpack_12, sample_unpack_16
        },
        {   sample_unpack_1_interleaved, sample_unpack_2_interleaved,
            sample_unpack_4_interleaved, sample_unpack_8_interleaved,
            sample_unpack_12, sample_unpack_16
        }};
        int num_planes = penum->num_planes;
        bool interleaved = (num_planes == 1 && penum->plane_depths[0] != penum->bps);
        int i;

        if (interleaved) {
            int num_components = penum->plane_depths[0] / penum->bps;

            for (i = 1; i < num_components; i++) {
                if (decode[0] != decode[i * 2 + 0] ||
                    decode[1] != decode[i * 2 + 1])
                    break;
            }
            if (i == num_components)
                interleaved = false; /* Use single table. */
        }
        penum->unpack = procs[interleaved][index_bps];

        if_debug1m('b', mem, "[b]unpack=%d\n", bps);
        /* Set up pixel0 for image class procedures. */
        penum->dda.pixel0 = penum->dda.strip;
        for (i = 0; i < gx_image_class_table_count; ++i)
            if ((penum->render = gx_image_class_table[i](penum)) != 0)
                break;
        if (i == gx_image_class_table_count) {
            /* No available class can handle this image. */
            gx_default_end_image(dev, (gx_image_enum_common_t *) penum,
                                 false);
            return_error(gs_error_rangecheck);
        }
    }
    if (penum->clip_image && pcpath) {  /* Set up the clipping device. */
        gx_device_clip *cdev =
            gs_alloc_struct(mem, gx_device_clip,
                            &st_device_clip, "image clipper");

        if (cdev == 0) {
            gx_default_end_image(dev,
                                 (gx_image_enum_common_t *) penum,
                                 false);
            return_error(gs_error_VMerror);
        }
        gx_make_clip_device_in_heap(cdev, pcpath, dev, mem);
        penum->clip_dev = cdev;
    }
    if (penum->use_rop) {       /* Set up the RasterOp source device. */
        gx_device_rop_texture *rtdev;

        code = gx_alloc_rop_texture_device(&rtdev, mem,
                                           "image RasterOp");
        if (code < 0) {
            gx_default_end_image(dev, (gx_image_enum_common_t *) penum,
                                 false);
            return code;
        }
        /* The 'target' must not be NULL for gx_make_rop_texture_device */
        if (!penum->clip_dev && !dev)
            return_error(gs_error_undefined);

        gx_make_rop_texture_device(rtdev,
                                   (penum->clip_dev != 0 ?
                                    (gx_device *) penum->clip_dev :
                                    dev), lop, pdcolor);
        gx_device_retain((gx_device *)rtdev, true);
        penum->rop_dev = rtdev;
    }
    return 0;
}

Project: ghostpdl
File: gxht_thresh.c

threshold_16_bit(byte *contone_ptr, byte *thresh_ptr, byte *ht_data)
{
    int j;

    for (j = 2; j > 0; j--) {
        byte h = 0;
        byte bit_init = 0x80;
        do {
            if (*contone_ptr++ < *thresh_ptr++) {
                h |=  bit_init;
            }
            bit_init >>= 1;
        } while (bit_init != 0);
        *ht_data++ = h;
    }
}