db-d2/primevul-codebert-embeddings huggingface dataset:
Commit message: Limit sprintf modes to 10 characters
CVE: CVE-2021-34552
CWE: CWE-120
Vulnerability description: Pillow through 8.2.0 and PIL (aka Python Imaging Library) through 1.1.7 allow an attacker to pass controlled parameters directly into a convert function to trigger a buffer overflow in Convert.c.

Project: Pillow
File: Convert.c

convert(
    Imaging imOut, Imaging imIn, const char *mode, ImagingPalette palette, int dither) {
    ImagingSectionCookie cookie;
    ImagingShuffler convert;
    int y;

    if (!imIn) {
        return (Imaging)ImagingError_ModeError();
    }

    if (!mode) {
        /* Map palette image to full depth */
        if (!imIn->palette) {
            return (Imaging)ImagingError_ModeError();
        }
        mode = imIn->palette->mode;
    } else {
        /* Same mode? */
        if (!strcmp(imIn->mode, mode)) {
            return ImagingCopy2(imOut, imIn);
        }
    }

    /* test for special conversions */

    if (strcmp(imIn->mode, "P") == 0 || strcmp(imIn->mode, "PA") == 0) {
        return frompalette(imOut, imIn, mode);
    }

    if (strcmp(mode, "P") == 0 || strcmp(mode, "PA") == 0) {
        return topalette(imOut, imIn, mode, palette, dither);
    }

    if (dither && strcmp(mode, "1") == 0) {
        return tobilevel(imOut, imIn, dither);
    }

    /* standard conversion machinery */

    convert = NULL;

    for (y = 0; converters[y].from; y++) {
        if (!strcmp(imIn->mode, converters[y].from) &&
            !strcmp(mode, converters[y].to)) {
            convert = converters[y].convert;
            break;
        }
    }

    if (!convert) {
#ifdef notdef
        return (Imaging)ImagingError_ValueError("conversion not supported");
#else
        static char buf[256];
        /* FIXME: may overflow if mode is too large */
        sprintf(buf, "conversion from %s to %s not supported", imIn->mode, mode);
        return (Imaging)ImagingError_ValueError(buf);
#endif
    }

    imOut = ImagingNew2Dirty(mode, imOut, imIn);
    if (!imOut) {
        return NULL;
    }

    ImagingSectionEnter(&cookie);
    for (y = 0; y < imIn->ysize; y++) {
        (*convert)((UINT8 *)imOut->image[y], (UINT8 *)imIn->image[y], imIn->xsize);
    }
    ImagingSectionLeave(&cookie);

    return imOut;
}
Commit message: force fraction to a limited resolution to finally solve those pesky numerical edge cases
CVE: CVE-2020-19498
CWE: CWE-703
Vulnerability description: Floating point exception in function Fraction in libheif 1.4.0, allows attackers to cause a Denial of Service or possibly other unspecified impacts.

Project: libheif
File: box.cc

Fraction::Fraction(int32_t num,int32_t den)
{
  int32_t g = gcd(num, den);

  // these strange tests are for catching the case that we divide -2147483648 by -1,
  // which would exceed the maximum positive value by one.

  if (num == std::numeric_limits<int32_t>::min() && g == -1) {
    num++;
  }

  if (den == std::numeric_limits<int32_t>::min() && g == -1) {
    den++;
  }


  numerator = num / g;
  denominator = den / g;


  // Reduce resolution of fraction until we are in a safe range.
  // We need this as adding fractions may lead to very large denominators
  // (e.g. 0x10000 * 0x10000 > 0x100000000 -> overflow, leading to integer 0)

  while (denominator > MAX_FRACTION_DENOMINATOR) {
    numerator >>= 1;
    denominator >>= 1;
  }
}
Commit message: cookie: cookie parser out of boundary memory access

The internal libcurl function called sanitize_cookie_path() that cleans
up the path element as given to it from a remote site or when read from
a file, did not properly validate the input. If given a path that
consisted of a single double-quote, libcurl would index a newly
allocated memory area with index -1 and assign a zero to it, thus
destroying heap memory it wasn't supposed to.

CVE-2015-3145

Bug: http://curl.haxx.se/docs/adv_20150422C.html
Reported-by: Hanno Böck
CVE: CVE-2015-3145
CWE: CWE-119
Vulnerability description: The sanitize_cookie_path function in cURL and libcurl 7.31.0 through 7.41.0 does not properly calculate an index, which allows remote attackers to cause a denial of service (out-of-bounds write and crash) or possibly have other unspecified impact via a cookie path containing only a double-quote character.

Project: curl
File: cookie.c

static char *sanitize_cookie_path(const char *cookie_path)
{
  size_t len;
  char *new_path = strdup(cookie_path);
  if(!new_path)
    return NULL;

  /* some stupid site sends path attribute with '"'. */
  if(new_path[0] == '\"') {
    memmove((void *)new_path, (const void *)(new_path + 1), strlen(new_path));
  }
  if(new_path[strlen(new_path) - 1] == '\"') {
    new_path[strlen(new_path) - 1] = 0x0;
  }

  /* RFC6265 5.2.4 The Path Attribute */
  if(new_path[0] != '/') {
    /* Let cookie-path be the default-path. */
    free(new_path);
    new_path = strdup("/");
    return new_path;
  }

  /* convert /hoge/ to /hoge */
  len = strlen(new_path);
  if(1 < len && new_path[len - 1] == '/') {
    new_path[len - 1] = 0x0;
  }

  return new_path;
}
Commit message: add several TIFF decoders and encoders
CVE: CVE-2016-0740
CWE: CWE-119
Vulnerability description: Buffer overflow in the ImagingLibTiffDecode function in libImaging/TiffDecode.c in Pillow before 3.1.1 allows remote attackers to overwrite memory via a crafted TIFF file.

Project: Pillow
File: decode.c

PyImaging_LibTiffDecoderNew(PyObject* self, PyObject* args)
{
    ImagingDecoderObject* decoder;
    char* mode;
    char* rawmode;
    char* compname;
    int compression;
    int fp;

    if (! PyArg_ParseTuple(args, "sssi", &mode, &rawmode, &compname, &fp))
        return NULL;

    TRACE(("new tiff decoder %s\n", compname));

    /* UNDONE -- we can probably do almost any arbitrary compression here,
     * since we're effective passing in the whole file in one shot and
     * getting back the data row by row. V2 maybe
     */

    if (strcasecmp(compname, "tiff_ccitt") == 0) {
        compression = COMPRESSION_CCITTRLE;

    } else if (strcasecmp(compname, "group3") == 0) {
        compression = COMPRESSION_CCITTFAX3;

    } else if (strcasecmp(compname, "group4") == 0) {
        compression = COMPRESSION_CCITTFAX4;

    } else if (strcasecmp(compname, "tiff_raw_16") == 0) {
        compression = COMPRESSION_CCITTRLEW;

    } else {
        PyErr_SetString(PyExc_ValueError, "unknown compession");
        return NULL;
    }

    decoder = PyImaging_DecoderNew(sizeof(TIFFSTATE));
    if (decoder == NULL)
        return NULL;

    if (get_unpacker(decoder, mode, rawmode) < 0)
        return NULL;

    if (! ImagingLibTiffInit(&decoder->state, compression, fp)) {
        Py_DECREF(decoder);
        PyErr_SetString(PyExc_RuntimeError, "tiff codec initialization failed");
        return NULL;
    }

    decoder->decode  = ImagingLibTiffDecode;

    return (PyObject*) decoder;
}
Commit message: VP8: Fix use-after-free in postproc.

The pointer in vp8 postproc refers to show_frame_mi which is only
updated on show frame. However, when there is a no-show frame which also
changes the size (thus new frame buffers allocated), show_frame_mi is
not updated with new frame buffer memory.

Change the pointer in postproc to mi which is always updated.

Bug: 842265
Change-Id: I33874f2112b39f74562cba528432b5f239e6a7bd
CVE: CVE-2019-9433
CWE: CWE-20
Vulnerability description: In libvpx, there is a possible information disclosure due to improper input validation. This could lead to remote information disclosure with no additional execution privileges needed. User interaction is needed for exploitation. Product: AndroidVersions: Android-10Android ID: A-80479354

Project: libvpx
File: postproc.c

void vp8_deblock(VP8_COMMON *cm, YV12_BUFFER_CONFIG *source,
                 YV12_BUFFER_CONFIG *post, int q, int low_var_thresh,
                 int flag) {
  double level = 6.0e-05 * q * q * q - .0067 * q * q + .306 * q + .0065;
  int ppl = (int)(level + .5);

  const MODE_INFO *mode_info_context = cm->show_frame_mi;
  int mbr, mbc;

  /* The pixel thresholds are adjusted according to if or not the macroblock
   * is a skipped block.  */
  unsigned char *ylimits = cm->pp_limits_buffer;
  unsigned char *uvlimits = cm->pp_limits_buffer + 16 * cm->mb_cols;
  (void)low_var_thresh;
  (void)flag;

  if (ppl > 0) {
    for (mbr = 0; mbr < cm->mb_rows; ++mbr) {
      unsigned char *ylptr = ylimits;
      unsigned char *uvlptr = uvlimits;
      for (mbc = 0; mbc < cm->mb_cols; ++mbc) {
        unsigned char mb_ppl;

        if (mode_info_context->mbmi.mb_skip_coeff) {
          mb_ppl = (unsigned char)ppl >> 1;
        } else {
          mb_ppl = (unsigned char)ppl;
        }

        memset(ylptr, mb_ppl, 16);
        memset(uvlptr, mb_ppl, 8);

        ylptr += 16;
        uvlptr += 8;
        mode_info_context++;
      }
      mode_info_context++;

      vpx_post_proc_down_and_across_mb_row(
          source->y_buffer + 16 * mbr * source->y_stride,
          post->y_buffer + 16 * mbr * post->y_stride, source->y_stride,
          post->y_stride, source->y_width, ylimits, 16);

      vpx_post_proc_down_and_across_mb_row(
          source->u_buffer + 8 * mbr * source->uv_stride,
          post->u_buffer + 8 * mbr * post->uv_stride, source->uv_stride,
          post->uv_stride, source->uv_width, uvlimits, 8);
      vpx_post_proc_down_and_across_mb_row(
          source->v_buffer + 8 * mbr * source->uv_stride,
          post->v_buffer + 8 * mbr * post->uv_stride, source->uv_stride,
          post->uv_stride, source->uv_width, uvlimits, 8);
    }
  } else {
    vp8_yv12_copy_frame(source, post);
  }
}
Commit message: drm/komeda: prevent memory leak in komeda_wb_connector_add

In komeda_wb_connector_add if drm_writeback_connector_init fails the
allocated memory for kwb_conn should be released.

Signed-off-by: Navid Emamdoost <navid.emamdoost@gmail.com>
Reviewed-by: James Qian Wang (Arm Technology China) <james.qian.wang@arm.com>
Signed-off-by: james qian wang (Arm Technology China) <james.qian.wang@arm.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20190925043031.32308-1-navid.emamdoost@gmail.com
CVE: CVE-2019-18810
CWE: CWE-401
Vulnerability description: A memory leak in the komeda_wb_connector_add() function in drivers/gpu/drm/arm/display/komeda/komeda_wb_connector.c in the Linux kernel before 5.3.8 allows attackers to cause a denial of service (memory consumption) by triggering drm_writeback_connector_init() failures, aka CID-a0ecd6fdbf5d.

Project: linux
File: komeda_wb_connector.c

static int komeda_wb_connector_add(struct komeda_kms_dev *kms,
				   struct komeda_crtc *kcrtc)
{
	struct komeda_dev *mdev = kms->base.dev_private;
	struct komeda_wb_connector *kwb_conn;
	struct drm_writeback_connector *wb_conn;
	u32 *formats, n_formats = 0;
	int err;

	if (!kcrtc->master->wb_layer)
		return 0;

	kwb_conn = kzalloc(sizeof(*kwb_conn), GFP_KERNEL);
	if (!kwb_conn)
		return -ENOMEM;

	kwb_conn->wb_layer = kcrtc->master->wb_layer;

	wb_conn = &kwb_conn->base;
	wb_conn->encoder.possible_crtcs = BIT(drm_crtc_index(&kcrtc->base));

	formats = komeda_get_layer_fourcc_list(&mdev->fmt_tbl,
					       kwb_conn->wb_layer->layer_type,
					       &n_formats);

	err = drm_writeback_connector_init(&kms->base, wb_conn,
					   &komeda_wb_connector_funcs,
					   &komeda_wb_encoder_helper_funcs,
					   formats, n_formats);
	komeda_put_fourcc_list(formats);
	if (err)
		return err;

	drm_connector_helper_add(&wb_conn->base, &komeda_wb_conn_helper_funcs);

	kcrtc->wb_conn = kwb_conn;

	return 0;
}
Commit message: Fix bug #69545 - avoid overflow when reading list
CVE: CVE-2015-4022
CWE: CWE-189
Vulnerability description: Integer overflow in the ftp_genlist function in ext/ftp/ftp.c in PHP before 5.4.41, 5.5.x before 5.5.25, and 5.6.x before 5.6.9 allows remote FTP servers to execute arbitrary code via a long reply to a LIST command, leading to a heap-based buffer overflow.

Project: php-src
File: ftp.c

ftp_genlist(ftpbuf_t *ftp, const char *cmd, const char *path TSRMLS_DC)
{
	php_stream	*tmpstream = NULL;
	databuf_t	*data = NULL;
	char		*ptr;
	int		ch, lastch;
	int		size, rcvd;
	int		lines;
	char		**ret = NULL;
	char		**entry;
	char		*text;


	if ((tmpstream = php_stream_fopen_tmpfile()) == NULL) {
		php_error_docref(NULL TSRMLS_CC, E_WARNING, "Unable to create temporary file.  Check permissions in temporary files directory.");
		return NULL;
	}

	if (!ftp_type(ftp, FTPTYPE_ASCII)) {
		goto bail;
	}

	if ((data = ftp_getdata(ftp TSRMLS_CC)) == NULL) {
		goto bail;
	}
	ftp->data = data;	

	if (!ftp_putcmd(ftp, cmd, path)) {
		goto bail;
	}
	if (!ftp_getresp(ftp) || (ftp->resp != 150 && ftp->resp != 125 && ftp->resp != 226)) {
		goto bail;
	}

	/* some servers don't open a ftp-data connection if the directory is empty */
	if (ftp->resp == 226) {
		ftp->data = data_close(ftp, data);
		php_stream_close(tmpstream);
		return ecalloc(1, sizeof(char*));
	}

	/* pull data buffer into tmpfile */
	if ((data = data_accept(data, ftp TSRMLS_CC)) == NULL) {
		goto bail;
	}
	size = 0;
	lines = 0;
	lastch = 0;
	while ((rcvd = my_recv(ftp, data->fd, data->buf, FTP_BUFSIZE))) {
		if (rcvd == -1) {
			goto bail;
		}

		php_stream_write(tmpstream, data->buf, rcvd);

		size += rcvd;
		for (ptr = data->buf; rcvd; rcvd--, ptr++) {
			if (*ptr == '\n' && lastch == '\r') {
				lines++;
			} else {
				size++;
			}
			lastch = *ptr;
		}
	}

	ftp->data = data_close(ftp, data);

	php_stream_rewind(tmpstream);

	ret = safe_emalloc((lines + 1), sizeof(char*), size);

	entry = ret;
	text = (char*) (ret + lines + 1);
	*entry = text;
	lastch = 0;
	while ((ch = php_stream_getc(tmpstream)) != EOF) {
		if (ch == '\n' && lastch == '\r') {
			*(text - 1) = 0;
			*++entry = text;
		} else {
			*text++ = ch;
		}
		lastch = ch;
	}
	*entry = NULL;

	php_stream_close(tmpstream);

	if (!ftp_getresp(ftp) || (ftp->resp != 226 && ftp->resp != 250)) {
		efree(ret);
		return NULL;
	}

	return ret;
bail:
	ftp->data = data_close(ftp, data);
	php_stream_close(tmpstream);
	if (ret)
		efree(ret);
	return NULL;
}
Commit message: Fix CVE-2014-6272 in Libevent 2.1

For this fix, we need to make sure that passing too-large inputs to
the evbuffer functions can't make us do bad things with the heap.

Also, lower the maximum chunk size to the lower of off_t, size_t maximum.

This is necessary since otherwise we could get into an infinite loop
if we make a chunk that 'misalign' cannot index into.
CVE: CVE-2014-6272
CWE: CWE-189
Vulnerability description: Multiple integer overflows in the evbuffer API in Libevent 1.4.x before 1.4.15, 2.0.x before 2.0.22, and 2.1.x before 2.1.5-beta allow context-dependent attackers to cause a denial of service or possibly have other unspecified impact via "insanely large inputs" to the (1) evbuffer_add, (2) evbuffer_expand, or (3) bufferevent_write function, which triggers a heap-based buffer overflow or an infinite loop.  NOTE: this identifier has been SPLIT per ADT3 due to different affected versions. See CVE-2015-6525 for the functions that are only affected in 2.0 and later.

Project: libevent
File: buffer.c

evbuffer_add(struct evbuffer *buf, const void *data_in, size_t datlen)
{
	struct evbuffer_chain *chain, *tmp;
	const unsigned char *data = data_in;
	size_t remain, to_alloc;
	int result = -1;

	EVBUFFER_LOCK(buf);

	if (buf->freeze_end) {
		goto done;
	}

	chain = buf->last;

	/* If there are no chains allocated for this buffer, allocate one
	 * big enough to hold all the data. */
	if (chain == NULL) {
		chain = evbuffer_chain_new(datlen);
		if (!chain)
			goto done;
		evbuffer_chain_insert(buf, chain);
	}

	if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) {
		remain = (size_t)(chain->buffer_len - chain->misalign - chain->off);
		if (remain >= datlen) {
			/* there's enough space to hold all the data in the
			 * current last chain */
			memcpy(chain->buffer + chain->misalign + chain->off,
			    data, datlen);
			chain->off += datlen;
			buf->total_len += datlen;
			buf->n_add_for_cb += datlen;
			goto out;
		} else if (!CHAIN_PINNED(chain) &&
		    evbuffer_chain_should_realign(chain, datlen)) {
			/* we can fit the data into the misalignment */
			evbuffer_chain_align(chain);

			memcpy(chain->buffer + chain->off, data, datlen);
			chain->off += datlen;
			buf->total_len += datlen;
			buf->n_add_for_cb += datlen;
			goto out;
		}
	} else {
		/* we cannot write any data to the last chain */
		remain = 0;
	}

	/* we need to add another chain */
	to_alloc = chain->buffer_len;
	if (to_alloc <= EVBUFFER_CHAIN_MAX_AUTO_SIZE/2)
		to_alloc <<= 1;
	if (datlen > to_alloc)
		to_alloc = datlen;
	tmp = evbuffer_chain_new(to_alloc);
	if (tmp == NULL)
		goto done;

	if (remain) {
		memcpy(chain->buffer + chain->misalign + chain->off,
		    data, remain);
		chain->off += remain;
		buf->total_len += remain;
		buf->n_add_for_cb += remain;
	}

	data += remain;
	datlen -= remain;

	memcpy(tmp->buffer, data, datlen);
	tmp->off = datlen;
	evbuffer_chain_insert(buf, tmp);
	buf->n_add_for_cb += datlen;

out:
	evbuffer_invoke_callbacks_(buf);
	result = 0;
done:
	EVBUFFER_UNLOCK(buf);
	return result;
}
Commit message: lower number of allowed compression loops in getLabelFromContent()
1000 is a bit high for a max 255 character qname
CVE: CVE-2015-1868
CWE: CWE-399
Vulnerability description: The label decompression functionality in PowerDNS Recursor 3.5.x, 3.6.x before 3.6.3, and 3.7.x before 3.7.2 and Authoritative (Auth) Server 3.2.x, 3.3.x before 3.3.2, and 3.4.x before 3.4.4 allows remote attackers to cause a denial of service (CPU consumption or crash) via a request with a name that refers to itself.

Project: pdns
File: dnsparser.cc

void PacketReader::getLabelFromContent(const vector<uint8_t>& content, uint16_t& frompos, string& ret, int recurs) 
{
  if(recurs > 1000) // the forward reference-check below should make this test 100% obsolete
    throw MOADNSException("Loop");
  // it is tempting to call reserve on ret, but it turns out it creates a malloc/free storm in the loop
  int pos = frompos;
  for(;;) {
    unsigned char labellen=content.at(frompos++);

    if(!labellen) {
      if(ret.empty())
              ret.append(1,'.');
      break;
    }
    else if((labellen & 0xc0) == 0xc0) {
      uint16_t offset=256*(labellen & ~0xc0) + (unsigned int)content.at(frompos++) - sizeof(dnsheader);
      //        cout<<"This is an offset, need to go to: "<<offset<<endl;

      if(offset >= pos-2)
        throw MOADNSException("forward reference during label decompression");
      return getLabelFromContent(content, offset, ret, ++recurs);
    }
    else if(labellen > 63) 
      throw MOADNSException("Overly long label during label decompression ("+lexical_cast<string>((unsigned int)labellen)+")");
    else {
      // XXX FIXME THIS MIGHT BE VERY SLOW!

      for(string::size_type n = 0 ; n < labellen; ++n, frompos++) {
        if(content.at(frompos)=='.' || content.at(frompos)=='\\') {
          ret.append(1, '\\');
          ret.append(1, content[frompos]);
        }
        else if(content.at(frompos)==' ') {
          ret+="\\032";
        }
        else 
          ret.append(1, content[frompos]);
      }
      ret.append(1,'.');
    }
  }
}
Commit message: drm/i915/gvt: Fix mmap range check

This is to fix missed mmap range check on vGPU bar2 region
and only allow to map vGPU allocated GMADDR range, which means
user space should support sparse mmap to get proper offset for
mmap vGPU aperture. And this takes care of actual pgoff in mmap
request as original code always does from beginning of vGPU
aperture.

Fixes: 659643f7d814 ("drm/i915/gvt/kvmgt: add vfio/mdev support to KVMGT")
Cc: "Monroy, Rodrigo Axel" <rodrigo.axel.monroy@intel.com>
Cc: "Orrala Contreras, Alfredo" <alfredo.orrala.contreras@intel.com>
Cc: stable@vger.kernel.org # v4.10+
Reviewed-by: Hang Yuan <hang.yuan@intel.com>
Signed-off-by: Zhenyu Wang <zhenyuw@linux.intel.com>
CVE: CVE-2019-11085
CWE: CWE-20
Vulnerability description: Insufficient input validation in Kernel Mode Driver in Intel(R) i915 Graphics for Linux before version 5.0 may allow an authenticated user to potentially enable escalation of privilege via local access.

Project: linux
File: kvmgt.c

static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma)
{
	unsigned int index;
	u64 virtaddr;
	unsigned long req_size, pgoff = 0;
	pgprot_t pg_prot;
	struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);

	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
	if (index >= VFIO_PCI_ROM_REGION_INDEX)
		return -EINVAL;

	if (vma->vm_end < vma->vm_start)
		return -EINVAL;
	if ((vma->vm_flags & VM_SHARED) == 0)
		return -EINVAL;
	if (index != VFIO_PCI_BAR2_REGION_INDEX)
		return -EINVAL;

	pg_prot = vma->vm_page_prot;
	virtaddr = vma->vm_start;
	req_size = vma->vm_end - vma->vm_start;
	pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;

	return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);
}
Commit message: Check index before writing to result->chars

Fixes #595
CVE: CVE-2018-12085
CWE: CWE-787
Vulnerability description: Liblouis 3.6.0 has a stack-based Buffer Overflow in the function parseChars in compileTranslationTable.c, a different vulnerability than CVE-2018-11440.

Project: liblouis
File: compileTranslationTable.c

parseChars(FileInfo *nested, CharsString *result, CharsString *token) {
	int in = 0;
	int out = 0;
	int lastOutSize = 0;
	int lastIn;
	unsigned int ch = 0;
	int numBytes = 0;
	unsigned int utf32 = 0;
	int k;
	while (in < token->length) {
		ch = token->chars[in++] & 0xff;
		if (ch < 128) {
			if (ch == '\\') { /* escape sequence */
				switch (ch = token->chars[in]) {
				case '\\':
					break;
				case 'e':
					ch = 0x1b;
					break;
				case 'f':
					ch = 12;
					break;
				case 'n':
					ch = 10;
					break;
				case 'r':
					ch = 13;
					break;
				case 's':
					ch = ' ';
					break;
				case 't':
					ch = 9;
					break;
				case 'v':
					ch = 11;
					break;
				case 'w':
					ch = ENDSEGMENT;
					break;
				case 34:
					ch = QUOTESUB;
					break;
				case 'X':
				case 'x':
					if (token->length - in > 4) {
						ch = hexValue(nested, &token->chars[in + 1], 4);
						in += 4;
					}
					break;
				case 'y':
				case 'Y':
					if (CHARSIZE == 2) {
					not32:
						compileError(nested,
								"liblouis has not been compiled for 32-bit Unicode");
						break;
					}
					if (token->length - in > 5) {
						ch = hexValue(nested, &token->chars[in + 1], 5);
						in += 5;
					}
					break;
				case 'z':
				case 'Z':
					if (CHARSIZE == 2) goto not32;
					if (token->length - in > 8) {
						ch = hexValue(nested, &token->chars[in + 1], 8);
						in += 8;
					}
					break;
				default:
					compileError(nested, "invalid escape sequence '\\%c'", ch);
					break;
				}
				in++;
			}
			result->chars[out++] = (widechar)ch;
			if (out >= MAXSTRING) {
				result->length = out;
				return 1;
			}
			continue;
		}
		lastOutSize = out;
		lastIn = in;
		for (numBytes = MAXBYTES - 1; numBytes > 0; numBytes--)
			if (ch >= first0Bit[numBytes]) break;
		utf32 = ch & (0XFF - first0Bit[numBytes]);
		for (k = 0; k < numBytes; k++) {
			if (in >= MAXSTRING) break;
			if (out >= MAXSTRING) {
				result->length = lastOutSize;
				return 1;
			}
			if (token->chars[in] < 128 || (token->chars[in] & 0x0040)) {
				compileWarning(nested, "invalid UTF-8. Assuming Latin-1.");
				result->chars[out++] = token->chars[lastIn];
				in = lastIn + 1;
				continue;
			}
			utf32 = (utf32 << 6) + (token->chars[in++] & 0x3f);
		}
		if (out >= MAXSTRING) {
			result->length = lastOutSize;
			return 1;
		}
		if (CHARSIZE == 2 && utf32 > 0xffff) utf32 = 0xffff;
		result->chars[out++] = (widechar)utf32;
	}
	result->length = out;
	return 1;
}