static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
					 struct task_struct *task,
					 bool cancel_all)
{
	struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
	struct io_uring_task *tctx = task ? task->io_uring : NULL;

	while (1) {
		enum io_wq_cancel cret;
		bool ret = false;

		if (!task) {
			ret |= io_uring_try_cancel_iowq(ctx);
		} else if (tctx && tctx->io_wq) {
			/*
			 * Cancels requests of all rings, not only @ctx, but
			 * it's fine as the task is in exit/exec.
			 */
			cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
					       &cancel, true);
			ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
		}

		/* SQPOLL thread does its own polling */
		if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
		    (ctx->sq_data && ctx->sq_data->thread == current)) {
			while (!list_empty_careful(&ctx->iopoll_list)) {
				io_iopoll_try_reap_events(ctx);
				ret = true;
			}
		}

		ret |= io_cancel_defer_files(ctx, task, cancel_all);
		ret |= io_poll_remove_all(ctx, task, cancel_all);
		ret |= io_kill_timeouts(ctx, task, cancel_all);
		if (task)
			ret |= io_run_task_work();
		if (!ret)
			break;
		cond_resched();
	}


static void __io_queue_sqe(struct io_kiocb *req)
{
	struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
	int ret;

issue_sqe:
	ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);

	/*
	 * We async punt it if the file wasn't marked NOWAIT, or if the file
	 * doesn't support non-blocking read/write attempts
	 */
	if (likely(!ret)) {
		/* drop submission reference */
		if (req->flags & REQ_F_COMPLETE_INLINE) {
			struct io_ring_ctx *ctx = req->ctx;
			struct io_comp_state *cs = &ctx->submit_state.comp;

			cs->reqs[cs->nr++] = req;
			if (cs->nr == ARRAY_SIZE(cs->reqs))
				io_submit_flush_completions(ctx);
		} else {
			io_put_req(req);
		}
	} else if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
		switch (io_arm_poll_handler(req)) {
		case IO_APOLL_READY:
			goto issue_sqe;
		case IO_APOLL_ABORTED:
			/*
			 * Queued up for async execution, worker will release
			 * submit reference when the iocb is actually submitted.
			 */
			io_queue_async_work(req);
			break;
		}
	} else {
		io_req_complete_failed(req, ret);
	}
	if (linked_timeout)
		io_queue_linked_timeout(linked_timeout);


static int __init io_uring_init(void)
{
#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
	BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
	BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
} while (0)

#define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \
	__BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename)
	BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64);
	BUILD_BUG_SQE_ELEM(0,  __u8,   opcode);
	BUILD_BUG_SQE_ELEM(1,  __u8,   flags);
	BUILD_BUG_SQE_ELEM(2,  __u16,  ioprio);
	BUILD_BUG_SQE_ELEM(4,  __s32,  fd);
	BUILD_BUG_SQE_ELEM(8,  __u64,  off);
	BUILD_BUG_SQE_ELEM(8,  __u64,  addr2);
	BUILD_BUG_SQE_ELEM(16, __u64,  addr);
	BUILD_BUG_SQE_ELEM(16, __u64,  splice_off_in);
	BUILD_BUG_SQE_ELEM(24, __u32,  len);
	BUILD_BUG_SQE_ELEM(28,     __kernel_rwf_t, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */   int, rw_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u32, rw_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fsync_flags);
	BUILD_BUG_SQE_ELEM(28, /* compat */ __u16,  poll_events);
	BUILD_BUG_SQE_ELEM(28, __u32,  poll32_events);
	BUILD_BUG_SQE_ELEM(28, __u32,  sync_range_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  msg_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  timeout_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  accept_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  cancel_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  open_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  statx_flags);
	BUILD_BUG_SQE_ELEM(28, __u32,  fadvise_advice);
	BUILD_BUG_SQE_ELEM(28, __u32,  splice_flags);
	BUILD_BUG_SQE_ELEM(32, __u64,  user_data);
	BUILD_BUG_SQE_ELEM(40, __u16,  buf_index);
	BUILD_BUG_SQE_ELEM(40, __u16,  buf_group);
	BUILD_BUG_SQE_ELEM(42, __u16,  personality);
	BUILD_BUG_SQE_ELEM(44, __s32,  splice_fd_in);

	BUILD_BUG_ON(sizeof(struct io_uring_files_update) !=
		     sizeof(struct io_uring_rsrc_update));
	BUILD_BUG_ON(sizeof(struct io_uring_rsrc_update) >
		     sizeof(struct io_uring_rsrc_update2));
	/* should fit into one byte */
	BUILD_BUG_ON(SQE_VALID_FLAGS >= (1 << 8));

	BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST);
	BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int));

	req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
				SLAB_ACCOUNT);
	return 0;


static void io_dismantle_req(struct io_kiocb *req)
{
	unsigned int flags = req->flags;

	if (io_req_needs_clean(req))
		io_clean_op(req);
	if (!(flags & REQ_F_FIXED_FILE))
		io_put_file(req->file);
	if (req->fixed_rsrc_refs)
		percpu_ref_put(req->fixed_rsrc_refs);
	if (req->async_data) {
		kfree(req->async_data);
		req->async_data = NULL;
	}
}


static void **io_alloc_page_table(size_t size)
{
	unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
	size_t init_size = size;
	void **table;

	table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL);
	if (!table)
		return NULL;

	for (i = 0; i < nr_tables; i++) {
		unsigned int this_size = min_t(size_t, size, PAGE_SIZE);

		table[i] = kzalloc(this_size, GFP_KERNEL);
		if (!table[i]) {
			io_free_page_table(table, init_size);
			return NULL;
		}
		size -= this_size;
	}
	return table;


 */
static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
			  const sigset_t __user *sig, size_t sigsz,
			  struct __kernel_timespec __user *uts)
{
	struct io_wait_queue iowq = {
		.wq = {
			.private	= current,
			.func		= io_wake_function,
			.entry		= LIST_HEAD_INIT(iowq.wq.entry),
		},
		.ctx		= ctx,
		.to_wait	= min_events,
	};
	struct io_rings *rings = ctx->rings;
	signed long timeout = MAX_SCHEDULE_TIMEOUT;
	int ret;

	do {
		io_cqring_overflow_flush(ctx, false);
		if (io_cqring_events(ctx) >= min_events)
			return 0;
		if (!io_run_task_work())
			break;
	} while (1);

	if (sig) {
#ifdef CONFIG_COMPAT
		if (in_compat_syscall())
			ret = set_compat_user_sigmask((const compat_sigset_t __user *)sig,
						      sigsz);
		else
#endif
			ret = set_user_sigmask(sig, sigsz);

		if (ret)
			return ret;
	}

	if (uts) {
		struct timespec64 ts;

		if (get_timespec64(&ts, uts))
			return -EFAULT;
		timeout = timespec64_to_jiffies(&ts);
	}

	iowq.nr_timeouts = atomic_read(&ctx->cq_timeouts);
	trace_io_uring_cqring_wait(ctx, min_events);
	do {
		/* if we can't even flush overflow, don't wait for more */
		if (!io_cqring_overflow_flush(ctx, false)) {
			ret = -EBUSY;
			break;
		}
		prepare_to_wait_exclusive(&ctx->cq_wait, &iowq.wq,
						TASK_INTERRUPTIBLE);
		ret = io_cqring_wait_schedule(ctx, &iowq, &timeout);
		finish_wait(&ctx->cq_wait, &iowq.wq);
		cond_resched();
	} while (ret > 0);

	restore_saved_sigmask_unless(ret == -EINTR);

	return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;


static int io_connect(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_async_connect __io, *io;
	unsigned file_flags;
	int ret;
	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;

	if (req->async_data) {
		io = req->async_data;
	} else {
		ret = move_addr_to_kernel(req->connect.addr,
						req->connect.addr_len,
						&__io.address);
		if (ret)
			goto out;
		io = &__io;
	}

	file_flags = force_nonblock ? O_NONBLOCK : 0;

	ret = __sys_connect_file(req->file, &io->address,
					req->connect.addr_len, file_flags);
	if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) {
		if (req->async_data)
			return -EAGAIN;
		if (io_alloc_async_data(req)) {
			ret = -ENOMEM;
			goto out;
		}
		memcpy(req->async_data, &__io, sizeof(__io));
		return -EAGAIN;
	}
	if (ret == -ERESTARTSYS)
		ret = -EINTR;
out:
	if (ret < 0)
		req_set_fail(req);
	__io_req_complete(req, issue_flags, ret, 0);
	return 0;
}


static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
{
	__s32 __user *fds = arg;
	int fd;

	if (ctx->cq_ev_fd)
		return -EBUSY;

	if (copy_from_user(&fd, fds, sizeof(*fds)))
		return -EFAULT;

	ctx->cq_ev_fd = eventfd_ctx_fdget(fd);
	if (IS_ERR(ctx->cq_ev_fd)) {
		int ret = PTR_ERR(ctx->cq_ev_fd);

		ctx->cq_ev_fd = NULL;
		return ret;
	}

	return 0;


static int run_decode_and_handler (BODY *b, STATE *s, handler_t handler, int plaintext)
{
  int origType;
  char *savePrefix = NULL;
  FILE *fp = NULL;
  BUFFER *tempfile = NULL;
  size_t tmplength = 0;
  LOFF_T tmpoffset = 0;
  int decode = 0;
  int rc = 0;

  fseeko (s->fpin, b->offset, SEEK_SET);

  /* see if we need to decode this part before processing it */
  if (b->encoding == ENCBASE64 || b->encoding == ENCQUOTEDPRINTABLE ||
      b->encoding == ENCUUENCODED || plaintext ||
      mutt_is_text_part (b))				/* text subtypes may
                                                         * require character
                                                         * set conversion even
                                                         * with 8bit encoding.
                                                         */
  {
    origType = b->type;

    if (!plaintext)
    {
      /* decode to a tempfile, saving the original destination */
      fp = s->fpout;
      tempfile = mutt_buffer_pool_get ();
      mutt_buffer_mktemp (tempfile);
      if ((s->fpout = safe_fopen (mutt_b2s (tempfile), "w")) == NULL)
      {
        mutt_error _("Unable to open temporary file!");
        dprint (1, (debugfile, "Can't open %s.\n", mutt_b2s (tempfile)));
        mutt_buffer_pool_release (&tempfile);
        return -1;
      }
      /* decoding the attachment changes the size and offset, so save a copy
       * of the "real" values now, and restore them after processing
       */
      tmplength = b->length;
      tmpoffset = b->offset;

      /* if we are decoding binary bodies, we don't want to prefix each
       * line with the prefix or else the data will get corrupted.
       */
      savePrefix = s->prefix;
      s->prefix = NULL;

      decode = 1;
    }
    else
      b->type = TYPETEXT;

    mutt_decode_attachment (b, s);

    if (decode)
    {
      b->length = ftello (s->fpout);
      b->offset = 0;
      safe_fclose (&s->fpout);

      /* restore final destination and substitute the tempfile for input */
      s->fpout = fp;
      fp = s->fpin;
      s->fpin = fopen (mutt_b2s (tempfile), "r");
      unlink (mutt_b2s (tempfile));
      mutt_buffer_pool_release (&tempfile);

      /* restore the prefix */
      s->prefix = savePrefix;
    }

    b->type = origType;
  }

  /* process the (decoded) body part */
  if (handler)
  {
    rc = handler (b, s);

    if (rc)
    {
      dprint (1, (debugfile, "Failed on attachment of type %s/%s.\n", TYPE(b), NONULL (b->subtype)));
    }

    if (decode)
    {
      b->length = tmplength;
      b->offset = tmpoffset;

      /* restore the original source stream */
      safe_fclose (&s->fpin);
      s->fpin = fp;
    }
  }
  s->flags |= MUTT_FIRSTDONE;

  return rc;
}


static int external_body_handler (BODY *b, STATE *s)
{
  const char *access_type;
  const char *expiration;
  time_t expire;

  access_type = mutt_get_parameter ("access-type", b->parameter);
  if (!access_type)
  {
    if (s->flags & MUTT_DISPLAY)
    {
      state_mark_attach (s);
      state_puts (_("[-- Error: message/external-body has no access-type parameter --]\n"), s);
      return 0;
    }
    else
      return -1;
  }

  expiration = mutt_get_parameter ("expiration", b->parameter);
  if (expiration)
    expire = mutt_parse_date (expiration, NULL);
  else
    expire = -1;

  if (!ascii_strcasecmp (access_type, "x-mutt-deleted"))
  {
    if (s->flags & (MUTT_DISPLAY|MUTT_PRINTING))
    {
      char *length;
      char pretty_size[10];

      state_mark_attach (s);
      state_printf (s, _("[-- This %s/%s attachment "),
                    TYPE(b->parts), b->parts->subtype);
      length = mutt_get_parameter ("length", b->parameter);
      if (length)
      {
	mutt_pretty_size (pretty_size, sizeof (pretty_size),
			  strtol (length, NULL, 10));
	state_printf (s, _("(size %s bytes) "), pretty_size);
      }
      state_puts (_("has been deleted --]\n"), s);

      if (expire != -1)
      {
	state_mark_attach (s);
	state_printf (s, _("[-- on %s --]\n"), expiration);
      }
      if (b->parts->filename)
      {
	state_mark_attach (s);
	state_printf (s, _("[-- name: %s --]\n"), b->parts->filename);
      }

      mutt_copy_hdr (s->fpin, s->fpout, ftello (s->fpin), b->parts->offset,
		     (option (OPTWEED) ? (CH_WEED | CH_REORDER) : 0) |
		     CH_DECODE , NULL);
    }
  }
  else if (expiration && expire < time(NULL))
  {
    if (s->flags & MUTT_DISPLAY)
    {
      state_mark_attach (s);
      state_printf (s, _("[-- This %s/%s attachment is not included, --]\n"),
		    TYPE(b->parts), b->parts->subtype);
      state_attach_puts (_("[-- and the indicated external source has --]\n"
			   "[-- expired. --]\n"), s);

      mutt_copy_hdr(s->fpin, s->fpout, ftello (s->fpin), b->parts->offset,
		    (option (OPTWEED) ? (CH_WEED | CH_REORDER) : 0) |
		    CH_DECODE | CH_DISPLAY, NULL);
    }
  }
  else
  {
    if (s->flags & MUTT_DISPLAY)
    {
      state_mark_attach (s);
      state_printf (s,
		    _("[-- This %s/%s attachment is not included, --]\n"),
		    TYPE (b->parts), b->parts->subtype);
      state_mark_attach (s);
      state_printf (s,
		    _("[-- and the indicated access-type %s is unsupported --]\n"),
		    access_type);
      mutt_copy_hdr (s->fpin, s->fpout, ftello (s->fpin), b->parts->offset,
		     (option (OPTWEED) ? (CH_WEED | CH_REORDER) : 0) |
		     CH_DECODE | CH_DISPLAY, NULL);
    }
  }

  return 0;
}


static NTSTATUS ldapsrv_DelRequest(struct ldapsrv_call *call)
{
	struct ldap_DelRequest *req = &call->request->r.DelRequest;
	struct ldap_Result *del_result;
	struct ldapsrv_reply *del_reply;
	TALLOC_CTX *local_ctx;
	struct ldb_context *samdb = call->conn->ldb;
	struct ldb_dn *dn;
	const char *errstr = NULL;
	int result = LDAP_SUCCESS;
	int ldb_ret;
	struct ldb_result *res = NULL;

	DEBUG(10, ("DelRequest"));
	DEBUGADD(10, (" dn: %s\n", req->dn));

	local_ctx = talloc_named(call, 0, "DelRequest local memory context");
	NT_STATUS_HAVE_NO_MEMORY(local_ctx);

	dn = ldb_dn_new(local_ctx, samdb, req->dn);
	NT_STATUS_HAVE_NO_MEMORY(dn);

	DEBUG(10, ("DelRequest: dn: [%s]\n", req->dn));

	del_reply = ldapsrv_init_reply(call, LDAP_TAG_DelResponse);
	NT_STATUS_HAVE_NO_MEMORY(del_reply);

	if (result == LDAP_SUCCESS) {
		res = talloc_zero(local_ctx, struct ldb_result);
		NT_STATUS_HAVE_NO_MEMORY(res);
		ldb_ret = ldapsrv_del_with_controls(call, dn, call->request->controls, res);
		result = map_ldb_error(local_ctx, ldb_ret, ldb_errstring(samdb),
				       &errstr);
	}

	del_result = &del_reply->msg->r.DelResponse;
	del_result->dn = NULL;
	if ((res != NULL) && (res->refs != NULL)) {
		del_result->resultcode = map_ldb_error(local_ctx,
						       LDB_ERR_REFERRAL, NULL,
						       &errstr);
		del_result->errormessage = (errstr?talloc_strdup(del_reply,errstr):NULL);
		del_result->referral = talloc_strdup(call, *res->refs);
	} else {
		del_result->resultcode = result;
		del_result->errormessage = (errstr?talloc_strdup(del_reply,errstr):NULL);
		del_result->referral = NULL;
	}

	talloc_free(local_ctx);

	return ldapsrv_queue_reply(call, del_reply);
}


static int ldap_server_search_callback(struct ldb_request *req, struct ldb_reply *ares)
{
	struct ldapsrv_context *ctx = talloc_get_type(req->context, struct ldapsrv_context);
	struct ldapsrv_call *call = ctx->call;
	struct ldb_context *ldb = call->conn->ldb;
	unsigned int j;
	struct ldapsrv_reply *ent_r = NULL;
	struct ldap_SearchResEntry *ent;
	int ret;
	NTSTATUS status;

	if (!ares) {
		return ldb_request_done(req, LDB_ERR_OPERATIONS_ERROR);
	}
	if (ares->error != LDB_SUCCESS) {
		return ldb_request_done(req, ares->error);
	}

	switch (ares->type) {
	case LDB_REPLY_ENTRY:
	{
		struct ldb_message *msg = ares->message;
		ent_r = ldapsrv_init_reply(call, LDAP_TAG_SearchResultEntry);
		if (ent_r == NULL) {
			return ldb_oom(ldb);
		}

		ctx->count++;

		/*
		 * Put the LDAP search response data under ent_r->msg
		 * so we can free that later once encoded
		 */
		talloc_steal(ent_r->msg, msg);

		ent = &ent_r->msg->r.SearchResultEntry;
		ent->dn = ldb_dn_get_extended_linearized(ent_r, msg->dn,
							 ctx->extended_type);
		ent->num_attributes = 0;
		ent->attributes = NULL;
		if (msg->num_elements == 0) {
			goto queue_reply;
		}
		ent->num_attributes = msg->num_elements;
		ent->attributes = talloc_array(ent_r, struct ldb_message_element, ent->num_attributes);
		if (ent->attributes == NULL) {
			return ldb_oom(ldb);
		}

		for (j=0; j < ent->num_attributes; j++) {
			ent->attributes[j].name = msg->elements[j].name;
			ent->attributes[j].num_values = 0;
			ent->attributes[j].values = NULL;
			if (ctx->attributesonly && (msg->elements[j].num_values == 0)) {
				continue;
			}
			ent->attributes[j].num_values = msg->elements[j].num_values;
			ent->attributes[j].values = msg->elements[j].values;
		}
queue_reply:
		status = ldapsrv_queue_reply(call, ent_r);
		if (NT_STATUS_EQUAL(status, NT_STATUS_FILE_TOO_LARGE)) {
			ret = ldb_request_done(req,
					       LDB_ERR_SIZE_LIMIT_EXCEEDED);
			ldb_asprintf_errstring(ldb,
					       "LDAP search response size "
					       "limited to %zu bytes\n",
					       LDAP_SERVER_MAX_REPLY_SIZE);
		} else if (!NT_STATUS_IS_OK(status)) {
			ret = ldb_request_done(req,
					       ldb_operr(ldb));
		} else {
			ret = LDB_SUCCESS;
		}
		break;
	}
	case LDB_REPLY_REFERRAL:
	{
		struct ldap_SearchResRef *ent_ref;

		/*
		 * TODO: This should be handled by the notification
		 * module not here
		 */
		if (call->notification.busy) {
			ret = LDB_SUCCESS;
			break;
		}

		ent_r = ldapsrv_init_reply(call, LDAP_TAG_SearchResultReference);
		if (ent_r == NULL) {
			return ldb_oom(ldb);
		}

		/*
		 * Put the LDAP referral data under ent_r->msg
		 * so we can free that later once encoded
		 */
		talloc_steal(ent_r->msg, ares->referral);

		ent_ref = &ent_r->msg->r.SearchResultReference;
		ent_ref->referral = ares->referral;

		status = ldapsrv_queue_reply(call, ent_r);
		if (!NT_STATUS_IS_OK(status)) {
			ret = LDB_ERR_OPERATIONS_ERROR;
		} else {
			ret = LDB_SUCCESS;
		}
		break;
	}
	case LDB_REPLY_DONE:
	{
		/*
		 * We don't queue the reply for this one, we let that
		 * happen outside
		 */
		ctx->controls = talloc_move(ctx, &ares->controls);

		TALLOC_FREE(ares);
		return ldb_request_done(req, LDB_SUCCESS);
	}
	default:
		/* Doesn't happen */
		ret = LDB_ERR_OPERATIONS_ERROR;
	}
	TALLOC_FREE(ares);

	return ret;
}


static NTSTATUS ldapsrv_AbandonRequest(struct ldapsrv_call *call)
{
	struct ldap_AbandonRequest *req = &call->request->r.AbandonRequest;
	struct ldapsrv_call *c = NULL;
	struct ldapsrv_call *n = NULL;

	DEBUG(10, ("AbandonRequest\n"));

	for (c = call->conn->pending_calls; c != NULL; c = n) {
		n = c->next;

		if (c->request->messageid != req->messageid) {
			continue;
		}

		DLIST_REMOVE(call->conn->pending_calls, c);
		TALLOC_FREE(c);
	}

	return NT_STATUS_OK;
}


handle_subscript(
    char_u	**arg,
    char_u	*name_start,
    typval_T	*rettv,
    evalarg_T	*evalarg,
    int		verbose)	// give error messages
{
    int		evaluate = evalarg != NULL
				      && (evalarg->eval_flags & EVAL_EVALUATE);
    int		ret = OK;
    dict_T	*selfdict = NULL;
    int		check_white = TRUE;
    int		getnext;
    char_u	*p;

    while (ret == OK)
    {
	// When at the end of the line and ".name" or "->{" or "->X" follows in
	// the next line then consume the line break.
	p = eval_next_non_blank(*arg, evalarg, &getnext);
	if (getnext
	    && ((rettv->v_type == VAR_DICT && *p == '.' && eval_isdictc(p[1]))
		|| (p[0] == '-' && p[1] == '>' && (p[2] == '{'
			|| ASCII_ISALPHA(in_vim9script() ? *skipwhite(p + 2)
								    : p[2])))))
	{
	    *arg = eval_next_line(evalarg);
	    p = *arg;
	    check_white = FALSE;
	}

	if (rettv->v_type == VAR_ANY)
	{
	    char_u	*exp_name;
	    int		cc;
	    int		idx;
	    ufunc_T	*ufunc;
	    type_T	*type;

	    // Found script from "import {name} as name", script item name must
	    // follow.  "rettv->vval.v_number" has the script ID.
	    if (**arg != '.')
	    {
		if (verbose)
		    semsg(_(e_expected_dot_after_name_str),
					name_start != NULL ? name_start: *arg);
		ret = FAIL;
		break;
	    }
	    ++*arg;
	    if (IS_WHITE_OR_NUL(**arg))
	    {
		if (verbose)
		    emsg(_(e_no_white_space_allowed_after_dot));
		ret = FAIL;
		break;
	    }

	    // isolate the name
	    exp_name = *arg;
	    while (eval_isnamec(**arg))
		++*arg;
	    cc = **arg;
	    **arg = NUL;

	    idx = find_exported(rettv->vval.v_number, exp_name, &ufunc, &type,
			    evalarg->eval_cctx, evalarg->eval_cstack, verbose);
	    **arg = cc;

	    if (idx < 0 && ufunc == NULL)
	    {
		ret = FAIL;
		break;
	    }
	    if (idx >= 0)
	    {
		scriptitem_T    *si = SCRIPT_ITEM(rettv->vval.v_number);
		svar_T		*sv = ((svar_T *)si->sn_var_vals.ga_data) + idx;

		copy_tv(sv->sv_tv, rettv);
	    }
	    else
	    {
		rettv->v_type = VAR_FUNC;
		rettv->vval.v_string = vim_strsave(ufunc->uf_name);
	    }
	    continue;
	}

	if ((**arg == '(' && (!evaluate || rettv->v_type == VAR_FUNC
			    || rettv->v_type == VAR_PARTIAL))
		    && (!check_white || !VIM_ISWHITE(*(*arg - 1))))
	{
	    ret = call_func_rettv(arg, evalarg, rettv, evaluate,
							       selfdict, NULL);

	    // Stop the expression evaluation when immediately aborting on
	    // error, or when an interrupt occurred or an exception was thrown
	    // but not caught.
	    if (aborting())
	    {
		if (ret == OK)
		    clear_tv(rettv);
		ret = FAIL;
	    }
	    dict_unref(selfdict);
	    selfdict = NULL;
	}
	else if (p[0] == '-' && p[1] == '>')
	{
	    if (in_vim9script())
		*arg = skipwhite(p + 2);
	    else
		*arg = p + 2;
	    if (ret == OK)
	    {
		if (VIM_ISWHITE(**arg))
		{
		    emsg(_(e_no_white_space_allowed_before_parenthesis));
		    ret = FAIL;
		}
		else if ((**arg == '{' && !in_vim9script()) || **arg == '(')
		    // expr->{lambda}() or expr->(lambda)()
		    ret = eval_lambda(arg, rettv, evalarg, verbose);
		else
		    // expr->name()
		    ret = eval_method(arg, rettv, evalarg, verbose);
	    }
	}
	// "." is ".name" lookup when we found a dict or when evaluating and
	// scriptversion is at least 2, where string concatenation is "..".
	else if (**arg == '['
		|| (**arg == '.' && (rettv->v_type == VAR_DICT
			|| (!evaluate
			    && (*arg)[1] != '.'
			    && !in_old_script(2)))))
	{
	    dict_unref(selfdict);
	    if (rettv->v_type == VAR_DICT)
	    {
		selfdict = rettv->vval.v_dict;
		if (selfdict != NULL)
		    ++selfdict->dv_refcount;
	    }
	    else
		selfdict = NULL;
	    if (eval_index(arg, rettv, evalarg, verbose) == FAIL)
	    {
		clear_tv(rettv);
		ret = FAIL;
	    }
	}
	else
	    break;
    }

    // Turn "dict.Func" into a partial for "Func" bound to "dict".
    // Don't do this when "Func" is already a partial that was bound
    // explicitly (pt_auto is FALSE).
    if (selfdict != NULL
	    && (rettv->v_type == VAR_FUNC
		|| (rettv->v_type == VAR_PARTIAL
		    && (rettv->vval.v_partial->pt_auto
			|| rettv->vval.v_partial->pt_dict == NULL))))
	selfdict = make_partial(selfdict, rettv);

    dict_unref(selfdict);
    return ret;
}


fill_evalarg_from_eap(evalarg_T *evalarg, exarg_T *eap, int skip)
{
    init_evalarg(evalarg);
    evalarg->eval_flags = skip ? 0 : EVAL_EVALUATE;
    if (eap != NULL)
    {
	evalarg->eval_cstack = eap->cstack;
	if (sourcing_a_script(eap))
	{
	    evalarg->eval_getline = eap->getline;
	    evalarg->eval_cookie = eap->cookie;
	}
    }
}


static void do_utfmoveback_invalid(compiler_common *common)
{
/* Goes one character back. */
DEFINE_COMPILER;
sljit_s32 i;
struct sljit_jump *jump;
struct sljit_jump *buffer_start_close;
struct sljit_label *exit_ok_label;
struct sljit_label *exit_invalid_label;
struct sljit_jump *exit_invalid[7];

sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);

OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3));
exit_invalid[0] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xc0);

/* Two-byte sequence. */
buffer_start_close = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(2));

OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x20);

OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1);
OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0);

/* Three-byte sequence. */
JUMPHERE(jump);
exit_invalid[1] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, -0x40);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(1));

OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0);
jump = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x10);

OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1);
OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0);

/* Four-byte sequence. */
JUMPHERE(jump);
OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0 - 0x80);
exit_invalid[2] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x40);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xf0);
exit_invalid[3] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x05);

exit_ok_label = LABEL();
OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 1);
OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0);

/* Two-byte sequence. */
JUMPHERE(buffer_start_close);
OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));

exit_invalid[4] = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));

OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xc0);
CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x20, exit_ok_label);

/* Three-byte sequence. */
OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
exit_invalid[5] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, -0x40);
exit_invalid[6] = CMP(SLJIT_LESS, STR_PTR, 0, TMP2, 0);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));

OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xe0);
CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x10, exit_ok_label);

/* Four-byte sequences are not possible. */

exit_invalid_label = LABEL();
sljit_set_label(exit_invalid[5], exit_invalid_label);
sljit_set_label(exit_invalid[6], exit_invalid_label);
OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(3));
OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0);

JUMPHERE(exit_invalid[4]);
/* -2 + 4 = 2 */
OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));

exit_invalid_label = LABEL();
for (i = 0; i < 4; i++)
  sljit_set_label(exit_invalid[i], exit_invalid_label);
OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, 0);
OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(4));
OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0);
}


static SLJIT_INLINE void fast_forward_newline(compiler_common *common)
{
DEFINE_COMPILER;
struct sljit_label *loop;
struct sljit_jump *lastchar = NULL;
struct sljit_jump *firstchar;
struct sljit_jump *quit = NULL;
struct sljit_jump *foundcr = NULL;
struct sljit_jump *notfoundnl;
jump_list *newline = NULL;

if (common->match_end_ptr != 0)
  {
  OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);
  OP1(SLJIT_MOV, STR_END, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);
  }

if (common->nltype == NLTYPE_FIXED && common->newline > 255)
  {
#ifdef JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD
  if (JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD && common->mode == PCRE2_JIT_COMPLETE)
    {
    if (HAS_VIRTUAL_REGISTERS)
      {
      OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
      }
    else
      {
      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, str));
      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin));
      }
    firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0);

    OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
    OP2U(SLJIT_SUB | SLJIT_SET_Z, STR_PTR, 0, TMP1, 0);
    OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_NOT_EQUAL);
#if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32
    OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT);
#endif
    OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP1, 0);

    fast_forward_char_pair_simd(common, 1, common->newline & 0xff, common->newline & 0xff, 0, (common->newline >> 8) & 0xff, (common->newline >> 8) & 0xff);
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(2));
    }
  else
#endif /* JIT_HAS_FAST_FORWARD_CHAR_PAIR_SIMD */
    {
    lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
    if (HAS_VIRTUAL_REGISTERS)
      {
      OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, begin));
      }
    else
      {
      OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, str));
      OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin));
      }
    firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0);

    OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(2));
    OP2U(SLJIT_SUB | SLJIT_SET_GREATER_EQUAL, STR_PTR, 0, TMP1, 0);
    OP_FLAGS(SLJIT_MOV, TMP2, 0, SLJIT_GREATER_EQUAL);
#if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32
    OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, UCHAR_SHIFT);
#endif
    OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, TMP2, 0);

    loop = LABEL();
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
    quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-2));
    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
    CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, loop);
    CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, loop);

    JUMPHERE(quit);
    JUMPHERE(lastchar);
    }

  JUMPHERE(firstchar);

  if (common->match_end_ptr != 0)
    OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
  return;
  }

if (HAS_VIRTUAL_REGISTERS)
  {
  OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, str));
  }
else
  OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, str));

/* Example: match /^/ to \r\n from offset 1. */
firstchar = CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, TMP2, 0);

if (common->nltype == NLTYPE_ANY)
  move_back(common, NULL, FALSE);
else
  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));

loop = LABEL();
common->ff_newline_shortcut = loop;

#ifdef JIT_HAS_FAST_FORWARD_CHAR_SIMD
if (JIT_HAS_FAST_FORWARD_CHAR_SIMD && (common->nltype == NLTYPE_FIXED || common->nltype == NLTYPE_ANYCRLF))
  {
  if (common->nltype == NLTYPE_ANYCRLF)
    {
    fast_forward_char_simd(common, CHAR_CR, CHAR_LF, 0);
    if (common->mode != PCRE2_JIT_COMPLETE)
      lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);

    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
    quit = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
    }
   else
    {
    fast_forward_char_simd(common, common->newline, common->newline, 0);

    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
    if (common->mode != PCRE2_JIT_COMPLETE)
      {
      OP2U(SLJIT_SUB | SLJIT_SET_GREATER, STR_PTR, 0, STR_END, 0);
      CMOV(SLJIT_GREATER, STR_PTR, STR_END, 0);
      }
    }
  }
else
#endif /* JIT_HAS_FAST_FORWARD_CHAR_SIMD */
  {
  read_char(common, common->nlmin, common->nlmax, NULL, READ_CHAR_NEWLINE);
  lastchar = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
  if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
    foundcr = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
  check_newlinechar(common, common->nltype, &newline, FALSE);
  set_jumps(newline, loop);
  }

if (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF)
  {
  if (quit == NULL)
    {
    quit = JUMP(SLJIT_JUMP);
    JUMPHERE(foundcr);
    }

  notfoundnl = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
  OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, CHAR_NL);
  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL);
#if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32
  OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT);
#endif
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
  JUMPHERE(notfoundnl);
  JUMPHERE(quit);
  }

if (lastchar)
  JUMPHERE(lastchar);
JUMPHERE(firstchar);

if (common->match_end_ptr != 0)
  OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
}


static void do_utfreadchar_invalid(compiler_common *common)
{
/* Slow decoding a UTF-16 character. TMP1 contains the first half
of the character (>= 0xd800). Return char value in TMP1. STR_PTR is
undefined for invalid characters. */
DEFINE_COMPILER;
struct sljit_jump *exit_invalid[3];

sljit_emit_fast_enter(compiler, RETURN_ADDR, 0);

/* TMP2 contains the high surrogate. */
exit_invalid[0] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0xdc00);
exit_invalid[1] = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
OP2(SLJIT_SHL, TMP2, 0, TMP2, 0, SLJIT_IMM, 10);
OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));

OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xdc00);
OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, SLJIT_IMM, 0x10000);
exit_invalid[2] = CMP(SLJIT_GREATER_EQUAL, TMP1, 0, SLJIT_IMM, 0x400);

OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, TMP2, 0);
OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0);

JUMPHERE(exit_invalid[0]);
JUMPHERE(exit_invalid[1]);
JUMPHERE(exit_invalid[2]);
OP1(SLJIT_MOV, TMP1, 0, SLJIT_IMM, INVALID_UTF_CHAR);
OP_SRC(SLJIT_FAST_RETURN, RETURN_ADDR, 0);
}


static void check_partial(compiler_common *common, BOOL force)
{
/* Checks whether a partial matching is occurred. Does not modify registers. */
DEFINE_COMPILER;
struct sljit_jump *jump = NULL;

SLJIT_ASSERT(!force || common->mode != PCRE2_JIT_COMPLETE);

if (common->mode == PCRE2_JIT_COMPLETE)
  return;

if (!force && !common->allow_empty_partial)
  jump = CMP(SLJIT_GREATER_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, STR_PTR, 0);
else if (common->mode == PCRE2_JIT_PARTIAL_SOFT)
  jump = CMP(SLJIT_EQUAL, SLJIT_MEM1(SLJIT_SP), common->start_used_ptr, SLJIT_IMM, -1);

if (common->mode == PCRE2_JIT_PARTIAL_SOFT)
  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, 0);
else
  {
  if (common->partialmatchlabel != NULL)
    JUMPTO(SLJIT_JUMP, common->partialmatchlabel);
  else
    add_jump(compiler, &common->partialmatch, JUMP(SLJIT_JUMP));
  }

if (jump != NULL)
  JUMPHERE(jump);
}


static SLJIT_INLINE struct sljit_label *mainloop_entry(compiler_common *common)
{
DEFINE_COMPILER;
struct sljit_label *mainloop;
struct sljit_label *newlinelabel = NULL;
struct sljit_jump *start;
struct sljit_jump *end = NULL;
struct sljit_jump *end2 = NULL;
#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
struct sljit_label *loop;
struct sljit_jump *jump;
#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */
jump_list *newline = NULL;
sljit_u32 overall_options = common->re->overall_options;
BOOL hascrorlf = (common->re->flags & PCRE2_HASCRORLF) != 0;
BOOL newlinecheck = FALSE;
BOOL readuchar = FALSE;

if (!(hascrorlf || (overall_options & PCRE2_FIRSTLINE) != 0)
    && (common->nltype == NLTYPE_ANY || common->nltype == NLTYPE_ANYCRLF || common->newline > 255))
  newlinecheck = TRUE;

SLJIT_ASSERT(common->abort_label == NULL);

if ((overall_options & PCRE2_FIRSTLINE) != 0)
  {
  /* Search for the end of the first line. */
  SLJIT_ASSERT(common->match_end_ptr != 0);
  OP1(SLJIT_MOV, TMP3, 0, STR_PTR, 0);

  if (common->nltype == NLTYPE_FIXED && common->newline > 255)
    {
    mainloop = LABEL();
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
    end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-1));
    OP1(MOV_UCHAR, TMP2, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
    CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, mainloop);
    CMPTO(SLJIT_NOT_EQUAL, TMP2, 0, SLJIT_IMM, common->newline & 0xff, mainloop);
    JUMPHERE(end);
    OP2(SLJIT_SUB, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
    }
  else
    {
    end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
    mainloop = LABEL();
    /* Continual stores does not cause data dependency. */
    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0);
    read_char(common, common->nlmin, common->nlmax, NULL, READ_CHAR_NEWLINE);
    check_newlinechar(common, common->nltype, &newline, TRUE);
    CMPTO(SLJIT_LESS, STR_PTR, 0, STR_END, 0, mainloop);
    JUMPHERE(end);
    OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, STR_PTR, 0);
    set_jumps(newline, LABEL());
    }

  OP1(SLJIT_MOV, STR_PTR, 0, TMP3, 0);
  }
else if ((overall_options & PCRE2_USE_OFFSET_LIMIT) != 0)
  {
  /* Check whether offset limit is set and valid. */
  SLJIT_ASSERT(common->match_end_ptr != 0);

  if (HAS_VIRTUAL_REGISTERS)
    {
    OP1(SLJIT_MOV, TMP1, 0, ARGUMENTS, 0);
    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(TMP1), SLJIT_OFFSETOF(jit_arguments, offset_limit));
    }
  else
    OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, offset_limit));

  OP1(SLJIT_MOV, TMP2, 0, STR_END, 0);
  end = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, (sljit_sw) PCRE2_UNSET);
  if (HAS_VIRTUAL_REGISTERS)
    OP1(SLJIT_MOV, TMP2, 0, ARGUMENTS, 0);
  else
    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(ARGUMENTS), SLJIT_OFFSETOF(jit_arguments, begin));

#if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32
  OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT);
#endif /* PCRE2_CODE_UNIT_WIDTH == [16|32] */
  if (HAS_VIRTUAL_REGISTERS)
    OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(jit_arguments, begin));

  OP2(SLJIT_ADD, TMP2, 0, TMP2, 0, TMP1, 0);
  end2 = CMP(SLJIT_LESS_EQUAL, TMP2, 0, STR_END, 0);
  OP1(SLJIT_MOV, TMP2, 0, STR_END, 0);
  JUMPHERE(end2);
  OP1(SLJIT_MOV, SLJIT_RETURN_REG, 0, SLJIT_IMM, PCRE2_ERROR_NOMATCH);
  add_jump(compiler, &common->abort, CMP(SLJIT_LESS, TMP2, 0, STR_PTR, 0));
  JUMPHERE(end);
  OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr, TMP2, 0);
  }

start = JUMP(SLJIT_JUMP);

if (newlinecheck)
  {
  newlinelabel = LABEL();
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  end = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
  OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, common->newline & 0xff);
  OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL);
#if PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32
  OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT);
#endif /* PCRE2_CODE_UNIT_WIDTH == [16|32] */
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
  end2 = JUMP(SLJIT_JUMP);
  }

mainloop = LABEL();

/* Increasing the STR_PTR here requires one less jump in the most common case. */
#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
if (common->utf && !common->invalid_utf) readuchar = TRUE;
#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */
if (newlinecheck) readuchar = TRUE;

if (readuchar)
  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);

if (newlinecheck)
  CMPTO(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff, newlinelabel);

OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
#if PCRE2_CODE_UNIT_WIDTH == 8
if (common->invalid_utf)
  {
  /* Skip continuation code units. */
  loop = LABEL();
  jump = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0x80);
  CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x40, loop);
  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  JUMPHERE(jump);
  }
else if (common->utf)
  {
  jump = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
  OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
  JUMPHERE(jump);
  }
#elif PCRE2_CODE_UNIT_WIDTH == 16
if (common->invalid_utf)
  {
  /* Skip continuation code units. */
  loop = LABEL();
  jump = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xdc00);
  CMPTO(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0x400, loop);
  OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  JUMPHERE(jump);
  }
else if (common->utf)
  {
  OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xd800);

  if (sljit_has_cpu_feature(SLJIT_HAS_CMOV))
    {
    OP2(SLJIT_ADD, TMP2, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
    OP2U(SLJIT_SUB | SLJIT_SET_LESS, TMP1, 0, SLJIT_IMM, 0x400);
    CMOV(SLJIT_LESS, STR_PTR, TMP2, 0);
    }
  else
    {
    OP2U(SLJIT_SUB | SLJIT_SET_LESS, TMP1, 0, SLJIT_IMM, 0x400);
    OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_LESS);
    OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, UCHAR_SHIFT);
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
    }
  }
#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16] */
#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 */
JUMPHERE(start);

if (newlinecheck)
  {
  JUMPHERE(end);
  JUMPHERE(end2);
  }

return mainloop;
}


static void fast_forward_first_char2(compiler_common *common, PCRE2_UCHAR char1, PCRE2_UCHAR char2, sljit_s32 offset)
{
DEFINE_COMPILER;
struct sljit_label *start;
struct sljit_jump *match;
struct sljit_jump *partial_quit;
PCRE2_UCHAR mask;
BOOL has_match_end = (common->match_end_ptr != 0);

SLJIT_ASSERT(common->mode == PCRE2_JIT_COMPLETE || offset == 0);

if (has_match_end)
  OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), common->match_end_ptr);

if (offset > 0)
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset));

if (has_match_end)
  {
  OP1(SLJIT_MOV, TMP3, 0, STR_END, 0);

  OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, IN_UCHARS(offset + 1));
  OP2U(SLJIT_SUB | SLJIT_SET_GREATER, STR_END, 0, TMP1, 0);
  CMOV(SLJIT_GREATER, STR_END, TMP1, 0);
  }

#ifdef JIT_HAS_FAST_FORWARD_CHAR_SIMD

if (JIT_HAS_FAST_FORWARD_CHAR_SIMD)
  {
  fast_forward_char_simd(common, char1, char2, offset);

  if (offset > 0)
    OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset));

  if (has_match_end)
    OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
  return;
  }

#endif

start = LABEL();

partial_quit = CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0);
if (common->mode == PCRE2_JIT_COMPLETE)
  add_jump(compiler, &common->failed_match, partial_quit);

OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));

if (char1 == char2)
  CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, char1, start);
else
  {
  mask = char1 ^ char2;
  if (is_powerof2(mask))
    {
    OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, mask);
    CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, char1 | mask, start);
    }
  else
    {
    match = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, char1);
    CMPTO(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, char2, start);
    JUMPHERE(match);
    }
  }

#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
if (common->utf && offset > 0)
  {
  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(-(offset + 1)));
  jumpto_if_not_utf_char_start(compiler, TMP1, start);
  }
#endif

OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(offset + 1));

if (common->mode != PCRE2_JIT_COMPLETE)
  JUMPHERE(partial_quit);

if (has_match_end)
  OP1(SLJIT_MOV, STR_END, 0, TMP3, 0);
}


static int detect_early_fail(compiler_common *common, PCRE2_SPTR cc, int *private_data_start,
   sljit_s32 depth, int start, BOOL fast_forward_allowed)
{
PCRE2_SPTR begin = cc;
PCRE2_SPTR next_alt;
PCRE2_SPTR end;
PCRE2_SPTR accelerated_start;
BOOL prev_fast_forward_allowed;
int result = 0;
int count;

SLJIT_ASSERT(*cc == OP_ONCE || *cc == OP_BRA || *cc == OP_CBRA);
SLJIT_ASSERT(*cc != OP_CBRA || common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] != 0);
SLJIT_ASSERT(start < EARLY_FAIL_ENHANCE_MAX);

next_alt = cc + GET(cc, 1);
if (*next_alt == OP_ALT)
  fast_forward_allowed = FALSE;

do
  {
  count = start;
  cc += 1 + LINK_SIZE + ((*cc == OP_CBRA) ? IMM2_SIZE : 0);

  while (TRUE)
    {
    accelerated_start = NULL;

    switch(*cc)
      {
      case OP_SOD:
      case OP_SOM:
      case OP_SET_SOM:
      case OP_NOT_WORD_BOUNDARY:
      case OP_WORD_BOUNDARY:
      case OP_EODN:
      case OP_EOD:
      case OP_CIRC:
      case OP_CIRCM:
      case OP_DOLL:
      case OP_DOLLM:
      /* Zero width assertions. */
      cc++;
      continue;

      case OP_NOT_DIGIT:
      case OP_DIGIT:
      case OP_NOT_WHITESPACE:
      case OP_WHITESPACE:
      case OP_NOT_WORDCHAR:
      case OP_WORDCHAR:
      case OP_ANY:
      case OP_ALLANY:
      case OP_ANYBYTE:
      case OP_NOT_HSPACE:
      case OP_HSPACE:
      case OP_NOT_VSPACE:
      case OP_VSPACE:
      fast_forward_allowed = FALSE;
      cc++;
      continue;

      case OP_ANYNL:
      case OP_EXTUNI:
      fast_forward_allowed = FALSE;
      if (count == 0)
        count = 1;
      cc++;
      continue;

      case OP_NOTPROP:
      case OP_PROP:
      fast_forward_allowed = FALSE;
      cc += 1 + 2;
      continue;

      case OP_CHAR:
      case OP_CHARI:
      case OP_NOT:
      case OP_NOTI:
      fast_forward_allowed = FALSE;
      cc += 2;
#ifdef SUPPORT_UNICODE
      if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
#endif
      continue;

      case OP_TYPESTAR:
      case OP_TYPEMINSTAR:
      case OP_TYPEPLUS:
      case OP_TYPEMINPLUS:
      case OP_TYPEPOSSTAR:
      case OP_TYPEPOSPLUS:
      /* The type or prop opcode is skipped in the next iteration. */
      cc += 1;

      if (cc[0] != OP_ANYNL && cc[0] != OP_EXTUNI)
        {
        accelerated_start = cc - 1;
        break;
        }

      if (count == 0)
        count = 1;
      fast_forward_allowed = FALSE;
      continue;

      case OP_TYPEUPTO:
      case OP_TYPEMINUPTO:
      case OP_TYPEEXACT:
      case OP_TYPEPOSUPTO:
      cc += IMM2_SIZE;
      /* Fall through */

      case OP_TYPEQUERY:
      case OP_TYPEMINQUERY:
      case OP_TYPEPOSQUERY:
      /* The type or prop opcode is skipped in the next iteration. */
      fast_forward_allowed = FALSE;
      if (count == 0)
        count = 1;
      cc += 1;
      continue;

      case OP_STAR:
      case OP_MINSTAR:
      case OP_PLUS:
      case OP_MINPLUS:
      case OP_POSSTAR:
      case OP_POSPLUS:

      case OP_STARI:
      case OP_MINSTARI:
      case OP_PLUSI:
      case OP_MINPLUSI:
      case OP_POSSTARI:
      case OP_POSPLUSI:

      case OP_NOTSTAR:
      case OP_NOTMINSTAR:
      case OP_NOTPLUS:
      case OP_NOTMINPLUS:
      case OP_NOTPOSSTAR:
      case OP_NOTPOSPLUS:

      case OP_NOTSTARI:
      case OP_NOTMINSTARI:
      case OP_NOTPLUSI:
      case OP_NOTMINPLUSI:
      case OP_NOTPOSSTARI:
      case OP_NOTPOSPLUSI:
      accelerated_start = cc;
      cc += 2;
#ifdef SUPPORT_UNICODE
      if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
#endif
      break;

      case OP_UPTO:
      case OP_MINUPTO:
      case OP_EXACT:
      case OP_POSUPTO:
      case OP_UPTOI:
      case OP_MINUPTOI:
      case OP_EXACTI:
      case OP_POSUPTOI:
      case OP_NOTUPTO:
      case OP_NOTMINUPTO:
      case OP_NOTEXACT:
      case OP_NOTPOSUPTO:
      case OP_NOTUPTOI:
      case OP_NOTMINUPTOI:
      case OP_NOTEXACTI:
      case OP_NOTPOSUPTOI:
      cc += IMM2_SIZE;
      /* Fall through */

      case OP_QUERY:
      case OP_MINQUERY:
      case OP_POSQUERY:
      case OP_QUERYI:
      case OP_MINQUERYI:
      case OP_POSQUERYI:
      case OP_NOTQUERY:
      case OP_NOTMINQUERY:
      case OP_NOTPOSQUERY:
      case OP_NOTQUERYI:
      case OP_NOTMINQUERYI:
      case OP_NOTPOSQUERYI:
      fast_forward_allowed = FALSE;
      if (count == 0)
        count = 1;
      cc += 2;
#ifdef SUPPORT_UNICODE
      if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
#endif
      continue;

      case OP_CLASS:
      case OP_NCLASS:
#if defined SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH != 8
      case OP_XCLASS:
      accelerated_start = cc;
      cc += ((*cc == OP_XCLASS) ? GET(cc, 1) : (unsigned int)(1 + (32 / sizeof(PCRE2_UCHAR))));
#else
      accelerated_start = cc;
      cc += (1 + (32 / sizeof(PCRE2_UCHAR)));
#endif

      switch (*cc)
        {
        case OP_CRSTAR:
        case OP_CRMINSTAR:
        case OP_CRPLUS:
        case OP_CRMINPLUS:
        case OP_CRPOSSTAR:
        case OP_CRPOSPLUS:
        cc++;
        break;

        case OP_CRRANGE:
        case OP_CRMINRANGE:
        case OP_CRPOSRANGE:
        cc += 2 * IMM2_SIZE;
        /* Fall through */
        case OP_CRQUERY:
        case OP_CRMINQUERY:
        case OP_CRPOSQUERY:
        cc++;
        if (count == 0)
          count = 1;
        /* Fall through */
        default:
        accelerated_start = NULL;
        fast_forward_allowed = FALSE;
        continue;
        }
      break;

      case OP_ONCE:
      case OP_BRA:
      case OP_CBRA:
      end = cc + GET(cc, 1);

      prev_fast_forward_allowed = fast_forward_allowed;
      fast_forward_allowed = FALSE;
      if (depth >= 4)
        break;

      end = bracketend(cc) - (1 + LINK_SIZE);
      if (*end != OP_KET || (*cc == OP_CBRA && common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0))
        break;

      count = detect_early_fail(common, cc, private_data_start, depth + 1, count, prev_fast_forward_allowed);

      if (PRIVATE_DATA(cc) != 0)
        common->private_data_ptrs[begin - common->start] = 1;

      if (count < EARLY_FAIL_ENHANCE_MAX)
        {
        cc = end + (1 + LINK_SIZE);
        continue;
        }
      break;

      case OP_KET:
      SLJIT_ASSERT(PRIVATE_DATA(cc) == 0);
      if (cc >= next_alt)
        break;
      cc += 1 + LINK_SIZE;
      continue;
      }

    if (accelerated_start != NULL)
      {
      if (count == 0)
        {
        count++;

        if (fast_forward_allowed)
          {
          common->fast_forward_bc_ptr = accelerated_start;
          common->private_data_ptrs[(accelerated_start + 1) - common->start] = ((*private_data_start) << 3) | type_skip;
          *private_data_start += sizeof(sljit_sw);
          }
        else
          {
          common->private_data_ptrs[(accelerated_start + 1) - common->start] = ((*private_data_start) << 3) | type_fail;

          if (common->early_fail_start_ptr == 0)
            common->early_fail_start_ptr = *private_data_start;

          *private_data_start += sizeof(sljit_sw);
          common->early_fail_end_ptr = *private_data_start;

          if (*private_data_start > SLJIT_MAX_LOCAL_SIZE)
            return EARLY_FAIL_ENHANCE_MAX;
          }
        }
      else
        {
        common->private_data_ptrs[(accelerated_start + 1) - common->start] = ((*private_data_start) << 3) | type_fail_range;

        if (common->early_fail_start_ptr == 0)
          common->early_fail_start_ptr = *private_data_start;

        *private_data_start += 2 * sizeof(sljit_sw);
        common->early_fail_end_ptr = *private_data_start;

        if (*private_data_start > SLJIT_MAX_LOCAL_SIZE)
          return EARLY_FAIL_ENHANCE_MAX;
        }

      /* Cannot be part of a repeat. */
      common->private_data_ptrs[begin - common->start] = 1;
      count++;

      if (count < EARLY_FAIL_ENHANCE_MAX)
        continue;
      }

    break;
    }

  if (*cc != OP_ALT && *cc != OP_KET)
    result = EARLY_FAIL_ENHANCE_MAX;
  else if (result < count)
    result = count;

  cc = next_alt;
  next_alt = cc + GET(cc, 1);
  }
while (*cc == OP_ALT);

return result;
}


static PCRE2_SPTR compile_char1_matchingpath(compiler_common *common, PCRE2_UCHAR type, PCRE2_SPTR cc, jump_list **backtracks, BOOL check_str_ptr)
{
DEFINE_COMPILER;
int length;
unsigned int c, oc, bit;
compare_context context;
struct sljit_jump *jump[3];
jump_list *end_list;
#ifdef SUPPORT_UNICODE
PCRE2_UCHAR propdata[5];
#endif /* SUPPORT_UNICODE */

switch(type)
  {
  case OP_NOT_DIGIT:
  case OP_DIGIT:
  /* Digits are usually 0-9, so it is worth to optimize them. */
  if (check_str_ptr)
    detect_partial_match(common, backtracks);
#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8
  if (common->utf && is_char7_bitset((const sljit_u8*)common->ctypes - cbit_length + cbit_digit, FALSE))
    read_char7_type(common, backtracks, type == OP_NOT_DIGIT);
  else
#endif
    read_char8_type(common, backtracks, type == OP_NOT_DIGIT);
    /* Flip the starting bit in the negative case. */
  OP2U(SLJIT_AND | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, ctype_digit);
  add_jump(compiler, backtracks, JUMP(type == OP_DIGIT ? SLJIT_ZERO : SLJIT_NOT_ZERO));
  return cc;

  case OP_NOT_WHITESPACE:
  case OP_WHITESPACE:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);
#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8
  if (common->utf && is_char7_bitset((const sljit_u8*)common->ctypes - cbit_length + cbit_space, FALSE))
    read_char7_type(common, backtracks, type == OP_NOT_WHITESPACE);
  else
#endif
    read_char8_type(common, backtracks, type == OP_NOT_WHITESPACE);
  OP2U(SLJIT_AND | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, ctype_space);
  add_jump(compiler, backtracks, JUMP(type == OP_WHITESPACE ? SLJIT_ZERO : SLJIT_NOT_ZERO));
  return cc;

  case OP_NOT_WORDCHAR:
  case OP_WORDCHAR:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);
#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8
  if (common->utf && is_char7_bitset((const sljit_u8*)common->ctypes - cbit_length + cbit_word, FALSE))
    read_char7_type(common, backtracks, type == OP_NOT_WORDCHAR);
  else
#endif
    read_char8_type(common, backtracks, type == OP_NOT_WORDCHAR);
  OP2U(SLJIT_AND | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, ctype_word);
  add_jump(compiler, backtracks, JUMP(type == OP_WORDCHAR ? SLJIT_ZERO : SLJIT_NOT_ZERO));
  return cc;

  case OP_ANY:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);
  read_char(common, common->nlmin, common->nlmax, backtracks, READ_CHAR_UPDATE_STR_PTR);
  if (common->nltype == NLTYPE_FIXED && common->newline > 255)
    {
    jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, (common->newline >> 8) & 0xff);
    end_list = NULL;
    if (common->mode != PCRE2_JIT_PARTIAL_HARD)
      add_jump(compiler, &end_list, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
    else
      check_str_end(common, &end_list);

    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, common->newline & 0xff));
    set_jumps(end_list, LABEL());
    JUMPHERE(jump[0]);
    }
  else
    check_newlinechar(common, common->nltype, backtracks, TRUE);
  return cc;

  case OP_ALLANY:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);
#ifdef SUPPORT_UNICODE
  if (common->utf)
    {
    if (common->invalid_utf)
      {
      read_char(common, 0, READ_CHAR_MAX, backtracks, READ_CHAR_UPDATE_STR_PTR);
      return cc;
      }

#if PCRE2_CODE_UNIT_WIDTH == 8 || PCRE2_CODE_UNIT_WIDTH == 16
    OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
#if PCRE2_CODE_UNIT_WIDTH == 8
    jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
    OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
#elif PCRE2_CODE_UNIT_WIDTH == 16
    jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xd800);
    OP2(SLJIT_AND, TMP1, 0, TMP1, 0, SLJIT_IMM, 0xfc00);
    OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, 0xd800);
    OP_FLAGS(SLJIT_MOV, TMP1, 0, SLJIT_EQUAL);
    OP2(SLJIT_SHL, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
#endif /* PCRE2_CODE_UNIT_WIDTH == 8 */
    JUMPHERE(jump[0]);
    return cc;
#endif /* PCRE2_CODE_UNIT_WIDTH == [8|16] */
    }
#endif /* SUPPORT_UNICODE */
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  return cc;

  case OP_ANYBYTE:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  return cc;

#ifdef SUPPORT_UNICODE
  case OP_NOTPROP:
  case OP_PROP:
  propdata[0] = XCL_HASPROP;
  propdata[1] = type == OP_NOTPROP ? XCL_NOTPROP : XCL_PROP;
  propdata[2] = cc[0];
  propdata[3] = cc[1];
  propdata[4] = XCL_END;
  if (check_str_ptr)
    detect_partial_match(common, backtracks);
  compile_xclass_matchingpath(common, propdata, backtracks);
  return cc + 2;
#endif

  case OP_ANYNL:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);
  read_char(common, common->bsr_nlmin, common->bsr_nlmax, NULL, 0);
  jump[0] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_CR);
  /* We don't need to handle soft partial matching case. */
  end_list = NULL;
  if (common->mode != PCRE2_JIT_PARTIAL_HARD)
    add_jump(compiler, &end_list, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));
  else
    check_str_end(common, &end_list);
  OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
  jump[1] = CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, CHAR_NL);
  OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
  jump[2] = JUMP(SLJIT_JUMP);
  JUMPHERE(jump[0]);
  check_newlinechar(common, common->bsr_nltype, backtracks, FALSE);
  set_jumps(end_list, LABEL());
  JUMPHERE(jump[1]);
  JUMPHERE(jump[2]);
  return cc;

  case OP_NOT_HSPACE:
  case OP_HSPACE:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);

  if (type == OP_NOT_HSPACE)
    read_char(common, 0x9, 0x3000, backtracks, READ_CHAR_UPDATE_STR_PTR);
  else
    read_char(common, 0x9, 0x3000, NULL, 0);

  add_jump(compiler, &common->hspace, JUMP(SLJIT_FAST_CALL));
  sljit_set_current_flags(compiler, SLJIT_SET_Z);
  add_jump(compiler, backtracks, JUMP(type == OP_NOT_HSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO));
  return cc;

  case OP_NOT_VSPACE:
  case OP_VSPACE:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);

  if (type == OP_NOT_VSPACE)
    read_char(common, 0xa, 0x2029, backtracks, READ_CHAR_UPDATE_STR_PTR);
  else
    read_char(common, 0xa, 0x2029, NULL, 0);

  add_jump(compiler, &common->vspace, JUMP(SLJIT_FAST_CALL));
  sljit_set_current_flags(compiler, SLJIT_SET_Z);
  add_jump(compiler, backtracks, JUMP(type == OP_NOT_VSPACE ? SLJIT_NOT_ZERO : SLJIT_ZERO));
  return cc;

#ifdef SUPPORT_UNICODE
  case OP_EXTUNI:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);

  SLJIT_ASSERT(TMP1 == SLJIT_R0 && STR_PTR == SLJIT_R1);
  OP1(SLJIT_MOV, SLJIT_R0, 0, ARGUMENTS, 0);

#if PCRE2_CODE_UNIT_WIDTH != 32
  sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS2(W, W, W), SLJIT_IMM,
    common->utf ? (common->invalid_utf ? SLJIT_FUNC_ADDR(do_extuni_utf_invalid) : SLJIT_FUNC_ADDR(do_extuni_utf)) : SLJIT_FUNC_ADDR(do_extuni_no_utf));
  if (common->invalid_utf)
    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0));
#else
  sljit_emit_icall(compiler, SLJIT_CALL, SLJIT_ARGS2(W, W, W), SLJIT_IMM,
    common->invalid_utf ? SLJIT_FUNC_ADDR(do_extuni_utf_invalid) : SLJIT_FUNC_ADDR(do_extuni_no_utf));
  if (!common->utf || common->invalid_utf)
    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, SLJIT_RETURN_REG, 0, SLJIT_IMM, 0));
#endif

  OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_RETURN_REG, 0);

  if (common->mode == PCRE2_JIT_PARTIAL_HARD)
    {
    jump[0] = CMP(SLJIT_LESS, SLJIT_RETURN_REG, 0, STR_END, 0);
    /* Since we successfully read a char above, partial matching must occure. */
    check_partial(common, TRUE);
    JUMPHERE(jump[0]);
    }
  return cc;
#endif

  case OP_CHAR:
  case OP_CHARI:
  length = 1;
#ifdef SUPPORT_UNICODE
  if (common->utf && HAS_EXTRALEN(*cc)) length += GET_EXTRALEN(*cc);
#endif

  if (check_str_ptr && common->mode != PCRE2_JIT_COMPLETE)
    detect_partial_match(common, backtracks);

  if (type == OP_CHAR || !char_has_othercase(common, cc) || char_get_othercase_bit(common, cc) != 0)
    {
    OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(length));
    if (length > 1 || (check_str_ptr && common->mode == PCRE2_JIT_COMPLETE))
      add_jump(compiler, backtracks, CMP(SLJIT_GREATER, STR_PTR, 0, STR_END, 0));

    context.length = IN_UCHARS(length);
    context.sourcereg = -1;
#if defined SLJIT_UNALIGNED && SLJIT_UNALIGNED
    context.ucharptr = 0;
#endif
    return byte_sequence_compare(common, type == OP_CHARI, cc, &context, backtracks);
    }

#ifdef SUPPORT_UNICODE
  if (common->utf)
    {
    GETCHAR(c, cc);
    }
  else
#endif
    c = *cc;

  SLJIT_ASSERT(type == OP_CHARI && char_has_othercase(common, cc));

  if (check_str_ptr && common->mode == PCRE2_JIT_COMPLETE)
    add_jump(compiler, backtracks, CMP(SLJIT_GREATER_EQUAL, STR_PTR, 0, STR_END, 0));

  oc = char_othercase(common, c);
  read_char(common, c < oc ? c : oc, c > oc ? c : oc, NULL, 0);

  SLJIT_ASSERT(!is_powerof2(c ^ oc));

  if (sljit_has_cpu_feature(SLJIT_HAS_CMOV))
    {
    OP2U(SLJIT_SUB | SLJIT_SET_Z, TMP1, 0, SLJIT_IMM, oc);
    CMOV(SLJIT_EQUAL, TMP1, SLJIT_IMM, c);
    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, c));
    }
  else
    {
    jump[0] = CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c);
    add_jump(compiler, backtracks, CMP(SLJIT_NOT_EQUAL, TMP1, 0, SLJIT_IMM, oc));
    JUMPHERE(jump[0]);
    }
  return cc + length;

  case OP_NOT:
  case OP_NOTI:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);

  length = 1;
#ifdef SUPPORT_UNICODE
  if (common->utf)
    {
#if PCRE2_CODE_UNIT_WIDTH == 8
    c = *cc;
    if (c < 128 && !common->invalid_utf)
      {
      OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(STR_PTR), 0);
      if (type == OP_NOT || !char_has_othercase(common, cc))
        add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c));
      else
        {
        /* Since UTF8 code page is fixed, we know that c is in [a-z] or [A-Z] range. */
        OP2(SLJIT_OR, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x20);
        add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP2, 0, SLJIT_IMM, c | 0x20));
        }
      /* Skip the variable-length character. */
      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
      jump[0] = CMP(SLJIT_LESS, TMP1, 0, SLJIT_IMM, 0xc0);
      OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)PRIV(utf8_table4) - 0xc0);
      OP2(SLJIT_ADD, STR_PTR, 0, STR_PTR, 0, TMP1, 0);
      JUMPHERE(jump[0]);
      return cc + 1;
      }
    else
#endif /* PCRE2_CODE_UNIT_WIDTH == 8 */
      {
      GETCHARLEN(c, cc, length);
      }
    }
  else
#endif /* SUPPORT_UNICODE */
    c = *cc;

  if (type == OP_NOT || !char_has_othercase(common, cc))
    {
    read_char(common, c, c, backtracks, READ_CHAR_UPDATE_STR_PTR);
    add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c));
    }
  else
    {
    oc = char_othercase(common, c);
    read_char(common, c < oc ? c : oc, c > oc ? c : oc, backtracks, READ_CHAR_UPDATE_STR_PTR);
    bit = c ^ oc;
    if (is_powerof2(bit))
      {
      OP2(SLJIT_OR, TMP1, 0, TMP1, 0, SLJIT_IMM, bit);
      add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c | bit));
      }
    else
      {
      add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, c));
      add_jump(compiler, backtracks, CMP(SLJIT_EQUAL, TMP1, 0, SLJIT_IMM, oc));
      }
    }
  return cc + length;

  case OP_CLASS:
  case OP_NCLASS:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);

#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8
  bit = (common->utf && is_char7_bitset((const sljit_u8 *)cc, type == OP_NCLASS)) ? 127 : 255;
  if (type == OP_NCLASS)
    read_char(common, 0, bit, backtracks, READ_CHAR_UPDATE_STR_PTR);
  else
    read_char(common, 0, bit, NULL, 0);
#else
  if (type == OP_NCLASS)
    read_char(common, 0, 255, backtracks, READ_CHAR_UPDATE_STR_PTR);
  else
    read_char(common, 0, 255, NULL, 0);
#endif

  if (optimize_class(common, (const sljit_u8 *)cc, type == OP_NCLASS, FALSE, backtracks))
    return cc + 32 / sizeof(PCRE2_UCHAR);

#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8
  jump[0] = NULL;
  if (common->utf)
    {
    jump[0] = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, bit);
    if (type == OP_CLASS)
      {
      add_jump(compiler, backtracks, jump[0]);
      jump[0] = NULL;
      }
    }
#elif PCRE2_CODE_UNIT_WIDTH != 8
  jump[0] = CMP(SLJIT_GREATER, TMP1, 0, SLJIT_IMM, 255);
  if (type == OP_CLASS)
    {
    add_jump(compiler, backtracks, jump[0]);
    jump[0] = NULL;
    }
#endif /* SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH == 8 */

  OP2(SLJIT_AND, TMP2, 0, TMP1, 0, SLJIT_IMM, 0x7);
  OP2(SLJIT_LSHR, TMP1, 0, TMP1, 0, SLJIT_IMM, 3);
  OP1(SLJIT_MOV_U8, TMP1, 0, SLJIT_MEM1(TMP1), (sljit_sw)cc);
  OP2(SLJIT_SHL, TMP2, 0, SLJIT_IMM, 1, TMP2, 0);
  OP2U(SLJIT_AND | SLJIT_SET_Z, TMP1, 0, TMP2, 0);
  add_jump(compiler, backtracks, JUMP(SLJIT_ZERO));

#if defined SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH != 8
  if (jump[0] != NULL)
    JUMPHERE(jump[0]);
#endif
  return cc + 32 / sizeof(PCRE2_UCHAR);

#if defined SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH == 16 || PCRE2_CODE_UNIT_WIDTH == 32
  case OP_XCLASS:
  if (check_str_ptr)
    detect_partial_match(common, backtracks);
  compile_xclass_matchingpath(common, cc + LINK_SIZE, backtracks);
  return cc + GET(cc, 0) - 1;
#endif
  }
SLJIT_UNREACHABLE();
return cc;
}


static __cold void **io_alloc_page_table(size_t size)
{
	unsigned i, nr_tables = DIV_ROUND_UP(size, PAGE_SIZE);
	size_t init_size = size;
	void **table;

	table = kcalloc(nr_tables, sizeof(*table), GFP_KERNEL_ACCOUNT);
	if (!table)
		return NULL;

	for (i = 0; i < nr_tables; i++) {
		unsigned int this_size = min_t(size_t, size, PAGE_SIZE);

		table[i] = kzalloc(this_size, GFP_KERNEL_ACCOUNT);
		if (!table[i]) {
			io_free_page_table(table, init_size);
			return NULL;
		}
		size -= this_size;
	}
	return table;


static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	const char __user *path;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->buf_index || sqe->splice_fd_in)
		return -EINVAL;
	if (req->flags & REQ_F_FIXED_FILE)
		return -EBADF;

	req->statx.dfd = READ_ONCE(sqe->fd);
	req->statx.mask = READ_ONCE(sqe->len);
	path = u64_to_user_ptr(READ_ONCE(sqe->addr));
	req->statx.buffer = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	req->statx.flags = READ_ONCE(sqe->statx_flags);

	req->statx.filename = getname_flags(path,
					getname_statx_lookup_flags(req->statx.flags),
					NULL);

	if (IS_ERR(req->statx.filename)) {
		int ret = PTR_ERR(req->statx.filename);

		req->statx.filename = NULL;
		return ret;
	}

	req->flags |= REQ_F_NEED_CLEANUP;
	return 0;
}


static inline bool req_ref_inc_not_zero(struct io_kiocb *req)
{
	WARN_ON_ONCE(!(req->flags & REQ_F_REFCOUNT));
	return atomic_inc_not_zero(&req->refs);
}


static inline struct file *io_file_get_fixed(struct io_kiocb *req, int fd,
					     unsigned int issue_flags)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct file *file = NULL;
	unsigned long file_ptr;

	if (issue_flags & IO_URING_F_UNLOCKED)
		mutex_lock(&ctx->uring_lock);

	if (unlikely((unsigned int)fd >= ctx->nr_user_files))
		goto out;
	fd = array_index_nospec(fd, ctx->nr_user_files);
	file_ptr = io_fixed_file_slot(&ctx->file_table, fd)->file_ptr;
	file = (struct file *) (file_ptr & FFS_MASK);
	file_ptr &= ~FFS_MASK;
	/* mask in overlapping REQ_F and FFS bits */
	req->flags |= (file_ptr << REQ_F_SUPPORT_NOWAIT_BIT);
	io_req_set_rsrc_node(req, ctx, 0);
out:
	if (issue_flags & IO_URING_F_UNLOCKED)
		mutex_unlock(&ctx->uring_lock);
	return file;


 */
static int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_timeout_rem *tr = &req->timeout_rem;
	struct io_ring_ctx *ctx = req->ctx;
	int ret;

	if (!(req->timeout_rem.flags & IORING_TIMEOUT_UPDATE)) {
		spin_lock(&ctx->completion_lock);
		spin_lock_irq(&ctx->timeout_lock);
		ret = io_timeout_cancel(ctx, tr->addr);
		spin_unlock_irq(&ctx->timeout_lock);
		spin_unlock(&ctx->completion_lock);
	} else {
		enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);

		spin_lock_irq(&ctx->timeout_lock);
		if (tr->ltimeout)
			ret = io_linked_timeout_update(ctx, tr->addr, &tr->ts, mode);
		else
			ret = io_timeout_update(ctx, tr->addr, &tr->ts, mode);
		spin_unlock_irq(&ctx->timeout_lock);
	}

	if (ret < 0)
		req_set_fail(req);
	io_req_complete_post(req, ret, 0);
	return 0;


static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	if (sqe->ioprio || sqe->buf_index || sqe->addr || sqe->splice_fd_in)
		return -EINVAL;
	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;

	req->fadvise.offset = READ_ONCE(sqe->off);
	req->fadvise.len = READ_ONCE(sqe->len);
	req->fadvise.advice = READ_ONCE(sqe->fadvise_advice);
	return 0;
}


static int io_unlinkat_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
{
	struct io_unlink *un = &req->unlink;
	const char __user *fname;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (sqe->ioprio || sqe->off || sqe->len || sqe->buf_index ||
	    sqe->splice_fd_in)
		return -EINVAL;
	if (unlikely(req->flags & REQ_F_FIXED_FILE))
		return -EBADF;

	un->dfd = READ_ONCE(sqe->fd);

	un->flags = READ_ONCE(sqe->unlink_flags);
	if (un->flags & ~AT_REMOVEDIR)
		return -EINVAL;

	fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
	un->filename = getname(fname);
	if (IS_ERR(un->filename))
		return PTR_ERR(un->filename);

	req->flags |= REQ_F_NEED_CLEANUP;
	return 0;
}


static int io_remove_buffers_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = &req->pbuf;
	u64 tmp;

	if (sqe->ioprio || sqe->rw_flags || sqe->addr || sqe->len || sqe->off ||
	    sqe->splice_fd_in)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -EINVAL;

	memset(p, 0, sizeof(*p));
	p->nbufs = tmp;
	p->bgid = READ_ONCE(sqe->buf_group);
	return 0;
}


static __cold void __io_free_req(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;

	io_req_put_rsrc(req, ctx);
	io_dismantle_req(req);
	io_put_task(req->task, 1);

	spin_lock(&ctx->completion_lock);
	wq_list_add_head(&req->comp_list, &ctx->locked_free_list);
	ctx->locked_free_nr++;
	spin_unlock(&ctx->completion_lock);
}


static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)
{
	struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb);

	if (kiocb->ki_flags & IOCB_WRITE)
		kiocb_end_write(req);
	if (unlikely(res != req->result)) {
		if (res == -EAGAIN && io_rw_should_reissue(req)) {
			req->flags |= REQ_F_REISSUE;
			return;
		}
		req->result = res;
	}

	/* order with io_iopoll_complete() checking ->iopoll_completed */
	smp_store_release(&req->iopoll_completed, 1);
}


static void *io_mem_alloc(size_t size)
{
	gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP;

	return (void *) __get_free_pages(gfp, get_order(size));


static int io_msg_ring_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
{
	if (unlikely(sqe->addr || sqe->ioprio || sqe->rw_flags ||
		     sqe->splice_fd_in || sqe->buf_index || sqe->personality))
		return -EINVAL;

	req->msg.user_data = READ_ONCE(sqe->off);
	req->msg.len = READ_ONCE(sqe->len);
	return 0;
}


static int io_timeout_remove_prep(struct io_kiocb *req,
				  const struct io_uring_sqe *sqe)
{
	struct io_timeout_rem *tr = &req->timeout_rem;

	if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL))
		return -EINVAL;
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->ioprio || sqe->buf_index || sqe->len || sqe->splice_fd_in)
		return -EINVAL;

	tr->ltimeout = false;
	tr->addr = READ_ONCE(sqe->addr);
	tr->flags = READ_ONCE(sqe->timeout_flags);
	if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) {
		if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
			return -EINVAL;
		if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
			tr->ltimeout = true;
		if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK|IORING_TIMEOUT_ABS))
			return -EINVAL;
		if (get_timespec64(&tr->ts, u64_to_user_ptr(sqe->addr2)))
			return -EFAULT;
		if (tr->ts.tv_sec < 0 || tr->ts.tv_nsec < 0)
			return -EINVAL;
	} else if (tr->flags) {
		/* timeout removal doesn't support flags */
		return -EINVAL;
	}

	return 0;


static inline void __io_req_set_refcount(struct io_kiocb *req, int nr)
{
	if (!(req->flags & REQ_F_REFCOUNT)) {
		req->flags |= REQ_F_REFCOUNT;
		atomic_set(&req->refs, nr);
	}
}


static inline void req_set_fail(struct io_kiocb *req)
{
	req->flags |= REQ_F_FAIL;
	if (req->flags & REQ_F_CQE_SKIP) {
		req->flags &= ~REQ_F_CQE_SKIP;
		req->flags |= REQ_F_SKIP_LINK_CQES;
	}
}


static int io_eventfd_unregister(struct io_ring_ctx *ctx)
{
	struct io_ev_fd *ev_fd;

	ev_fd = rcu_dereference_protected(ctx->io_ev_fd,
					lockdep_is_held(&ctx->uring_lock));
	if (ev_fd) {
		ctx->has_evfd = false;
		rcu_assign_pointer(ctx->io_ev_fd, NULL);
		call_rcu(&ev_fd->rcu, io_eventfd_put);
		return 0;
	}

	return -ENXIO;


/* Returns true if we found and killed one or more timeouts */
static __cold bool io_kill_timeouts(struct io_ring_ctx *ctx,
				    struct task_struct *tsk, bool cancel_all)
{
	struct io_kiocb *req, *tmp;
	int canceled = 0;

	spin_lock(&ctx->completion_lock);
	spin_lock_irq(&ctx->timeout_lock);
	list_for_each_entry_safe(req, tmp, &ctx->timeout_list, timeout.list) {
		if (io_match_task(req, tsk, cancel_all)) {
			io_kill_timeout(req, -ECANCELED);
			canceled++;
		}
	}
	spin_unlock_irq(&ctx->timeout_lock);
	if (canceled != 0)
		io_commit_cqring(ctx);
	spin_unlock(&ctx->completion_lock);
	if (canceled != 0)
		io_cqring_ev_posted(ctx);
	return canceled != 0;


static inline void io_req_put_rsrc(struct io_kiocb *req, struct io_ring_ctx *ctx)
{
	if (req->fixed_rsrc_refs)
		percpu_ref_put(req->fixed_rsrc_refs);
}


static __cold void io_drain_req(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_defer_entry *de;
	int ret;
	u32 seq = io_get_sequence(req);

	/* Still need defer if there is pending req in defer list. */
	spin_lock(&ctx->completion_lock);
	if (!req_need_defer(req, seq) && list_empty_careful(&ctx->defer_list)) {
		spin_unlock(&ctx->completion_lock);
queue:
		ctx->drain_active = false;
		io_req_task_queue(req);
		return;
	}
	spin_unlock(&ctx->completion_lock);

	ret = io_req_prep_async(req);
	if (ret) {
fail:
		io_req_complete_failed(req, ret);
		return;
	}
	io_prep_async_link(req);
	de = kmalloc(sizeof(*de), GFP_KERNEL);
	if (!de) {
		ret = -ENOMEM;
		goto fail;
	}

	spin_lock(&ctx->completion_lock);
	if (!req_need_defer(req, seq) && list_empty(&ctx->defer_list)) {
		spin_unlock(&ctx->completion_lock);
		kfree(de);
		goto queue;
	}

	trace_io_uring_defer(ctx, req, req->user_data, req->opcode);
	de->req = req;
	de->seq = seq;
	list_add_tail(&de->list, &ctx->defer_list);
	spin_unlock(&ctx->completion_lock);


static void tctx_task_work(struct callback_head *cb)
{
	bool uring_locked = false;
	struct io_ring_ctx *ctx = NULL;
	struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
						  task_work);

	while (1) {
		struct io_wq_work_node *node1, *node2;

		if (!tctx->task_list.first &&
		    !tctx->prior_task_list.first && uring_locked)
			io_submit_flush_completions(ctx);

		spin_lock_irq(&tctx->task_lock);
		node1 = tctx->prior_task_list.first;
		node2 = tctx->task_list.first;
		INIT_WQ_LIST(&tctx->task_list);
		INIT_WQ_LIST(&tctx->prior_task_list);
		if (!node2 && !node1)
			tctx->task_running = false;
		spin_unlock_irq(&tctx->task_lock);
		if (!node2 && !node1)
			break;

		if (node1)
			handle_prev_tw_list(node1, &ctx, &uring_locked);

		if (node2)
			handle_tw_list(node2, &ctx, &uring_locked);
		cond_resched();
	}

	ctx_flush_and_put(ctx, &uring_locked);

	/* relaxed read is enough as only the task itself sets ->in_idle */
	if (unlikely(atomic_read(&tctx->in_idle)))
		io_uring_drop_tctx_refs(current);
}


static void io_req_task_queue_reissue(struct io_kiocb *req)
{
	req->io_task_work.func = io_queue_async_work;
	io_req_task_work_add(req, false);
}


	__must_hold(&ctx->completion_lock)
	__must_hold(&ctx->timeout_lock)
{
	struct io_kiocb *req = io_timeout_extract(ctx, user_data);

	if (IS_ERR(req))
		return PTR_ERR(req);
	io_req_task_queue_fail(req, -ECANCELED);
	return 0;


static __cold int io_uring_mmap(struct file *file, struct vm_area_struct *vma)
{
	size_t sz = vma->vm_end - vma->vm_start;
	unsigned long pfn;
	void *ptr;

	ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
	if (IS_ERR(ptr))
		return PTR_ERR(ptr);

	pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
	return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);


static void handle_tw_list(struct io_wq_work_node *node,
			   struct io_ring_ctx **ctx, bool *locked)
{
	do {
		struct io_wq_work_node *next = node->next;
		struct io_kiocb *req = container_of(node, struct io_kiocb,
						    io_task_work.node);

		prefetch(container_of(next, struct io_kiocb, io_task_work.node));

		if (req->ctx != *ctx) {
			ctx_flush_and_put(*ctx, locked);
			*ctx = req->ctx;
			/* if not contended, grab and improve batching */
			*locked = mutex_trylock(&(*ctx)->uring_lock);
			percpu_ref_get(&(*ctx)->refs);
		}
		req->io_task_work.func(req, locked);
		node = next;
	} while (node);
}


static int io_send(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_sr_msg *sr = &req->sr_msg;
	struct msghdr msg;
	struct iovec iov;
	struct socket *sock;
	unsigned flags;
	int min_ret = 0;
	int ret;

	sock = sock_from_file(req->file);
	if (unlikely(!sock))
		return -ENOTSOCK;

	ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter);
	if (unlikely(ret))
		return ret;

	msg.msg_name = NULL;
	msg.msg_control = NULL;
	msg.msg_controllen = 0;
	msg.msg_namelen = 0;

	flags = req->sr_msg.msg_flags;
	if (issue_flags & IO_URING_F_NONBLOCK)
		flags |= MSG_DONTWAIT;
	if (flags & MSG_WAITALL)
		min_ret = iov_iter_count(&msg.msg_iter);

	msg.msg_flags = flags;
	ret = sock_sendmsg(sock, &msg);
	if (ret < min_ret) {
		if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
			return -EAGAIN;
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
		req_set_fail(req);
	}
	__io_req_complete(req, issue_flags, ret, 0);
	return 0;
}


static inline void io_get_task_refs(int nr)
{
	struct io_uring_task *tctx = current->io_uring;

	tctx->cached_refs -= nr;
	if (unlikely(tctx->cached_refs < 0))
		io_task_refs_refill(tctx);
}


	__must_hold(&ctx->uring_lock)
{
	struct io_wq_work_node *node, *prev;
	struct io_submit_state *state = &ctx->submit_state;

	if (state->flush_cqes) {
		spin_lock(&ctx->completion_lock);
		wq_list_for_each(node, prev, &state->compl_reqs) {
			struct io_kiocb *req = container_of(node, struct io_kiocb,
						    comp_list);

			if (!(req->flags & REQ_F_CQE_SKIP))
				__io_fill_cqe_req(req, req->result, req->cflags);
			if ((req->flags & REQ_F_POLLED) && req->apoll) {
				struct async_poll *apoll = req->apoll;

				if (apoll->double_poll)
					kfree(apoll->double_poll);
				list_add(&apoll->poll.wait.entry,
						&ctx->apoll_cache);
				req->flags &= ~REQ_F_POLLED;
			}
		}

		io_commit_cqring(ctx);
		spin_unlock(&ctx->completion_lock);
		io_cqring_ev_posted(ctx);
		state->flush_cqes = false;
	}

	io_free_batch_list(ctx, state->compl_reqs.first);
	INIT_WQ_LIST(&state->compl_reqs);
}


static inline int io_iter_do_read(struct io_kiocb *req, struct iov_iter *iter)
{
	if (likely(req->file->f_op->read_iter))
		return call_read_iter(req->file, &req->rw.kiocb, iter);
	else if (req->file->f_op->read)
		return loop_rw_iter(READ, req, iter);
	else
		return -EINVAL;
}


static void _exchange_nodes(RRBNode *node_a, RRBNode *node_b) {
	if (!node_a || !node_b) {
		return;
	}
	RRBNode node_a_tmp, node_b_tmp;
	memcpy (&node_a_tmp, node_a, sizeof (RRBNode));
	memcpy (&node_b_tmp, node_b, sizeof (RRBNode));
	node_a->link[0] = node_b_tmp.link[0];
	node_a->link[1] = node_b_tmp.link[1];
	node_a->red = node_b_tmp.red;
	node_b->link[0] = node_a_tmp.link[0];
	node_b->link[1] = node_a_tmp.link[1];
	node_b->red = node_a_tmp.red;
	if (node_a->parent == node_b->parent) {
		if (node_a->parent) {
			if (node_a->parent->link[0] == node_a) {
				node_a->parent->link[0] = node_b;
				node_a->parent->link[1] = node_a;
			} else {
				node_a->parent->link[1] = node_b;
				node_a->parent->link[0] = node_a;
			}
		}
		if (node_a->link[0]) {
			node_a->link[0]->parent = node_a;
		}
		if (node_a->link[1]) {
			node_a->link[1]->parent = node_a;
		}
		if (node_b->link[0]) {
			node_b->link[0]->parent = node_b;
		}
		if (node_b->link[1]) {
			node_b->link[1]->parent = node_b;
		}
		return;
	}
	RRBNode *parent_a = node_a->parent;
	RRBNode *parent_b = node_b->parent;
	if (parent_a) {
		if (parent_a->link[0] == node_a) {
			parent_a->link[0] = node_b;
		} else {
			parent_a->link[1] = node_b;
		}
	}
	node_b->parent = parent_a;
	if (parent_b) {
		if (parent_b->link[0] == node_b) {
			parent_b->link[0] = node_a;
		} else {
			parent_b->link[1] = node_a;
		}
	}
	node_a->parent = parent_b;
	if (node_a->link[0]) {
		node_a->link[0]->parent = node_a;
	}
	if (node_a->link[1]) {
		node_a->link[1]->parent = node_a;
	}
	if (node_b->link[0]) {
		node_b->link[0]->parent = node_b;
	}
	if (node_b->link[1]) {
		node_b->link[1]->parent = node_b;
	}
}


R_API char *r_bin_java_get_item_name_from_cp_item_list(RList *cp_list, RBinJavaCPTypeObj *obj, int depth) {
	/*
	Given a constant pool object Class, FieldRef, MethodRef, or InterfaceMethodRef
	return the actual descriptor string.
	@param cp_list: RList of RBinJavaCPTypeObj *
	@param obj object to look up the name for
	@rvalue ut8* (user frees) or NULL
	*/
	if (!obj || !cp_list || depth < 0) {
		return NULL;
	}
	switch (obj->tag) {
	case R_BIN_JAVA_CP_NAMEANDTYPE:
		return r_bin_java_get_utf8_from_cp_item_list (
			cp_list, obj->info.cp_name_and_type.name_idx);
	case R_BIN_JAVA_CP_CLASS:
		return r_bin_java_get_utf8_from_cp_item_list (
			cp_list, obj->info.cp_class.name_idx);
	// XXX - Probably not good form, but they are the same memory structure
	case R_BIN_JAVA_CP_FIELDREF:
	case R_BIN_JAVA_CP_INTERFACEMETHOD_REF:
	case R_BIN_JAVA_CP_METHODREF:
		obj = r_bin_java_get_item_from_cp_item_list (
			cp_list, obj->info.cp_method.name_and_type_idx);
		return r_bin_java_get_item_name_from_cp_item_list (
			cp_list, obj, depth - 1);
	default:
		return NULL;
	case 0:
		IFDBG eprintf ("Invalid 0 tag in the constant pool\n");
		return NULL;
	}
	return NULL;
}


static char *convert_string(const char *bytes, ut32 len) {
	ut32 idx = 0, pos = 0;
	ut32 str_sz = 32 * len + 1;
	char *cpy_buffer = len > 0 ? malloc (str_sz) : NULL;
	if (!cpy_buffer) {
		return cpy_buffer;
	}
	// 4x is the increase from byte to \xHH where HH represents hexed byte
	memset (cpy_buffer, 0, str_sz);
	while (idx < len && pos < len) {
		if (char_must_be_escaped (bytes[idx])) {
			if (pos + 2 < len) {
				free (cpy_buffer);
				return NULL;
			}
			sprintf (cpy_buffer + pos, "\\x%02x", bytes[idx]);
			pos += 4;
		} else {
			cpy_buffer[pos] = bytes[idx];
			pos++;
		}
		idx++;
	}
	return cpy_buffer;
}


static int x25_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
{
	struct sock *sk = sock->sk;
	struct x25_sock *x25 = x25_sk(sk);
	DECLARE_SOCKADDR(struct sockaddr_x25 *, usx25, msg->msg_name);
	struct sockaddr_x25 sx25;
	struct sk_buff *skb;
	unsigned char *asmptr;
	int noblock = msg->msg_flags & MSG_DONTWAIT;
	size_t size;
	int qbit = 0, rc = -EINVAL;

	lock_sock(sk);
	if (msg->msg_flags & ~(MSG_DONTWAIT|MSG_OOB|MSG_EOR|MSG_CMSG_COMPAT))
		goto out;

	/* we currently don't support segmented records at the user interface */
	if (!(msg->msg_flags & (MSG_EOR|MSG_OOB)))
		goto out;

	rc = -EADDRNOTAVAIL;
	if (sock_flag(sk, SOCK_ZAPPED))
		goto out;

	rc = -EPIPE;
	if (sk->sk_shutdown & SEND_SHUTDOWN) {
		send_sig(SIGPIPE, current, 0);
		goto out;
	}

	rc = -ENETUNREACH;
	if (!x25->neighbour)
		goto out;

	if (usx25) {
		rc = -EINVAL;
		if (msg->msg_namelen < sizeof(sx25))
			goto out;
		memcpy(&sx25, usx25, sizeof(sx25));
		rc = -EISCONN;
		if (strcmp(x25->dest_addr.x25_addr, sx25.sx25_addr.x25_addr))
			goto out;
		rc = -EINVAL;
		if (sx25.sx25_family != AF_X25)
			goto out;
	} else {
		/*
		 *	FIXME 1003.1g - if the socket is like this because
		 *	it has become closed (not started closed) we ought
		 *	to SIGPIPE, EPIPE;
		 */
		rc = -ENOTCONN;
		if (sk->sk_state != TCP_ESTABLISHED)
			goto out;

		sx25.sx25_family = AF_X25;
		sx25.sx25_addr   = x25->dest_addr;
	}

	/* Sanity check the packet size */
	if (len > 65535) {
		rc = -EMSGSIZE;
		goto out;
	}

	SOCK_DEBUG(sk, "x25_sendmsg: sendto: Addresses built.\n");

	/* Build a packet */
	SOCK_DEBUG(sk, "x25_sendmsg: sendto: building packet.\n");

	if ((msg->msg_flags & MSG_OOB) && len > 32)
		len = 32;

	size = len + X25_MAX_L2_LEN + X25_EXT_MIN_LEN;

	release_sock(sk);
	skb = sock_alloc_send_skb(sk, size, noblock, &rc);
	lock_sock(sk);
	if (!skb)
		goto out;
	X25_SKB_CB(skb)->flags = msg->msg_flags;

	skb_reserve(skb, X25_MAX_L2_LEN + X25_EXT_MIN_LEN);

	/*
	 *	Put the data on the end
	 */
	SOCK_DEBUG(sk, "x25_sendmsg: Copying user data\n");

	skb_reset_transport_header(skb);
	skb_put(skb, len);

	rc = memcpy_from_msg(skb_transport_header(skb), msg, len);
	if (rc)
		goto out_kfree_skb;

	/*
	 *	If the Q BIT Include socket option is in force, the first
	 *	byte of the user data is the logical value of the Q Bit.
	 */
	if (test_bit(X25_Q_BIT_FLAG, &x25->flags)) {
		if (!pskb_may_pull(skb, 1))
			goto out_kfree_skb;

		qbit = skb->data[0];
		skb_pull(skb, 1);
	}

	/*
	 *	Push down the X.25 header
	 */
	SOCK_DEBUG(sk, "x25_sendmsg: Building X.25 Header.\n");

	if (msg->msg_flags & MSG_OOB) {
		if (x25->neighbour->extended) {
			asmptr    = skb_push(skb, X25_STD_MIN_LEN);
			*asmptr++ = ((x25->lci >> 8) & 0x0F) | X25_GFI_EXTSEQ;
			*asmptr++ = (x25->lci >> 0) & 0xFF;
			*asmptr++ = X25_INTERRUPT;
		} else {
			asmptr    = skb_push(skb, X25_STD_MIN_LEN);
			*asmptr++ = ((x25->lci >> 8) & 0x0F) | X25_GFI_STDSEQ;
			*asmptr++ = (x25->lci >> 0) & 0xFF;
			*asmptr++ = X25_INTERRUPT;
		}
	} else {
		if (x25->neighbour->extended) {
			/* Build an Extended X.25 header */
			asmptr    = skb_push(skb, X25_EXT_MIN_LEN);
			*asmptr++ = ((x25->lci >> 8) & 0x0F) | X25_GFI_EXTSEQ;
			*asmptr++ = (x25->lci >> 0) & 0xFF;
			*asmptr++ = X25_DATA;
			*asmptr++ = X25_DATA;
		} else {
			/* Build an Standard X.25 header */
			asmptr    = skb_push(skb, X25_STD_MIN_LEN);
			*asmptr++ = ((x25->lci >> 8) & 0x0F) | X25_GFI_STDSEQ;
			*asmptr++ = (x25->lci >> 0) & 0xFF;
			*asmptr++ = X25_DATA;
		}

		if (qbit)
			skb->data[0] |= X25_Q_BIT;
	}

	SOCK_DEBUG(sk, "x25_sendmsg: Built header.\n");
	SOCK_DEBUG(sk, "x25_sendmsg: Transmitting buffer\n");

	rc = -ENOTCONN;
	if (sk->sk_state != TCP_ESTABLISHED)
		goto out_kfree_skb;

	if (msg->msg_flags & MSG_OOB)
		skb_queue_tail(&x25->interrupt_out_queue, skb);
	else {
		rc = x25_output(sk, skb);
		len = rc;
		if (rc < 0)
			kfree_skb(skb);
		else if (test_bit(X25_Q_BIT_FLAG, &x25->flags))
			len++;
	}

	x25_kick(sk);
	rc = len;
out:
	release_sock(sk);
	return rc;
out_kfree_skb:
	kfree_skb(skb);
	goto out;
}


struct sock *x25_find_socket(unsigned int lci, struct x25_neigh *nb)
{
	struct sock *s;

	read_lock_bh(&x25_list_lock);
	s = __x25_find_socket(lci, nb);
	read_unlock_bh(&x25_list_lock);
	return s;
}


static struct sock *x25_find_listener(struct x25_address *addr,
					struct sk_buff *skb)
{
	struct sock *s;
	struct sock *next_best;

	read_lock_bh(&x25_list_lock);
	next_best = NULL;

	sk_for_each(s, &x25_list)
		if ((!strcmp(addr->x25_addr,
			x25_sk(s)->source_addr.x25_addr) ||
				!strcmp(x25_sk(s)->source_addr.x25_addr,
					null_x25_address.x25_addr)) &&
					s->sk_state == TCP_LISTEN) {
			/*
			 * Found a listening socket, now check the incoming
			 * call user data vs this sockets call user data
			 */
			if (x25_sk(s)->cudmatchlength > 0 &&
				skb->len >= x25_sk(s)->cudmatchlength) {
				if((memcmp(x25_sk(s)->calluserdata.cuddata,
					skb->data,
					x25_sk(s)->cudmatchlength)) == 0) {
					sock_hold(s);
					goto found;
				 }
			} else
				next_best = s;
		}
	if (next_best) {
		s = next_best;
		sock_hold(s);
		goto found;
	}
	s = NULL;
found:
	read_unlock_bh(&x25_list_lock);
	return s;
}


static void x25_insert_socket(struct sock *sk)
{
	write_lock_bh(&x25_list_lock);
	sk_add_node(sk, &x25_list);
	write_unlock_bh(&x25_list_lock);
}


static int x25_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
{
	struct sock *sk = sock->sk;
	struct sockaddr_x25 *addr = (struct sockaddr_x25 *)uaddr;
	int len, i, rc = 0;

	if (addr_len != sizeof(struct sockaddr_x25) ||
	    addr->sx25_family != AF_X25 ||
	    strnlen(addr->sx25_addr.x25_addr, X25_ADDR_LEN) == X25_ADDR_LEN) {
		rc = -EINVAL;
		goto out;
	}

	/* check for the null_x25_address */
	if (strcmp(addr->sx25_addr.x25_addr, null_x25_address.x25_addr)) {

		len = strlen(addr->sx25_addr.x25_addr);
		for (i = 0; i < len; i++) {
			if (!isdigit(addr->sx25_addr.x25_addr[i])) {
				rc = -EINVAL;
				goto out;
			}
		}
	}

	lock_sock(sk);
	if (sock_flag(sk, SOCK_ZAPPED)) {
		x25_sk(sk)->source_addr = addr->sx25_addr;
		x25_insert_socket(sk);
		sock_reset_flag(sk, SOCK_ZAPPED);
	} else {
		rc = -EINVAL;
	}
	release_sock(sk);
	SOCK_DEBUG(sk, "x25_bind: socket is bound\n");
out:
	return rc;
}


WriteCompressedCellArrayField(mat_t *mat, matvar_t *matvar, z_streamp z)
{
    mat_uint32_t comp_buf[512];
    mat_uint32_t uncomp_buf[512];
    int buf_size = 512;
    size_t byteswritten = 0, field_buf_size;

    if ( NULL == matvar || NULL == mat || NULL == z )
        return 0;

    memset(&uncomp_buf, 0, sizeof(uncomp_buf));
    uncomp_buf[0] = MAT_T_MATRIX;
    if ( MAT_C_EMPTY != matvar->class_type ) {
        int err = GetCellArrayFieldBufSize(matvar, &field_buf_size);
        if ( err || field_buf_size > UINT32_MAX )
            return 0;

        uncomp_buf[1] = field_buf_size;
    } else {
        uncomp_buf[1] = 0;
    }
    z->next_in = ZLIB_BYTE_PTR(uncomp_buf);
    z->avail_in = 8;
    do {
        z->next_out = ZLIB_BYTE_PTR(comp_buf);
        z->avail_out = buf_size * sizeof(*comp_buf);
        deflate(z, Z_NO_FLUSH);
        byteswritten +=
            fwrite(comp_buf, 1, buf_size * sizeof(*comp_buf) - z->avail_out, (FILE *)mat->fp);
    } while ( z->avail_out == 0 );

    byteswritten += WriteCompressedTypeArrayFlags(mat, matvar, z);
    return byteswritten;
}


static gif_result gif_initialise_frame(gif_animation *gif)
{
        int frame;
        gif_frame *temp_buf;

        const unsigned char *gif_data, *gif_end;
        ssize_t gif_bytes;
        unsigned int flags = 0;
        unsigned int width, height, offset_x, offset_y;
        ssize_t block_size, colour_table_size;
        bool first_image = true;
        gif_result return_value;
        bool premature_eof = false;

        /* Get the frame to decode and our data position */
        frame = gif->frame_count;

        /* Get our buffer position etc. */
        gif_data = (const unsigned char *)(gif->gif_data + gif->buffer_position);
        gif_end = (const unsigned char *)(gif->gif_data + gif->buffer_size);
        gif_bytes = (gif_end - gif_data);

        /* Check if we've finished */
        if ((gif_bytes > 0) && (gif_data[0] == GIF_TRAILER)) {
                return GIF_OK;
        }

        /* Check if there is enough data remaining. The shortest block of data
         * is a 4-byte comment extension + 1-byte block terminator + 1-byte gif
         * trailer
         */
        if (gif_bytes < 6) {
                return GIF_INSUFFICIENT_DATA;
        }

        /* We could theoretically get some junk data that gives us millions of
         * frames, so we ensure that we don't have a silly number
         */
        if (frame > 262143)
                return GIF_FRAME_DATA_ERROR;

        /* Get some memory to store our pointers in etc. */
        if ((int)gif->frame_holders <= frame) {
                /* Allocate more memory */
                temp_buf = (gif_frame *)realloc(gif->frames, (frame + 1) * sizeof(gif_frame));
                if (temp_buf == NULL) {
                        return GIF_INSUFFICIENT_MEMORY;
                }
                gif->frames = temp_buf;
                gif->frame_holders = frame + 1;
        }

        /* Store our frame pointer. We would do it when allocating except we
         * start off with one frame allocated so we can always use realloc.
         */
        gif->frames[frame].frame_pointer = gif->buffer_position;
        gif->frames[frame].display = false;
        gif->frames[frame].virgin = true;
        gif->frames[frame].disposal_method = 0;
        gif->frames[frame].transparency = false;
        gif->frames[frame].frame_delay = 0;
        gif->frames[frame].redraw_required = false;

        /* Invalidate any previous decoding we have of this frame */
        if (gif->decoded_frame == frame) {
                gif->decoded_frame = GIF_INVALID_FRAME;
        }

        /* We pretend to initialise the frames, but really we just skip over
         * all the data contained within. This is all basically a cut down
         * version of gif_decode_frame that doesn't have any of the LZW bits in
         * it.
         */

        /* Initialise any extensions */
        gif->buffer_position = gif_data - gif->gif_data;
        return_value = gif_initialise_frame_extensions(gif, frame);
        if (return_value != GIF_OK) {
                return return_value;
        }
        gif_data = (gif->gif_data + gif->buffer_position);
        gif_bytes = (gif_end - gif_data);

        /* Check if we've finished */
        if ((gif_bytes = (gif_end - gif_data)) < 1) {
                return GIF_INSUFFICIENT_FRAME_DATA;
        }

        if (gif_data[0] == GIF_TRAILER) {
                gif->buffer_position = (gif_data - gif->gif_data);
                gif->frame_count = frame + 1;
                return GIF_OK;
        }

        /* If we're not done, there should be an image descriptor */
        if (gif_data[0] != GIF_IMAGE_SEPARATOR) {
                return GIF_FRAME_DATA_ERROR;
        }

        /* Do some simple boundary checking */
        if (gif_bytes < 10) {
                return GIF_INSUFFICIENT_FRAME_DATA;
        }
        offset_x = gif_data[1] | (gif_data[2] << 8);
        offset_y = gif_data[3] | (gif_data[4] << 8);
        width = gif_data[5] | (gif_data[6] << 8);
        height = gif_data[7] | (gif_data[8] << 8);

        /* Set up the redraw characteristics. We have to check for extending
         * the area due to multi-image frames.
         */
        if (!first_image) {
                if (gif->frames[frame].redraw_x > offset_x) {
                        gif->frames[frame].redraw_width += (gif->frames[frame].redraw_x - offset_x);
                        gif->frames[frame].redraw_x = offset_x;
                }

                if (gif->frames[frame].redraw_y > offset_y) {
                        gif->frames[frame].redraw_height += (gif->frames[frame].redraw_y - offset_y);
                        gif->frames[frame].redraw_y = offset_y;
                }

                if ((offset_x + width) > (gif->frames[frame].redraw_x + gif->frames[frame].redraw_width)) {
                        gif->frames[frame].redraw_width = (offset_x + width) - gif->frames[frame].redraw_x;
                }

                if ((offset_y + height) > (gif->frames[frame].redraw_y + gif->frames[frame].redraw_height)) {
                        gif->frames[frame].redraw_height = (offset_y + height) - gif->frames[frame].redraw_y;
                }
        } else {
                first_image = false;
                gif->frames[frame].redraw_x = offset_x;
                gif->frames[frame].redraw_y = offset_y;
                gif->frames[frame].redraw_width = width;
                gif->frames[frame].redraw_height = height;
        }

        /* if we are clearing the background then we need to redraw enough to
         * cover the previous frame too
         */
        gif->frames[frame].redraw_required = ((gif->frames[frame].disposal_method == GIF_FRAME_CLEAR) ||
                                                (gif->frames[frame].disposal_method == GIF_FRAME_RESTORE));

        /* Boundary checking - shouldn't ever happen except with junk data */
        if (gif_initialise_sprite(gif, (offset_x + width), (offset_y + height))) {
                return GIF_INSUFFICIENT_MEMORY;
        }

        /* Decode the flags */
        flags = gif_data[9];
        colour_table_size = 2 << (flags & GIF_COLOUR_TABLE_SIZE_MASK);

        /* Move our data onwards and remember we've got a bit of this frame */
        gif_data += 10;
        gif_bytes = (gif_end - gif_data);
        gif->frame_count_partial = frame + 1;

        /* Skip the local colour table */
        if (flags & GIF_COLOUR_TABLE_MASK) {
                gif_data += 3 * colour_table_size;
                if ((gif_bytes = (gif_end - gif_data)) < 0) {
                        return GIF_INSUFFICIENT_FRAME_DATA;
                }
        }

        /* Ensure we have a correct code size */
        if (gif_bytes < 1) {
                return GIF_INSUFFICIENT_FRAME_DATA;
        }
        if (gif_data[0] > LZW_CODE_MAX) {
                return GIF_DATA_ERROR;
        }

        /* Move our pointer to the actual image data */
        gif_data++;
        --gif_bytes;

        /* Repeatedly skip blocks until we get a zero block or run out of data
         * These blocks of image data are processed later by gif_decode_frame()
         */
        block_size = 0;
        while (block_size != 1) {
                if (gif_bytes < 1) return GIF_INSUFFICIENT_FRAME_DATA;
                block_size = gif_data[0] + 1;
                /* Check if the frame data runs off the end of the file	*/
                if ((ssize_t)(gif_bytes - block_size) < 0) {
                        /* Try to recover by signaling the end of the gif.
                         * Once we get garbage data, there is no logical way to
                         * determine where the next frame is.  It's probably
                         * better to partially load the gif than not at all.
                         */
                        if (gif_bytes >= 2) {
                                gif_bytes = 1;
                                ++gif_data;
                                premature_eof = true;
                                break;
                        } else {
                                return GIF_INSUFFICIENT_FRAME_DATA;
                        }
                } else {
                        gif_bytes -= block_size;
                        gif_data += block_size;
                }
        }

        /* Add the frame and set the display flag */
        gif->buffer_position = gif_data - gif->gif_data;
        gif->frame_count = frame + 1;
        gif->frames[frame].display = true;

        /* Check if we've finished */
        if (gif_bytes < 1) {
                return GIF_INSUFFICIENT_FRAME_DATA;
        } else {
                if (premature_eof || gif_data[0] == GIF_TRAILER) {
                        return GIF_OK;
                }
        }
        return GIF_WORKING;
}