static void shadow_walk_init_using_root(struct kvm_shadow_walk_iterator *iterator,
					struct kvm_vcpu *vcpu, hpa_t root,
					u64 addr)
{
	iterator->addr = addr;
	iterator->shadow_addr = root;
	iterator->level = vcpu->arch.mmu->shadow_root_level;

	if (iterator->level >= PT64_ROOT_4LEVEL &&
	    vcpu->arch.mmu->root_level < PT64_ROOT_4LEVEL &&
	    !vcpu->arch.mmu->direct_map)
		iterator->level = PT32E_ROOT_LEVEL;

	if (iterator->level == PT32E_ROOT_LEVEL) {
		/*
		 * prev_root is currently only used for 64-bit hosts. So only
		 * the active root_hpa is valid here.
		 */
		BUG_ON(root != vcpu->arch.mmu->root.hpa);

		iterator->shadow_addr
			= vcpu->arch.mmu->pae_root[(addr >> 30) & 3];
		iterator->shadow_addr &= PT64_BASE_ADDR_MASK;
		--iterator->level;
		if (!iterator->shadow_addr)
			iterator->level = 0;
	}
}


static bool pte_list_destroy(struct kvm *kvm, struct kvm_rmap_head *rmap_head)
{
	struct pte_list_desc *desc, *next;
	int i;

	if (!rmap_head->val)
		return false;

	if (!(rmap_head->val & 1)) {
		mmu_spte_clear_track_bits(kvm, (u64 *)rmap_head->val);
		goto out;
	}

	desc = (struct pte_list_desc *)(rmap_head->val & ~1ul);

	for (; desc; desc = next) {
		for (i = 0; i < desc->spte_count; i++)
			mmu_spte_clear_track_bits(kvm, desc->sptes[i]);
		next = desc->more;
		mmu_free_pte_list_desc(desc);
	}
out:
	/* rmap_head is meaningless now, remember to reset it */
	rmap_head->val = 0;
	return true;
}


void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu)
{
	int i;
	struct kvm_mmu_page *sp;

	if (vcpu->arch.mmu->direct_map)
		return;

	if (!VALID_PAGE(vcpu->arch.mmu->root.hpa))
		return;

	vcpu_clear_mmio_info(vcpu, MMIO_GVA_ANY);

	if (vcpu->arch.mmu->root_level >= PT64_ROOT_4LEVEL) {
		hpa_t root = vcpu->arch.mmu->root.hpa;
		sp = to_shadow_page(root);

		if (!is_unsync_root(root))
			return;

		write_lock(&vcpu->kvm->mmu_lock);
		mmu_sync_children(vcpu, sp, true);
		write_unlock(&vcpu->kvm->mmu_lock);
		return;
	}

	write_lock(&vcpu->kvm->mmu_lock);

	for (i = 0; i < 4; ++i) {
		hpa_t root = vcpu->arch.mmu->pae_root[i];

		if (IS_VALID_PAE_ROOT(root)) {
			root &= PT64_BASE_ADDR_MASK;
			sp = to_shadow_page(root);
			mmu_sync_children(vcpu, sp, true);
		}
	}

	write_unlock(&vcpu->kvm->mmu_lock);
}


void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
			    gva_t gva, hpa_t root_hpa)
{
	int i;

	/* It's actually a GPA for vcpu->arch.guest_mmu.  */
	if (mmu != &vcpu->arch.guest_mmu) {
		/* INVLPG on a non-canonical address is a NOP according to the SDM.  */
		if (is_noncanonical_address(gva, vcpu))
			return;

		static_call(kvm_x86_flush_tlb_gva)(vcpu, gva);
	}

	if (!mmu->invlpg)
		return;

	if (root_hpa == INVALID_PAGE) {
		mmu->invlpg(vcpu, gva, mmu->root.hpa);

		/*
		 * INVLPG is required to invalidate any global mappings for the VA,
		 * irrespective of PCID. Since it would take us roughly similar amount
		 * of work to determine whether any of the prev_root mappings of the VA
		 * is marked global, or to just sync it blindly, so we might as well
		 * just always sync it.
		 *
		 * Mappings not reachable via the current cr3 or the prev_roots will be
		 * synced when switching to that cr3, so nothing needs to be done here
		 * for them.
		 */
		for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
			if (VALID_PAGE(mmu->prev_roots[i].hpa))
				mmu->invlpg(vcpu, gva, mmu->prev_roots[i].hpa);
	} else {
		mmu->invlpg(vcpu, gva, root_hpa);
	}
}


static bool spte_clear_dirty(u64 *sptep)
{
	u64 spte = *sptep;

	rmap_printk("spte %p %llx\n", sptep, *sptep);

	MMU_WARN_ON(!spte_ad_enabled(spte));
	spte &= ~shadow_dirty_mask;
	return mmu_spte_update(sptep, spte);
}


static void direct_pte_prefetch(struct kvm_vcpu *vcpu, u64 *sptep)
{
	struct kvm_mmu_page *sp;

	sp = sptep_to_sp(sptep);

	/*
	 * Without accessed bits, there's no way to distinguish between
	 * actually accessed translations and prefetched, so disable pte
	 * prefetch if accessed bits aren't available.
	 */
	if (sp_ad_disabled(sp))
		return;

	if (sp->role.level > PG_LEVEL_4K)
		return;

	/*
	 * If addresses are being invalidated, skip prefetching to avoid
	 * accidentally prefetching those addresses.
	 */
	if (unlikely(vcpu->kvm->mmu_notifier_count))
		return;

	__direct_pte_prefetch(vcpu, sp, sptep);
}


static unsigned kvm_page_table_hashfn(gfn_t gfn)
{
	return hash_64(gfn, KVM_MMU_HASH_SHIFT);
}


static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn)
{
	int ret = 0;

	if (!kvm_vcpu_is_visible_gfn(vcpu, root_gfn)) {
		kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
		ret = 1;
	}

	return ret;
}


static bool cached_root_find_without_current(struct kvm *kvm, struct kvm_mmu *mmu,
					     gpa_t new_pgd,
					     union kvm_mmu_page_role new_role)
{
	uint i;

	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
		if (is_root_usable(&mmu->prev_roots[i], new_pgd, new_role))
			goto hit;

	return false;

hit:
	swap(mmu->root, mmu->prev_roots[i]);
	/* Bubble up the remaining roots.  */
	for (; i < KVM_MMU_NUM_PREV_ROOTS - 1; i++)
		mmu->prev_roots[i] = mmu->prev_roots[i + 1];
	mmu->prev_roots[i].hpa = INVALID_PAGE;
	return true;
}


static struct pte_list_desc *mmu_alloc_pte_list_desc(struct kvm_vcpu *vcpu)
{
	return kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_pte_list_desc_cache);
}


void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
				   const struct kvm_memory_slot *memslot)
{
	bool flush = false;

	if (kvm_memslots_have_rmaps(kvm)) {
		write_lock(&kvm->mmu_lock);
		/*
		 * Clear dirty bits only on 4k SPTEs since the legacy MMU only
		 * support dirty logging at a 4k granularity.
		 */
		flush = slot_handle_level_4k(kvm, memslot, __rmap_clear_dirty, false);
		write_unlock(&kvm->mmu_lock);
	}

	if (is_tdp_mmu_enabled(kvm)) {
		read_lock(&kvm->mmu_lock);
		flush |= kvm_tdp_mmu_clear_dirty_slot(kvm, memslot);
		read_unlock(&kvm->mmu_lock);
	}

	/*
	 * It's also safe to flush TLBs out of mmu lock here as currently this
	 * function is only used for dirty logging, in which case flushing TLB
	 * out of mmu lock also guarantees no dirty pages will be lost in
	 * dirty_bitmap.
	 */
	if (flush)
		kvm_arch_flush_remote_tlbs_memslot(kvm, memslot);
}


int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
				u64 fault_address, char *insn, int insn_len)
{
	int r = 1;
	u32 flags = vcpu->arch.apf.host_apf_flags;

#ifndef CONFIG_X86_64
	/* A 64-bit CR2 should be impossible on 32-bit KVM. */
	if (WARN_ON_ONCE(fault_address >> 32))
		return -EFAULT;
#endif

	vcpu->arch.l1tf_flush_l1d = true;
	if (!flags) {
		trace_kvm_page_fault(fault_address, error_code);

		if (kvm_event_needs_reinjection(vcpu))
			kvm_mmu_unprotect_page_virt(vcpu, fault_address);
		r = kvm_mmu_page_fault(vcpu, fault_address, error_code, insn,
				insn_len);
	} else if (flags & KVM_PV_REASON_PAGE_NOT_PRESENT) {
		vcpu->arch.apf.host_apf_flags = 0;
		local_irq_disable();
		kvm_async_pf_task_wait_schedule(fault_address);
		local_irq_enable();
	} else {
		WARN_ONCE(1, "Unexpected host async PF flags: %x\n", flags);
	}

	return r;
}


static int handle_mmio_page_fault(struct kvm_vcpu *vcpu, u64 addr, bool direct)
{
	u64 spte;
	bool reserved;

	if (mmio_info_in_cache(vcpu, addr, direct))
		return RET_PF_EMULATE;

	reserved = get_mmio_spte(vcpu, addr, &spte);
	if (WARN_ON(reserved))
		return -EINVAL;

	if (is_mmio_spte(spte)) {
		gfn_t gfn = get_mmio_spte_gfn(spte);
		unsigned int access = get_mmio_spte_access(spte);

		if (!check_mmio_spte(vcpu, spte))
			return RET_PF_INVALID;

		if (direct)
			addr = 0;

		trace_handle_mmio_page_fault(addr, gfn, access);
		vcpu_cache_mmio_info(vcpu, addr, gfn, access);
		return RET_PF_EMULATE;
	}

	/*
	 * If the page table is zapped by other cpus, let CPU fault again on
	 * the address.
	 */
	return RET_PF_RETRY;
}


static int set_nx_huge_pages_recovery_param(const char *val, const struct kernel_param *kp)
{
	bool was_recovery_enabled, is_recovery_enabled;
	uint old_period, new_period;
	int err;

	was_recovery_enabled = calc_nx_huge_pages_recovery_period(&old_period);

	err = param_set_uint(val, kp);
	if (err)
		return err;

	is_recovery_enabled = calc_nx_huge_pages_recovery_period(&new_period);

	if (is_recovery_enabled &&
	    (!was_recovery_enabled || old_period > new_period)) {
		struct kvm *kvm;

		mutex_lock(&kvm_lock);

		list_for_each_entry(kvm, &vm_list, vm_list)
			wake_up_process(kvm->arch.nx_lpage_recovery_thread);

		mutex_unlock(&kvm_lock);
	}

	return err;
}


static void kvm_mmu_unlink_parents(struct kvm_mmu_page *sp)
{
	u64 *sptep;
	struct rmap_iterator iter;

	while ((sptep = rmap_get_first(&sp->parent_ptes, &iter)))
		drop_parent_pte(sp, sptep);
}


static int set_nx_huge_pages(const char *val, const struct kernel_param *kp)
{
	bool old_val = nx_huge_pages;
	bool new_val;

	/* In "auto" mode deploy workaround only if CPU has the bug. */
	if (sysfs_streq(val, "off"))
		new_val = 0;
	else if (sysfs_streq(val, "force"))
		new_val = 1;
	else if (sysfs_streq(val, "auto"))
		new_val = get_nx_auto_mode();
	else if (strtobool(val, &new_val) < 0)
		return -EINVAL;

	__set_nx_huge_pages(new_val);

	if (new_val != old_val) {
		struct kvm *kvm;

		mutex_lock(&kvm_lock);

		list_for_each_entry(kvm, &vm_list, vm_list) {
			mutex_lock(&kvm->slots_lock);
			kvm_mmu_zap_all_fast(kvm);
			mutex_unlock(&kvm->slots_lock);

			wake_up_process(kvm->arch.nx_lpage_recovery_thread);
		}
		mutex_unlock(&kvm_lock);
	}

	return 0;
}


static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu)
{
	struct kvm_mmu_role_regs regs = {
		.cr0 = kvm_read_cr0_bits(vcpu, KVM_MMU_CR0_ROLE_BITS),
		.cr4 = kvm_read_cr4_bits(vcpu, KVM_MMU_CR4_ROLE_BITS),
		.efer = vcpu->arch.efer,
	};

	return regs;
}


static void update_gfn_disallow_lpage_count(const struct kvm_memory_slot *slot,
					    gfn_t gfn, int count)
{
	struct kvm_lpage_info *linfo;
	int i;

	for (i = PG_LEVEL_2M; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) {
		linfo = lpage_info_slot(gfn, slot, i);
		linfo->disallow_lpage += count;
		WARN_ON(linfo->disallow_lpage < 0);
	}
}


slot_handle_level_range(struct kvm *kvm, const struct kvm_memory_slot *memslot,
			slot_level_handler fn, int start_level, int end_level,
			gfn_t start_gfn, gfn_t end_gfn, bool flush_on_yield,
			bool flush)
{
	struct slot_rmap_walk_iterator iterator;

	for_each_slot_rmap_range(memslot, start_level, end_level, start_gfn,
			end_gfn, &iterator) {
		if (iterator.rmap)
			flush |= fn(kvm, iterator.rmap, memslot);

		if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
			if (flush && flush_on_yield) {
				kvm_flush_remote_tlbs_with_address(kvm,
						start_gfn,
						iterator.gfn - start_gfn + 1);
				flush = false;
			}
			cond_resched_rwlock_write(&kvm->mmu_lock);
		}
	}

	return flush;
}


static struct kvm_lpage_info *lpage_info_slot(gfn_t gfn,
		const struct kvm_memory_slot *slot, int level)
{
	unsigned long idx;

	idx = gfn_to_index(gfn, slot->base_gfn, level);
	return &slot->arch.lpage_info[level - 2][idx];
}


void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu)
{
	unsigned long roots_to_free = 0;
	int i;

	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++)
		if (is_unsync_root(vcpu->arch.mmu->prev_roots[i].hpa))
			roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i);

	/* sync prev_roots by simply freeing them */
	kvm_mmu_free_roots(vcpu->kvm, vcpu->arch.mmu, roots_to_free);
}


int kvm_cpu_dirty_log_size(void)
{
	return kvm_x86_ops.cpu_dirty_log_size;
}


static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
			     struct kvm_mmu_page *sp)
{
	u64 spte;

	BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != PT_WRITABLE_MASK);

	spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));

	mmu_spte_set(sptep, spte);

	mmu_page_add_parent_pte(vcpu, sp, sptep);

	if (sp->unsync_children || sp->unsync)
		mark_unsync(sptep);
}


static int mmu_spte_clear_track_bits(struct kvm *kvm, u64 *sptep)
{
	kvm_pfn_t pfn;
	u64 old_spte = *sptep;
	int level = sptep_to_sp(sptep)->role.level;

	if (!is_shadow_present_pte(old_spte) ||
	    !spte_has_volatile_bits(old_spte))
		__update_clear_spte_fast(sptep, 0ull);
	else
		old_spte = __update_clear_spte_slow(sptep, 0ull);

	if (!is_shadow_present_pte(old_spte))
		return old_spte;

	kvm_update_page_stats(kvm, level, -1);

	pfn = spte_to_pfn(old_spte);

	/*
	 * KVM does not hold the refcount of the page used by
	 * kvm mmu, before reclaiming the page, we should
	 * unmap it from mmu first.
	 */
	WARN_ON(!kvm_is_reserved_pfn(pfn) && !page_count(pfn_to_page(pfn)));

	if (is_accessed_spte(old_spte))
		kvm_set_pfn_accessed(pfn);

	if (is_dirty_spte(old_spte))
		kvm_set_pfn_dirty(pfn);

	return old_spte;
}


static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator,
			     struct kvm_vcpu *vcpu, u64 addr)
{
	shadow_walk_init_using_root(iterator, vcpu, vcpu->arch.mmu->root.hpa,
				    addr);
}


auto_format(
    int		trailblank,	// when TRUE also format with trailing blank
    int		prev_line)	// may start in previous line
{
    pos_T	pos;
    colnr_T	len;
    char_u	*old;
    char_u	*new, *pnew;
    int		wasatend;
    int		cc;

    if (!has_format_option(FO_AUTO))
	return;

    pos = curwin->w_cursor;
    old = ml_get_curline();

    // may remove added space
    check_auto_format(FALSE);

    // Don't format in Insert mode when the cursor is on a trailing blank, the
    // user might insert normal text next.  Also skip formatting when "1" is
    // in 'formatoptions' and there is a single character before the cursor.
    // Otherwise the line would be broken and when typing another non-white
    // next they are not joined back together.
    wasatend = (pos.col == (colnr_T)STRLEN(old));
    if (*old != NUL && !trailblank && wasatend)
    {
	dec_cursor();
	cc = gchar_cursor();
	if (!WHITECHAR(cc) && curwin->w_cursor.col > 0
					  && has_format_option(FO_ONE_LETTER))
	    dec_cursor();
	cc = gchar_cursor();
	if (WHITECHAR(cc))
	{
	    curwin->w_cursor = pos;
	    return;
	}
	curwin->w_cursor = pos;
    }

    // With the 'c' flag in 'formatoptions' and 't' missing: only format
    // comments.
    if (has_format_option(FO_WRAP_COMS) && !has_format_option(FO_WRAP)
				&& get_leader_len(old, NULL, FALSE, TRUE) == 0)
	return;

    // May start formatting in a previous line, so that after "x" a word is
    // moved to the previous line if it fits there now.  Only when this is not
    // the start of a paragraph.
    if (prev_line && !paragraph_start(curwin->w_cursor.lnum))
    {
	--curwin->w_cursor.lnum;
	if (u_save_cursor() == FAIL)
	    return;
    }

    // Do the formatting and restore the cursor position.  "saved_cursor" will
    // be adjusted for the text formatting.
    saved_cursor = pos;
    format_lines((linenr_T)-1, FALSE);
    curwin->w_cursor = saved_cursor;
    saved_cursor.lnum = 0;

    if (curwin->w_cursor.lnum > curbuf->b_ml.ml_line_count)
    {
	// "cannot happen"
	curwin->w_cursor.lnum = curbuf->b_ml.ml_line_count;
	coladvance((colnr_T)MAXCOL);
    }
    else
	check_cursor_col();

    // Insert mode: If the cursor is now after the end of the line while it
    // previously wasn't, the line was broken.  Because of the rule above we
    // need to add a space when 'w' is in 'formatoptions' to keep a paragraph
    // formatted.
    if (!wasatend && has_format_option(FO_WHITE_PAR))
    {
	new = ml_get_curline();
	len = (colnr_T)STRLEN(new);
	if (curwin->w_cursor.col == len)
	{
	    pnew = vim_strnsave(new, len + 2);
	    pnew[len] = ' ';
	    pnew[len + 1] = NUL;
	    ml_replace(curwin->w_cursor.lnum, pnew, FALSE);
	    // remove the space later
	    did_add_space = TRUE;
	}
	else
	    // may remove added space
	    check_auto_format(FALSE);
    }

    check_cursor();
}


fex_format(
    linenr_T	lnum,
    long	count,
    int		c)	// character to be inserted
{
    int		use_sandbox = was_set_insecurely((char_u *)"formatexpr",
								   OPT_LOCAL);
    int		r;
    char_u	*fex;
    sctx_T	save_sctx = current_sctx;

    // Set v:lnum to the first line number and v:count to the number of lines.
    // Set v:char to the character to be inserted (can be NUL).
    set_vim_var_nr(VV_LNUM, lnum);
    set_vim_var_nr(VV_COUNT, count);
    set_vim_var_char(c);

    // Make a copy, the option could be changed while calling it.
    fex = vim_strsave(curbuf->b_p_fex);
    if (fex == NULL)
	return 0;
    current_sctx = curbuf->b_p_script_ctx[BV_FEX];

    // Evaluate the function.
    if (use_sandbox)
	++sandbox;
    r = (int)eval_to_number(fex);
    if (use_sandbox)
	--sandbox;

    set_vim_var_string(VV_CHAR, NULL, -1);
    vim_free(fex);
    current_sctx = save_sctx;

    return r;
}


nv_gv_cmd(cmdarg_T *cap)
{
    pos_T	tpos;
    int		i;

    if (checkclearop(cap->oap))
	return;

    if (curbuf->b_visual.vi_start.lnum == 0
	    || curbuf->b_visual.vi_start.lnum > curbuf->b_ml.ml_line_count
	    || curbuf->b_visual.vi_end.lnum == 0)
    {
	beep_flush();
	return;
    }

    // set w_cursor to the start of the Visual area, tpos to the end
    if (VIsual_active)
    {
	i = VIsual_mode;
	VIsual_mode = curbuf->b_visual.vi_mode;
	curbuf->b_visual.vi_mode = i;
# ifdef FEAT_EVAL
	curbuf->b_visual_mode_eval = i;
# endif
	i = curwin->w_curswant;
	curwin->w_curswant = curbuf->b_visual.vi_curswant;
	curbuf->b_visual.vi_curswant = i;

	tpos = curbuf->b_visual.vi_end;
	curbuf->b_visual.vi_end = curwin->w_cursor;
	curwin->w_cursor = curbuf->b_visual.vi_start;
	curbuf->b_visual.vi_start = VIsual;
    }
    else
    {
	VIsual_mode = curbuf->b_visual.vi_mode;
	curwin->w_curswant = curbuf->b_visual.vi_curswant;
	tpos = curbuf->b_visual.vi_end;
	curwin->w_cursor = curbuf->b_visual.vi_start;
    }

    VIsual_active = TRUE;
    VIsual_reselect = TRUE;

    // Set Visual to the start and w_cursor to the end of the Visual
    // area.  Make sure they are on an existing character.
    check_cursor();
    VIsual = curwin->w_cursor;
    curwin->w_cursor = tpos;
    check_cursor();
    update_topline();

    // When called from normal "g" command: start Select mode when
    // 'selectmode' contains "cmd".  When called for K_SELECT, always
    // start Select mode.
    if (cap->arg)
    {
	VIsual_select = TRUE;
	VIsual_select_reg = 0;
    }
    else
	may_start_select('c');
    setmouse();
#ifdef FEAT_CLIPBOARD
    // Make sure the clipboard gets updated.  Needed because start and
    // end are still the same, and the selection needs to be owned
    clip_star.vmode = NUL;
#endif
    redraw_curbuf_later(INVERTED);
    showmode();
}


check_scrollbind(linenr_T topline_diff, long leftcol_diff)
{
    int		want_ver;
    int		want_hor;
    win_T	*old_curwin = curwin;
    buf_T	*old_curbuf = curbuf;
    int		old_VIsual_select = VIsual_select;
    int		old_VIsual_active = VIsual_active;
    colnr_T	tgt_leftcol = curwin->w_leftcol;
    long	topline;
    long	y;

    // check 'scrollopt' string for vertical and horizontal scroll options
    want_ver = (vim_strchr(p_sbo, 'v') && topline_diff != 0);
#ifdef FEAT_DIFF
    want_ver |= old_curwin->w_p_diff;
#endif
    want_hor = (vim_strchr(p_sbo, 'h') && (leftcol_diff || topline_diff != 0));

    // loop through the scrollbound windows and scroll accordingly
    VIsual_select = VIsual_active = 0;
    FOR_ALL_WINDOWS(curwin)
    {
	curbuf = curwin->w_buffer;
	// skip original window  and windows with 'noscrollbind'
	if (curwin != old_curwin && curwin->w_p_scb)
	{
	    // do the vertical scroll
	    if (want_ver)
	    {
#ifdef FEAT_DIFF
		if (old_curwin->w_p_diff && curwin->w_p_diff)
		{
		    diff_set_topline(old_curwin, curwin);
		}
		else
#endif
		{
		    curwin->w_scbind_pos += topline_diff;
		    topline = curwin->w_scbind_pos;
		    if (topline > curbuf->b_ml.ml_line_count)
			topline = curbuf->b_ml.ml_line_count;
		    if (topline < 1)
			topline = 1;

		    y = topline - curwin->w_topline;
		    if (y > 0)
			scrollup(y, FALSE);
		    else
			scrolldown(-y, FALSE);
		}

		redraw_later(VALID);
		cursor_correct();
		curwin->w_redr_status = TRUE;
	    }

	    // do the horizontal scroll
	    if (want_hor && curwin->w_leftcol != tgt_leftcol)
	    {
		curwin->w_leftcol = tgt_leftcol;
		leftcol_changed();
	    }
	}
    }

    // reset current-window
    VIsual_select = old_VIsual_select;
    VIsual_active = old_VIsual_active;
    curwin = old_curwin;
    curbuf = old_curbuf;
}


nv_wordcmd(cmdarg_T *cap)
{
    int		n;
    int		word_end;
    int		flag = FALSE;
    pos_T	startpos = curwin->w_cursor;

    // Set inclusive for the "E" and "e" command.
    if (cap->cmdchar == 'e' || cap->cmdchar == 'E')
	word_end = TRUE;
    else
	word_end = FALSE;
    cap->oap->inclusive = word_end;

    // "cw" and "cW" are a special case.
    if (!word_end && cap->oap->op_type == OP_CHANGE)
    {
	n = gchar_cursor();
	if (n != NUL)			// not an empty line
	{
	    if (VIM_ISWHITE(n))
	    {
		// Reproduce a funny Vi behaviour: "cw" on a blank only
		// changes one character, not all blanks until the start of
		// the next word.  Only do this when the 'w' flag is included
		// in 'cpoptions'.
		if (cap->count1 == 1 && vim_strchr(p_cpo, CPO_CW) != NULL)
		{
		    cap->oap->inclusive = TRUE;
		    cap->oap->motion_type = MCHAR;
		    return;
		}
	    }
	    else
	    {
		// This is a little strange. To match what the real Vi does,
		// we effectively map 'cw' to 'ce', and 'cW' to 'cE', provided
		// that we are not on a space or a TAB.  This seems impolite
		// at first, but it's really more what we mean when we say
		// 'cw'.
		// Another strangeness: When standing on the end of a word
		// "ce" will change until the end of the next word, but "cw"
		// will change only one character! This is done by setting
		// flag.
		cap->oap->inclusive = TRUE;
		word_end = TRUE;
		flag = TRUE;
	    }
	}
    }

    cap->oap->motion_type = MCHAR;
    curwin->w_set_curswant = TRUE;
    if (word_end)
	n = end_word(cap->count1, cap->arg, flag, FALSE);
    else
	n = fwd_word(cap->count1, cap->arg, cap->oap->op_type != OP_NOP);

    // Don't leave the cursor on the NUL past the end of line. Unless we
    // didn't move it forward.
    if (LT_POS(startpos, curwin->w_cursor))
	adjust_cursor(cap->oap);

    if (n == FAIL && cap->oap->op_type == OP_NOP)
	clearopbeep(cap->oap);
    else
    {
	adjust_for_sel(cap);
#ifdef FEAT_FOLDING
	if ((fdo_flags & FDO_HOR) && KeyTyped && cap->oap->op_type == OP_NOP)
	    foldOpenCursor();
#endif
    }
}


nv_put_opt(cmdarg_T *cap, int fix_indent)
{
    int		regname = 0;
    void	*reg1 = NULL, *reg2 = NULL;
    int		empty = FALSE;
    int		was_visual = FALSE;
    int		dir;
    int		flags = 0;
    int		keep_registers = FALSE;

    if (cap->oap->op_type != OP_NOP)
    {
#ifdef FEAT_DIFF
	// "dp" is ":diffput"
	if (cap->oap->op_type == OP_DELETE && cap->cmdchar == 'p')
	{
	    clearop(cap->oap);
	    nv_diffgetput(TRUE, cap->opcount);
	}
	else
#endif
	clearopbeep(cap->oap);
    }
#ifdef FEAT_JOB_CHANNEL
    else if (bt_prompt(curbuf) && !prompt_curpos_editable())
    {
	clearopbeep(cap->oap);
    }
#endif
    else
    {
	if (fix_indent)
	{
	    dir = (cap->cmdchar == ']' && cap->nchar == 'p')
							 ? FORWARD : BACKWARD;
	    flags |= PUT_FIXINDENT;
	}
	else
	    dir = (cap->cmdchar == 'P'
		    || ((cap->cmdchar == 'g' || cap->cmdchar == 'z')
			&& cap->nchar == 'P')) ? BACKWARD : FORWARD;
	prep_redo_cmd(cap);
	if (cap->cmdchar == 'g')
	    flags |= PUT_CURSEND;
	else if (cap->cmdchar == 'z')
	    flags |= PUT_BLOCK_INNER;

	if (VIsual_active)
	{
	    // Putting in Visual mode: The put text replaces the selected
	    // text.  First delete the selected text, then put the new text.
	    // Need to save and restore the registers that the delete
	    // overwrites if the old contents is being put.
	    was_visual = TRUE;
	    regname = cap->oap->regname;
	    keep_registers = cap->cmdchar == 'P';
#ifdef FEAT_CLIPBOARD
	    adjust_clip_reg(&regname);
#endif
	   if (regname == 0 || regname == '"'
				     || VIM_ISDIGIT(regname) || regname == '-'
#ifdef FEAT_CLIPBOARD
		    || (clip_unnamed && (regname == '*' || regname == '+'))
#endif

		    )
	    {
		// The delete is going to overwrite the register we want to
		// put, save it first.
		reg1 = get_register(regname, TRUE);
	    }

	    // Now delete the selected text. Avoid messages here.
	    cap->cmdchar = 'd';
	    cap->nchar = NUL;
	    cap->oap->regname = keep_registers ? '_' : NUL;
	    ++msg_silent;
	    nv_operator(cap);
	    do_pending_operator(cap, 0, FALSE);
	    empty = (curbuf->b_ml.ml_flags & ML_EMPTY);
	    --msg_silent;

	    // delete PUT_LINE_BACKWARD;
	    cap->oap->regname = regname;

	    if (reg1 != NULL)
	    {
		// Delete probably changed the register we want to put, save
		// it first. Then put back what was there before the delete.
		reg2 = get_register(regname, FALSE);
		put_register(regname, reg1);
	    }

	    // When deleted a linewise Visual area, put the register as
	    // lines to avoid it joined with the next line.  When deletion was
	    // characterwise, split a line when putting lines.
	    if (VIsual_mode == 'V')
		flags |= PUT_LINE;
	    else if (VIsual_mode == 'v')
		flags |= PUT_LINE_SPLIT;
	    if (VIsual_mode == Ctrl_V && dir == FORWARD)
		flags |= PUT_LINE_FORWARD;
	    dir = BACKWARD;
	    if ((VIsual_mode != 'V'
			&& curwin->w_cursor.col < curbuf->b_op_start.col)
		    || (VIsual_mode == 'V'
			&& curwin->w_cursor.lnum < curbuf->b_op_start.lnum))
		// cursor is at the end of the line or end of file, put
		// forward.
		dir = FORWARD;
	    // May have been reset in do_put().
	    VIsual_active = TRUE;
	}
	do_put(cap->oap->regname, NULL, dir, cap->count1, flags);

	// If a register was saved, put it back now.
	if (reg2 != NULL)
	    put_register(regname, reg2);

	// What to reselect with "gv"?  Selecting the just put text seems to
	// be the most useful, since the original text was removed.
	if (was_visual)
	{
	    curbuf->b_visual.vi_start = curbuf->b_op_start;
	    curbuf->b_visual.vi_end = curbuf->b_op_end;
	    // need to adjust cursor position
	    if (*p_sel == 'e')
		inc(&curbuf->b_visual.vi_end);
	}

	// When all lines were selected and deleted do_put() leaves an empty
	// line that needs to be deleted now.
	if (empty && *ml_get(curbuf->b_ml.ml_line_count) == NUL)
	{
	    ml_delete_flags(curbuf->b_ml.ml_line_count, ML_DEL_MESSAGE);
	    deleted_lines(curbuf->b_ml.ml_line_count + 1, 1);

	    // If the cursor was in that line, move it to the end of the last
	    // line.
	    if (curwin->w_cursor.lnum > curbuf->b_ml.ml_line_count)
	    {
		curwin->w_cursor.lnum = curbuf->b_ml.ml_line_count;
		coladvance((colnr_T)MAXCOL);
	    }
	}
	auto_format(FALSE, TRUE);
    }
}


set_cursor_for_append_to_line(void)
{
    curwin->w_set_curswant = TRUE;
    if (get_ve_flags() == VE_ALL)
    {
	int save_State = State;

	// Pretend Insert mode here to allow the cursor on the
	// character past the end of the line
	State = MODE_INSERT;
	coladvance((colnr_T)MAXCOL);
	State = save_State;
    }
    else
	curwin->w_cursor.col += (colnr_T)STRLEN(ml_get_cursor());
}


get_visual_text(
    cmdarg_T	*cap,
    char_u	**pp,	    // return: start of selected text
    int		*lenp)	    // return: length of selected text
{
    if (VIsual_mode != 'V')
	unadjust_for_sel();
    if (VIsual.lnum != curwin->w_cursor.lnum)
    {
	if (cap != NULL)
	    clearopbeep(cap->oap);
	return FAIL;
    }
    if (VIsual_mode == 'V')
    {
	*pp = ml_get_curline();
	*lenp = (int)STRLEN(*pp);
    }
    else
    {
	if (LT_POS(curwin->w_cursor, VIsual))
	{
	    *pp = ml_get_pos(&curwin->w_cursor);
	    *lenp = VIsual.col - curwin->w_cursor.col + 1;
	}
	else
	{
	    *pp = ml_get_pos(&VIsual);
	    *lenp = curwin->w_cursor.col - VIsual.col + 1;
	}
	if (**pp == NUL)
	    *lenp = 0;
	if (*lenp > 0)
	{
	    if (has_mbyte)
		// Correct the length to include all bytes of the last
		// character.
		*lenp += (*mb_ptr2len)(*pp + (*lenp - 1)) - 1;
	    else if ((*pp)[*lenp - 1] == NUL)
		// Do not include a trailing NUL.
		*lenp -= 1;
	}
    }
    reset_VIsual_and_resel();
    return OK;
}


end_visual_mode_keep_button()
{
#ifdef FEAT_CLIPBOARD
    // If we are using the clipboard, then remember what was selected in case
    // we need to paste it somewhere while we still own the selection.
    // Only do this when the clipboard is already owned.  Don't want to grab
    // the selection when hitting ESC.
    if (clip_star.available && clip_star.owned)
	clip_auto_select();

# if defined(FEAT_EVAL)
    // Emit a TextYankPost for the automatic copy of the selection into the
    // star and/or plus register.
    if (has_textyankpost())
    {
	if (clip_isautosel_star())
	    call_yank_do_autocmd('*');
	if (clip_isautosel_plus())
	    call_yank_do_autocmd('+');
    }
# endif
#endif

    VIsual_active = FALSE;
    setmouse();
    mouse_dragging = 0;

    // Save the current VIsual area for '< and '> marks, and "gv"
    curbuf->b_visual.vi_mode = VIsual_mode;
    curbuf->b_visual.vi_start = VIsual;
    curbuf->b_visual.vi_end = curwin->w_cursor;
    curbuf->b_visual.vi_curswant = curwin->w_curswant;
#ifdef FEAT_EVAL
    curbuf->b_visual_mode_eval = VIsual_mode;
#endif
    if (!virtual_active())
	curwin->w_cursor.coladd = 0;
    may_clear_cmdline();

    adjust_cursor_eol();
    may_trigger_modechanged();
}


undo_read(bufinfo_T *bi, char_u *buffer, size_t size)
{
    int retval = OK;

#ifdef FEAT_CRYPT
    if (bi->bi_buffer != NULL)
    {
	int	size_todo = (int)size;
	char_u	*p = buffer;

	while (size_todo > 0)
	{
	    size_t n;

	    if (bi->bi_used >= bi->bi_avail)
	    {
		n = fread(bi->bi_buffer, 1, (size_t)CRYPT_BUF_SIZE, bi->bi_fp);
		if (n == 0)
		{
		    retval = FAIL;
		    break;
		}
		bi->bi_avail = n;
		bi->bi_used = 0;
		crypt_decode_inplace(bi->bi_state, bi->bi_buffer, bi->bi_avail, FALSE);
	    }
	    n = size_todo;
	    if (n > bi->bi_avail - bi->bi_used)
		n = bi->bi_avail - bi->bi_used;
	    mch_memmove(p, bi->bi_buffer + bi->bi_used, n);
	    bi->bi_used += n;
	    size_todo -= (int)n;
	    p += n;
	}
    }
    else
#endif
    if (fread(buffer, size, 1, bi->bi_fp) != 1)
	retval = FAIL;

    if (retval == FAIL)
	// Error may be checked for only later.  Fill with zeros,
	// so that the reader won't use garbage.
	vim_memset(buffer, 0, size);
    return retval;
}


do_cmdline(
    char_u	*cmdline,
    char_u	*(*fgetline)(int, void *, int, getline_opt_T),
    void	*cookie,		// argument for fgetline()
    int		flags)
{
    char_u	*next_cmdline;		// next cmd to execute
    char_u	*cmdline_copy = NULL;	// copy of cmd line
    int		used_getline = FALSE;	// used "fgetline" to obtain command
    static int	recursive = 0;		// recursive depth
    int		msg_didout_before_start = 0;
    int		count = 0;		// line number count
    int		did_inc = FALSE;	// incremented RedrawingDisabled
    int		retval = OK;
#ifdef FEAT_EVAL
    cstack_T	cstack;			// conditional stack
    garray_T	lines_ga;		// keep lines for ":while"/":for"
    int		current_line = 0;	// active line in lines_ga
    int		current_line_before = 0;
    char_u	*fname = NULL;		// function or script name
    linenr_T	*breakpoint = NULL;	// ptr to breakpoint field in cookie
    int		*dbg_tick = NULL;	// ptr to dbg_tick field in cookie
    struct dbg_stuff debug_saved;	// saved things for debug mode
    int		initial_trylevel;
    msglist_T	**saved_msg_list = NULL;
    msglist_T	*private_msg_list = NULL;

    // "fgetline" and "cookie" passed to do_one_cmd()
    char_u	*(*cmd_getline)(int, void *, int, getline_opt_T);
    void	*cmd_cookie;
    struct loop_cookie cmd_loop_cookie;
    void	*real_cookie;
    int		getline_is_func;
#else
# define cmd_getline fgetline
# define cmd_cookie cookie
#endif
    static int	call_depth = 0;		// recursiveness
#ifdef FEAT_EVAL
    // For every pair of do_cmdline()/do_one_cmd() calls, use an extra memory
    // location for storing error messages to be converted to an exception.
    // This ensures that the do_errthrow() call in do_one_cmd() does not
    // combine the messages stored by an earlier invocation of do_one_cmd()
    // with the command name of the later one.  This would happen when
    // BufWritePost autocommands are executed after a write error.
    saved_msg_list = msg_list;
    msg_list = &private_msg_list;
#endif

    // It's possible to create an endless loop with ":execute", catch that
    // here.  The value of 200 allows nested function calls, ":source", etc.
    // Allow 200 or 'maxfuncdepth', whatever is larger.
    if (call_depth >= 200
#ifdef FEAT_EVAL
	    && call_depth >= p_mfd
#endif
	    )
    {
	emsg(_(e_command_too_recursive));
#ifdef FEAT_EVAL
	// When converting to an exception, we do not include the command name
	// since this is not an error of the specific command.
	do_errthrow((cstack_T *)NULL, (char_u *)NULL);
	msg_list = saved_msg_list;
#endif
	return FAIL;
    }
    ++call_depth;

#ifdef FEAT_EVAL
    CLEAR_FIELD(cstack);
    cstack.cs_idx = -1;
    ga_init2(&lines_ga, sizeof(wcmd_T), 10);

    real_cookie = getline_cookie(fgetline, cookie);

    // Inside a function use a higher nesting level.
    getline_is_func = getline_equal(fgetline, cookie, get_func_line);
    if (getline_is_func && ex_nesting_level == func_level(real_cookie))
	++ex_nesting_level;

    // Get the function or script name and the address where the next breakpoint
    // line and the debug tick for a function or script are stored.
    if (getline_is_func)
    {
	fname = func_name(real_cookie);
	breakpoint = func_breakpoint(real_cookie);
	dbg_tick = func_dbg_tick(real_cookie);
    }
    else if (getline_equal(fgetline, cookie, getsourceline))
    {
	fname = SOURCING_NAME;
	breakpoint = source_breakpoint(real_cookie);
	dbg_tick = source_dbg_tick(real_cookie);
    }

    /*
     * Initialize "force_abort"  and "suppress_errthrow" at the top level.
     */
    if (!recursive)
    {
	force_abort = FALSE;
	suppress_errthrow = FALSE;
    }

    /*
     * If requested, store and reset the global values controlling the
     * exception handling (used when debugging).  Otherwise clear it to avoid
     * a bogus compiler warning when the optimizer uses inline functions...
     */
    if (flags & DOCMD_EXCRESET)
	save_dbg_stuff(&debug_saved);
    else
	CLEAR_FIELD(debug_saved);

    initial_trylevel = trylevel;

    /*
     * "did_throw" will be set to TRUE when an exception is being thrown.
     */
    did_throw = FALSE;
#endif
    /*
     * "did_emsg" will be set to TRUE when emsg() is used, in which case we
     * cancel the whole command line, and any if/endif or loop.
     * If force_abort is set, we cancel everything.
     */
#ifdef FEAT_EVAL
    did_emsg_cumul += did_emsg;
#endif
    did_emsg = FALSE;

    /*
     * KeyTyped is only set when calling vgetc().  Reset it here when not
     * calling vgetc() (sourced command lines).
     */
    if (!(flags & DOCMD_KEYTYPED)
			       && !getline_equal(fgetline, cookie, getexline))
	KeyTyped = FALSE;

    /*
     * Continue executing command lines:
     * - when inside an ":if", ":while" or ":for"
     * - for multiple commands on one line, separated with '|'
     * - when repeating until there are no more lines (for ":source")
     */
    next_cmdline = cmdline;
    do
    {
#ifdef FEAT_EVAL
	getline_is_func = getline_equal(fgetline, cookie, get_func_line);
#endif

	// stop skipping cmds for an error msg after all endif/while/for
	if (next_cmdline == NULL
#ifdef FEAT_EVAL
		&& !force_abort
		&& cstack.cs_idx < 0
		&& !(getline_is_func && func_has_abort(real_cookie))
#endif
							)
	{
#ifdef FEAT_EVAL
	    did_emsg_cumul += did_emsg;
#endif
	    did_emsg = FALSE;
	}

	/*
	 * 1. If repeating a line in a loop, get a line from lines_ga.
	 * 2. If no line given: Get an allocated line with fgetline().
	 * 3. If a line is given: Make a copy, so we can mess with it.
	 */

#ifdef FEAT_EVAL
	// 1. If repeating, get a previous line from lines_ga.
	if (cstack.cs_looplevel > 0 && current_line < lines_ga.ga_len)
	{
	    // Each '|' separated command is stored separately in lines_ga, to
	    // be able to jump to it.  Don't use next_cmdline now.
	    VIM_CLEAR(cmdline_copy);

	    // Check if a function has returned or, unless it has an unclosed
	    // try conditional, aborted.
	    if (getline_is_func)
	    {
# ifdef FEAT_PROFILE
		if (do_profiling == PROF_YES)
		    func_line_end(real_cookie);
# endif
		if (func_has_ended(real_cookie))
		{
		    retval = FAIL;
		    break;
		}
	    }
#ifdef FEAT_PROFILE
	    else if (do_profiling == PROF_YES
			    && getline_equal(fgetline, cookie, getsourceline))
		script_line_end();
#endif

	    // Check if a sourced file hit a ":finish" command.
	    if (source_finished(fgetline, cookie))
	    {
		retval = FAIL;
		break;
	    }

	    // If breakpoints have been added/deleted need to check for it.
	    if (breakpoint != NULL && dbg_tick != NULL
						   && *dbg_tick != debug_tick)
	    {
		*breakpoint = dbg_find_breakpoint(
				getline_equal(fgetline, cookie, getsourceline),
							fname, SOURCING_LNUM);
		*dbg_tick = debug_tick;
	    }

	    next_cmdline = ((wcmd_T *)(lines_ga.ga_data))[current_line].line;
	    SOURCING_LNUM = ((wcmd_T *)(lines_ga.ga_data))[current_line].lnum;

	    // Did we encounter a breakpoint?
	    if (breakpoint != NULL && *breakpoint != 0
					      && *breakpoint <= SOURCING_LNUM)
	    {
		dbg_breakpoint(fname, SOURCING_LNUM);
		// Find next breakpoint.
		*breakpoint = dbg_find_breakpoint(
			       getline_equal(fgetline, cookie, getsourceline),
							fname, SOURCING_LNUM);
		*dbg_tick = debug_tick;
	    }
# ifdef FEAT_PROFILE
	    if (do_profiling == PROF_YES)
	    {
		if (getline_is_func)
		    func_line_start(real_cookie, SOURCING_LNUM);
		else if (getline_equal(fgetline, cookie, getsourceline))
		    script_line_start();
	    }
# endif
	}
#endif

	// 2. If no line given, get an allocated line with fgetline().
	if (next_cmdline == NULL)
	{
	    /*
	     * Need to set msg_didout for the first line after an ":if",
	     * otherwise the ":if" will be overwritten.
	     */
	    if (count == 1 && getline_equal(fgetline, cookie, getexline))
		msg_didout = TRUE;
	    if (fgetline == NULL || (next_cmdline = fgetline(':', cookie,
#ifdef FEAT_EVAL
		    cstack.cs_idx < 0 ? 0 : (cstack.cs_idx + 1) * 2
#else
		    0
#endif
		    , in_vim9script() ? GETLINE_CONCAT_CONTBAR
					       : GETLINE_CONCAT_CONT)) == NULL)
	    {
		// Don't call wait_return for aborted command line.  The NULL
		// returned for the end of a sourced file or executed function
		// doesn't do this.
		if (KeyTyped && !(flags & DOCMD_REPEAT))
		    need_wait_return = FALSE;
		retval = FAIL;
		break;
	    }
	    used_getline = TRUE;

	    /*
	     * Keep the first typed line.  Clear it when more lines are typed.
	     */
	    if (flags & DOCMD_KEEPLINE)
	    {
		vim_free(repeat_cmdline);
		if (count == 0)
		    repeat_cmdline = vim_strsave(next_cmdline);
		else
		    repeat_cmdline = NULL;
	    }
	}

	// 3. Make a copy of the command so we can mess with it.
	else if (cmdline_copy == NULL)
	{
	    next_cmdline = vim_strsave(next_cmdline);
	    if (next_cmdline == NULL)
	    {
		emsg(_(e_out_of_memory));
		retval = FAIL;
		break;
	    }
	}
	cmdline_copy = next_cmdline;

#ifdef FEAT_EVAL
	/*
	 * Inside a while/for loop, and when the command looks like a ":while"
	 * or ":for", the line is stored, because we may need it later when
	 * looping.
	 *
	 * When there is a '|' and another command, it is stored separately,
	 * because we need to be able to jump back to it from an
	 * :endwhile/:endfor.
	 *
	 * Pass a different "fgetline" function to do_one_cmd() below,
	 * that it stores lines in or reads them from "lines_ga".  Makes it
	 * possible to define a function inside a while/for loop and handles
	 * line continuation.
	 */
	if ((cstack.cs_looplevel > 0 || has_loop_cmd(next_cmdline)))
	{
	    cmd_getline = get_loop_line;
	    cmd_cookie = (void *)&cmd_loop_cookie;
	    cmd_loop_cookie.lines_gap = &lines_ga;
	    cmd_loop_cookie.current_line = current_line;
	    cmd_loop_cookie.getline = fgetline;
	    cmd_loop_cookie.cookie = cookie;
	    cmd_loop_cookie.repeating = (current_line < lines_ga.ga_len);

	    // Save the current line when encountering it the first time.
	    if (current_line == lines_ga.ga_len
		    && store_loop_line(&lines_ga, next_cmdline) == FAIL)
	    {
		retval = FAIL;
		break;
	    }
	    current_line_before = current_line;
	}
	else
	{
	    cmd_getline = fgetline;
	    cmd_cookie = cookie;
	}

	did_endif = FALSE;
#endif

	if (count++ == 0)
	{
	    /*
	     * All output from the commands is put below each other, without
	     * waiting for a return. Don't do this when executing commands
	     * from a script or when being called recursive (e.g. for ":e
	     * +command file").
	     */
	    if (!(flags & DOCMD_NOWAIT) && !recursive)
	    {
		msg_didout_before_start = msg_didout;
		msg_didany = FALSE; // no output yet
		msg_start();
		msg_scroll = TRUE;  // put messages below each other
		++no_wait_return;   // don't wait for return until finished
		++RedrawingDisabled;
		did_inc = TRUE;
	    }
	}

	if ((p_verbose >= 15 && SOURCING_NAME != NULL) || p_verbose >= 16)
	    msg_verbose_cmd(SOURCING_LNUM, cmdline_copy);

	/*
	 * 2. Execute one '|' separated command.
	 *    do_one_cmd() will return NULL if there is no trailing '|'.
	 *    "cmdline_copy" can change, e.g. for '%' and '#' expansion.
	 */
	++recursive;
	next_cmdline = do_one_cmd(&cmdline_copy, flags,
#ifdef FEAT_EVAL
				&cstack,
#endif
				cmd_getline, cmd_cookie);
	--recursive;

#ifdef FEAT_EVAL
	if (cmd_cookie == (void *)&cmd_loop_cookie)
	    // Use "current_line" from "cmd_loop_cookie", it may have been
	    // incremented when defining a function.
	    current_line = cmd_loop_cookie.current_line;
#endif

	if (next_cmdline == NULL)
	{
	    VIM_CLEAR(cmdline_copy);

	    /*
	     * If the command was typed, remember it for the ':' register.
	     * Do this AFTER executing the command to make :@: work.
	     */
	    if (getline_equal(fgetline, cookie, getexline)
						  && new_last_cmdline != NULL)
	    {
		vim_free(last_cmdline);
		last_cmdline = new_last_cmdline;
		new_last_cmdline = NULL;
	    }
	}
	else
	{
	    // need to copy the command after the '|' to cmdline_copy, for the
	    // next do_one_cmd()
	    STRMOVE(cmdline_copy, next_cmdline);
	    next_cmdline = cmdline_copy;
	}

#ifdef FEAT_EVAL
	// reset did_emsg for a function that is not aborted by an error
	if (did_emsg && !force_abort
		&& getline_equal(fgetline, cookie, get_func_line)
					      && !func_has_abort(real_cookie))
	{
	    // did_emsg_cumul is not set here
	    did_emsg = FALSE;
	}

	if (cstack.cs_looplevel > 0)
	{
	    ++current_line;

	    /*
	     * An ":endwhile", ":endfor" and ":continue" is handled here.
	     * If we were executing commands, jump back to the ":while" or
	     * ":for".
	     * If we were not executing commands, decrement cs_looplevel.
	     */
	    if (cstack.cs_lflags & (CSL_HAD_CONT | CSL_HAD_ENDLOOP))
	    {
		cstack.cs_lflags &= ~(CSL_HAD_CONT | CSL_HAD_ENDLOOP);

		// Jump back to the matching ":while" or ":for".  Be careful
		// not to use a cs_line[] from an entry that isn't a ":while"
		// or ":for": It would make "current_line" invalid and can
		// cause a crash.
		if (!did_emsg && !got_int && !did_throw
			&& cstack.cs_idx >= 0
			&& (cstack.cs_flags[cstack.cs_idx]
						      & (CSF_WHILE | CSF_FOR))
			&& cstack.cs_line[cstack.cs_idx] >= 0
			&& (cstack.cs_flags[cstack.cs_idx] & CSF_ACTIVE))
		{
		    current_line = cstack.cs_line[cstack.cs_idx];
						// remember we jumped there
		    cstack.cs_lflags |= CSL_HAD_LOOP;
		    line_breakcheck();		// check if CTRL-C typed

		    // Check for the next breakpoint at or after the ":while"
		    // or ":for".
		    if (breakpoint != NULL)
		    {
			*breakpoint = dbg_find_breakpoint(
			       getline_equal(fgetline, cookie, getsourceline),
									fname,
			   ((wcmd_T *)lines_ga.ga_data)[current_line].lnum-1);
			*dbg_tick = debug_tick;
		    }
		}
		else
		{
		    // can only get here with ":endwhile" or ":endfor"
		    if (cstack.cs_idx >= 0)
			rewind_conditionals(&cstack, cstack.cs_idx - 1,
				   CSF_WHILE | CSF_FOR, &cstack.cs_looplevel);
		}
	    }

	    /*
	     * For a ":while" or ":for" we need to remember the line number.
	     */
	    else if (cstack.cs_lflags & CSL_HAD_LOOP)
	    {
		cstack.cs_lflags &= ~CSL_HAD_LOOP;
		cstack.cs_line[cstack.cs_idx] = current_line_before;
	    }
	}

	// Check for the next breakpoint after a watchexpression
	if (breakpoint != NULL && has_watchexpr())
	{
	    *breakpoint = dbg_find_breakpoint(FALSE, fname, SOURCING_LNUM);
	    *dbg_tick = debug_tick;
	}

	/*
	 * When not inside any ":while" loop, clear remembered lines.
	 */
	if (cstack.cs_looplevel == 0)
	{
	    if (lines_ga.ga_len > 0)
	    {
		SOURCING_LNUM =
		       ((wcmd_T *)lines_ga.ga_data)[lines_ga.ga_len - 1].lnum;
		free_cmdlines(&lines_ga);
	    }
	    current_line = 0;
	}

	/*
	 * A ":finally" makes did_emsg, got_int, and did_throw pending for
	 * being restored at the ":endtry".  Reset them here and set the
	 * ACTIVE and FINALLY flags, so that the finally clause gets executed.
	 * This includes the case where a missing ":endif", ":endwhile" or
	 * ":endfor" was detected by the ":finally" itself.
	 */
	if (cstack.cs_lflags & CSL_HAD_FINA)
	{
	    cstack.cs_lflags &= ~CSL_HAD_FINA;
	    report_make_pending(cstack.cs_pending[cstack.cs_idx]
		    & (CSTP_ERROR | CSTP_INTERRUPT | CSTP_THROW),
		    did_throw ? (void *)current_exception : NULL);
	    did_emsg = got_int = did_throw = FALSE;
	    cstack.cs_flags[cstack.cs_idx] |= CSF_ACTIVE | CSF_FINALLY;
	}

	// Update global "trylevel" for recursive calls to do_cmdline() from
	// within this loop.
	trylevel = initial_trylevel + cstack.cs_trylevel;

	/*
	 * If the outermost try conditional (across function calls and sourced
	 * files) is aborted because of an error, an interrupt, or an uncaught
	 * exception, cancel everything.  If it is left normally, reset
	 * force_abort to get the non-EH compatible abortion behavior for
	 * the rest of the script.
	 */
	if (trylevel == 0 && !did_emsg && !got_int && !did_throw)
	    force_abort = FALSE;

	// Convert an interrupt to an exception if appropriate.
	(void)do_intthrow(&cstack);
#endif // FEAT_EVAL

    }
    /*
     * Continue executing command lines when:
     * - no CTRL-C typed, no aborting error, no exception thrown or try
     *   conditionals need to be checked for executing finally clauses or
     *   catching an interrupt exception
     * - didn't get an error message or lines are not typed
     * - there is a command after '|', inside a :if, :while, :for or :try, or
     *   looping for ":source" command or function call.
     */
    while (!((got_int
#ifdef FEAT_EVAL
		    || (did_emsg && (force_abort || in_vim9script()))
		    || did_throw
#endif
	     )
#ifdef FEAT_EVAL
		&& cstack.cs_trylevel == 0
#endif
	    )
	    && !(did_emsg
#ifdef FEAT_EVAL
		// Keep going when inside try/catch, so that the error can be
		// deal with, except when it is a syntax error, it may cause
		// the :endtry to be missed.
		&& (cstack.cs_trylevel == 0 || did_emsg_syntax)
#endif
		&& used_getline
			    && (getline_equal(fgetline, cookie, getexmodeline)
			       || getline_equal(fgetline, cookie, getexline)))
	    && (next_cmdline != NULL
#ifdef FEAT_EVAL
			|| cstack.cs_idx >= 0
#endif
			|| (flags & DOCMD_REPEAT)));

    vim_free(cmdline_copy);
    did_emsg_syntax = FALSE;
#ifdef FEAT_EVAL
    free_cmdlines(&lines_ga);
    ga_clear(&lines_ga);

    if (cstack.cs_idx >= 0)
    {
	/*
	 * If a sourced file or executed function ran to its end, report the
	 * unclosed conditional.
	 * In Vim9 script do not give a second error, executing aborts after
	 * the first one.
	 */
	if (!got_int && !did_throw && !aborting()
		&& !(did_emsg && in_vim9script())
		&& ((getline_equal(fgetline, cookie, getsourceline)
			&& !source_finished(fgetline, cookie))
		    || (getline_equal(fgetline, cookie, get_func_line)
					    && !func_has_ended(real_cookie))))
	{
	    if (cstack.cs_flags[cstack.cs_idx] & CSF_TRY)
		emsg(_(e_missing_endtry));
	    else if (cstack.cs_flags[cstack.cs_idx] & CSF_WHILE)
		emsg(_(e_missing_endwhile));
	    else if (cstack.cs_flags[cstack.cs_idx] & CSF_FOR)
		emsg(_(e_missing_endfor));
	    else
		emsg(_(e_missing_endif));
	}

	/*
	 * Reset "trylevel" in case of a ":finish" or ":return" or a missing
	 * ":endtry" in a sourced file or executed function.  If the try
	 * conditional is in its finally clause, ignore anything pending.
	 * If it is in a catch clause, finish the caught exception.
	 * Also cleanup any "cs_forinfo" structures.
	 */
	do
	{
	    int idx = cleanup_conditionals(&cstack, 0, TRUE);

	    if (idx >= 0)
		--idx;	    // remove try block not in its finally clause
	    rewind_conditionals(&cstack, idx, CSF_WHILE | CSF_FOR,
							&cstack.cs_looplevel);
	}
	while (cstack.cs_idx >= 0);
	trylevel = initial_trylevel;
    }

    // If a missing ":endtry", ":endwhile", ":endfor", or ":endif" or a memory
    // lack was reported above and the error message is to be converted to an
    // exception, do this now after rewinding the cstack.
    do_errthrow(&cstack, getline_equal(fgetline, cookie, get_func_line)
				  ? (char_u *)"endfunction" : (char_u *)NULL);

    if (trylevel == 0)
    {
	// Just in case did_throw got set but current_exception wasn't.
	if (current_exception == NULL)
	    did_throw = FALSE;

	/*
	 * When an exception is being thrown out of the outermost try
	 * conditional, discard the uncaught exception, disable the conversion
	 * of interrupts or errors to exceptions, and ensure that no more
	 * commands are executed.
	 */
	if (did_throw)
	    handle_did_throw();

	/*
	 * On an interrupt or an aborting error not converted to an exception,
	 * disable the conversion of errors to exceptions.  (Interrupts are not
	 * converted anymore, here.) This enables also the interrupt message
	 * when force_abort is set and did_emsg unset in case of an interrupt
	 * from a finally clause after an error.
	 */
	else if (got_int || (did_emsg && force_abort))
	    suppress_errthrow = TRUE;
    }

    /*
     * The current cstack will be freed when do_cmdline() returns.  An uncaught
     * exception will have to be rethrown in the previous cstack.  If a function
     * has just returned or a script file was just finished and the previous
     * cstack belongs to the same function or, respectively, script file, it
     * will have to be checked for finally clauses to be executed due to the
     * ":return" or ":finish".  This is done in do_one_cmd().
     */
    if (did_throw)
	need_rethrow = TRUE;
    if ((getline_equal(fgetline, cookie, getsourceline)
		&& ex_nesting_level > source_level(real_cookie))
	    || (getline_equal(fgetline, cookie, get_func_line)
		&& ex_nesting_level > func_level(real_cookie) + 1))
    {
	if (!did_throw)
	    check_cstack = TRUE;
    }
    else
    {
	// When leaving a function, reduce nesting level.
	if (getline_equal(fgetline, cookie, get_func_line))
	    --ex_nesting_level;
	/*
	 * Go to debug mode when returning from a function in which we are
	 * single-stepping.
	 */
	if ((getline_equal(fgetline, cookie, getsourceline)
		    || getline_equal(fgetline, cookie, get_func_line))
		&& ex_nesting_level + 1 <= debug_break_level)
	    do_debug(getline_equal(fgetline, cookie, getsourceline)
		    ? (char_u *)_("End of sourced file")
		    : (char_u *)_("End of function"));
    }

    /*
     * Restore the exception environment (done after returning from the
     * debugger).
     */
    if (flags & DOCMD_EXCRESET)
	restore_dbg_stuff(&debug_saved);

    msg_list = saved_msg_list;

    // Cleanup if "cs_emsg_silent_list" remains.
    if (cstack.cs_emsg_silent_list != NULL)
    {
	eslist_T *elem, *temp;

	for (elem = cstack.cs_emsg_silent_list; elem != NULL; elem = temp)
	{
	    temp = elem->next;
	    vim_free(elem);
	}
    }
#endif // FEAT_EVAL

    /*
     * If there was too much output to fit on the command line, ask the user to
     * hit return before redrawing the screen. With the ":global" command we do
     * this only once after the command is finished.
     */
    if (did_inc)
    {
	--RedrawingDisabled;
	--no_wait_return;
	msg_scroll = FALSE;

	/*
	 * When just finished an ":if"-":else" which was typed, no need to
	 * wait for hit-return.  Also for an error situation.
	 */
	if (retval == FAIL
#ifdef FEAT_EVAL
		|| (did_endif && KeyTyped && !did_emsg)
#endif
					    )
	{
	    need_wait_return = FALSE;
	    msg_didany = FALSE;		// don't wait when restarting edit
	}
	else if (need_wait_return)
	{
	    /*
	     * The msg_start() above clears msg_didout. The wait_return we do
	     * here should not overwrite the command that may be shown before
	     * doing that.
	     */
	    msg_didout |= msg_didout_before_start;
	    wait_return(FALSE);
	}
    }

#ifdef FEAT_EVAL
    did_endif = FALSE;  // in case do_cmdline used recursively
#else
    /*
     * Reset if_level, in case a sourced script file contains more ":if" than
     * ":endif" (could be ":if x | foo | endif").
     */
    if_level = 0;
#endif

    --call_depth;
    return retval;
}


ex_startinsert(exarg_T *eap)
{
    if (eap->forceit)
    {
	// cursor line can be zero on startup
	if (!curwin->w_cursor.lnum)
	    curwin->w_cursor.lnum = 1;
	set_cursor_for_append_to_line();
    }
#ifdef FEAT_TERMINAL
    // Ignore this when running in an active terminal.
    if (term_job_running(curbuf->b_term))
	return;
#endif

    // Ignore the command when already in Insert mode.  Inserting an
    // expression register that invokes a function can do this.
    if (State & MODE_INSERT)
	return;

    if (eap->cmdidx == CMD_startinsert)
	restart_edit = 'a';
    else if (eap->cmdidx == CMD_startreplace)
	restart_edit = 'R';
    else
	restart_edit = 'V';

    if (!eap->forceit)
    {
	if (eap->cmdidx == CMD_startinsert)
	    restart_edit = 'i';
	curwin->w_curswant = 0;	    // avoid MAXCOL
    }

    if (VIsual_active)
	showmode();
}


compileBrailleIndicator(FileInfo *file, const char *ermsg, TranslationTableOpcode opcode,
		TranslationTableOffset *ruleOffset, int noback, int nofor,
		TranslationTableHeader **table) {
	CharsString token;
	CharsString cells;
	if (!getToken(file, &token, ermsg)) return 0;
	if (!parseDots(file, &cells, &token)) return 0;
	return addRule(
			file, opcode, NULL, &cells, 0, 0, ruleOffset, NULL, noback, nofor, table);
}


_lou_findOpcodeNumber(const char *toFind) {
	/* Used by tools such as lou_debug */
	static TranslationTableOpcode lastOpcode = 0;
	TranslationTableOpcode opcode = lastOpcode;
	int length = (int)strlen(toFind);
	do {
		if (length == opcodeLengths[opcode] &&
				strcasecmp(toFind, opcodeNames[opcode]) == 0) {
			lastOpcode = opcode;
			return opcode;
		}
		opcode++;
		if (opcode >= CTO_None) opcode = 0;
	} while (opcode != lastOpcode);
	return CTO_None;
}


setDefaults(TranslationTableHeader *table) {
	for (int i = 0; i < 3; i++)
		if (!table->emphRules[i][lenPhraseOffset])
			table->emphRules[i][lenPhraseOffset] = 4;
	if (table->numPasses == 0) table->numPasses = 1;
	return 1;
}


passGetString(CharsString *passLine, int *passLinepos, CharsString *string,
		const FileInfo *file) {
	string->length = 0;
	while (1) {
		if ((*passLinepos >= passLine->length) || !passLine->chars[*passLinepos]) {
			compileError(file, "unterminated string");
			return 0;
		}
		if (passLine->chars[*passLinepos] == 34) break;
		if (passLine->chars[*passLinepos] == QUOTESUB)
			string->chars[string->length++] = 34;
		else
			string->chars[string->length++] = passLine->chars[*passLinepos];
		(*passLinepos)++;
	}
	string->chars[string->length] = 0;
	(*passLinepos)++;
	return 1;
}


_lou_allocMem(AllocBuf buffer, int index, int srcmax, int destmax) {
	if (srcmax < 1024) srcmax = 1024;
	if (destmax < 1024) destmax = 1024;
	switch (buffer) {
	case alloc_typebuf:
		if (destmax > sizeTypebuf) {
			if (typebuf != NULL) free(typebuf);
			// TODO: should this be srcmax?
			typebuf = malloc((destmax + 4) * sizeof(formtype));
			if (!typebuf) _lou_outOfMemory();
			sizeTypebuf = destmax;
		}
		return typebuf;

	case alloc_wordBuffer:

		if (wordBuffer != NULL) free(wordBuffer);
		wordBuffer = calloc(srcmax + 4, sizeof(unsigned int));
		if (wordBuffer == NULL) _lou_outOfMemory();
		return wordBuffer;

	case alloc_emphasisBuffer:

		if (emphasisBuffer != NULL) free(emphasisBuffer);
		emphasisBuffer = calloc(srcmax + 4, sizeof(EmphasisInfo));
		if (emphasisBuffer == NULL) _lou_outOfMemory();
		return emphasisBuffer;

	case alloc_destSpacing:
		if (destmax > sizeDestSpacing) {
			if (destSpacing != NULL) free(destSpacing);
			destSpacing = malloc(destmax + 4);
			if (!destSpacing) _lou_outOfMemory();
			sizeDestSpacing = destmax;
		}
		return destSpacing;
	case alloc_passbuf:
		if (index < 0 || index >= MAXPASSBUF) {
			_lou_logMessage(LOU_LOG_FATAL, "Index out of bounds: %d\n", index);
			exit(3);
		}
		if (destmax > sizePassbuf[index]) {
			if (passbuf[index] != NULL) free(passbuf[index]);
			passbuf[index] = malloc((destmax + 4) * CHARSIZE);
			if (!passbuf[index]) _lou_outOfMemory();
			sizePassbuf[index] = destmax;
		}
		return passbuf[index];
	case alloc_posMapping1: {
		int mapSize;
		if (srcmax >= destmax)
			mapSize = srcmax;
		else
			mapSize = destmax;
		if (mapSize > sizePosMapping1) {
			if (posMapping1 != NULL) free(posMapping1);
			posMapping1 = malloc((mapSize + 4) * sizeof(int));
			if (!posMapping1) _lou_outOfMemory();
			sizePosMapping1 = mapSize;
		}
	}
		return posMapping1;
	case alloc_posMapping2: {
		int mapSize;
		if (srcmax >= destmax)
			mapSize = srcmax;
		else
			mapSize = destmax;
		if (mapSize > sizePosMapping2) {
			if (posMapping2 != NULL) free(posMapping2);
			posMapping2 = malloc((mapSize + 4) * sizeof(int));
			if (!posMapping2) _lou_outOfMemory();
			sizePosMapping2 = mapSize;
		}
	}
		return posMapping2;
	case alloc_posMapping3: {
		int mapSize;
		if (srcmax >= destmax)
			mapSize = srcmax;
		else
			mapSize = destmax;
		if (mapSize > sizePosMapping3) {
			if (posMapping3 != NULL) free(posMapping3);
			posMapping3 = malloc((mapSize + 4) * sizeof(int));
			if (!posMapping3) _lou_outOfMemory();
			sizePosMapping3 = mapSize;
		}
	}
		return posMapping3;
	default:
		return NULL;
	}
}


putDots(const FileInfo *file, widechar d, TranslationTableHeader **table) {
	/* See if a dot pattern is in the appropriate table. If not, insert it. In either
	 * case, return a pointer to it. */
	TranslationTableCharacter *character;
	TranslationTableOffset offset;
	if ((character = getDots(d, *table))) return character;
	if (!allocateSpaceInTranslationTable(file, &offset, sizeof(*character), table))
		return NULL;
	character = (TranslationTableCharacter *)&(*table)->ruleArea[offset];
	memset(character, 0, sizeof(*character));
	character->sourceFile = file->sourceFile;
	character->sourceLine = file->lineNumber;
	character->value = d;
	const unsigned long int charHash = _lou_charHash(d);
	const TranslationTableOffset bucket = (*table)->dots[charHash];
	if (!bucket)
		(*table)->dots[charHash] = offset;
	else {
		TranslationTableCharacter *oldchar =
				(TranslationTableCharacter *)&(*table)->ruleArea[bucket];
		while (oldchar->next)
			oldchar = (TranslationTableCharacter *)&(*table)->ruleArea[oldchar->next];
		oldchar->next = offset;
	}
	return character;
}


allocateSpaceInTranslationTable(const FileInfo *file, TranslationTableOffset *offset,
		int size, TranslationTableHeader **table) {
	/* allocate memory for table and expand previously allocated memory if necessary */
	int spaceNeeded = ((size + OFFSETSIZE - 1) / OFFSETSIZE) * OFFSETSIZE;
	TranslationTableOffset newTableSize = (*table)->bytesUsed + spaceNeeded;
	TranslationTableOffset tableSize = (*table)->tableSize;
	if (newTableSize > tableSize) {
		TranslationTableHeader *newTable;
		newTableSize += (newTableSize / OFFSETSIZE);
		newTable = realloc(*table, newTableSize);
		if (!newTable) {
			compileError(file, "Not enough memory for translation table.");
			_lou_outOfMemory();
		}
		memset(((unsigned char *)newTable) + tableSize, 0, newTableSize - tableSize);
		/* update references to the old table */
		{
			TranslationTableChainEntry *entry;
			for (entry = translationTableChain; entry != NULL; entry = entry->next)
				if (entry->table == *table)
					entry->table = (TranslationTableHeader *)newTable;
		}
		newTable->tableSize = newTableSize;
		*table = newTable;
	}
	if (offset != NULL) {
		*offset = ((*table)->bytesUsed - sizeof(**table)) / OFFSETSIZE;
		(*table)->bytesUsed += spaceNeeded;
	}
	return 1;
}


getDots(widechar d, TranslationTableHeader *table) {
	const TranslationTableOffset bucket = table->dots[_lou_charHash(d)];
	TranslationTableOffset offset = bucket;
	while (offset) {
		TranslationTableCharacter *character =
				(TranslationTableCharacter *)&table->ruleArea[offset];
		if (character->value == d) return character;
		offset = character->next;
	}
	return NULL;
}


static void io_req_caches_free(struct io_ring_ctx *ctx)
{
	struct io_submit_state *state = &ctx->submit_state;
	int nr = 0;

	mutex_lock(&ctx->uring_lock);
	io_flush_cached_locked_reqs(ctx, state);

	while (!io_req_cache_empty(ctx)) {
		struct io_wq_work_node *node;
		struct io_kiocb *req;

		node = wq_stack_extract(&state->free_list);
		req = container_of(node, struct io_kiocb, comp_list);
		kmem_cache_free(req_cachep, req);
		nr++;
	}
	if (nr)
		percpu_ref_put_many(&ctx->refs, nr);
	mutex_unlock(&ctx->uring_lock);


					  struct io_cancel_data *cd)
	__must_hold(&ctx->completion_lock)
{
	struct io_kiocb *req;
	int i;

	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list;

		list = &ctx->cancel_hash[i];
		hlist_for_each_entry(req, list, hash_node) {
			if (!(cd->flags & IORING_ASYNC_CANCEL_ANY) &&
			    req->file != cd->file)
				continue;
			if (cd->seq == req->work.cancel_seq)
				continue;
			req->work.cancel_seq = cd->seq;
			return req;
		}
	}
	return NULL;


static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct kiocb *kiocb = &req->rw.kiocb;
	unsigned ioprio;
	int ret;

	kiocb->ki_pos = READ_ONCE(sqe->off);

	ioprio = READ_ONCE(sqe->ioprio);
	if (ioprio) {
		ret = ioprio_check_cap(ioprio);
		if (ret)
			return ret;

		kiocb->ki_ioprio = ioprio;
	} else {
		kiocb->ki_ioprio = get_current_ioprio();
	}

	req->imu = NULL;
	req->rw.addr = READ_ONCE(sqe->addr);
	req->rw.len = READ_ONCE(sqe->len);
	req->rw.flags = READ_ONCE(sqe->rw_flags);
	/* used for fixed read/write too - just read unconditionally */
	req->buf_index = READ_ONCE(sqe->buf_index);
	return 0;
}


static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
{
	bool all_flushed, posted;
	size_t cqe_size = sizeof(struct io_uring_cqe);

	if (!force && __io_cqring_events(ctx) == ctx->cq_entries)
		return false;

	if (ctx->flags & IORING_SETUP_CQE32)
		cqe_size <<= 1;

	posted = false;
	spin_lock(&ctx->completion_lock);
	while (!list_empty(&ctx->cq_overflow_list)) {
		struct io_uring_cqe *cqe = io_get_cqe(ctx);
		struct io_overflow_cqe *ocqe;

		if (!cqe && !force)
			break;
		ocqe = list_first_entry(&ctx->cq_overflow_list,
					struct io_overflow_cqe, list);
		if (cqe)
			memcpy(cqe, &ocqe->cqe, cqe_size);
		else
			io_account_cq_overflow(ctx);

		posted = true;
		list_del(&ocqe->list);
		kfree(ocqe);
	}

	all_flushed = list_empty(&ctx->cq_overflow_list);
	if (all_flushed) {
		clear_bit(IO_CHECK_CQ_OVERFLOW_BIT, &ctx->check_cq);
		atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
	}

	io_commit_cqring(ctx);
	spin_unlock(&ctx->completion_lock);
	if (posted)
		io_cqring_ev_posted(ctx);
	return all_flushed;
}


static bool io_match_linked(struct io_kiocb *head)
{
	struct io_kiocb *req;

	io_for_each_link(req, head) {
		if (req->flags & REQ_F_INFLIGHT)
			return true;
	}
	return false;
}


void io_uring_cmd_complete_in_task(struct io_uring_cmd *ioucmd,
			void (*task_work_cb)(struct io_uring_cmd *))
{
	struct io_kiocb *req = container_of(ioucmd, struct io_kiocb, uring_cmd);

	req->uring_cmd.task_work_cb = task_work_cb;
	req->io_task_work.func = io_uring_cmd_work;
	io_req_task_prio_work_add(req);
}


static int io_files_update_prep(struct io_kiocb *req,
				const struct io_uring_sqe *sqe)
{
	if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
		return -EINVAL;
	if (sqe->rw_flags || sqe->splice_fd_in)
		return -EINVAL;

	req->rsrc_update.offset = READ_ONCE(sqe->off);
	req->rsrc_update.nr_args = READ_ONCE(sqe->len);
	if (!req->rsrc_update.nr_args)
		return -EINVAL;
	req->rsrc_update.arg = READ_ONCE(sqe->addr);
	return 0;


static noinline struct io_uring_cqe *__io_get_cqe(struct io_ring_ctx *ctx)
{
	struct io_rings *rings = ctx->rings;
	unsigned int off = ctx->cached_cq_tail & (ctx->cq_entries - 1);
	unsigned int shift = 0;
	unsigned int free, queued, len;

	if (ctx->flags & IORING_SETUP_CQE32)
		shift = 1;

	/* userspace may cheat modifying the tail, be safe and do min */
	queued = min(__io_cqring_events(ctx), ctx->cq_entries);
	free = ctx->cq_entries - queued;
	/* we need a contiguous range, limit based on the current array offset */
	len = min(free, ctx->cq_entries - off);
	if (!len)
		return NULL;

	ctx->cached_cq_tail++;
	ctx->cqe_cached = &rings->cqes[off];
	ctx->cqe_sentinel = ctx->cqe_cached + len;
	ctx->cqe_cached++;
	return &rings->cqes[off << shift];
}


static int io_provide_buffers_prep(struct io_kiocb *req,
				   const struct io_uring_sqe *sqe)
{
	unsigned long size, tmp_check;
	struct io_provide_buf *p = &req->pbuf;
	u64 tmp;

	if (sqe->rw_flags || sqe->splice_fd_in)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -E2BIG;
	p->nbufs = tmp;
	p->addr = READ_ONCE(sqe->addr);
	p->len = READ_ONCE(sqe->len);

	if (check_mul_overflow((unsigned long)p->len, (unsigned long)p->nbufs,
				&size))
		return -EOVERFLOW;
	if (check_add_overflow((unsigned long)p->addr, size, &tmp_check))
		return -EOVERFLOW;

	size = (unsigned long)p->len * p->nbufs;
	if (!access_ok(u64_to_user_ptr(p->addr), size))
		return -EFAULT;

	p->bgid = READ_ONCE(sqe->buf_group);
	tmp = READ_ONCE(sqe->off);
	if (tmp > USHRT_MAX)
		return -E2BIG;
	p->bid = tmp;
	return 0;
}


static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_uring_rsrc_update2 up;
	int ret;

	up.offset = req->rsrc_update.offset;
	up.data = req->rsrc_update.arg;
	up.nr = 0;
	up.tags = 0;
	up.resv = 0;
	up.resv2 = 0;

	if (req->rsrc_update.offset == IORING_FILE_INDEX_ALLOC) {
		ret = io_files_update_with_index_alloc(req, issue_flags);
	} else {
		io_ring_submit_lock(ctx, issue_flags);
		ret = __io_register_rsrc_update(ctx, IORING_RSRC_FILE,
				&up, req->rsrc_update.nr_args);
		io_ring_submit_unlock(ctx, issue_flags);
	}

	if (ret < 0)
		req_set_fail(req);
	__io_req_complete(req, issue_flags, ret, 0);
	return 0;


 */
static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
{
	unsigned head, mask = ctx->sq_entries - 1;
	unsigned sq_idx = ctx->cached_sq_head++ & mask;

	/*
	 * The cached sq head (or cq tail) serves two purposes:
	 *
	 * 1) allows us to batch the cost of updating the user visible
	 *    head updates.
	 * 2) allows the kernel side to track the head on its own, even
	 *    though the application is the one updating it.
	 */
	head = READ_ONCE(ctx->sq_array[sq_idx]);
	if (likely(head < ctx->sq_entries)) {
		/* double index for 128-byte SQEs, twice as long */
		if (ctx->flags & IORING_SETUP_SQE128)
			head <<= 1;
		return &ctx->sq_sqes[head];
	}

	/* drop invalid entries */
	ctx->cq_extra--;
	WRITE_ONCE(ctx->rings->sq_dropped,
		   READ_ONCE(ctx->rings->sq_dropped) + 1);
	return NULL;


static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
				     s32 res, u32 cflags)
{
	ctx->cq_extra++;
	trace_io_uring_complete(ctx, NULL, user_data, res, cflags, 0, 0);
	return __io_fill_cqe(ctx, user_data, res, cflags);
}


static int io_files_update_with_index_alloc(struct io_kiocb *req,
					    unsigned int issue_flags)
{
	__s32 __user *fds = u64_to_user_ptr(req->rsrc_update.arg);
	unsigned int done;
	struct file *file;
	int ret, fd;

	for (done = 0; done < req->rsrc_update.nr_args; done++) {
		if (copy_from_user(&fd, &fds[done], sizeof(fd))) {
			ret = -EFAULT;
			break;
		}

		file = fget(fd);
		if (!file) {
			ret = -EBADF;
			break;
		}
		ret = io_fixed_fd_install(req, issue_flags, file,
					  IORING_FILE_INDEX_ALLOC);
		if (ret < 0)
			break;
		if (copy_to_user(&fds[done], &ret, sizeof(ret))) {
			ret = -EFAULT;
			__io_close_fixed(req, issue_flags, ret);
			break;
		}
	}

	if (done)
		return done;
	return ret;


static inline void io_put_req(struct io_kiocb *req)
{
	if (req_ref_put_and_test(req)) {
		io_queue_next(req);
		io_free_req(req);
	}
}


static int io_shutdown_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
{
	if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
		     sqe->buf_index || sqe->splice_fd_in))
		return -EINVAL;

	req->shutdown.how = READ_ONCE(sqe->len);
	return 0;
}


static int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_provide_buf *p = &req->pbuf;
	struct io_ring_ctx *ctx = req->ctx;
	struct io_buffer_list *bl;
	int ret = 0;

	io_ring_submit_lock(ctx, issue_flags);

	ret = -ENOENT;
	bl = io_buffer_get_list(ctx, p->bgid);
	if (bl) {
		ret = -EINVAL;
		/* can't use provide/remove buffers command on mapped buffers */
		if (!bl->buf_nr_pages)
			ret = __io_remove_buffers(ctx, bl, p->nbufs);
	}
	if (ret < 0)
		req_set_fail(req);

	/* complete before unlock, IOPOLL may need the lock */
	__io_req_complete(req, issue_flags, ret, 0);
	io_ring_submit_unlock(ctx, issue_flags);
	return 0;
}


static inline void __io_fill_cqe32_req(struct io_kiocb *req, s32 res, u32 cflags,
				u64 extra1, u64 extra2)
{
	struct io_ring_ctx *ctx = req->ctx;
	struct io_uring_cqe *cqe;

	if (WARN_ON_ONCE(!(ctx->flags & IORING_SETUP_CQE32)))
		return;
	if (req->flags & REQ_F_CQE_SKIP)
		return;

	trace_io_uring_complete(ctx, req, req->cqe.user_data, res, cflags,
				extra1, extra2);

	/*
	 * If we can't get a cq entry, userspace overflowed the
	 * submission (by quite a lot). Increment the overflow count in
	 * the ring.
	 */
	cqe = io_get_cqe(ctx);
	if (likely(cqe)) {
		WRITE_ONCE(cqe->user_data, req->cqe.user_data);
		WRITE_ONCE(cqe->res, res);
		WRITE_ONCE(cqe->flags, cflags);
		WRITE_ONCE(cqe->big_cqe[0], extra1);
		WRITE_ONCE(cqe->big_cqe[1], extra2);
		return;
	}

	io_cqring_event_overflow(ctx, req->cqe.user_data, res, cflags, extra1, extra2);
}


static __cold void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
						struct task_struct *task,
						bool cancel_all)
{
	struct io_task_cancel cancel = { .task = task, .all = cancel_all, };
	struct io_uring_task *tctx = task ? task->io_uring : NULL;

	/* failed during ring init, it couldn't have issued any requests */
	if (!ctx->rings)
		return;

	while (1) {
		enum io_wq_cancel cret;
		bool ret = false;

		if (!task) {
			ret |= io_uring_try_cancel_iowq(ctx);
		} else if (tctx && tctx->io_wq) {
			/*
			 * Cancels requests of all rings, not only @ctx, but
			 * it's fine as the task is in exit/exec.
			 */
			cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
					       &cancel, true);
			ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
		}

		/* SQPOLL thread does its own polling */
		if ((!(ctx->flags & IORING_SETUP_SQPOLL) && cancel_all) ||
		    (ctx->sq_data && ctx->sq_data->thread == current)) {
			while (!wq_list_empty(&ctx->iopoll_list)) {
				io_iopoll_try_reap_events(ctx);
				ret = true;
			}
		}

		ret |= io_cancel_defer_files(ctx, task, cancel_all);
		ret |= io_poll_remove_all(ctx, task, cancel_all);
		ret |= io_kill_timeouts(ctx, task, cancel_all);
		if (task)
			ret |= io_run_task_work();
		if (!ret)
			break;
		cond_resched();
	}


static inline void io_poll_execute(struct io_kiocb *req, int res,
		__poll_t events)
{
	if (io_poll_get_ownership(req))
		__io_poll_execute(req, res, events);


static int io_buffer_add_list(struct io_ring_ctx *ctx,
			      struct io_buffer_list *bl, unsigned int bgid)
{
	bl->bgid = bgid;
	if (bgid < BGID_ARRAY)
		return 0;

	return xa_err(xa_store(&ctx->io_bl_xa, bgid, bl, GFP_KERNEL));
}


static int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
{
	struct io_async_msghdr iomsg, *kmsg;
	struct io_sr_msg *sr = &req->sr_msg;
	struct socket *sock;
	unsigned int cflags;
	unsigned flags;
	int ret, min_ret = 0;
	bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;

	sock = sock_from_file(req->file);
	if (unlikely(!sock))
		return -ENOTSOCK;

	if (req_has_async_data(req)) {
		kmsg = req->async_data;
	} else {
		ret = io_recvmsg_copy_hdr(req, &iomsg);
		if (ret)
			return ret;
		kmsg = &iomsg;
	}

	if (!(req->flags & REQ_F_POLLED) &&
	    (sr->flags & IORING_RECVSEND_POLL_FIRST))
		return io_setup_async_msg(req, kmsg);

	if (io_do_buffer_select(req)) {
		void __user *buf;

		buf = io_buffer_select(req, &sr->len, issue_flags);
		if (!buf)
			return -ENOBUFS;
		kmsg->fast_iov[0].iov_base = buf;
		kmsg->fast_iov[0].iov_len = sr->len;
		iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1,
				sr->len);
	}

	flags = sr->msg_flags;
	if (force_nonblock)
		flags |= MSG_DONTWAIT;
	if (flags & MSG_WAITALL)
		min_ret = iov_iter_count(&kmsg->msg.msg_iter);

	kmsg->msg.msg_get_inq = 1;
	ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, kmsg->uaddr, flags);
	if (ret < min_ret) {
		if (ret == -EAGAIN && force_nonblock)
			return io_setup_async_msg(req, kmsg);
		if (ret == -ERESTARTSYS)
			ret = -EINTR;
		if (ret > 0 && io_net_retry(sock, flags)) {
			sr->done_io += ret;
			req->flags |= REQ_F_PARTIAL_IO;
			return io_setup_async_msg(req, kmsg);
		}
		req_set_fail(req);
	} else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
		req_set_fail(req);
	}

	/* fast path, check for non-NULL to avoid function call */
	if (kmsg->free_iov)
		kfree(kmsg->free_iov);
	req->flags &= ~REQ_F_NEED_CLEANUP;
	if (ret >= 0)
		ret += sr->done_io;
	else if (sr->done_io)
		ret = sr->done_io;
	cflags = io_put_kbuf(req, issue_flags);
	if (kmsg->msg.msg_inq)
		cflags |= IORING_CQE_F_SOCK_NONEMPTY;
	__io_req_complete(req, issue_flags, ret, cflags);
	return 0;
}


static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	if (sqe->off || sqe->addr || sqe->len || sqe->buf_index)
		return -EINVAL;
	if (req->flags & REQ_F_FIXED_FILE)
		return -EBADF;

	req->close.fd = READ_ONCE(sqe->fd);
	req->close.file_slot = READ_ONCE(sqe->file_index);
	req->close.flags = READ_ONCE(sqe->close_flags);
	if (req->close.flags & ~IORING_CLOSE_FD_AND_FILE_SLOT)
		return -EINVAL;
	if (!(req->close.flags & IORING_CLOSE_FD_AND_FILE_SLOT) &&
	    req->close.file_slot && req->close.fd)
		return -EINVAL;

	return 0;
}


static int io_renameat_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
{
	struct io_rename *ren = &req->rename;
	const char __user *oldf, *newf;

	if (sqe->buf_index || sqe->splice_fd_in)
		return -EINVAL;
	if (unlikely(req->flags & REQ_F_FIXED_FILE))
		return -EBADF;

	ren->old_dfd = READ_ONCE(sqe->fd);
	oldf = u64_to_user_ptr(READ_ONCE(sqe->addr));
	newf = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	ren->new_dfd = READ_ONCE(sqe->len);
	ren->flags = READ_ONCE(sqe->rename_flags);

	ren->oldpath = getname(oldf);
	if (IS_ERR(ren->oldpath))
		return PTR_ERR(ren->oldpath);

	ren->newpath = getname(newf);
	if (IS_ERR(ren->newpath)) {
		putname(ren->oldpath);
		return PTR_ERR(ren->newpath);
	}

	req->flags |= REQ_F_NEED_CLEANUP;
	return 0;
}


static inline void io_req_track_inflight(struct io_kiocb *req)
{
	if (!(req->flags & REQ_F_INFLIGHT)) {
		req->flags |= REQ_F_INFLIGHT;
		atomic_inc(&current->io_uring->inflight_tracked);
	}
}


	__must_hold(&req->ctx->completion_lock)
{
	struct io_kiocb *nxt, *link = req->link;
	bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;

	req->link = NULL;
	while (link) {
		long res = -ECANCELED;

		if (link->flags & REQ_F_FAIL)
			res = link->cqe.res;

		nxt = link->link;
		link->link = NULL;

		trace_io_uring_fail_link(req->ctx, req, req->cqe.user_data,
					req->opcode, link);

		if (ignore_cqes)
			link->flags |= REQ_F_CQE_SKIP;
		else
			link->flags &= ~REQ_F_CQE_SKIP;
		__io_req_complete_post(link, res, 0);
		link = nxt;
	}
}


static struct io_buffer_list *io_buffer_get_list(struct io_ring_ctx *ctx,
						 unsigned int bgid)
{
	if (ctx->io_bl && bgid < BGID_ARRAY)
		return &ctx->io_bl[bgid];

	return xa_load(&ctx->io_bl_xa, bgid);
}


static int io_poll_remove_prep(struct io_kiocb *req,
			       const struct io_uring_sqe *sqe)
{
	struct io_poll_update *upd = &req->poll_update;
	u32 flags;

	if (sqe->buf_index || sqe->splice_fd_in)
		return -EINVAL;
	flags = READ_ONCE(sqe->len);
	if (flags & ~(IORING_POLL_UPDATE_EVENTS | IORING_POLL_UPDATE_USER_DATA |
		      IORING_POLL_ADD_MULTI))
		return -EINVAL;
	/* meaningless without update */
	if (flags == IORING_POLL_ADD_MULTI)
		return -EINVAL;

	upd->old_user_data = READ_ONCE(sqe->addr);
	upd->update_events = flags & IORING_POLL_UPDATE_EVENTS;
	upd->update_user_data = flags & IORING_POLL_UPDATE_USER_DATA;

	upd->new_user_data = READ_ONCE(sqe->off);
	if (!upd->update_user_data && upd->new_user_data)
		return -EINVAL;
	if (upd->update_events)
		upd->events = io_poll_parse_events(sqe, flags);
	else if (sqe->poll32_events)
		return -EINVAL;

	return 0;


static void __io_req_find_next_prep(struct io_kiocb *req)
{
	struct io_ring_ctx *ctx = req->ctx;
	bool posted;

	spin_lock(&ctx->completion_lock);
	posted = io_disarm_next(req);
	io_commit_cqring(ctx);
	spin_unlock(&ctx->completion_lock);
	if (posted)
		io_cqring_ev_posted(ctx);
}


static int io_fsetxattr(struct io_kiocb *req, unsigned int issue_flags)
{
	int ret;

	if (issue_flags & IO_URING_F_NONBLOCK)
		return -EAGAIN;

	ret = __io_setxattr(req, issue_flags, &req->file->f_path);
	io_xattr_finish(req, ret);

	return 0;
}


static void io_req_tw_post(struct io_kiocb *req, bool *locked)
{
	io_req_complete_post(req, req->cqe.res, req->cqe.flags);
}


static int io_symlinkat_prep(struct io_kiocb *req,
			    const struct io_uring_sqe *sqe)
{
	struct io_symlink *sl = &req->symlink;
	const char __user *oldpath, *newpath;

	if (sqe->len || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)
		return -EINVAL;
	if (unlikely(req->flags & REQ_F_FIXED_FILE))
		return -EBADF;

	sl->new_dfd = READ_ONCE(sqe->fd);
	oldpath = u64_to_user_ptr(READ_ONCE(sqe->addr));
	newpath = u64_to_user_ptr(READ_ONCE(sqe->addr2));

	sl->oldpath = getname(oldpath);
	if (IS_ERR(sl->oldpath))
		return PTR_ERR(sl->oldpath);

	sl->newpath = getname(newpath);
	if (IS_ERR(sl->newpath)) {
		putname(sl->oldpath);
		return PTR_ERR(sl->newpath);
	}

	req->flags |= REQ_F_NEED_CLEANUP;
	return 0;
}


static void __io_rsrc_put_work(struct io_rsrc_node *ref_node)
{
	struct io_rsrc_data *rsrc_data = ref_node->rsrc_data;
	struct io_ring_ctx *ctx = rsrc_data->ctx;
	struct io_rsrc_put *prsrc, *tmp;

	list_for_each_entry_safe(prsrc, tmp, &ref_node->rsrc_list, list) {
		list_del(&prsrc->list);

		if (prsrc->tag) {
			if (ctx->flags & IORING_SETUP_IOPOLL)
				mutex_lock(&ctx->uring_lock);

			spin_lock(&ctx->completion_lock);
			io_fill_cqe_aux(ctx, prsrc->tag, 0, 0);
			io_commit_cqring(ctx);
			spin_unlock(&ctx->completion_lock);
			io_cqring_ev_posted(ctx);

			if (ctx->flags & IORING_SETUP_IOPOLL)
				mutex_unlock(&ctx->uring_lock);
		}

		rsrc_data->do_put(ctx, prsrc);
		kfree(prsrc);
	}

	io_rsrc_node_destroy(ref_node);
	if (atomic_dec_and_test(&rsrc_data->refs))
		complete(&rsrc_data->done);


static int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_poll_iocb *poll = &req->poll;
	u32 flags;

	if (sqe->buf_index || sqe->off || sqe->addr)
		return -EINVAL;
	flags = READ_ONCE(sqe->len);
	if (flags & ~IORING_POLL_ADD_MULTI)
		return -EINVAL;
	if ((flags & IORING_POLL_ADD_MULTI) && (req->flags & REQ_F_CQE_SKIP))
		return -EINVAL;

	io_req_set_refcount(req);
	req->apoll_events = poll->events = io_poll_parse_events(sqe, flags);
	return 0;


static int io_timeout_prep(struct io_kiocb *req,
			   const struct io_uring_sqe *sqe)
{
	return __io_timeout_prep(req, sqe, false);


int nft_parse_register_load(const struct nlattr *attr, u8 *sreg, u32 len)
{
	u32 reg;
	int err;

	err = nft_parse_register(attr, &reg);
	if (err < 0)
		return err;

	err = nft_validate_register_load(reg, len);
	if (err < 0)
		return err;

	*sreg = reg;
	return 0;
}


static void __nft_reg_track_clobber(struct nft_regs_track *track, u8 dreg)
{
	int i;

	for (i = track->regs[dreg].num_reg; i > 0; i--)
		__nft_reg_track_cancel(track, dreg - i);
}


static void nf_tables_set_notify(const struct nft_ctx *ctx,
				 const struct nft_set *set, int event,
			         gfp_t gfp_flags)
{
	struct nftables_pernet *nft_net = nft_pernet(ctx->net);
	u32 portid = ctx->portid;
	struct sk_buff *skb;
	u16 flags = 0;
	int err;

	if (!ctx->report &&
	    !nfnetlink_has_listeners(ctx->net, NFNLGRP_NFTABLES))
		return;

	skb = nlmsg_new(NLMSG_GOODSIZE, gfp_flags);
	if (skb == NULL)
		goto err;

	if (ctx->flags & (NLM_F_CREATE | NLM_F_EXCL))
		flags |= ctx->flags & (NLM_F_CREATE | NLM_F_EXCL);

	err = nf_tables_fill_set(skb, ctx, set, event, flags);
	if (err < 0) {
		kfree_skb(skb);
		goto err;
	}

	nft_notify_enqueue(skb, ctx->report, &nft_net->notify_list);
	return;
err:
	nfnetlink_set_err(ctx->net, portid, NFNLGRP_NFTABLES, -ENOBUFS);
}


int nft_set_elem_expr_clone(const struct nft_ctx *ctx, struct nft_set *set,
			    struct nft_expr *expr_array[])
{
	struct nft_expr *expr;
	int err, i, k;

	for (i = 0; i < set->num_exprs; i++) {
		expr = kzalloc(set->exprs[i]->ops->size, GFP_KERNEL_ACCOUNT);
		if (!expr)
			goto err_expr;

		err = nft_expr_clone(expr, set->exprs[i]);
		if (err < 0) {
			nft_expr_destroy(ctx, expr);
			goto err_expr;
		}
		expr_array[i] = expr;
	}

	return 0;

err_expr:
	for (k = i - 1; k >= 0; k--)
		nft_expr_destroy(ctx, expr_array[k]);

	return -ENOMEM;
}


static int nft_parse_register(const struct nlattr *attr, u32 *preg)
{
	unsigned int reg;

	reg = ntohl(nla_get_be32(attr));
	switch (reg) {
	case NFT_REG_VERDICT...NFT_REG_4:
		*preg = reg * NFT_REG_SIZE / NFT_REG32_SIZE;
		break;
	case NFT_REG32_00...NFT_REG32_15:
		*preg = reg + NFT_REG_SIZE / NFT_REG32_SIZE - NFT_REG32_00;
		break;
	default:
		return -ERANGE;
	}

	return 0;
}


static void *qemu_ram_ptr_length(struct uc_struct *uc, RAMBlock *ram_block, ram_addr_t addr,
                                 hwaddr *size, bool lock)
{
    RAMBlock *block = ram_block;
    if (*size == 0) {
        return NULL;
    }

    if (block == NULL) {
        block = qemu_get_ram_block(uc, addr);
        addr -= block->offset;
    }
    *size = MIN(*size, block->max_length - addr);

    return ramblock_ptr(block, addr);
}


MemoryRegionSection *iotlb_to_section(CPUState *cpu,
                                      hwaddr index, MemTxAttrs attrs)
{
#ifdef TARGET_ARM
    struct uc_struct *uc = cpu->uc;
#endif
    int asidx = cpu_asidx_from_attrs(cpu, attrs);
    CPUAddressSpace *cpuas = &cpu->cpu_ases[asidx];
    AddressSpaceDispatch *d = cpuas->memory_dispatch;
    MemoryRegionSection *sections = d->map.sections;

    return &sections[index & ~TARGET_PAGE_MASK];
}


void cpu_exec_init_all(struct uc_struct *uc)
{
    /* The data structures we set up here depend on knowing the page size,
     * so no more changes can be made after this point.
     * In an ideal world, nothing we did before we had finished the
     * machine setup would care about the target page size, and we could
     * do this much later, rather than requiring board models to state
     * up front what their requirements are.
     */
    finalize_target_page_bits(uc);
    memory_map_init(uc);
    io_mem_init(uc);
}


static void listener_add_address_space(MemoryListener *listener,
                                       AddressSpace *as)
{
    FlatView *view;
    FlatRange *fr;

    if (listener->begin) {
        listener->begin(listener);
    }

    view = address_space_get_flatview(as);
    FOR_EACH_FLAT_RANGE(fr, view) {
        MemoryRegionSection section = section_from_flat_range(fr, view);

        if (listener->region_add) {
            listener->region_add(listener, &section);
        }
    }
    if (listener->commit) {
        listener->commit(listener);
    }
}


static void test_map_wrap(void)
{
    uc_engine *uc;

    OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc));

    uc_assert_err(UC_ERR_ARG,
                  uc_mem_map(uc, 0xfffffffffffff000, 0x2000, UC_PROT_ALL));

    OK(uc_close(uc));
}


void cpu_check_watchpoint(CPUState *cpu, vaddr addr, vaddr len,
                          MemTxAttrs attrs, int flags, uintptr_t ra)
{
}


static void memory_region_update_container_subregions(MemoryRegion *subregion)
{
    MemoryRegion *mr = subregion->container;
    MemoryRegion *other;

    memory_region_transaction_begin();

    QTAILQ_FOREACH(other, &mr->subregions, subregions_link) {
        QTAILQ_INSERT_BEFORE(other, subregion, subregions_link);
        goto done;
    }
    QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link);

done:
    mr->uc->memory_region_update_pending = true;
    memory_region_transaction_commit(mr);
}


static MemTxResult memory_region_dispatch_read1(struct uc_struct *uc, MemoryRegion *mr,
                                                hwaddr addr,
                                                uint64_t *pval,
                                                unsigned size,
                                                MemTxAttrs attrs)
{
    *pval = 0;

    if (mr->ops->read) {
        return access_with_adjusted_size(uc, addr, pval, size,
                                         mr->ops->impl.min_access_size,
                                         mr->ops->impl.max_access_size,
                                         memory_region_read_accessor,
                                         mr, attrs);
    } else {
        return access_with_adjusted_size(uc, addr, pval, size,
                                         mr->ops->impl.min_access_size,
                                         mr->ops->impl.max_access_size,
                                         memory_region_read_with_attrs_accessor,
                                         mr, attrs);
    }
}


RAMBlock *qemu_ram_alloc_from_ptr(struct uc_struct *uc, ram_addr_t size, void *host,
                                   MemoryRegion *mr)
{
    RAMBlock *new_block;
    ram_addr_t max_size = size;

    size = HOST_PAGE_ALIGN(uc, size);
    max_size = HOST_PAGE_ALIGN(uc, max_size);
    new_block = g_malloc0(sizeof(*new_block));
    if (new_block == NULL)
        return NULL;
    new_block->mr = mr;
    new_block->used_length = size;
    new_block->max_length = max_size;
    assert(max_size >= size);
    new_block->page_size = uc->qemu_real_host_page_size;
    new_block->host = host;
    if (host) {
        new_block->flags |= RAM_PREALLOC;
    }

    uc->invalid_addr = UC_ERR_OK;
    ram_block_add(mr->uc, new_block);

    if (uc->invalid_error != UC_ERR_OK) {
        g_free(new_block);
        return NULL;
    }

    return new_block;
}


static inline MemTxResult address_space_write_rom_internal(AddressSpace *as,
                                                           hwaddr addr,
                                                           MemTxAttrs attrs,
                                                           const void *ptr,
                                                           hwaddr len,
                                                           enum write_rom_type type)
{
    hwaddr l;
    uint8_t *ram_ptr;
    hwaddr addr1;
    MemoryRegion *mr;
    const uint8_t *buf = ptr;

    while (len > 0) {
        l = len;
        mr = address_space_translate(as, addr, &addr1, &l, true, attrs);

        if (!memory_region_is_ram(mr)) {
            l = memory_access_size(mr, l, addr1);
        } else {
            /* ROM/RAM case */
            ram_ptr = qemu_map_ram_ptr(as->uc, mr->ram_block, addr1);
            switch (type) {
            case WRITE_DATA:
                memcpy(ram_ptr, buf, l);
                break;
            case FLUSH_CACHE:
                flush_icache_range((uintptr_t)ram_ptr, (uintptr_t)ram_ptr + l);
                break;
            }
        }
        len -= l;
        buf += l;
        addr += l;
    }
    return MEMTX_OK;
}


void memory_region_del_subregion(MemoryRegion *mr,
                                 MemoryRegion *subregion)
{
    memory_region_transaction_begin();
    assert(subregion->container == mr);
    subregion->container = NULL;
    QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link);
    mr->uc->memory_region_update_pending = true;
    memory_region_transaction_commit(mr);
}


MemoryRegion *memory_map(struct uc_struct *uc, hwaddr begin, size_t size, uint32_t perms)
{
    MemoryRegion *ram = g_new(MemoryRegion, 1);

    memory_region_init_ram(uc, ram, size, perms);
    if (ram->addr == -1 || !ram->ram_block) {
        // out of memory
        g_free(ram);
        return NULL;
    }

    memory_region_add_subregion(uc->system_memory, begin, ram);

    if (uc->cpu) {
        tlb_flush(uc->cpu);
    }

    return ram;
}


static FlatView *flatview_new(MemoryRegion *mr_root)
{
    FlatView *view;

    view = g_new0(FlatView, 1);
    view->ref = 1;
    view->root = mr_root;

    return view;
}


static MemoryRegionSection address_space_translate_iommu(IOMMUMemoryRegion *iommu_mr,
                                                         hwaddr *xlat,
                                                         hwaddr *plen_out,
                                                         hwaddr *page_mask_out,
                                                         bool is_write,
                                                         bool is_mmio,
                                                         AddressSpace **target_as,
                                                         MemTxAttrs attrs)
{
    MemoryRegionSection *section;
    hwaddr page_mask = (hwaddr)-1;
    MemoryRegion *mr = MEMORY_REGION(iommu_mr);

    do {
        hwaddr addr = *xlat;
        IOMMUMemoryRegionClass *imrc = memory_region_get_iommu_class_nocheck(iommu_mr);
        int iommu_idx = 0;
        IOMMUTLBEntry iotlb;

        if (imrc->attrs_to_index) {
            iommu_idx = imrc->attrs_to_index(iommu_mr, attrs);
        }

        iotlb = imrc->translate(iommu_mr, addr, is_write ?
                                IOMMU_WO : IOMMU_RO, iommu_idx);

        if (!(iotlb.perm & (1 << is_write))) {
            goto unassigned;
        }

        addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
                | (addr & iotlb.addr_mask));
        page_mask &= iotlb.addr_mask;
        *plen_out = MIN(*plen_out, (addr | iotlb.addr_mask) - addr + 1);
        *target_as = iotlb.target_as;

        section = address_space_translate_internal(
                address_space_to_dispatch(iotlb.target_as), addr, xlat,
                plen_out, is_mmio);

        iommu_mr = memory_region_get_iommu(section->mr);
    } while (unlikely(iommu_mr));

    if (page_mask_out) {
        *page_mask_out = page_mask;
    }
    return *section;

unassigned:
    return (MemoryRegionSection) { .mr = &(mr->uc->io_mem_unassigned) };
}


static bool flatview_access_valid(struct uc_struct *uc, FlatView *fv, hwaddr addr, hwaddr len,
                                  bool is_write, MemTxAttrs attrs)
{
    MemoryRegion *mr;
    hwaddr l, xlat;

    while (len > 0) {
        l = len;
        mr = flatview_translate(uc, fv, addr, &xlat, &l, is_write, attrs);
        if (!memory_access_is_direct(mr, is_write)) {
            l = memory_access_size(mr, l, addr);
            if (!memory_region_access_valid(uc, mr, xlat, l, is_write, attrs)) {
                return false;
            }
        }

        len -= l;
        addr += l;
    }
    return true;
}


int qemu_target_page_bits(struct uc_struct *uc)
{
    return TARGET_PAGE_BITS;
}


int safer_openat(int dirfd, const char *path, int flags) {
	assert(path && path[0]);
	flags |= O_NOFOLLOW;

	int fd = -1;

#ifdef __NR_openat2 // kernel 5.6 or better
	struct open_how oh;
	memset(&oh, 0, sizeof(oh));
	oh.flags = flags;
	oh.resolve = RESOLVE_NO_SYMLINKS;
	fd = syscall(__NR_openat2, dirfd, path, &oh, sizeof(struct open_how));
	if (fd != -1 || errno != ENOSYS)
		return fd;
#endif

	// openat2 syscall is not available, traverse path and
	// check each component if it is a symbolic link or not
	char *dup = strdup(path);
	if (!dup)
		errExit("strdup");
	char *tok = strtok(dup, "/");
	if (!tok) { // nothing to do, path is the root directory
		free(dup);
		return openat(dirfd, path, flags);
	}
	char *last_tok = EMPTY_STRING;

	int parentfd;
	if (path[0] == '/')
		parentfd = open("/", O_PATH|O_CLOEXEC);
	else
		parentfd = fcntl(dirfd, F_DUPFD_CLOEXEC, 0);
	if (parentfd == -1)
		errExit("open/fcntl");

	while (1) {
		// open path component, assuming it is a directory; this fails with ENOTDIR if it is a symbolic link
		// if token is a single dot, the directory referred to by parentfd is reopened
		fd = openat(parentfd, tok, O_PATH|O_DIRECTORY|O_NOFOLLOW|O_CLOEXEC);
		if (fd == -1) {
			// if the following token is NULL, the current token is the final path component
			// try again to open it, this time using the passed flags, and return -1 or the descriptor
			last_tok = tok;
			tok = strtok(NULL, "/");
			if (!tok)
				fd = openat(parentfd, last_tok, flags);
			close(parentfd);
			free(dup);
			return fd;
		}
		// move on to next path component
		last_tok = tok;
		tok = strtok(NULL, "/");
		if (!tok)
			break;
		close(parentfd);
		parentfd = fd;
	}
	// getting here when the last path component exists and is of file type directory
	// reopen it using the passed flags
	close(fd);
	fd = openat(parentfd, last_tok, flags);
	close(parentfd);
	free(dup);
	return fd;
}


void set_profile_run_file(pid_t pid, const char *fname) {
	char *runfile;
	if (asprintf(&runfile, "%s/%d", RUN_FIREJAIL_PROFILE_DIR, pid) == -1)
		errExit("asprintf");

	EUID_ROOT();
	// the file is deleted first
	FILE *fp = fopen(runfile, "we");
	if (!fp) {
		fprintf(stderr, "Error: cannot create %s\n", runfile);
		exit(1);
	}
	fprintf(fp, "%s\n", fname);

	// mode and ownership
	SET_PERMS_STREAM(fp, 0, 0, 0644);
	fclose(fp);
	EUID_USER();
	free(runfile);
}


void delete_run_files(pid_t pid) {
	delete_sandbox_run_file(pid);
	delete_bandwidth_run_file(pid);
	delete_network_run_file(pid);
	delete_name_run_file(pid);
	delete_x11_run_file(pid);
	delete_profile_run_file(pid);
}


static int is_cgroup_path(const char *fname) {
	// path starts with /sys/fs/cgroup
	if (strncmp(fname, "/sys/fs/cgroup", 14) != 0)
		return 0;

	// no .. traversal
	char *ptr = strstr(fname, "..");
	if (ptr)
		return 0;

	return 1;
}


static void print_cpu(ProcessHandle process) {
	FILE *fp = process_fopen(process, "status");

#define MAXBUF 4096
	char buf[MAXBUF];
	while (fgets(buf, MAXBUF, fp)) {
		if (strncmp(buf, "Cpus_allowed_list:", 18) == 0) {
			printf("  %s", buf);
			fflush(0);
			fclose(fp);
			return;
		}
	}
	fclose(fp);
}


static void delete_name_run_file(pid_t pid) {
	char *fname;
	if (asprintf(&fname, "%s/%d", RUN_FIREJAIL_NAME_DIR, pid) == -1)
		errExit("asprintf");
	int rv = unlink(fname);
	(void) rv;
	free(fname);
}


void fs_private_tmp(void) {
	EUID_ASSERT();
	if (arg_debug)
		printf("Generate private-tmp whitelist commands\n");

	// check XAUTHORITY file, KDE keeps it under /tmp
	const char *xauth = env_get("XAUTHORITY");
	if (xauth) {
		char *rp = realpath(xauth, NULL);
		if (rp && strncmp(rp, "/tmp/", 5) == 0) {
			char *cmd;
			if (asprintf(&cmd, "whitelist %s", rp) == -1)
				errExit("asprintf");
			profile_check_line(cmd, 0, NULL);
			profile_add(cmd); // profile_add does not duplicate the string
		}
		if (rp)
			free(rp);
	}

	// whitelist x11 directory
	profile_add("whitelist /tmp/.X11-unix");
	profile_add("read-only /tmp/.X11-unix");

	// whitelist sndio directory
	profile_add("whitelist /tmp/sndio");

	// whitelist any pulse* file in /tmp directory
	// some distros use PulseAudio sockets under /tmp instead of the socket in /urn/user
	DIR *dir;
	if (!(dir = opendir("/tmp"))) {
		// sleep 2 seconds and try again
		sleep(2);
		if (!(dir = opendir("/tmp"))) {
			return;
		}
	}

	struct dirent *entry;
	while ((entry = readdir(dir))) {
		if (strncmp(entry->d_name, "pulse-", 6) == 0) {
			char *cmd;
			if (asprintf(&cmd, "whitelist /tmp/%s", entry->d_name) == -1)
				errExit("asprintf");
			profile_check_line(cmd, 0, NULL);
			profile_add(cmd); // profile_add does not duplicate the string
		}
	}
	closedir(dir);
}


int *str_to_int_array(const char *str, size_t *sz) {
	assert(str && sz);

	size_t curr_sz = 0;
	size_t arr_sz = 16;
	int *rv = malloc(arr_sz * sizeof(int));
	if (!rv)
		errExit("malloc");

	char *dup = strdup(str);
	if (!dup)
		errExit("strdup");
	char *tok = strtok(dup, ",");
	if (!tok) {
		free(dup);
		free(rv);
		goto errout;
	}

	while (tok) {
		char *end;
		long val = strtol(tok, &end, 10);
		if (end == tok || *end != '\0' || val < INT_MIN || val > INT_MAX) {
			free(dup);
			free(rv);
			goto errout;
		}

		if (curr_sz == arr_sz) {
			arr_sz *= 2;
			rv = realloc(rv, arr_sz * sizeof(int));
			if (!rv)
				errExit("realloc");
		}
		rv[curr_sz++] = val;

		tok = strtok(NULL, ",");
	}
	free(dup);

	*sz = curr_sz;
	return rv;

errout:
	*sz = 0;
	return NULL;
}


int read_pid(const char *name, pid_t *pid) {
	char *endptr;
	errno = 0;
	long int pidtmp = strtol(name, &endptr, 10);
	if ((errno == ERANGE && (pidtmp == LONG_MAX || pidtmp == LONG_MIN))
		|| (errno != 0 && pidtmp == 0)) {
		return extract_pid(name,pid);
	}
	// endptr points to '\0' char in name if the entire string is valid
	if (endptr == NULL || endptr[0]!='\0') {
		return extract_pid(name,pid);
	}
	*pid =(pid_t)pidtmp;
	return 0;
}


static void extract_cgroup(ProcessHandle sandbox) {
	int fd = process_rootfs_open(sandbox, RUN_CGROUP_CFG);
	if (fd < 0)
		return; // not configured

	FILE *fp = fdopen(fd, "r");
	if (!fp)
		errExit("fdopen");

	char buf[BUFLEN];
	if (fgets(buf, BUFLEN, fp)) {
		cfg.cgroup = strdup(buf);
		if (!cfg.cgroup)
			errExit("strdup");
	}
	fclose(fp);
}


char *do_replace_cntrl_chars(char *str, char c) {
	if (str) {
		size_t i;
		for (i = 0; str[i]; i++) {
			if (iscntrl((unsigned char) str[i]))
				str[i] = c;
		}
	}
	return str;
}


int is_link(const char *fname) {
	assert(fname);
	if (*fname == '\0')
		return 0;

	// remove trailing '/' if any
	char *tmp = strdup(fname);
	if (!tmp)
		errExit("strdup");
	trim_trailing_slash_or_dot(tmp);

	char c;
	ssize_t rv = readlink_as_user(tmp, &c, 1);
	free(tmp);

	return (rv != -1);
}


int has_handler(pid_t pid, int signal) {
	if (signal > 0 && signal <= SIGRTMAX) {
		char *fname;
		if (asprintf(&fname, "/proc/%d/status", pid) == -1)
			errExit("asprintf");
		EUID_ROOT();
		FILE *fp = fopen(fname, "re");
		EUID_USER();
		free(fname);
		if (fp) {
			char buf[MAXBUF];
			while (fgets(buf, MAXBUF, fp)) {
				if (strncmp(buf, "SigCgt:", 7) == 0) {
					unsigned long long val;
					if (sscanf(buf + 7, "%llx", &val) != 1) {
						fprintf(stderr, "Error: cannot read /proc file\n");
						exit(1);
					}
					val >>= (signal - 1);
					val &= 1ULL;
					fclose(fp);
					return val;  // 1 if process has a handler for the signal, else 0
				}
			}
			fclose(fp);
		}
	}
	return 0;
}


static void do_set_cgroup(const char *fname, pid_t pid) {
	FILE *fp = fopen(fname, "ae");
	if (!fp) {
		fwarning("cannot open %s for writing: %s\n", fname, strerror(errno));
		return;
	}

	int rv = fprintf(fp, "%d\n", pid);
	(void) rv;
	fclose(fp);
}


static void fix_std_streams(void) {
	fix_single_std_fd(0, "/dev/full", O_RDONLY|O_NOFOLLOW);
	fix_single_std_fd(1, "/dev/null", O_WRONLY|O_NOFOLLOW);
	fix_single_std_fd(2, "/dev/null", O_WRONLY|O_NOFOLLOW);
}