get_work_q(struct Slapi_op_stack **op_stack_obj)
{
    struct Slapi_work_q *tmp = NULL;
    work_q_item *wqitem;

    slapi_log_err(SLAPI_LOG_TRACE, "get_work_q", "=>\n");
    if (head_work_q == NULL) {
        slapi_log_err(SLAPI_LOG_TRACE, "get_work_q", "The work queue is empty.\n");
        return NULL;
    }

    tmp = head_work_q;
    if (head_work_q == tail_work_q) {
        tail_work_q = NULL;
    }
    head_work_q = tmp->next_work_item;

    wqitem = tmp->work_item;
    *op_stack_obj = tmp->op_stack_obj;
    PR_AtomicDecrement(&work_q_size); /* decrement q size */
    /* Free the memory used by the item found. */
    destroy_work_q(&tmp);

    return (wqitem);
}


conn_buffered_data_avail_nolock(Connection *conn, int *conn_closed)
{
    if ((conn->c_sd == SLAPD_INVALID_SOCKET) || (conn->c_flags & CONN_FLAG_CLOSING)) {
        /* connection is closed - ignore the buffer */
        *conn_closed = 1;
        return 0;
    } else {
        *conn_closed = 0;
        return conn->c_private->c_buffer_bytes - conn->c_private->c_buffer_offset;
    }
}


connection_remove_operation_ext(Slapi_PBlock *pb, Connection *conn, Operation *op)
{
    connection_remove_operation(conn, op);
    void *op_stack_elem = slapi_pblock_get_op_stack_elem(pb);
    connection_done_operation(conn, op_stack_elem);
    slapi_pblock_set(pb, SLAPI_OPERATION, NULL);
    slapi_pblock_init(pb);
}


static void bpf_release_orig_filter(struct bpf_prog *fp)
{
	struct sock_fprog_kern *fprog = fp->orig_prog;

	if (fprog) {
		kfree(fprog->filter);
		kfree(fprog);
	}
}


static void *__xdp_map_lookup_elem(struct bpf_map *map, u32 index)
{
	switch (map->map_type) {
	case BPF_MAP_TYPE_DEVMAP:
		return __dev_map_lookup_elem(map, index);
	case BPF_MAP_TYPE_CPUMAP:
		return __cpu_map_lookup_elem(map, index);
	default:
		return NULL;
	}
}


void bpf_prog_array_free(struct bpf_prog_array __rcu *progs)
{
	if (!progs ||
	    progs == (struct bpf_prog_array __rcu *)&empty_prog_array.hdr)
		return;
	kfree_rcu(progs, rcu);
}


void __weak bpf_jit_free(struct bpf_prog *fp)
{
	if (fp->jited) {
		struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp);

		bpf_jit_binary_unlock_ro(hdr);
		bpf_jit_binary_free(hdr);

		WARN_ON_ONCE(!bpf_prog_kallsyms_verify_off(fp));
	}

	bpf_prog_unlock_free(fp);
}


static bool sk_msg_is_valid_access(int off, int size,
				   enum bpf_access_type type,
				   const struct bpf_prog *prog,
				   struct bpf_insn_access_aux *info)
{
	if (type == BPF_WRITE)
		return false;

	switch (off) {
	case offsetof(struct sk_msg_md, data):
		info->reg_type = PTR_TO_PACKET;
		break;
	case offsetof(struct sk_msg_md, data_end):
		info->reg_type = PTR_TO_PACKET_END;
		break;
	}

	if (off < 0 || off >= sizeof(struct sk_msg_md))
		return false;
	if (off % size != 0)
		return false;
	if (size != sizeof(__u64))
		return false;

	return true;
}


static bool bpf_check_basics_ok(const struct sock_filter *filter,
				unsigned int flen)
{
	if (filter == NULL)
		return false;
	if (flen == 0 || flen > BPF_MAXINSNS)
		return false;

	return true;
}


BPF_CALL_2(bpf_xdp_adjust_meta, struct xdp_buff *, xdp, int, offset)
{
	void *meta = xdp->data_meta + offset;
	unsigned long metalen = xdp->data - meta;

	if (xdp_data_meta_unsupported(xdp))
		return -ENOTSUPP;
	if (unlikely(meta < xdp->data_hard_start ||
		     meta > xdp->data))
		return -EINVAL;
	if (unlikely((metalen & (sizeof(__u32) - 1)) ||
		     (metalen > 32)))
		return -EACCES;

	xdp->data_meta = meta;

	return 0;
}


static bool sock_filter_is_valid_access(int off, int size,
					enum bpf_access_type type,
					const struct bpf_prog *prog,
					struct bpf_insn_access_aux *info)
{
	if (off < 0 || off >= sizeof(struct bpf_sock))
		return false;
	if (off % size != 0)
		return false;
	if (!__sock_filter_check_attach_type(off, type,
					     prog->expected_attach_type))
		return false;
	if (!__sock_filter_check_size(off, size, info))
		return false;
	return true;
}


static int bpf_try_make_head_writable(struct sk_buff *skb)
{
	return bpf_try_make_writable(skb, skb_headlen(skb));
}


BPF_CALL_2(bpf_sock_ops_cb_flags_set, struct bpf_sock_ops_kern *, bpf_sock,
	   int, argval)
{
	struct sock *sk = bpf_sock->sk;
	int val = argval & BPF_SOCK_OPS_ALL_CB_FLAGS;

	if (!IS_ENABLED(CONFIG_INET) || !sk_fullsock(sk))
		return -EINVAL;

	if (val)
		tcp_sk(sk)->bpf_sock_ops_cb_flags = val;

	return argval & (~BPF_SOCK_OPS_ALL_CB_FLAGS);
}


bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
		     unsigned int alignment,
		     bpf_jit_fill_hole_t bpf_fill_ill_insns)
{
	struct bpf_binary_header *hdr;
	unsigned int size, hole, start;

	/* Most of BPF filters are really small, but if some of them
	 * fill a page, allow at least 128 extra bytes to insert a
	 * random section of illegal instructions.
	 */
	size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE);
	hdr = module_alloc(size);
	if (hdr == NULL)
		return NULL;

	/* Fill space with illegal/arch-dep instructions. */
	bpf_fill_ill_insns(hdr, size);

	hdr->pages = size / PAGE_SIZE;
	hole = min_t(unsigned int, size - (proglen + sizeof(*hdr)),
		     PAGE_SIZE - sizeof(*hdr));
	start = (get_random_int() % hole) & ~(alignment - 1);

	/* Leave a random number of instructions before BPF code. */
	*image_ptr = &hdr->image[start];

	return hdr;
}


BPF_CALL_2(bpf_msg_apply_bytes, struct sk_msg_buff *, msg, u32, bytes)
{
	msg->apply_bytes = bytes;
	return 0;
}


BPF_CALL_3(bpf_bind, struct bpf_sock_addr_kern *, ctx, struct sockaddr *, addr,
	   int, addr_len)
{
#ifdef CONFIG_INET
	struct sock *sk = ctx->sk;
	int err;

	/* Binding to port can be expensive so it's prohibited in the helper.
	 * Only binding to IP is supported.
	 */
	err = -EINVAL;
	if (addr->sa_family == AF_INET) {
		if (addr_len < sizeof(struct sockaddr_in))
			return err;
		if (((struct sockaddr_in *)addr)->sin_port != htons(0))
			return err;
		return __inet_bind(sk, addr, addr_len, true, false);
#if IS_ENABLED(CONFIG_IPV6)
	} else if (addr->sa_family == AF_INET6) {
		if (addr_len < SIN6_LEN_RFC2133)
			return err;
		if (((struct sockaddr_in6 *)addr)->sin6_port != htons(0))
			return err;
		/* ipv6_bpf_stub cannot be NULL, since it's called from
		 * bpf_cgroup_inet6_connect hook and ipv6 is already loaded
		 */
		return ipv6_bpf_stub->inet6_bind(sk, addr, addr_len, true, false);
#endif /* CONFIG_IPV6 */
	}
#endif /* CONFIG_INET */

	return -EAFNOSUPPORT;
}


int bpf_prog_array_copy_info(struct bpf_prog_array __rcu *array,
			     u32 *prog_ids, u32 request_cnt,
			     u32 *prog_cnt)
{
	struct bpf_prog **prog;
	u32 cnt = 0;

	if (array)
		cnt = bpf_prog_array_length(array);

	*prog_cnt = cnt;

	/* return early if user requested only program count or nothing to copy */
	if (!request_cnt || !cnt)
		return 0;

	/* this function is called under trace/bpf_trace.c: bpf_event_mutex */
	prog = rcu_dereference_check(array, 1)->progs;
	return bpf_prog_array_copy_core(prog, prog_ids, request_cnt) ? -ENOSPC
								     : 0;
}


struct sock *do_sk_redirect_map(struct sk_buff *skb)
{
	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
	struct sock *sk = NULL;

	if (tcb->bpf.map) {
		sk = __sock_map_lookup_elem(tcb->bpf.map, tcb->bpf.key);

		tcb->bpf.key = 0;
		tcb->bpf.map = NULL;
	}

	return sk;
}


static int __reuseport_attach_prog(struct bpf_prog *prog, struct sock *sk)
{
	struct bpf_prog *old_prog;
	int err;

	if (bpf_prog_size(prog->len) > sysctl_optmem_max)
		return -ENOMEM;

	if (sk_unhashed(sk) && sk->sk_reuseport) {
		err = reuseport_alloc(sk);
		if (err)
			return err;
	} else if (!rcu_access_pointer(sk->sk_reuseport_cb)) {
		/* The socket wasn't bound with SO_REUSEPORT */
		return -EINVAL;
	}

	old_prog = reuseport_attach_prog(sk, prog);
	if (old_prog)
		bpf_prog_destroy(old_prog);

	return 0;
}


static u32 sk_msg_convert_ctx_access(enum bpf_access_type type,
				     const struct bpf_insn *si,
				     struct bpf_insn *insn_buf,
				     struct bpf_prog *prog, u32 *target_size)
{
	struct bpf_insn *insn = insn_buf;

	switch (si->off) {
	case offsetof(struct sk_msg_md, data):
		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg_buff, data),
				      si->dst_reg, si->src_reg,
				      offsetof(struct sk_msg_buff, data));
		break;
	case offsetof(struct sk_msg_md, data_end):
		*insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg_buff, data_end),
				      si->dst_reg, si->src_reg,
				      offsetof(struct sk_msg_buff, data_end));
		break;
	}

	return insn - insn_buf;
}


struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *prog)
{
	struct bpf_insn insn_buff[16], aux[2];
	struct bpf_prog *clone, *tmp;
	int insn_delta, insn_cnt;
	struct bpf_insn *insn;
	int i, rewritten;

	if (!bpf_jit_blinding_enabled(prog) || prog->blinded)
		return prog;

	clone = bpf_prog_clone_create(prog, GFP_USER);
	if (!clone)
		return ERR_PTR(-ENOMEM);

	insn_cnt = clone->len;
	insn = clone->insnsi;

	for (i = 0; i < insn_cnt; i++, insn++) {
		/* We temporarily need to hold the original ld64 insn
		 * so that we can still access the first part in the
		 * second blinding run.
		 */
		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW) &&
		    insn[1].code == 0)
			memcpy(aux, insn, sizeof(aux));

		rewritten = bpf_jit_blind_insn(insn, aux, insn_buff);
		if (!rewritten)
			continue;

		tmp = bpf_patch_insn_single(clone, i, insn_buff, rewritten);
		if (!tmp) {
			/* Patching may have repointed aux->prog during
			 * realloc from the original one, so we need to
			 * fix it up here on error.
			 */
			bpf_jit_prog_release_other(prog, clone);
			return ERR_PTR(-ENOMEM);
		}

		clone = tmp;
		insn_delta = rewritten - 1;

		/* Walk new program and skip insns we just inserted. */
		insn = clone->insnsi + i + insn_delta;
		insn_cnt += insn_delta;
		i        += insn_delta;
	}

	clone->blinded = 1;
	return clone;
}


static int bpf_skb_generic_pop(struct sk_buff *skb, u32 off, u32 len)
{
	/* skb_ensure_writable() is not needed here, as we're
	 * already working on an uncloned skb.
	 */
	if (unlikely(!pskb_may_pull(skb, off + len)))
		return -ENOMEM;

	skb_postpull_rcsum(skb, skb->data + off, len);
	memmove(skb->data + len, skb->data, off);
	__skb_pull(skb, len);

	return 0;
}


void xdp_do_flush_map(void)
{
	struct redirect_info *ri = this_cpu_ptr(&redirect_info);
	struct bpf_map *map = ri->map_to_flush;

	ri->map_to_flush = NULL;
	if (map) {
		switch (map->map_type) {
		case BPF_MAP_TYPE_DEVMAP:
			__dev_map_flush(map);
			break;
		case BPF_MAP_TYPE_CPUMAP:
			__cpu_map_flush(map);
			break;
		default:
			break;
		}
	}
}


static inline int max_evtchn_port(struct kvm *kvm)
{
	if (IS_ENABLED(CONFIG_64BIT) && kvm->arch.xen.long_mode)
		return EVTCHN_2L_NR_CHANNELS;
	else
		return COMPAT_EVTCHN_2L_NR_CHANNELS;
}


void kvm_arch_flush_shadow_all(struct kvm *kvm)
{
	kvm_mmu_zap_all(kvm);
}


void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0)
{
	if ((cr0 ^ old_cr0) & X86_CR0_PG) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_async_pf_hash_reset(vcpu);
	}

	if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS)
		kvm_mmu_reset_context(vcpu);

	if (((cr0 ^ old_cr0) & X86_CR0_CD) &&
	    kvm_arch_has_noncoherent_dma(vcpu->kvm) &&
	    !kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
		kvm_zap_gfn_range(vcpu->kvm, 0, ~0ULL);
}


static int kvm_write_guest_virt_helper(gva_t addr, void *val, unsigned int bytes,
				      struct kvm_vcpu *vcpu, u32 access,
				      struct x86_exception *exception)
{
	struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
	void *data = val;
	int r = X86EMUL_CONTINUE;

	while (bytes) {
		gpa_t gpa = mmu->gva_to_gpa(vcpu, mmu, addr, access, exception);
		unsigned offset = addr & (PAGE_SIZE-1);
		unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset);
		int ret;

		if (gpa == UNMAPPED_GVA)
			return X86EMUL_PROPAGATE_FAULT;
		ret = kvm_vcpu_write_guest(vcpu, gpa, data, towrite);
		if (ret < 0) {
			r = X86EMUL_IO_NEEDED;
			goto out;
		}

		bytes -= towrite;
		data += towrite;
		addr += towrite;
	}
out:
	return r;
}


static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt)
{
	kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt));
}


static int complete_fast_pio_out_port_0x7e(struct kvm_vcpu *vcpu)
{
	vcpu->arch.pio.count = 0;
	return 1;
}


int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
		    int reason, bool has_error_code, u32 error_code)
{
	struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt;
	int ret;

	init_emulate_ctxt(vcpu);

	ret = emulator_task_switch(ctxt, tss_selector, idt_index, reason,
				   has_error_code, error_code);
	if (ret) {
		vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
		vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
		vcpu->run->internal.ndata = 0;
		return 0;
	}

	kvm_rip_write(vcpu, ctxt->eip);
	kvm_set_rflags(vcpu, ctxt->eflags);
	return 1;
}


static bool emulator_guest_has_movbe(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_MOVBE);
}


int kvm_emulate_halt_noskip(struct kvm_vcpu *vcpu)
{
	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_HALTED, KVM_EXIT_HLT);
}


void kvm_xen_update_runstate_guest(struct kvm_vcpu *v, int state)
{
	struct kvm_vcpu_xen *vx = &v->arch.xen;
	uint64_t state_entry_time;
	unsigned int offset;

	kvm_xen_update_runstate(v, state);

	if (!vx->runstate_set)
		return;

	BUILD_BUG_ON(sizeof(struct compat_vcpu_runstate_info) != 0x2c);

	offset = offsetof(struct compat_vcpu_runstate_info, state_entry_time);
#ifdef CONFIG_X86_64
	/*
	 * The only difference is alignment of uint64_t in 32-bit.
	 * So the first field 'state' is accessed directly using
	 * offsetof() (where its offset happens to be zero), while the
	 * remaining fields which are all uint64_t, start at 'offset'
	 * which we tweak here by adding 4.
	 */
	BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state_entry_time) !=
		     offsetof(struct compat_vcpu_runstate_info, state_entry_time) + 4);
	BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, time) !=
		     offsetof(struct compat_vcpu_runstate_info, time) + 4);

	if (v->kvm->arch.xen.long_mode)
		offset = offsetof(struct vcpu_runstate_info, state_entry_time);
#endif
	/*
	 * First write the updated state_entry_time at the appropriate
	 * location determined by 'offset'.
	 */
	state_entry_time = vx->runstate_entry_time;
	state_entry_time |= XEN_RUNSTATE_UPDATE;

	BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, state_entry_time) !=
		     sizeof(state_entry_time));
	BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state_entry_time) !=
		     sizeof(state_entry_time));

	if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache,
					  &state_entry_time, offset,
					  sizeof(state_entry_time)))
		return;
	smp_wmb();

	/*
	 * Next, write the new runstate. This is in the *same* place
	 * for 32-bit and 64-bit guests, asserted here for paranoia.
	 */
	BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state) !=
		     offsetof(struct compat_vcpu_runstate_info, state));
	BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, state) !=
		     sizeof(vx->current_runstate));
	BUILD_BUG_ON(sizeof_field(struct compat_vcpu_runstate_info, state) !=
		     sizeof(vx->current_runstate));

	if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache,
					  &vx->current_runstate,
					  offsetof(struct vcpu_runstate_info, state),
					  sizeof(vx->current_runstate)))
		return;

	/*
	 * Write the actual runstate times immediately after the
	 * runstate_entry_time.
	 */
	BUILD_BUG_ON(offsetof(struct vcpu_runstate_info, state_entry_time) !=
		     offsetof(struct vcpu_runstate_info, time) - sizeof(u64));
	BUILD_BUG_ON(offsetof(struct compat_vcpu_runstate_info, state_entry_time) !=
		     offsetof(struct compat_vcpu_runstate_info, time) - sizeof(u64));
	BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) !=
		     sizeof_field(struct compat_vcpu_runstate_info, time));
	BUILD_BUG_ON(sizeof_field(struct vcpu_runstate_info, time) !=
		     sizeof(vx->runstate_times));

	if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache,
					  &vx->runstate_times[0],
					  offset + sizeof(u64),
					  sizeof(vx->runstate_times)))
		return;

	smp_wmb();

	/*
	 * Finally, clear the XEN_RUNSTATE_UPDATE bit in the guest's
	 * runstate_entry_time field.
	 */

	state_entry_time &= ~XEN_RUNSTATE_UPDATE;
	if (kvm_write_guest_offset_cached(v->kvm, &v->arch.xen.runstate_cache,
					  &state_entry_time, offset,
					  sizeof(state_entry_time)))
		return;
}


static bool emulator_guest_has_long_mode(struct x86_emulate_ctxt *ctxt)
{
	return guest_cpuid_has(emul_to_vcpu(ctxt), X86_FEATURE_LM);
}


static int kvm_ioctl_get_supported_hv_cpuid(struct kvm_vcpu *vcpu,
					    struct kvm_cpuid2 __user *cpuid_arg)
{
	struct kvm_cpuid2 cpuid;
	int r;

	r = -EFAULT;
	if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
		return r;

	r = kvm_get_hv_cpuid(vcpu, &cpuid, cpuid_arg->entries);
	if (r)
		return r;

	r = -EFAULT;
	if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
		return r;

	return 0;
}


static void emulator_get_idt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	static_call(kvm_x86_get_idt)(emul_to_vcpu(ctxt), dt);
}


static void __get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
	if (vcpu->arch.emulate_regs_need_sync_to_vcpu) {
		/*
		 * We are here if userspace calls get_regs() in the middle of
		 * instruction emulation. Registers state needs to be copied
		 * back from emulation context to vcpu. Userspace shouldn't do
		 * that usually, but some bad designed PV devices (vmware
		 * backdoor interface) need this to work
		 */
		emulator_writeback_register_cache(vcpu->arch.emulate_ctxt);
		vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
	}
	regs->rax = kvm_rax_read(vcpu);
	regs->rbx = kvm_rbx_read(vcpu);
	regs->rcx = kvm_rcx_read(vcpu);
	regs->rdx = kvm_rdx_read(vcpu);
	regs->rsi = kvm_rsi_read(vcpu);
	regs->rdi = kvm_rdi_read(vcpu);
	regs->rsp = kvm_rsp_read(vcpu);
	regs->rbp = kvm_rbp_read(vcpu);
#ifdef CONFIG_X86_64
	regs->r8 = kvm_r8_read(vcpu);
	regs->r9 = kvm_r9_read(vcpu);
	regs->r10 = kvm_r10_read(vcpu);
	regs->r11 = kvm_r11_read(vcpu);
	regs->r12 = kvm_r12_read(vcpu);
	regs->r13 = kvm_r13_read(vcpu);
	regs->r14 = kvm_r14_read(vcpu);
	regs->r15 = kvm_r15_read(vcpu);
#endif

	regs->rip = kvm_rip_read(vcpu);
	regs->rflags = kvm_get_rflags(vcpu);
}


static int kvm_get_msr_ignored_check(struct kvm_vcpu *vcpu,
				     u32 index, u64 *data, bool host_initiated)
{
	int ret = __kvm_get_msr(vcpu, index, data, host_initiated);

	if (ret == KVM_MSR_RET_INVALID) {
		/* Unconditionally clear *data for simplicity */
		*data = 0;
		if (kvm_msr_ignored_check(index, 0, false))
			ret = 0;
	}

	return ret;
}


void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
				 struct kvm_async_pf *work)
{
	struct kvm_lapic_irq irq = {
		.delivery_mode = APIC_DM_FIXED,
		.vector = vcpu->arch.apf.vec
	};

	if (work->wakeup_all)
		work->arch.token = ~0; /* broadcast wakeup */
	else
		kvm_del_async_pf_gfn(vcpu, work->arch.gfn);
	trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa);

	if ((work->wakeup_all || work->notpresent_injected) &&
	    kvm_pv_async_pf_enabled(vcpu) &&
	    !apf_put_user_ready(vcpu, work->arch.token)) {
		vcpu->arch.apf.pageready_pending = true;
		kvm_apic_set_irq(vcpu, &irq, NULL);
	}

	vcpu->arch.apf.halted = false;
	vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
}


static int kvm_arch_tsc_get_attr(struct kvm_vcpu *vcpu,
				 struct kvm_device_attr *attr)
{
	u64 __user *uaddr = (u64 __user *)(unsigned long)attr->addr;
	int r;

	if ((u64)(unsigned long)uaddr != attr->addr)
		return -EFAULT;

	switch (attr->attr) {
	case KVM_VCPU_TSC_OFFSET:
		r = -EFAULT;
		if (put_user(vcpu->arch.l1_tsc_offset, uaddr))
			break;
		r = 0;
		break;
	default:
		r = -ENXIO;
	}

	return r;
}


static s64 get_kvmclock_base_ns(void)
{
	/* Master clock not used, so we can just use CLOCK_BOOTTIME.  */
	return ktime_get_boottime_ns();
}


static void kvm_free_vcpus(struct kvm *kvm)
{
	unsigned long i;
	struct kvm_vcpu *vcpu;

	/*
	 * Unpin any mmu pages first.
	 */
	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_clear_async_pf_completion_queue(vcpu);
		kvm_unload_vcpu_mmu(vcpu);
	}

	kvm_destroy_vcpus(kvm);
}


int kvm_arch_init(void *opaque)
{
	struct kvm_x86_init_ops *ops = opaque;
	int r;

	if (kvm_x86_ops.hardware_enable) {
		pr_err("kvm: already loaded vendor module '%s'\n", kvm_x86_ops.name);
		r = -EEXIST;
		goto out;
	}

	if (!ops->cpu_has_kvm_support()) {
		pr_err_ratelimited("kvm: no hardware support for '%s'\n",
				   ops->runtime_ops->name);
		r = -EOPNOTSUPP;
		goto out;
	}
	if (ops->disabled_by_bios()) {
		pr_err_ratelimited("kvm: support for '%s' disabled by bios\n",
				   ops->runtime_ops->name);
		r = -EOPNOTSUPP;
		goto out;
	}

	/*
	 * KVM explicitly assumes that the guest has an FPU and
	 * FXSAVE/FXRSTOR. For example, the KVM_GET_FPU explicitly casts the
	 * vCPU's FPU state as a fxregs_state struct.
	 */
	if (!boot_cpu_has(X86_FEATURE_FPU) || !boot_cpu_has(X86_FEATURE_FXSR)) {
		printk(KERN_ERR "kvm: inadequate fpu\n");
		r = -EOPNOTSUPP;
		goto out;
	}

	r = -ENOMEM;

	x86_emulator_cache = kvm_alloc_emulator_cache();
	if (!x86_emulator_cache) {
		pr_err("kvm: failed to allocate cache for x86 emulator\n");
		goto out;
	}

	user_return_msrs = alloc_percpu(struct kvm_user_return_msrs);
	if (!user_return_msrs) {
		printk(KERN_ERR "kvm: failed to allocate percpu kvm_user_return_msrs\n");
		goto out_free_x86_emulator_cache;
	}
	kvm_nr_uret_msrs = 0;

	r = kvm_mmu_module_init();
	if (r)
		goto out_free_percpu;

	kvm_timer_init();

	perf_register_guest_info_callbacks(&kvm_guest_cbs);

	if (boot_cpu_has(X86_FEATURE_XSAVE)) {
		host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
		supported_xcr0 = host_xcr0 & KVM_SUPPORTED_XCR0;
	}

	if (pi_inject_timer == -1)
		pi_inject_timer = housekeeping_enabled(HK_FLAG_TIMER);
#ifdef CONFIG_X86_64
	pvclock_gtod_register_notifier(&pvclock_gtod_notifier);

	if (hypervisor_is_type(X86_HYPER_MS_HYPERV))
		set_hv_tscchange_cb(kvm_hyperv_tsc_notifier);
#endif

	return 0;

out_free_percpu:
	free_percpu(user_return_msrs);
out_free_x86_emulator_cache:
	kmem_cache_destroy(x86_emulator_cache);
out:
	return r;
}


static void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter)
{
	u32 i;

	if (!msr_filter)
		return;

	for (i = 0; i < msr_filter->count; i++)
		kfree(msr_filter->ranges[i].bitmap);

	kfree(msr_filter);
}


static int emulator_pio_out(struct kvm_vcpu *vcpu, int size,
			    unsigned short port, const void *val,
			    unsigned int count)
{
	int ret;

	memcpy(vcpu->arch.pio_data, val, size * count);
	trace_kvm_pio(KVM_PIO_OUT, port, size, count, vcpu->arch.pio_data);
	ret = emulator_pio_in_out(vcpu, size, port, count, false);
	if (ret)
                vcpu->arch.pio.count = 0;

        return ret;
}


static int emulator_set_msr(struct x86_emulate_ctxt *ctxt,
			    u32 msr_index, u64 data)
{
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	int r;

	r = kvm_set_msr(vcpu, msr_index, data);

	if (r && kvm_msr_user_space(vcpu, msr_index, KVM_EXIT_X86_WRMSR, data,
				    complete_emulated_msr_access, r)) {
		/* Bounce to user space */
		return X86EMUL_IO_NEEDED;
	}

	return r;
}


u64 get_kvmclock_ns(struct kvm *kvm)
{
	struct kvm_clock_data data;

	get_kvmclock(kvm, &data);
	return data.clock;
}


static void kvmclock_update_fn(struct work_struct *work)
{
	unsigned long i;
	struct delayed_work *dwork = to_delayed_work(work);
	struct kvm_arch *ka = container_of(dwork, struct kvm_arch,
					   kvmclock_update_work);
	struct kvm *kvm = container_of(ka, struct kvm, arch);
	struct kvm_vcpu *vcpu;

	kvm_for_each_vcpu(i, vcpu, kvm) {
		kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
		kvm_vcpu_kick(vcpu);
	}
}


int kvm_xen_write_hypercall_page(struct kvm_vcpu *vcpu, u64 data)
{
	struct kvm *kvm = vcpu->kvm;
	u32 page_num = data & ~PAGE_MASK;
	u64 page_addr = data & PAGE_MASK;
	bool lm = is_long_mode(vcpu);

	/* Latch long_mode for shared_info pages etc. */
	vcpu->kvm->arch.xen.long_mode = lm;

	/*
	 * If Xen hypercall intercept is enabled, fill the hypercall
	 * page with VMCALL/VMMCALL instructions since that's what
	 * we catch. Else the VMM has provided the hypercall pages
	 * with instructions of its own choosing, so use those.
	 */
	if (kvm_xen_hypercall_enabled(kvm)) {
		u8 instructions[32];
		int i;

		if (page_num)
			return 1;

		/* mov imm32, %eax */
		instructions[0] = 0xb8;

		/* vmcall / vmmcall */
		kvm_x86_ops.patch_hypercall(vcpu, instructions + 5);

		/* ret */
		instructions[8] = 0xc3;

		/* int3 to pad */
		memset(instructions + 9, 0xcc, sizeof(instructions) - 9);

		for (i = 0; i < PAGE_SIZE / sizeof(instructions); i++) {
			*(u32 *)&instructions[1] = i;
			if (kvm_vcpu_write_guest(vcpu,
						 page_addr + (i * sizeof(instructions)),
						 instructions, sizeof(instructions)))
				return 1;
		}
	} else {
		/*
		 * Note, truncation is a non-issue as 'lm' is guaranteed to be
		 * false for a 32-bit kernel, i.e. when hva_t is only 4 bytes.
		 */
		hva_t blob_addr = lm ? kvm->arch.xen_hvm_config.blob_addr_64
				     : kvm->arch.xen_hvm_config.blob_addr_32;
		u8 blob_size = lm ? kvm->arch.xen_hvm_config.blob_size_64
				  : kvm->arch.xen_hvm_config.blob_size_32;
		u8 *page;

		if (page_num >= blob_size)
			return 1;

		blob_addr += page_num * PAGE_SIZE;

		page = memdup_user((u8 __user *)blob_addr, PAGE_SIZE);
		if (IS_ERR(page))
			return PTR_ERR(page);

		if (kvm_vcpu_write_guest(vcpu, page_addr, page, PAGE_SIZE)) {
			kfree(page);
			return 1;
		}
	}
	return 0;
}


static void emulator_get_gdt(struct x86_emulate_ctxt *ctxt, struct desc_ptr *dt)
{
	static_call(kvm_x86_get_gdt)(emul_to_vcpu(ctxt), dt);
}


static void kvm_hyperv_tsc_notifier(void)
{
	struct kvm *kvm;
	int cpu;

	mutex_lock(&kvm_lock);
	list_for_each_entry(kvm, &vm_list, vm_list)
		kvm_make_mclock_inprogress_request(kvm);

	/* no guest entries from this point */
	hyperv_stop_tsc_emulation();

	/* TSC frequency always matches when on Hyper-V */
	for_each_present_cpu(cpu)
		per_cpu(cpu_tsc_khz, cpu) = tsc_khz;
	kvm_max_guest_tsc_khz = tsc_khz;

	list_for_each_entry(kvm, &vm_list, vm_list) {
		__kvm_start_pvclock_update(kvm);
		pvclock_update_vm_gtod_copy(kvm);
		kvm_end_pvclock_update(kvm);
	}

	mutex_unlock(&kvm_lock);
}


bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
{
	if (READ_ONCE(vcpu->arch.pv.pv_unhalted))
		return true;

	if (kvm_test_request(KVM_REQ_NMI, vcpu) ||
		kvm_test_request(KVM_REQ_SMI, vcpu) ||
		 kvm_test_request(KVM_REQ_EVENT, vcpu))
		return true;

	return kvm_arch_dy_has_pending_interrupt(vcpu);
}


static inline bool apf_pageready_slot_free(struct kvm_vcpu *vcpu)
{
	unsigned int offset = offsetof(struct kvm_vcpu_pv_apf_data, token);
	u32 val;

	if (kvm_read_guest_offset_cached(vcpu->kvm, &vcpu->arch.apf.data,
					 &val, offset, sizeof(val)))
		return false;

	return !val;
}


static int kvm_vm_ioctl_set_tss_addr(struct kvm *kvm, unsigned long addr)
{
	int ret;

	if (addr > (unsigned int)(-3 * PAGE_SIZE))
		return -EINVAL;
	ret = static_call(kvm_x86_set_tss_addr)(kvm, addr);
	return ret;
}


static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
				     unsigned long addr,
				     const void *old,
				     const void *new,
				     unsigned int bytes,
				     struct x86_exception *exception)
{
	struct kvm_host_map map;
	struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
	u64 page_line_mask;
	gpa_t gpa;
	char *kaddr;
	bool exchanged;

	/* guests cmpxchg8b have to be emulated atomically */
	if (bytes > 8 || (bytes & (bytes - 1)))
		goto emul_write;

	gpa = kvm_mmu_gva_to_gpa_write(vcpu, addr, NULL);

	if (gpa == UNMAPPED_GVA ||
	    (gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE)
		goto emul_write;

	/*
	 * Emulate the atomic as a straight write to avoid #AC if SLD is
	 * enabled in the host and the access splits a cache line.
	 */
	if (boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT))
		page_line_mask = ~(cache_line_size() - 1);
	else
		page_line_mask = PAGE_MASK;

	if (((gpa + bytes - 1) & page_line_mask) != (gpa & page_line_mask))
		goto emul_write;

	if (kvm_vcpu_map(vcpu, gpa_to_gfn(gpa), &map))
		goto emul_write;

	kaddr = map.hva + offset_in_page(gpa);

	switch (bytes) {
	case 1:
		exchanged = CMPXCHG_TYPE(u8, kaddr, old, new);
		break;
	case 2:
		exchanged = CMPXCHG_TYPE(u16, kaddr, old, new);
		break;
	case 4:
		exchanged = CMPXCHG_TYPE(u32, kaddr, old, new);
		break;
	case 8:
		exchanged = CMPXCHG64(kaddr, old, new);
		break;
	default:
		BUG();
	}

	kvm_vcpu_unmap(vcpu, &map, true);

	if (!exchanged)
		return X86EMUL_CMPXCHG_FAILED;

	kvm_page_track_write(vcpu, gpa, new, bytes);

	return X86EMUL_CONTINUE;

emul_write:
	printk_once(KERN_WARNING "kvm: emulating exchange as write\n");

	return emulator_write_emulated(ctxt, addr, new, bytes, exception);
}


static void record_steal_time(struct kvm_vcpu *vcpu)
{
	struct gfn_to_hva_cache *ghc = &vcpu->arch.st.cache;
	struct kvm_steal_time __user *st;
	struct kvm_memslots *slots;
	u64 steal;
	u32 version;

	if (kvm_xen_msr_enabled(vcpu->kvm)) {
		kvm_xen_runstate_set_running(vcpu);
		return;
	}

	if (!(vcpu->arch.st.msr_val & KVM_MSR_ENABLED))
		return;

	if (WARN_ON_ONCE(current->mm != vcpu->kvm->mm))
		return;

	slots = kvm_memslots(vcpu->kvm);

	if (unlikely(slots->generation != ghc->generation ||
		     kvm_is_error_hva(ghc->hva) || !ghc->memslot)) {
		gfn_t gfn = vcpu->arch.st.msr_val & KVM_STEAL_VALID_BITS;

		/* We rely on the fact that it fits in a single page. */
		BUILD_BUG_ON((sizeof(*st) - 1) & KVM_STEAL_VALID_BITS);

		if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gfn, sizeof(*st)) ||
		    kvm_is_error_hva(ghc->hva) || !ghc->memslot)
			return;
	}

	st = (struct kvm_steal_time __user *)ghc->hva;
	/*
	 * Doing a TLB flush here, on the guest's behalf, can avoid
	 * expensive IPIs.
	 */
	if (guest_pv_has(vcpu, KVM_FEATURE_PV_TLB_FLUSH)) {
		u8 st_preempted = 0;
		int err = -EFAULT;

		if (!user_access_begin(st, sizeof(*st)))
			return;

		asm volatile("1: xchgb %0, %2\n"
			     "xor %1, %1\n"
			     "2:\n"
			     _ASM_EXTABLE_UA(1b, 2b)
			     : "+q" (st_preempted),
			       "+&r" (err),
			       "+m" (st->preempted));
		if (err)
			goto out;

		user_access_end();

		vcpu->arch.st.preempted = 0;

		trace_kvm_pv_tlb_flush(vcpu->vcpu_id,
				       st_preempted & KVM_VCPU_FLUSH_TLB);
		if (st_preempted & KVM_VCPU_FLUSH_TLB)
			kvm_vcpu_flush_tlb_guest(vcpu);

		if (!user_access_begin(st, sizeof(*st)))
			goto dirty;
	} else {
		if (!user_access_begin(st, sizeof(*st)))
			return;

		unsafe_put_user(0, &st->preempted, out);
		vcpu->arch.st.preempted = 0;
	}

	unsafe_get_user(version, &st->version, out);
	if (version & 1)
		version += 1;  /* first time write, random junk */

	version += 1;
	unsafe_put_user(version, &st->version, out);

	smp_wmb();

	unsafe_get_user(steal, &st->steal, out);
	steal += current->sched_info.run_delay -
		vcpu->arch.st.last_steal;
	vcpu->arch.st.last_steal = current->sched_info.run_delay;
	unsafe_put_user(steal, &st->steal, out);

	version += 1;
	unsafe_put_user(version, &st->version, out);

 out:
	user_access_end();
 dirty:
	mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa));
}


static void emulator_get_dr(struct x86_emulate_ctxt *ctxt, int dr,
			    unsigned long *dest)
{
	kvm_get_dr(emul_to_vcpu(ctxt), dr, dest);
}


static void kvm_vcpu_flush_tlb_guest(struct kvm_vcpu *vcpu)
{
	++vcpu->stat.tlb_flush;

	if (!tdp_enabled) {
		/*
		 * A TLB flush on behalf of the guest is equivalent to
		 * INVPCID(all), toggling CR4.PGE, etc., which requires
		 * a forced sync of the shadow page tables.  Ensure all the
		 * roots are synced and the guest TLB in hardware is clean.
		 */
		kvm_mmu_sync_roots(vcpu);
		kvm_mmu_sync_prev_roots(vcpu);
	}

	static_call(kvm_x86_tlb_flush_guest)(vcpu);
}


int kvm_emulate_ap_reset_hold(struct kvm_vcpu *vcpu)
{
	int ret = kvm_skip_emulated_instruction(vcpu);

	return __kvm_emulate_halt(vcpu, KVM_MP_STATE_AP_RESET_HOLD,
					KVM_EXIT_AP_RESET_HOLD) && ret;
}


long kvm_arch_vcpu_ioctl(struct file *filp,
			 unsigned int ioctl, unsigned long arg)
{
	struct kvm_vcpu *vcpu = filp->private_data;
	void __user *argp = (void __user *)arg;
	int r;
	union {
		struct kvm_sregs2 *sregs2;
		struct kvm_lapic_state *lapic;
		struct kvm_xsave *xsave;
		struct kvm_xcrs *xcrs;
		void *buffer;
	} u;

	vcpu_load(vcpu);

	u.buffer = NULL;
	switch (ioctl) {
	case KVM_GET_LAPIC: {
		r = -EINVAL;
		if (!lapic_in_kernel(vcpu))
			goto out;
		u.lapic = kzalloc(sizeof(struct kvm_lapic_state),
				GFP_KERNEL_ACCOUNT);

		r = -ENOMEM;
		if (!u.lapic)
			goto out;
		r = kvm_vcpu_ioctl_get_lapic(vcpu, u.lapic);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, u.lapic, sizeof(struct kvm_lapic_state)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_LAPIC: {
		r = -EINVAL;
		if (!lapic_in_kernel(vcpu))
			goto out;
		u.lapic = memdup_user(argp, sizeof(*u.lapic));
		if (IS_ERR(u.lapic)) {
			r = PTR_ERR(u.lapic);
			goto out_nofree;
		}

		r = kvm_vcpu_ioctl_set_lapic(vcpu, u.lapic);
		break;
	}
	case KVM_INTERRUPT: {
		struct kvm_interrupt irq;

		r = -EFAULT;
		if (copy_from_user(&irq, argp, sizeof(irq)))
			goto out;
		r = kvm_vcpu_ioctl_interrupt(vcpu, &irq);
		break;
	}
	case KVM_NMI: {
		r = kvm_vcpu_ioctl_nmi(vcpu);
		break;
	}
	case KVM_SMI: {
		r = kvm_vcpu_ioctl_smi(vcpu);
		break;
	}
	case KVM_SET_CPUID: {
		struct kvm_cpuid __user *cpuid_arg = argp;
		struct kvm_cpuid cpuid;

		/*
		 * KVM does not correctly handle changing guest CPUID after KVM_RUN, as
		 * MAXPHYADDR, GBPAGES support, AMD reserved bit behavior, etc.. aren't
		 * tracked in kvm_mmu_page_role.  As a result, KVM may miss guest page
		 * faults due to reusing SPs/SPTEs.  In practice no sane VMM mucks with
		 * the core vCPU model on the fly, so fail.
		 */
		r = -EINVAL;
		if (vcpu->arch.last_vmentry_cpu != -1)
			goto out;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid(vcpu, &cpuid, cpuid_arg->entries);
		break;
	}
	case KVM_SET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		/*
		 * KVM_SET_CPUID{,2} after KVM_RUN is forbidded, see the comment in
		 * KVM_SET_CPUID case above.
		 */
		r = -EINVAL;
		if (vcpu->arch.last_vmentry_cpu != -1)
			goto out;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
			goto out;
		r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid,
					      cpuid_arg->entries);
		break;
	}
	case KVM_GET_CPUID2: {
		struct kvm_cpuid2 __user *cpuid_arg = argp;
		struct kvm_cpuid2 cpuid;

		r = -EFAULT;
		if (copy_from_user(&cpuid, cpuid_arg, sizeof(cpuid)))
			goto out;
		r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid,
					      cpuid_arg->entries);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(cpuid_arg, &cpuid, sizeof(cpuid)))
			goto out;
		r = 0;
		break;
	}
	case KVM_GET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
		r = msr_io(vcpu, argp, do_get_msr, 1);
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
		break;
	}
	case KVM_SET_MSRS: {
		int idx = srcu_read_lock(&vcpu->kvm->srcu);
		r = msr_io(vcpu, argp, do_set_msr, 0);
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
		break;
	}
	case KVM_TPR_ACCESS_REPORTING: {
		struct kvm_tpr_access_ctl tac;

		r = -EFAULT;
		if (copy_from_user(&tac, argp, sizeof(tac)))
			goto out;
		r = vcpu_ioctl_tpr_access_reporting(vcpu, &tac);
		if (r)
			goto out;
		r = -EFAULT;
		if (copy_to_user(argp, &tac, sizeof(tac)))
			goto out;
		r = 0;
		break;
	};
	case KVM_SET_VAPIC_ADDR: {
		struct kvm_vapic_addr va;
		int idx;

		r = -EINVAL;
		if (!lapic_in_kernel(vcpu))
			goto out;
		r = -EFAULT;
		if (copy_from_user(&va, argp, sizeof(va)))
			goto out;
		idx = srcu_read_lock(&vcpu->kvm->srcu);
		r = kvm_lapic_set_vapic_addr(vcpu, va.vapic_addr);
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
		break;
	}
	case KVM_X86_SETUP_MCE: {
		u64 mcg_cap;

		r = -EFAULT;
		if (copy_from_user(&mcg_cap, argp, sizeof(mcg_cap)))
			goto out;
		r = kvm_vcpu_ioctl_x86_setup_mce(vcpu, mcg_cap);
		break;
	}
	case KVM_X86_SET_MCE: {
		struct kvm_x86_mce mce;

		r = -EFAULT;
		if (copy_from_user(&mce, argp, sizeof(mce)))
			goto out;
		r = kvm_vcpu_ioctl_x86_set_mce(vcpu, &mce);
		break;
	}
	case KVM_GET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		kvm_vcpu_ioctl_x86_get_vcpu_events(vcpu, &events);

		r = -EFAULT;
		if (copy_to_user(argp, &events, sizeof(struct kvm_vcpu_events)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_VCPU_EVENTS: {
		struct kvm_vcpu_events events;

		r = -EFAULT;
		if (copy_from_user(&events, argp, sizeof(struct kvm_vcpu_events)))
			break;

		r = kvm_vcpu_ioctl_x86_set_vcpu_events(vcpu, &events);
		break;
	}
	case KVM_GET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		kvm_vcpu_ioctl_x86_get_debugregs(vcpu, &dbgregs);

		r = -EFAULT;
		if (copy_to_user(argp, &dbgregs,
				 sizeof(struct kvm_debugregs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_DEBUGREGS: {
		struct kvm_debugregs dbgregs;

		r = -EFAULT;
		if (copy_from_user(&dbgregs, argp,
				   sizeof(struct kvm_debugregs)))
			break;

		r = kvm_vcpu_ioctl_x86_set_debugregs(vcpu, &dbgregs);
		break;
	}
	case KVM_GET_XSAVE: {
		u.xsave = kzalloc(sizeof(struct kvm_xsave), GFP_KERNEL_ACCOUNT);
		r = -ENOMEM;
		if (!u.xsave)
			break;

		kvm_vcpu_ioctl_x86_get_xsave(vcpu, u.xsave);

		r = -EFAULT;
		if (copy_to_user(argp, u.xsave, sizeof(struct kvm_xsave)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XSAVE: {
		u.xsave = memdup_user(argp, sizeof(*u.xsave));
		if (IS_ERR(u.xsave)) {
			r = PTR_ERR(u.xsave);
			goto out_nofree;
		}

		r = kvm_vcpu_ioctl_x86_set_xsave(vcpu, u.xsave);
		break;
	}
	case KVM_GET_XCRS: {
		u.xcrs = kzalloc(sizeof(struct kvm_xcrs), GFP_KERNEL_ACCOUNT);
		r = -ENOMEM;
		if (!u.xcrs)
			break;

		kvm_vcpu_ioctl_x86_get_xcrs(vcpu, u.xcrs);

		r = -EFAULT;
		if (copy_to_user(argp, u.xcrs,
				 sizeof(struct kvm_xcrs)))
			break;
		r = 0;
		break;
	}
	case KVM_SET_XCRS: {
		u.xcrs = memdup_user(argp, sizeof(*u.xcrs));
		if (IS_ERR(u.xcrs)) {
			r = PTR_ERR(u.xcrs);
			goto out_nofree;
		}

		r = kvm_vcpu_ioctl_x86_set_xcrs(vcpu, u.xcrs);
		break;
	}
	case KVM_SET_TSC_KHZ: {
		u32 user_tsc_khz;

		r = -EINVAL;
		user_tsc_khz = (u32)arg;

		if (kvm_has_tsc_control &&
		    user_tsc_khz >= kvm_max_guest_tsc_khz)
			goto out;

		if (user_tsc_khz == 0)
			user_tsc_khz = tsc_khz;

		if (!kvm_set_tsc_khz(vcpu, user_tsc_khz))
			r = 0;

		goto out;
	}
	case KVM_GET_TSC_KHZ: {
		r = vcpu->arch.virtual_tsc_khz;
		goto out;
	}
	case KVM_KVMCLOCK_CTRL: {
		r = kvm_set_guest_paused(vcpu);
		goto out;
	}
	case KVM_ENABLE_CAP: {
		struct kvm_enable_cap cap;

		r = -EFAULT;
		if (copy_from_user(&cap, argp, sizeof(cap)))
			goto out;
		r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
		break;
	}
	case KVM_GET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		u32 user_data_size;

		r = -EINVAL;
		if (!kvm_x86_ops.nested_ops->get_state)
			break;

		BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size));
		r = -EFAULT;
		if (get_user(user_data_size, &user_kvm_nested_state->size))
			break;

		r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state,
						     user_data_size);
		if (r < 0)
			break;

		if (r > user_data_size) {
			if (put_user(r, &user_kvm_nested_state->size))
				r = -EFAULT;
			else
				r = -E2BIG;
			break;
		}

		r = 0;
		break;
	}
	case KVM_SET_NESTED_STATE: {
		struct kvm_nested_state __user *user_kvm_nested_state = argp;
		struct kvm_nested_state kvm_state;
		int idx;

		r = -EINVAL;
		if (!kvm_x86_ops.nested_ops->set_state)
			break;

		r = -EFAULT;
		if (copy_from_user(&kvm_state, user_kvm_nested_state, sizeof(kvm_state)))
			break;

		r = -EINVAL;
		if (kvm_state.size < sizeof(kvm_state))
			break;

		if (kvm_state.flags &
		    ~(KVM_STATE_NESTED_RUN_PENDING | KVM_STATE_NESTED_GUEST_MODE
		      | KVM_STATE_NESTED_EVMCS | KVM_STATE_NESTED_MTF_PENDING
		      | KVM_STATE_NESTED_GIF_SET))
			break;

		/* nested_run_pending implies guest_mode.  */
		if ((kvm_state.flags & KVM_STATE_NESTED_RUN_PENDING)
		    && !(kvm_state.flags & KVM_STATE_NESTED_GUEST_MODE))
			break;

		idx = srcu_read_lock(&vcpu->kvm->srcu);
		r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state);
		srcu_read_unlock(&vcpu->kvm->srcu, idx);
		break;
	}
	case KVM_GET_SUPPORTED_HV_CPUID:
		r = kvm_ioctl_get_supported_hv_cpuid(vcpu, argp);
		break;
#ifdef CONFIG_KVM_XEN
	case KVM_XEN_VCPU_GET_ATTR: {
		struct kvm_xen_vcpu_attr xva;

		r = -EFAULT;
		if (copy_from_user(&xva, argp, sizeof(xva)))
			goto out;
		r = kvm_xen_vcpu_get_attr(vcpu, &xva);
		if (!r && copy_to_user(argp, &xva, sizeof(xva)))
			r = -EFAULT;
		break;
	}
	case KVM_XEN_VCPU_SET_ATTR: {
		struct kvm_xen_vcpu_attr xva;

		r = -EFAULT;
		if (copy_from_user(&xva, argp, sizeof(xva)))
			goto out;
		r = kvm_xen_vcpu_set_attr(vcpu, &xva);
		break;
	}
#endif
	case KVM_GET_SREGS2: {
		u.sregs2 = kzalloc(sizeof(struct kvm_sregs2), GFP_KERNEL);
		r = -ENOMEM;
		if (!u.sregs2)
			goto out;
		__get_sregs2(vcpu, u.sregs2);
		r = -EFAULT;
		if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2)))
			goto out;
		r = 0;
		break;
	}
	case KVM_SET_SREGS2: {
		u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2));
		if (IS_ERR(u.sregs2)) {
			r = PTR_ERR(u.sregs2);
			u.sregs2 = NULL;
			goto out;
		}
		r = __set_sregs2(vcpu, u.sregs2);
		break;
	}
	case KVM_HAS_DEVICE_ATTR:
	case KVM_GET_DEVICE_ATTR:
	case KVM_SET_DEVICE_ATTR:
		r = kvm_vcpu_ioctl_device_attr(vcpu, ioctl, argp);
		break;
	default:
		r = -EINVAL;
	}
out:
	kfree(u.buffer);
out_nofree:
	vcpu_put(vcpu);
	return r;
}


static int unix_stream_sendmsg(struct socket *sock, struct msghdr *msg,
			       size_t len)
{
	struct sock *sk = sock->sk;
	struct sock *other = NULL;
	int err, size;
	struct sk_buff *skb;
	int sent = 0;
	struct scm_cookie scm;
	bool fds_sent = false;
	int data_len;

	wait_for_unix_gc();
	err = scm_send(sock, msg, &scm, false);
	if (err < 0)
		return err;

	err = -EOPNOTSUPP;
	if (msg->msg_flags&MSG_OOB)
		goto out_err;

	if (msg->msg_namelen) {
		err = sk->sk_state == TCP_ESTABLISHED ? -EISCONN : -EOPNOTSUPP;
		goto out_err;
	} else {
		err = -ENOTCONN;
		other = unix_peer(sk);
		if (!other)
			goto out_err;
	}

	if (sk->sk_shutdown & SEND_SHUTDOWN)
		goto pipe_err;

	while (sent < len) {
		size = len - sent;

		/* Keep two messages in the pipe so it schedules better */
		size = min_t(int, size, (sk->sk_sndbuf >> 1) - 64);

		/* allow fallback to order-0 allocations */
		size = min_t(int, size, SKB_MAX_HEAD(0) + UNIX_SKB_FRAGS_SZ);

		data_len = max_t(int, 0, size - SKB_MAX_HEAD(0));

		data_len = min_t(size_t, size, PAGE_ALIGN(data_len));

		skb = sock_alloc_send_pskb(sk, size - data_len, data_len,
					   msg->msg_flags & MSG_DONTWAIT, &err,
					   get_order(UNIX_SKB_FRAGS_SZ));
		if (!skb)
			goto out_err;

		/* Only send the fds in the first buffer */
		err = unix_scm_to_skb(&scm, skb, !fds_sent);
		if (err < 0) {
			kfree_skb(skb);
			goto out_err;
		}
		fds_sent = true;

		skb_put(skb, size - data_len);
		skb->data_len = data_len;
		skb->len = size;
		err = skb_copy_datagram_from_iter(skb, 0, &msg->msg_iter, size);
		if (err) {
			kfree_skb(skb);
			goto out_err;
		}

		unix_state_lock(other);

		if (sock_flag(other, SOCK_DEAD) ||
		    (other->sk_shutdown & RCV_SHUTDOWN))
			goto pipe_err_free;

		maybe_add_creds(skb, sock, other);
		scm_stat_add(other, skb);
		skb_queue_tail(&other->sk_receive_queue, skb);
		unix_state_unlock(other);
		other->sk_data_ready(other);
		sent += size;
	}

	scm_destroy(&scm);

	return sent;

pipe_err_free:
	unix_state_unlock(other);
	kfree_skb(skb);
pipe_err:
	if (sent == 0 && !(msg->msg_flags&MSG_NOSIGNAL))
		send_sig(SIGPIPE, current, 0);
	err = -EPIPE;
out_err:
	scm_destroy(&scm);
	return sent ? : err;
}


static struct sock *unix_find_other(struct net *net,
				    struct sockaddr_un *sunname, int len,
				    int type, unsigned int hash, int *error)
{
	struct sock *u;
	struct path path;
	int err = 0;

	if (sunname->sun_path[0]) {
		struct inode *inode;
		err = kern_path(sunname->sun_path, LOOKUP_FOLLOW, &path);
		if (err)
			goto fail;
		inode = d_backing_inode(path.dentry);
		err = path_permission(&path, MAY_WRITE);
		if (err)
			goto put_fail;

		err = -ECONNREFUSED;
		if (!S_ISSOCK(inode->i_mode))
			goto put_fail;
		u = unix_find_socket_byinode(inode);
		if (!u)
			goto put_fail;

		if (u->sk_type == type)
			touch_atime(&path);

		path_put(&path);

		err = -EPROTOTYPE;
		if (u->sk_type != type) {
			sock_put(u);
			goto fail;
		}
	} else {
		err = -ECONNREFUSED;
		u = unix_find_socket_byname(net, sunname, len, type ^ hash);
		if (u) {
			struct dentry *dentry;
			dentry = unix_sk(u)->path.dentry;
			if (dentry)
				touch_atime(&unix_sk(u)->path);
		} else
			goto fail;
	}
	return u;

put_fail:
	path_put(&path);
fail:
	*error = err;
	return NULL;
}


static int unix_stream_recvmsg(struct socket *sock, struct msghdr *msg,
			       size_t size, int flags)
{
	struct unix_stream_read_state state = {
		.recv_actor = unix_stream_read_actor,
		.socket = sock,
		.msg = msg,
		.size = size,
		.flags = flags
	};

	return unix_stream_read_generic(&state, true);
}


static int unix_seq_show(struct seq_file *seq, void *v)
{

	if (v == SEQ_START_TOKEN)
		seq_puts(seq, "Num       RefCount Protocol Flags    Type St "
			 "Inode Path\n");
	else {
		struct sock *s = v;
		struct unix_sock *u = unix_sk(s);
		unix_state_lock(s);

		seq_printf(seq, "%pK: %08X %08X %08X %04X %02X %5lu",
			s,
			refcount_read(&s->sk_refcnt),
			0,
			s->sk_state == TCP_LISTEN ? __SO_ACCEPTCON : 0,
			s->sk_type,
			s->sk_socket ?
			(s->sk_state == TCP_ESTABLISHED ? SS_CONNECTED : SS_UNCONNECTED) :
			(s->sk_state == TCP_ESTABLISHED ? SS_CONNECTING : SS_DISCONNECTING),
			sock_i_ino(s));

		if (u->addr) {	// under unix_table_lock here
			int i, len;
			seq_putc(seq, ' ');

			i = 0;
			len = u->addr->len - sizeof(short);
			if (!UNIX_ABSTRACT(s))
				len--;
			else {
				seq_putc(seq, '@');
				i++;
			}
			for ( ; i < len; i++)
				seq_putc(seq, u->addr->name->sun_path[i] ?:
					 '@');
		}
		unix_state_unlock(s);
		seq_putc(seq, '\n');
	}

	return 0;
}


	__releases(&unix_sk(other)->lock)
{
	struct unix_sock *u = unix_sk(other);
	int sched;
	DEFINE_WAIT(wait);

	prepare_to_wait_exclusive(&u->peer_wait, &wait, TASK_INTERRUPTIBLE);

	sched = !sock_flag(other, SOCK_DEAD) &&
		!(other->sk_shutdown & RCV_SHUTDOWN) &&
		unix_recvq_full(other);

	unix_state_unlock(other);

	if (sched)
		timeo = schedule_timeout(timeo);

	finish_wait(&u->peer_wait, &wait);
	return timeo;
}


static void unix_sock_destructor(struct sock *sk)
{
	struct unix_sock *u = unix_sk(sk);

	skb_queue_purge(&sk->sk_receive_queue);

	WARN_ON(refcount_read(&sk->sk_wmem_alloc));
	WARN_ON(!sk_unhashed(sk));
	WARN_ON(sk->sk_socket);
	if (!sock_flag(sk, SOCK_DEAD)) {
		pr_info("Attempt to release alive unix socket: %p\n", sk);
		return;
	}

	if (u->addr)
		unix_release_addr(u->addr);

	atomic_long_dec(&unix_nr_socks);
	local_bh_disable();
	sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
	local_bh_enable();
#ifdef UNIX_REFCNT_DEBUG
	pr_debug("UNIX %p is destroyed, %ld are still alive.\n", sk,
		atomic_long_read(&unix_nr_socks));
#endif
}


static int reencrypt_add_moved_segment(struct luks2_hdr *hdr, struct luks2_reencrypt *rh)
{
	int s = LUKS2_segment_first_unused_id(hdr);

	if (!rh->jobj_segment_moved)
		return 0;

	if (s < 0)
		return s;

	if (json_object_object_add_by_uint(LUKS2_get_segments_jobj(hdr), s, json_object_get(rh->jobj_segment_moved))) {
		json_object_put(rh->jobj_segment_moved);
		return -EINVAL;
	}

	return 0;
}


static int _activate_loopaes(struct crypt_device *cd,
	const char *name,
	char *buffer,
	size_t buffer_size,
	uint32_t flags)
{
	int r;
	unsigned int key_count = 0;
	struct volume_key *vk = NULL;

	r = LOOPAES_parse_keyfile(cd, &vk, cd->u.loopaes.hdr.hash, &key_count,
				  buffer, buffer_size);

	if (!r && name)
		r = LOOPAES_activate(cd, name, cd->u.loopaes.cipher, key_count,
				     vk, flags);

	crypt_free_volume_key(vk);

	return r;
}


static json_object *reencrypt_make_post_segments_backward(struct crypt_device *cd,
	struct luks2_hdr *hdr,
	struct luks2_reencrypt *rh,
	uint64_t data_offset)
{
	int reenc_seg;
	uint64_t fixed_length;

	json_object *jobj_new_seg_after, *jobj_old_seg,
		    *jobj_segs_post = json_object_new_object();

	if (!rh->jobj_segs_hot || !jobj_segs_post)
		goto err;

	reenc_seg = json_segments_segment_in_reencrypt(rh->jobj_segs_hot);
	if (reenc_seg < 0)
		return NULL;

	jobj_old_seg = json_segments_get_segment(rh->jobj_segs_hot, reenc_seg - 1);
	if (jobj_old_seg)
		json_object_object_add_by_uint(jobj_segs_post, reenc_seg - 1, json_object_get(jobj_old_seg));
	if (rh->fixed_length && rh->offset) {
		fixed_length = rh->device_size - rh->offset;
		jobj_new_seg_after = reencrypt_make_segment_new(cd, hdr, rh, data_offset, rh->offset, rh->offset, &fixed_length);
	} else
		jobj_new_seg_after = reencrypt_make_segment_new(cd, hdr, rh, data_offset, rh->offset, rh->offset, NULL);
	if (!jobj_new_seg_after)
		goto err;
	json_object_object_add_by_uint(jobj_segs_post, reenc_seg, jobj_new_seg_after);

	return jobj_segs_post;
err:
	json_object_put(jobj_segs_post);
	return NULL;
}


void *crypt_get_hdr(struct crypt_device *cd, const char *type)
{
	/* If requested type differs, ignore it */
	if (strcmp(cd->type, type))
		return NULL;

	if (isPLAIN(cd->type))
		return &cd->u.plain;

	if (isLUKS1(cd->type))
		return &cd->u.luks1.hdr;

	if (isLUKS2(cd->type))
		return &cd->u.luks2.hdr;

	if (isLOOPAES(cd->type))
		return &cd->u.loopaes;

	if (isVERITY(cd->type))
		return &cd->u.verity;

	if (isTCRYPT(cd->type))
		return &cd->u.tcrypt;

	return NULL;
}


static json_object *reencrypt_make_hot_segments_forward(struct crypt_device *cd,
		struct luks2_hdr *hdr,
		struct luks2_reencrypt *rh,
		uint64_t device_size,
		uint64_t data_offset)
{
	json_object *jobj_segs_hot, *jobj_reenc_seg, *jobj_old_seg, *jobj_new_seg;
	uint64_t fixed_length, tmp = rh->offset + rh->length;
	unsigned int sg = 0;

	jobj_segs_hot = json_object_new_object();
	if (!jobj_segs_hot)
		return NULL;

	if (rh->offset) {
		jobj_new_seg = reencrypt_make_segment_new(cd, hdr, rh, data_offset, 0, 0, &rh->offset);
		if (!jobj_new_seg)
			goto err;
		json_object_object_add_by_uint(jobj_segs_hot, sg++, jobj_new_seg);
	}

	jobj_reenc_seg = reencrypt_make_segment_reencrypt(cd, hdr, rh, data_offset, rh->offset, rh->offset, &rh->length);
	if (!jobj_reenc_seg)
		goto err;

	json_object_object_add_by_uint(jobj_segs_hot, sg++, jobj_reenc_seg);

	if (tmp < device_size) {
		fixed_length = device_size - tmp;
		jobj_old_seg = reencrypt_make_segment_old(cd, hdr, rh, data_offset + rh->data_shift, rh->offset + rh->length, rh->fixed_length ? &fixed_length : NULL);
		if (!jobj_old_seg)
			goto err;
		json_object_object_add_by_uint(jobj_segs_hot, sg, jobj_old_seg);
	}

	return jobj_segs_hot;
err:
	json_object_put(jobj_segs_hot);
	return NULL;
}


static int _check_header_data_overlap(struct crypt_device *cd, const char *name)
{
	if (!name || !isLUKS(cd->type))
		return 0;

	if (device_is_identical(crypt_data_device(cd), crypt_metadata_device(cd)) <= 0)
		return 0;

	/* FIXME: check real header size */
	if (crypt_get_data_offset(cd) == 0) {
		log_err(cd, _("Device header overlaps with data area."));
		return -EINVAL;
	}

	return 0;
}


int crypt_resume_by_volume_key(struct crypt_device *cd,
	const char *name,
	const char *volume_key,
	size_t volume_key_size)
{
	struct volume_key *vk = NULL;
	int r;

	if (!name || !volume_key)
		return -EINVAL;

	log_dbg(cd, "Resuming volume %s by volume key.", name);

	if ((r = onlyLUKS(cd)))
		return r;

	r = dm_status_suspended(cd, name);
	if (r < 0)
		return r;

	if (!r) {
		log_err(cd, _("Volume %s is not suspended."), name);
		return -EINVAL;
	}

	vk = crypt_alloc_volume_key(volume_key_size, volume_key);
	if (!vk)
		return -ENOMEM;

	if (isLUKS1(cd->type))
		r = LUKS_verify_volume_key(&cd->u.luks1.hdr, vk);
	else if (isLUKS2(cd->type))
		r = LUKS2_digest_verify_by_segment(cd, &cd->u.luks2.hdr, CRYPT_DEFAULT_SEGMENT, vk);
	else
		r = -EINVAL;
	if (r == -EPERM || r == -ENOENT)
		log_err(cd, _("Volume key does not match the volume."));

	if (r >= 0)
		r = resume_by_volume_key(cd, vk, name);

	crypt_free_volume_key(vk);
	return r;
}


uint64_t crypt_get_iv_offset(struct crypt_device *cd)
{
	if (!cd)
		return 0;

	if (isPLAIN(cd->type))
		return cd->u.plain.hdr.skip;

	if (isLOOPAES(cd->type))
		return cd->u.loopaes.hdr.skip;

	if (isTCRYPT(cd->type))
		return TCRYPT_get_iv_offset(cd, &cd->u.tcrypt.hdr, &cd->u.tcrypt.params);

	return 0;
}


static void crypt_set_null_type(struct crypt_device *cd)
{
	if (!cd->type)
		return;

	free(cd->type);
	cd->type = NULL;
	cd->u.none.active_name = NULL;
	cd->data_offset = 0;
	cd->metadata_size = 0;
	cd->keyslots_size = 0;
}


int LUKS2_keyslot_wipe(struct crypt_device *cd,
	struct luks2_hdr *hdr,
	int keyslot,
	int wipe_area_only)
{
	struct device *device = crypt_metadata_device(cd);
	uint64_t area_offset, area_length;
	int r;
	json_object *jobj_keyslot, *jobj_keyslots;
	const keyslot_handler *h;

	h = LUKS2_keyslot_handler(cd, keyslot);

	if (!json_object_object_get_ex(hdr->jobj, "keyslots", &jobj_keyslots))
		return -EINVAL;

	jobj_keyslot = LUKS2_get_keyslot_jobj(hdr, keyslot);
	if (!jobj_keyslot)
		return -ENOENT;

	if (wipe_area_only)
		log_dbg(cd, "Wiping keyslot %d area only.", keyslot);

	r = LUKS2_device_write_lock(cd, hdr, device);
	if (r)
		return r;

	/* secure deletion of possible key material in keyslot area */
	r = crypt_keyslot_area(cd, keyslot, &area_offset, &area_length);
	if (r && r != -ENOENT)
		goto out;

	if (!r) {
		r = crypt_wipe_device(cd, device, CRYPT_WIPE_SPECIAL, area_offset,
			      area_length, area_length, NULL, NULL);
		if (r) {
			if (r == -EACCES) {
				log_err(cd, _("Cannot write to device %s, permission denied."),
					device_path(device));
				r = -EINVAL;
			} else
				log_err(cd, _("Cannot wipe device %s."), device_path(device));
			goto out;
		}
	}

	if (wipe_area_only)
		goto out;

	/* Slot specific wipe */
	if (h) {
		r = h->wipe(cd, keyslot);
		if (r < 0)
			goto out;
	} else
		log_dbg(cd, "Wiping keyslot %d without specific-slot handler loaded.", keyslot);

	json_object_object_del_by_uint(jobj_keyslots, keyslot);

	r = LUKS2_hdr_write(cd, hdr);
out:
	device_write_unlock(cd, crypt_metadata_device(cd));
	return r;
}


const char *crypt_get_uuid(struct crypt_device *cd)
{
	if (!cd)
		return NULL;

	if (isLUKS1(cd->type))
		return cd->u.luks1.hdr.uuid;

	if (isLUKS2(cd->type))
		return cd->u.luks2.hdr.uuid;

	if (isVERITY(cd->type))
		return cd->u.verity.uuid;

	if (isBITLK(cd->type))
		return cd->u.bitlk.params.guid;

	return NULL;
}


static int reencrypt_lock_internal(struct crypt_device *cd, const char *uuid, struct crypt_lock_handle **reencrypt_lock)
{
	int r;
	char *lock_resource;

	if (!crypt_metadata_locking_enabled()) {
		*reencrypt_lock = NULL;
		return 0;
	}

	r = asprintf(&lock_resource, "LUKS2-reencryption-%s", uuid);
	if (r < 0)
		return -ENOMEM;
	if (r < 20) {
		free(lock_resource);
		return -EINVAL;
	}

	r = crypt_write_lock(cd, lock_resource, false, reencrypt_lock);

	free(lock_resource);

	return r;
}


int crypt_get_integrity_key_size(struct crypt_device *cd)
{
	if (isINTEGRITY(cd->type))
		return INTEGRITY_key_size(cd, crypt_get_integrity(cd));

	if (isLUKS2(cd->type))
		return INTEGRITY_key_size(cd, crypt_get_integrity(cd));

	return 0;
}


crypt_token_info crypt_token_status(struct crypt_device *cd, int token, const char **type)
{
	if (_onlyLUKS2(cd, CRYPT_CD_QUIET | CRYPT_CD_UNRESTRICTED, 0))
		return CRYPT_TOKEN_INVALID;

	return LUKS2_token_status(cd, &cd->u.luks2.hdr, token, type);
}


static size_t segment_crypt_serialize(json_object *jobj_segment, uint8_t *buffer)
{
	struct jtype j[] = {
		{ JSTR, jobj_segment, "type" },
		{ JU64, jobj_segment, "offset" },
		{ JX64, jobj_segment, "size" },
		{ JU64, jobj_segment, "iv_tweak" },
		{ JSTR, jobj_segment, "encryption" },
		{ JU32, jobj_segment, "sector_size" },
		{}
	};
	return srs(j, buffer);
}


const char *crypt_get_type(struct crypt_device *cd)
{
	return cd ? cd->type : NULL;
}


int LUKS2_reencrypt_lock_by_dm_uuid(struct crypt_device *cd, const char *dm_uuid,
	struct crypt_lock_handle **reencrypt_lock)
{
	int r;
	char hdr_uuid[37];
	const char *uuid = crypt_get_uuid(cd);

	if (!dm_uuid)
		return -EINVAL;

	if (!uuid) {
		r = snprintf(hdr_uuid, sizeof(hdr_uuid), "%.8s-%.4s-%.4s-%.4s-%.12s",
			 dm_uuid + 6, dm_uuid + 14, dm_uuid + 18, dm_uuid + 22, dm_uuid + 26);
		if (r < 0 || (size_t)r != (sizeof(hdr_uuid) - 1))
			return -EINVAL;
	} else if (crypt_uuid_cmp(dm_uuid, uuid))
		return -EINVAL;

	return reencrypt_lock_internal(cd, uuid, reencrypt_lock);
}


static int _crypt_load_luks2(struct crypt_device *cd, int reload, int repair)
{
	int r;
	char *type = NULL;
	struct luks2_hdr hdr2 = {};

	log_dbg(cd, "%soading LUKS2 header (repair %sabled).", reload ? "Rel" : "L", repair ? "en" : "dis");

	r = LUKS2_hdr_read(cd, &hdr2, repair);
	if (r)
		return r;

	if (!reload && !(type = strdup(CRYPT_LUKS2))) {
		r = -ENOMEM;
		goto out;
	}

	if (verify_pbkdf_params(cd, &cd->pbkdf)) {
		r = init_pbkdf_type(cd, NULL, CRYPT_LUKS2);
		if (r)
			goto out;
	}

	if (reload) {
		LUKS2_hdr_free(cd, &cd->u.luks2.hdr);
		free(cd->u.luks2.keyslot_cipher);
	} else
		cd->type = type;

	r = 0;
	memcpy(&cd->u.luks2.hdr, &hdr2, sizeof(hdr2));
	cd->u.luks2.keyslot_cipher = NULL;
	cd->u.luks2.rh = NULL;

out:
	if (r) {
		free(type);
		LUKS2_hdr_free(cd, &hdr2);
	}
	return r;
}


static size_t backup_segments_serialize(struct luks2_hdr *hdr, uint8_t *buffer)
{
	json_object *jobj_segment;
	size_t l, len = 0;

	jobj_segment = LUKS2_get_segment_by_flag(hdr, "backup-previous");
	if (!jobj_segment || !(l = segment_serialize(jobj_segment, buffer)))
		return 0;
	len += l;
	if (buffer)
		buffer += l;

	jobj_segment = LUKS2_get_segment_by_flag(hdr, "backup-final");
	if (!jobj_segment || !(l = segment_serialize(jobj_segment, buffer)))
		return 0;
	len += l;
	if (buffer)
		buffer += l;

	jobj_segment = LUKS2_get_segment_by_flag(hdr, "backup-moved-segment");
	if (jobj_segment) {
		if (!(l = segment_serialize(jobj_segment, buffer)))
			return 0;
		len += l;
	}

	return len;
}


int crypt_get_debug_level(void)
{
	return _debug_level;
}


static int isBITLK(const char *type)
{
	return (type && !strcmp(CRYPT_BITLK, type));
}


int crypt_keyslot_add_by_key(struct crypt_device *cd,
	int keyslot,
	const char *volume_key,
	size_t volume_key_size,
	const char *passphrase,
	size_t passphrase_size,
	uint32_t flags)
{
	int digest, r;
	struct luks2_keyslot_params params;
	struct volume_key *vk = NULL;

	if (!passphrase || ((flags & CRYPT_VOLUME_KEY_NO_SEGMENT) &&
			    (flags & CRYPT_VOLUME_KEY_SET)))
		return -EINVAL;

	log_dbg(cd, "Adding new keyslot %d with volume key %sassigned to a crypt segment.",
		keyslot, flags & CRYPT_VOLUME_KEY_NO_SEGMENT ? "un" : "");

	if ((r = onlyLUKS2(cd)))
		return r;

	/* new volume key assignment */
	if ((flags & CRYPT_VOLUME_KEY_SET) && crypt_keyslot_status(cd, keyslot) > CRYPT_SLOT_INACTIVE)
		return verify_and_update_segment_digest(cd, &cd->u.luks2.hdr,
			keyslot, volume_key, volume_key_size, passphrase, passphrase_size);

	r = keyslot_verify_or_find_empty(cd, &keyslot);
	if (r < 0)
		return r;

	if (volume_key)
		vk = crypt_alloc_volume_key(volume_key_size, volume_key);
	else if (flags & CRYPT_VOLUME_KEY_NO_SEGMENT)
		vk = crypt_generate_volume_key(cd, volume_key_size);
	else if (cd->volume_key)
		vk = crypt_alloc_volume_key(cd->volume_key->keylength, cd->volume_key->key);
	else
		return -EINVAL;

	if (!vk)
		return -ENOMEM;

	/* if key matches volume key digest tear down new vk flag */
	digest = LUKS2_digest_verify_by_segment(cd, &cd->u.luks2.hdr, CRYPT_DEFAULT_SEGMENT, vk);
	if (digest >= 0)
		flags &= ~CRYPT_VOLUME_KEY_SET;

	/* if key matches any existing digest, do not create new digest */
	if (digest < 0 && (flags & CRYPT_VOLUME_KEY_DIGEST_REUSE))
		digest = LUKS2_digest_any_matching(cd, &cd->u.luks2.hdr, vk);

	/* no segment flag or new vk flag requires new key digest */
	if (flags & (CRYPT_VOLUME_KEY_NO_SEGMENT | CRYPT_VOLUME_KEY_SET)) {
		if (digest < 0 || !(flags & CRYPT_VOLUME_KEY_DIGEST_REUSE))
			digest = LUKS2_digest_create(cd, "pbkdf2", &cd->u.luks2.hdr, vk);
	}

	r = digest;
	if (r < 0) {
		log_err(cd, _("Volume key does not match the volume."));
		goto out;
	}

	r = LUKS2_keyslot_params_default(cd, &cd->u.luks2.hdr, &params);
	if (r < 0) {
		log_err(cd, _("Failed to initialize default LUKS2 keyslot parameters."));
		goto out;
	}

	r = LUKS2_digest_assign(cd, &cd->u.luks2.hdr, keyslot, digest, 1, 0);
	if (r < 0) {
		log_err(cd, _("Failed to assign keyslot %d to digest."), keyslot);
		goto out;
	}

	r = LUKS2_keyslot_store(cd, &cd->u.luks2.hdr, keyslot,
				passphrase, passphrase_size, vk, &params);

	if (r >= 0 && (flags & CRYPT_VOLUME_KEY_SET))
		r = update_volume_key_segment_digest(cd, &cd->u.luks2.hdr, digest, 1);
out:
	crypt_free_volume_key(vk);
	if (r < 0) {
		_luks2_reload(cd);
		return r;
	}
	return keyslot;
}


static json_object *reencrypt_make_segment_new(struct crypt_device *cd,
		struct luks2_hdr *hdr,
		const struct luks2_reencrypt *rh,
		uint64_t data_offset,
		uint64_t segment_offset,
		uint64_t iv_offset,
		const uint64_t *segment_length)
{
	switch (rh->mode) {
	case CRYPT_REENCRYPT_REENCRYPT:
	case CRYPT_REENCRYPT_ENCRYPT:
		return json_segment_create_crypt(data_offset + segment_offset,
						  crypt_get_iv_offset(cd) + (iv_offset >> SECTOR_SHIFT),
						  segment_length,
						  reencrypt_segment_cipher_new(hdr),
						  reencrypt_get_sector_size_new(hdr), 0);
	case CRYPT_REENCRYPT_DECRYPT:
		return json_segment_create_linear(data_offset + segment_offset, segment_length, 0);
	}

	return NULL;
}


static void crypt_free_type(struct crypt_device *cd)
{
	if (isPLAIN(cd->type)) {
		free(CONST_CAST(void*)cd->u.plain.hdr.hash);
		free(cd->u.plain.cipher);
		free(cd->u.plain.cipher_spec);
	} else if (isLUKS2(cd->type)) {
		LUKS2_reencrypt_free(cd, cd->u.luks2.rh);
		LUKS2_hdr_free(cd, &cd->u.luks2.hdr);
		free(cd->u.luks2.keyslot_cipher);
	} else if (isLUKS1(cd->type)) {
		free(cd->u.luks1.cipher_spec);
	} else if (isLOOPAES(cd->type)) {
		free(CONST_CAST(void*)cd->u.loopaes.hdr.hash);
		free(cd->u.loopaes.cipher);
		free(cd->u.loopaes.cipher_spec);
	} else if (isVERITY(cd->type)) {
		free(CONST_CAST(void*)cd->u.verity.hdr.hash_name);
		free(CONST_CAST(void*)cd->u.verity.hdr.data_device);
		free(CONST_CAST(void*)cd->u.verity.hdr.hash_device);
		free(CONST_CAST(void*)cd->u.verity.hdr.fec_device);
		free(CONST_CAST(void*)cd->u.verity.hdr.salt);
		free(CONST_CAST(void*)cd->u.verity.root_hash);
		free(cd->u.verity.uuid);
		device_free(cd, cd->u.verity.fec_device);
	} else if (isINTEGRITY(cd->type)) {
		free(CONST_CAST(void*)cd->u.integrity.params.integrity);
		free(CONST_CAST(void*)cd->u.integrity.params.journal_integrity);
		free(CONST_CAST(void*)cd->u.integrity.params.journal_crypt);
		crypt_free_volume_key(cd->u.integrity.journal_crypt_key);
		crypt_free_volume_key(cd->u.integrity.journal_mac_key);
	} else if (isBITLK(cd->type)) {
		free(cd->u.bitlk.cipher_spec);
		BITLK_bitlk_metadata_free(&cd->u.bitlk.params);
	} else if (!cd->type) {
		free(cd->u.none.active_name);
		cd->u.none.active_name = NULL;
	}

	crypt_set_null_type(cd);
}


static int verify_and_update_segment_digest(struct crypt_device *cd,
		struct luks2_hdr *hdr, int keyslot,
		const char *volume_key, size_t volume_key_size,
		const char *password, size_t password_size)
{
	int digest, r;
	struct volume_key *vk = NULL;

	if (keyslot < 0 || (volume_key && !volume_key_size))
		return -EINVAL;

	if (volume_key)
		vk = crypt_alloc_volume_key(volume_key_size, volume_key);
	else {
		r = LUKS2_keyslot_open(cd, keyslot, CRYPT_ANY_SEGMENT, password, password_size, &vk);
		if (r != keyslot) {
			r = -EINVAL;
			goto out;
		}
	}

	if (!vk)
		return -ENOMEM;

	/* check volume_key (param) digest matches keyslot digest */
	r = LUKS2_digest_verify(cd, hdr, vk, keyslot);
	if (r < 0)
		goto out;
	digest = r;

	/* nothing to do, volume key in keyslot is already assigned to default segment */
	r = LUKS2_digest_verify_by_segment(cd, hdr, CRYPT_DEFAULT_SEGMENT, vk);
	if (r >= 0)
		goto out;

	/* FIXME: check new volume key is usable with current default segment */

	r = update_volume_key_segment_digest(cd, &cd->u.luks2.hdr, digest, 1);
	if (r)
		log_err(cd, _("Failed to assign keyslot %u as the new volume key."), keyslot);
out:
	crypt_free_volume_key(vk);
	return r < 0 ? r : keyslot;
}


int crypt_deactivate_by_name(struct crypt_device *cd, const char *name, uint32_t flags)
{
	struct crypt_device *fake_cd = NULL;
	struct luks2_hdr *hdr2 = NULL;
	struct crypt_dm_active_device dmd = {};
	int r;
	uint32_t get_flags = DM_ACTIVE_DEVICE | DM_ACTIVE_UUID | DM_ACTIVE_HOLDERS;

	if (!name)
		return -EINVAL;

	if ((flags & CRYPT_DEACTIVATE_DEFERRED) && (flags & CRYPT_DEACTIVATE_DEFERRED_CANCEL))
		return -EINVAL;

	log_dbg(cd, "Deactivating volume %s.", name);

	if (!cd) {
		r = crypt_init_by_name(&fake_cd, name);
		if (r < 0)
			return r;
		cd = fake_cd;
	}

	/* skip holders detection and early abort when some flags raised */
	if (flags & (CRYPT_DEACTIVATE_FORCE | CRYPT_DEACTIVATE_DEFERRED | CRYPT_DEACTIVATE_DEFERRED_CANCEL))
		get_flags &= ~DM_ACTIVE_HOLDERS;

	switch (crypt_status(cd, name)) {
		case CRYPT_ACTIVE:
		case CRYPT_BUSY:
			if (flags & CRYPT_DEACTIVATE_DEFERRED_CANCEL) {
				r = dm_cancel_deferred_removal(name);
				if (r < 0)
					log_err(cd, _("Could not cancel deferred remove from device %s."), name);
				break;
			}

			r = dm_query_device(cd, name, get_flags, &dmd);
			if (r >= 0) {
				if (dmd.holders) {
					log_err(cd, _("Device %s is still in use."), name);
					r = -EBUSY;
					break;
				}
			}

			if (isLUKS2(cd->type))
				hdr2 = crypt_get_hdr(cd, CRYPT_LUKS2);

			if ((dmd.uuid && !strncmp(CRYPT_LUKS2, dmd.uuid, sizeof(CRYPT_LUKS2)-1)) || hdr2)
				r = LUKS2_deactivate(cd, name, hdr2, &dmd, flags);
			else if (isTCRYPT(cd->type))
				r = TCRYPT_deactivate(cd, name, flags);
			else
				r = dm_remove_device(cd, name, flags);
			if (r < 0 && crypt_status(cd, name) == CRYPT_BUSY) {
				log_err(cd, _("Device %s is still in use."), name);
				r = -EBUSY;
			}
			break;
		case CRYPT_INACTIVE:
			log_err(cd, _("Device %s is not active."), name);
			r = -ENODEV;
			break;
		default:
			log_err(cd, _("Invalid device %s."), name);
			r = -EINVAL;
	}

	dm_targets_free(cd, &dmd);
	free(CONST_CAST(void*)dmd.uuid);
	crypt_free(fake_cd);

	return r;
}


static uint64_t reencrypt_data_shift(struct luks2_hdr *hdr)
{
	json_object *jobj_keyslot, *jobj_area, *jobj_data_shift;
	int ks = LUKS2_find_keyslot(hdr, "reencrypt");

	if (ks < 0)
		return 0;

	jobj_keyslot = LUKS2_get_keyslot_jobj(hdr, ks);

	json_object_object_get_ex(jobj_keyslot, "area", &jobj_area);
	if (!json_object_object_get_ex(jobj_area, "shift_size", &jobj_data_shift))
		return 0;

	return crypt_jobj_get_uint64(jobj_data_shift);
}


static int reencrypt_recover_segment(struct crypt_device *cd,
	struct luks2_hdr *hdr,
	struct luks2_reencrypt *rh,
	struct volume_key *vks)
{
	struct volume_key *vk_old, *vk_new;
	size_t count, s;
	ssize_t read, w;
	unsigned resilience;
	uint64_t area_offset, area_length, area_length_read, crash_iv_offset,
		 data_offset = crypt_get_data_offset(cd) << SECTOR_SHIFT;
	int devfd, r, new_sector_size, old_sector_size, rseg = json_segments_segment_in_reencrypt(rh->jobj_segs_hot);
	char *checksum_tmp = NULL, *data_buffer = NULL;
	struct crypt_storage_wrapper *cw1 = NULL, *cw2 = NULL;

	resilience = rh->rp.type;

	if (rseg < 0 || rh->length < 512)
		return -EINVAL;

	vk_new = crypt_volume_key_by_id(vks, rh->digest_new);
	if (!vk_new && rh->mode != CRYPT_REENCRYPT_DECRYPT)
		return -EINVAL;
	vk_old = crypt_volume_key_by_id(vks, rh->digest_old);
	if (!vk_old && rh->mode != CRYPT_REENCRYPT_ENCRYPT)
		return -EINVAL;
	old_sector_size = json_segment_get_sector_size(reencrypt_segment_old(hdr));
	new_sector_size = json_segment_get_sector_size(reencrypt_segment_new(hdr));
	if (rh->mode == CRYPT_REENCRYPT_DECRYPT)
		crash_iv_offset = rh->offset >> SECTOR_SHIFT; /* TODO: + old iv_tweak */
	else
		crash_iv_offset = json_segment_get_iv_offset(json_segments_get_segment(rh->jobj_segs_hot, rseg));

	log_dbg(cd, "crash_offset: %" PRIu64 ", crash_length: %" PRIu64 ",  crash_iv_offset: %" PRIu64, data_offset + rh->offset, rh->length, crash_iv_offset);

	r = crypt_storage_wrapper_init(cd, &cw2, crypt_data_device(cd),
			data_offset + rh->offset, crash_iv_offset, new_sector_size,
			reencrypt_segment_cipher_new(hdr), vk_new, 0);
	if (r) {
		log_err(cd, _("Failed to initialize new segment storage wrapper."));
		return r;
	}

	if (LUKS2_keyslot_area(hdr, rh->reenc_keyslot, &area_offset, &area_length)) {
		r = -EINVAL;
		goto out;
	}

	if (posix_memalign((void**)&data_buffer, device_alignment(crypt_data_device(cd)), rh->length)) {
		r = -ENOMEM;
		goto out;
	}

	switch (resilience) {
	case  REENC_PROTECTION_CHECKSUM:
		log_dbg(cd, "Checksums based recovery.");

		r = crypt_storage_wrapper_init(cd, &cw1, crypt_data_device(cd),
				data_offset + rh->offset, crash_iv_offset, old_sector_size,
				reencrypt_segment_cipher_old(hdr), vk_old, 0);
		if (r) {
			log_err(cd, _("Failed to initialize old segment storage wrapper."));
			goto out;
		}

		count = rh->length / rh->alignment;
		area_length_read = count * rh->rp.p.csum.hash_size;
		if (area_length_read > area_length) {
			log_dbg(cd, "Internal error in calculated area_length.");
			r = -EINVAL;
			goto out;
		}

		checksum_tmp = malloc(rh->rp.p.csum.hash_size);
		if (!checksum_tmp) {
			r = -ENOMEM;
			goto out;
		}

		/* TODO: lock for read */
		devfd = device_open(cd, crypt_metadata_device(cd), O_RDONLY);
		if (devfd < 0)
			goto out;

		/* read old data checksums */
		read = read_lseek_blockwise(devfd, device_block_size(cd, crypt_metadata_device(cd)),
					device_alignment(crypt_metadata_device(cd)), rh->rp.p.csum.checksums, area_length_read, area_offset);
		if (read < 0 || (size_t)read != area_length_read) {
			log_err(cd, _("Failed to read checksums for current hotzone."));
			r = -EINVAL;
			goto out;
		}

		read = crypt_storage_wrapper_read(cw2, 0, data_buffer, rh->length);
		if (read < 0 || (size_t)read != rh->length) {
			log_err(cd, _("Failed to read hotzone area starting at %" PRIu64 "."), rh->offset + data_offset);
			r = -EINVAL;
			goto out;
		}

		for (s = 0; s < count; s++) {
			if (crypt_hash_write(rh->rp.p.csum.ch, data_buffer + (s * rh->alignment), rh->alignment)) {
				log_dbg(cd, "Failed to write hash.");
				r = EINVAL;
				goto out;
			}
			if (crypt_hash_final(rh->rp.p.csum.ch, checksum_tmp, rh->rp.p.csum.hash_size)) {
				log_dbg(cd, "Failed to finalize hash.");
				r = EINVAL;
				goto out;
			}
			if (!memcmp(checksum_tmp, (char *)rh->rp.p.csum.checksums + (s * rh->rp.p.csum.hash_size), rh->rp.p.csum.hash_size)) {
				log_dbg(cd, "Sector %zu (size %zu, offset %zu) needs recovery", s, rh->alignment, s * rh->alignment);
				if (crypt_storage_wrapper_decrypt(cw1, s * rh->alignment, data_buffer + (s * rh->alignment), rh->alignment)) {
					log_err(cd, _("Failed to decrypt sector %zu."), s);
					r = -EINVAL;
					goto out;
				}
				w = crypt_storage_wrapper_encrypt_write(cw2, s * rh->alignment, data_buffer + (s * rh->alignment), rh->alignment);
				if (w < 0 || (size_t)w != rh->alignment) {
					log_err(cd, _("Failed to recover sector %zu."), s);
					r = -EINVAL;
					goto out;
				}
			}
		}

		r = 0;
		break;
	case  REENC_PROTECTION_JOURNAL:
		log_dbg(cd, "Journal based recovery.");

		/* FIXME: validation candidate */
		if (rh->length > area_length) {
			r = -EINVAL;
			log_dbg(cd, "Invalid journal size.");
			goto out;
		}

		/* TODO locking */
		r = crypt_storage_wrapper_init(cd, &cw1, crypt_metadata_device(cd),
				area_offset, crash_iv_offset, old_sector_size,
				reencrypt_segment_cipher_old(hdr), vk_old, 0);
		if (r) {
			log_err(cd, _("Failed to initialize old segment storage wrapper."));
			goto out;
		}
		read = crypt_storage_wrapper_read_decrypt(cw1, 0, data_buffer, rh->length);
		if (read < 0 || (size_t)read != rh->length) {
			log_dbg(cd, "Failed to read journaled data.");
			r = -EIO;
			/* may content plaintext */
			crypt_safe_memzero(data_buffer, rh->length);
			goto out;
		}
		read = crypt_storage_wrapper_encrypt_write(cw2, 0, data_buffer, rh->length);
		/* may content plaintext */
		crypt_safe_memzero(data_buffer, rh->length);
		if (read < 0 || (size_t)read != rh->length) {
			log_dbg(cd, "recovery write failed.");
			r = -EINVAL;
			goto out;
		}

		r = 0;
		break;
	case  REENC_PROTECTION_DATASHIFT:
		log_dbg(cd, "Data shift based recovery.");

		if (rseg == 0) {
			r = crypt_storage_wrapper_init(cd, &cw1, crypt_data_device(cd),
					json_segment_get_offset(rh->jobj_segment_moved, 0), 0, 0,
					reencrypt_segment_cipher_old(hdr), NULL, 0);
		} else
			r = crypt_storage_wrapper_init(cd, &cw1, crypt_data_device(cd),
					data_offset + rh->offset - rh->data_shift, 0, 0,
					reencrypt_segment_cipher_old(hdr), NULL, 0);
		if (r) {
			log_err(cd, _("Failed to initialize old segment storage wrapper."));
			goto out;
		}

		read = crypt_storage_wrapper_read_decrypt(cw1, 0, data_buffer, rh->length);
		if (read < 0 || (size_t)read != rh->length) {
			log_dbg(cd, "Failed to read data.");
			r = -EIO;
			/* may content plaintext */
			crypt_safe_memzero(data_buffer, rh->length);
			goto out;
		}

		read = crypt_storage_wrapper_encrypt_write(cw2, 0, data_buffer, rh->length);
		/* may content plaintext */
		crypt_safe_memzero(data_buffer, rh->length);
		if (read < 0 || (size_t)read != rh->length) {
			log_dbg(cd, "recovery write failed.");
			r = -EINVAL;
			goto out;
		}
		r = 0;
		break;
	default:
		r = -EINVAL;
	}

	if (!r)
		rh->read = rh->length;
out:
	free(data_buffer);
	free(checksum_tmp);
	crypt_storage_wrapper_destroy(cw1);
	crypt_storage_wrapper_destroy(cw2);

	return r;
}


static size_t segment_linear_serialize(json_object *jobj_segment, uint8_t *buffer)
{
	struct jtype j[] = {
		{ JSTR, jobj_segment, "type" },
		{ JU64, jobj_segment, "offset" },
		{ JX64, jobj_segment, "size" },
		{}
	};
	return srs(j, buffer);
}


int crypt_dump(struct crypt_device *cd)
{
	if (!cd)
		return -EINVAL;
	if (isLUKS1(cd->type))
		return _luks_dump(cd);
	else if (isLUKS2(cd->type))
		return LUKS2_hdr_dump(cd, &cd->u.luks2.hdr);
	else if (isVERITY(cd->type))
		return _verity_dump(cd);
	else if (isTCRYPT(cd->type))
		return TCRYPT_dump(cd, &cd->u.tcrypt.hdr, &cd->u.tcrypt.params);
	else if (isINTEGRITY(cd->type))
		return INTEGRITY_dump(cd, crypt_data_device(cd), 0);
	else if (isBITLK(cd->type))
		return BITLK_dump(cd, crypt_data_device(cd), &cd->u.bitlk.params);

	log_err(cd, _("Dump operation is not supported for this device type."));
	return -EINVAL;
}


int crypt_volume_key_load_in_keyring(struct crypt_device *cd, struct volume_key *vk)
{
	int r;
	const char *type_name = key_type_name(LOGON_KEY);

	if (!vk || !cd || !type_name)
		return -EINVAL;

	if (!vk->key_description) {
		log_dbg(cd, "Invalid key description");
		return -EINVAL;
	}

	log_dbg(cd, "Loading key (%zu bytes, type %s) in thread keyring.", vk->keylength, type_name);

	r = keyring_add_key_in_thread_keyring(LOGON_KEY, vk->key_description, vk->key, vk->keylength);
	if (r) {
		log_dbg(cd, "keyring_add_key_in_thread_keyring failed (error %d)", r);
		log_err(cd, _("Failed to load key in kernel keyring."));
	} else
		crypt_set_key_in_keyring(cd, 1);

	return r;
}


int LUKS2_keyslots_validate(struct crypt_device *cd, json_object *hdr_jobj)
{
	const keyslot_handler *h;
	int keyslot;
	json_object *jobj_keyslots, *jobj_type;

	if (!json_object_object_get_ex(hdr_jobj, "keyslots", &jobj_keyslots))
		return -EINVAL;

	json_object_object_foreach(jobj_keyslots, slot, val) {
		keyslot = atoi(slot);
		json_object_object_get_ex(val, "type", &jobj_type);
		h = LUKS2_keyslot_handler_type(cd, json_object_get_string(jobj_type));
		if (!h)
			continue;
		if (h->validate && h->validate(cd, val)) {
			log_dbg(cd, "Keyslot type %s validation failed on keyslot %d.", h->name, keyslot);
			return -EINVAL;
		}

		if (!strcmp(h->name, "luks2") && LUKS2_get_keyslot_digests_count(hdr_jobj, keyslot) != 1) {
			log_dbg(cd, "Keyslot %d is not assigned to exactly 1 digest.", keyslot);
			return -EINVAL;
		}
	}

	return 0;
}


static int isLUKS(const char *type)
{
	return (isLUKS2(type) || isLUKS1(type));
}


int LUKS2_reencrypt_lock(struct crypt_device *cd, struct crypt_lock_handle **reencrypt_lock)
{
	if (!cd || !crypt_get_type(cd) || strcmp(crypt_get_type(cd), CRYPT_LUKS2))
		return -EINVAL;

	return reencrypt_lock_internal(cd, crypt_get_uuid(cd), reencrypt_lock);
}


static int LUKS2_keyslot_open_priority(struct crypt_device *cd,
	struct luks2_hdr *hdr,
	crypt_keyslot_priority priority,
	const char *password,
	size_t password_len,
	int segment,
	struct volume_key **vk)
{
	json_object *jobj_keyslots, *jobj;
	crypt_keyslot_priority slot_priority;
	int keyslot, r = -ENOENT;

	json_object_object_get_ex(hdr->jobj, "keyslots", &jobj_keyslots);

	json_object_object_foreach(jobj_keyslots, slot, val) {
		if (!json_object_object_get_ex(val, "priority", &jobj))
			slot_priority = CRYPT_SLOT_PRIORITY_NORMAL;
		else
			slot_priority = json_object_get_int(jobj);

		keyslot = atoi(slot);
		if (slot_priority != priority) {
			log_dbg(cd, "Keyslot %d priority %d != %d (required), skipped.",
				keyslot, slot_priority, priority);
			continue;
		}

		r = LUKS2_open_and_verify(cd, hdr, keyslot, segment, password, password_len, vk);

		/* Do not retry for errors that are no -EPERM or -ENOENT,
		   former meaning password wrong, latter key slot unusable for segment */
		if ((r != -EPERM) && (r != -ENOENT))
			break;
	}

	return r;
}


static int dmcrypt_keyring_bug(void)
{
	uint64_t kversion;

	if (kernel_version(&kversion))
		return 1;
	return kversion < compact_version(4,15,0,0);
}


static json_object *reencrypt_make_segment_old(struct crypt_device *cd,
		struct luks2_hdr *hdr,
		const struct luks2_reencrypt *rh,
		uint64_t data_offset,
		uint64_t segment_offset,
		const uint64_t *segment_length)
{
	json_object *jobj_old_seg = NULL;

	switch (rh->mode) {
	case CRYPT_REENCRYPT_REENCRYPT:
	case CRYPT_REENCRYPT_DECRYPT:
		jobj_old_seg = json_segment_create_crypt(data_offset + segment_offset,
						    crypt_get_iv_offset(cd) + (segment_offset >> SECTOR_SHIFT),
						    segment_length,
						    reencrypt_segment_cipher_old(hdr),
						    reencrypt_get_sector_size_old(hdr),
						    0);
		break;
	case CRYPT_REENCRYPT_ENCRYPT:
		jobj_old_seg = json_segment_create_linear(data_offset + segment_offset, segment_length, 0);
	}

	return jobj_old_seg;
}


static int onlyLUKS(struct crypt_device *cd)
{
	return _onlyLUKS(cd, 0);
}


static size_t std_opcode_args_count(const RzBinDwarfLineHeader *hdr, ut8 opcode) {
	if (!opcode || opcode > hdr->opcode_base - 1 || !hdr->std_opcode_lengths) {
		return 0;
	}
	return hdr->std_opcode_lengths[opcode - 1];
}


RzBinSection *getsection(RzBinFile *binfile, const char *sn) {
	rz_return_val_if_fail(binfile && sn, NULL);
	RzListIter *iter;
	RzBinSection *section = NULL;
	RzBinObject *o = binfile->o;
	if (!o || !o->sections) {
		return NULL;
	}
	rz_list_foreach (o->sections, iter, section) {
		if (!section->name) {
			continue;
		}
		if (strstr(section->name, sn)) {
			return section;
		}
	}
	return NULL;
}


static const ut8 *parse_line_header(
	RzBinFile *bf, const ut8 *buf, const ut8 *buf_end,
	RzBinDwarfLineHeader *hdr, ut64 offset_cur, bool big_endian) {
	rz_return_val_if_fail(hdr && bf && buf && buf_end, NULL);

	hdr->offset = offset_cur;
	hdr->is_64bit = false;
	hdr->unit_length = dwarf_read_initial_length(&hdr->is_64bit, big_endian, &buf, buf_end);
	hdr->version = READ16(buf);

	if (hdr->version == 5) {
		hdr->address_size = READ8(buf);
		hdr->segment_selector_size = READ8(buf);
	}

	hdr->header_length = dwarf_read_offset(hdr->is_64bit, big_endian, &buf, buf_end);

	const ut8 *tmp_buf = buf; // So I can skip parsing DWARF 5 headers for now

	if (buf_end - buf < 8) {
		return NULL;
	}
	hdr->min_inst_len = READ8(buf);
	if (hdr->version >= 4) {
		hdr->max_ops_per_inst = READ8(buf);
	}
	hdr->default_is_stmt = READ8(buf);
	hdr->line_base = (st8)READ8(buf);
	hdr->line_range = READ8(buf);
	hdr->opcode_base = READ8(buf);

	hdr->file_names = NULL;

	if (hdr->opcode_base > 1) {
		hdr->std_opcode_lengths = calloc(sizeof(ut8), hdr->opcode_base - 1);
		for (size_t i = 1; i < hdr->opcode_base; i++) {
			if (buf + 2 > buf_end) {
				hdr->opcode_base = i;
				break;
			}
			hdr->std_opcode_lengths[i - 1] = READ8(buf);
		}
	} else {
		hdr->std_opcode_lengths = NULL;
	}
	// TODO finish parsing of source files out of DWARF 5 header
	// for now we skip
	if (hdr->version == 5) {
		tmp_buf += hdr->header_length;
		return tmp_buf;
	}

	if (hdr->version <= 4) {
		buf = parse_line_header_source(bf, buf, buf_end, hdr);
	} else {
		buf = NULL;
	}

	return buf;
}


RZ_API RzBinDwarfLineInfo *rz_bin_dwarf_parse_line(RzBinFile *binfile, RZ_NULLABLE RzBinDwarfDebugInfo *info, RzBinDwarfLineInfoMask mask) {
	rz_return_val_if_fail(binfile, NULL);
	RzBinSection *section = getsection(binfile, "debug_line");
	if (!section) {
		return NULL;
	}
	ut64 len = section->size;
	if (len < 1) {
		return NULL;
	}
	ut8 *buf = RZ_NEWS0(ut8, len + 1);
	if (!buf) {
		return NULL;
	}
	int ret = rz_buf_read_at(binfile->buf, section->paddr, buf, len);
	if (ret != len) {
		free(buf);
		return NULL;
	}
	// Actually parse the section
	RzBinDwarfLineInfo *r = parse_line_raw(binfile, buf, len, mask, binfile->o && binfile->o->info && binfile->o->info->big_endian, info);
	free(buf);
	return r;
}


static int expand_debug_abbrev(RzBinDwarfDebugAbbrev *da) {
	RzBinDwarfAbbrevDecl *tmp;

	if (!da || da->capacity == 0 || da->capacity != da->count) {
		return -EINVAL;
	}

	tmp = (RzBinDwarfAbbrevDecl *)realloc(da->decls,
		da->capacity * 2 * sizeof(RzBinDwarfAbbrevDecl));

	if (!tmp) {
		return -ENOMEM;
	}
	memset((ut8 *)tmp + da->capacity * sizeof(RzBinDwarfAbbrevDecl),
		0, da->capacity * sizeof(RzBinDwarfAbbrevDecl));

	da->decls = tmp;
	da->capacity *= 2;

	return 0;
}


static int expand_cu(RzBinDwarfCompUnit *cu) {
	RzBinDwarfDie *tmp;

	if (!cu || cu->capacity == 0 || cu->capacity != cu->count) {
		return -EINVAL;
	}

	tmp = (RzBinDwarfDie *)realloc(cu->dies,
		cu->capacity * 2 * sizeof(RzBinDwarfDie));
	if (!tmp) {
		return -ENOMEM;
	}

	memset((ut8 *)tmp + cu->capacity * sizeof(RzBinDwarfDie),
		0, cu->capacity * sizeof(RzBinDwarfDie));
	cu->dies = tmp;
	cu->capacity *= 2;

	return 0;
}


static void free_loc_table_list(RzBinDwarfLocList *loc_list) {
	RzListIter *iter;
	RzBinDwarfLocRange *range;
	rz_list_foreach (loc_list->list, iter, range) {
		free(range->expression->data);
		free(range->expression);
		free(range);
	}
	rz_list_free(loc_list->list);
	free(loc_list);
}


RZ_API const char *rz_bin_dwarf_get_attr_form_name(ut64 form_code) {
	if (form_code < DW_FORM_addr || form_code > DW_FORM_addrx4) {
		return NULL;
	}
	return dwarf_attr_form_encodings[form_code];
}


RZ_API RzBinDwarfLineFileCache rz_bin_dwarf_line_header_new_file_cache(const RzBinDwarfLineHeader *hdr) {
	return RZ_NEWS0(char *, hdr->file_names_count);
}


static int abbrev_cmp(const void *a, const void *b) {
	const RzBinDwarfAbbrevDecl *first = a;
	const RzBinDwarfAbbrevDecl *second = b;

	if (first->offset > second->offset) {
		return 1;
	} else if (first->offset < second->offset) {
		return -1;
	} else {
		return 0;
	}
}


static void free_ht_comp_dir(HtUPKv *kv) {
	free(kv->value);
}


static inline ut64 dwarf_read_address(size_t size, bool big_endian, const ut8 **buf, const ut8 *buf_end) {
	ut64 result;
	switch (size) {
	case 2:
		result = READ16(*buf);
		break;
	case 4:
		result = READ32(*buf);
		break;
	case 8:
		result = READ64(*buf);
		break;
	default:
		result = 0;
		*buf += size;
		eprintf("Weird dwarf address size: %zu.", size);
	}
	return result;
}


libxlDomainStartRestore(libxlDriverPrivate *driver,
                        virDomainObj *vm,
                        bool start_paused,
                        int restore_fd,
                        uint32_t restore_ver)
{
    return libxlDomainStart(driver, vm, start_paused,
                            restore_fd, restore_ver);
}


libxlDomainDefPostParse(virDomainDef *def,
                        unsigned int parseFlags G_GNUC_UNUSED,
                        void *opaque G_GNUC_UNUSED,
                        void *parseOpaque G_GNUC_UNUSED)
{
    /* Xen PV domains always have a PV console, so add one to the domain config
     * via post-parse callback if not explicitly specified in the XML. */
    if (def->os.type != VIR_DOMAIN_OSTYPE_HVM && def->nconsoles == 0) {
        virDomainChrDef *chrdef;

        if (!(chrdef = virDomainChrDefNew(NULL)))
            return -1;

        chrdef->source->type = VIR_DOMAIN_CHR_TYPE_PTY;
        chrdef->deviceType = VIR_DOMAIN_CHR_DEVICE_TYPE_CONSOLE;
        chrdef->target.port = 0;
        chrdef->targetType = VIR_DOMAIN_CHR_CONSOLE_TARGET_TYPE_XEN;

        def->consoles = g_new0(virDomainChrDef *, 1);
        def->nconsoles = 1;
        def->consoles[0] = chrdef;
    }

    /* add implicit input devices */
    if (xenDomainDefAddImplicitInputDevice(def) < 0)
        return -1;

    /* For x86_64 HVM */
    if (def->os.type == VIR_DOMAIN_OSTYPE_HVM &&
        def->os.arch == VIR_ARCH_X86_64) {
        /* always enable pae */
        def->features[VIR_DOMAIN_FEATURE_PAE] = VIR_TRISTATE_SWITCH_ON;

        /* if vnuma is effective enable acpi */
        if (virDomainNumaGetNodeCount(def->numa) > 0)
            def->features[VIR_DOMAIN_FEATURE_ACPI] = VIR_TRISTATE_SWITCH_ON;
    }

    /* add implicit balloon device */
    if (def->memballoon == NULL) {
        virDomainMemballoonDef *memballoon;
        memballoon = g_new0(virDomainMemballoonDef,
                            1);

        memballoon->model = VIR_DOMAIN_MEMBALLOON_MODEL_XEN;
        def->memballoon = memballoon;
    }

    /* add implicit xenbus device */
    if (virDomainControllerFindByType(def, VIR_DOMAIN_CONTROLLER_TYPE_XENBUS) == -1)
        if (virDomainDefAddController(def, VIR_DOMAIN_CONTROLLER_TYPE_XENBUS, -1, -1) == NULL)
            return -1;

    return 0;
}


libxlDomainSaveImageOpen(libxlDriverPrivate *driver,
                         const char *from,
                         virDomainDef **ret_def,
                         libxlSavefileHeader *ret_hdr)
{
    int fd;
    virDomainDef *def = NULL;
    libxlSavefileHeader hdr;
    g_autofree char *xml = NULL;

    if ((fd = virFileOpenAs(from, O_RDONLY, 0, -1, -1, 0)) < 0) {
        virReportSystemError(-fd,
                             _("Failed to open domain image file '%s'"), from);
        goto error;
    }

    if (saferead(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) {
        virReportError(VIR_ERR_OPERATION_FAILED,
                       "%s", _("failed to read libxl header"));
        goto error;
    }

    if (memcmp(hdr.magic, LIBXL_SAVE_MAGIC, sizeof(hdr.magic))) {
        virReportError(VIR_ERR_INVALID_ARG, "%s", _("image magic is incorrect"));
        goto error;
    }

    if (hdr.version > LIBXL_SAVE_VERSION) {
        virReportError(VIR_ERR_OPERATION_FAILED,
                       _("image version is not supported (%d > %d)"),
                       hdr.version, LIBXL_SAVE_VERSION);
        goto error;
    }

    if (hdr.xmlLen <= 0) {
        virReportError(VIR_ERR_OPERATION_FAILED,
                       _("invalid XML length: %d"), hdr.xmlLen);
        goto error;
    }

    xml = g_new0(char, hdr.xmlLen);

    if (saferead(fd, xml, hdr.xmlLen) != hdr.xmlLen) {
        virReportError(VIR_ERR_OPERATION_FAILED, "%s", _("failed to read XML"));
        goto error;
    }

    if (!(def = virDomainDefParseString(xml, driver->xmlopt, NULL,
                                        VIR_DOMAIN_DEF_PARSE_INACTIVE |
                                        VIR_DOMAIN_DEF_PARSE_SKIP_VALIDATE)))
        goto error;

    *ret_def = def;
    *ret_hdr = hdr;

    return fd;

 error:
    virDomainDefFree(def);
    VIR_FORCE_CLOSE(fd);
    return -1;
}


libxlDomainDestroyInternal(libxlDriverPrivate *driver,
                           virDomainObj *vm)
{
    g_autoptr(libxlDriverConfig) cfg = libxlDriverConfigGet(driver);
    libxlDomainObjPrivate *priv = vm->privateData;
    int ret = -1;

    if (priv->deathW) {
        libxl_evdisable_domain_death(cfg->ctx, priv->deathW);
        priv->deathW = NULL;
    }

    /* Unlock virDomainObj during destroy, which can take considerable
     * time on large memory domains.
     */
    virObjectUnlock(vm);
    ret = libxl_domain_destroy(cfg->ctx, vm->def->id, NULL);
    virObjectLock(vm);

    return ret;
}


libvirt_vmessage(xentoollog_logger *logger_in,
                 xentoollog_level level,
                 int errnoval,
                 const char *context,
                 const char *format,
                 va_list args)
{
    xentoollog_logger_libvirt *lg = (xentoollog_logger_libvirt *)logger_in;
    FILE *logFile = lg->defaultLogFile;
    char timestamp[VIR_TIME_STRING_BUFLEN];
    g_autofree char *message = NULL;
    char *start, *end;

    VIR_DEBUG("libvirt_vmessage: context='%s' format='%s'", context, format);

    if (level < lg->minLevel)
        return;

    message = g_strdup_vprintf(format, args);

    /* Should we print to a domain-specific log file? */
    if ((start = strstr(message, ": Domain ")) &&
        (end = strstr(start + 9, ":"))) {
        FILE *domainLogFile;

        VIR_DEBUG("Found domain log message");

        start = start + 9;
        *end = '\0';

        virMutexLock(&lg->tableLock);
        domainLogFile = virHashLookup(lg->files, start);
        virMutexUnlock(&lg->tableLock);
        if (domainLogFile)
            logFile = domainLogFile;

        *end = ':';
    }

    /* Do the actual print to the log file */
    if (virTimeStringNowRaw(timestamp) < 0)
        timestamp[0] = '\0';

    fprintf(logFile, "%s: ", timestamp);
    if (context)
        fprintf(logFile, "%s: ", context);

    fprintf(logFile, "%s", message);

    if (errnoval >= 0)
        fprintf(logFile, ": %s", g_strerror(errnoval));

    fputc('\n', logFile);
    fflush(logFile);
}


static int __init cgroup_no_v1(char *str)
{
	struct cgroup_subsys *ss;
	char *token;
	int i;

	while ((token = strsep(&str, ",")) != NULL) {
		if (!*token)
			continue;

		if (!strcmp(token, "all")) {
			cgroup_no_v1_mask = U16_MAX;
			continue;
		}

		if (!strcmp(token, "named")) {
			cgroup_no_v1_named = true;
			continue;
		}

		for_each_subsys(ss, i) {
			if (strcmp(token, ss->name) &&
			    strcmp(token, ss->legacy_name))
				continue;

			cgroup_no_v1_mask |= 1 << i;
		}
	}
	return 1;
}


static int pidlist_uniq(pid_t *list, int length)
{
	int src, dest = 1;

	/*
	 * we presume the 0th element is unique, so i starts at 1. trivial
	 * edge cases first; no work needs to be done for either
	 */
	if (length == 0 || length == 1)
		return length;
	/* src and dest walk down the list; dest counts unique elements */
	for (src = 1; src < length; src++) {
		/* find next unique element */
		while (list[src] == list[src-1]) {
			src++;
			if (src == length)
				goto after;
		}
		/* dest always points to where the next unique element goes */
		list[dest] = list[src];
		dest++;
	}
after:
	return dest;
}


static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self,
                                                  MirrorBlockJob *s,
                                                  uint64_t offset,
                                                  uint64_t bytes)
{
    uint64_t self_start_chunk = offset / s->granularity;
    uint64_t self_end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
    uint64_t self_nb_chunks = self_end_chunk - self_start_chunk;

    while (find_next_bit(s->in_flight_bitmap, self_end_chunk,
                         self_start_chunk) < self_end_chunk &&
           s->ret >= 0)
    {
        MirrorOp *op;

        QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
            uint64_t op_start_chunk = op->offset / s->granularity;
            uint64_t op_nb_chunks = DIV_ROUND_UP(op->offset + op->bytes,
                                                 s->granularity) -
                                    op_start_chunk;

            if (op == self) {
                continue;
            }

            if (ranges_overlap(self_start_chunk, self_nb_chunks,
                               op_start_chunk, op_nb_chunks))
            {
                if (self) {
                    /*
                     * If the operation is already (indirectly) waiting for us,
                     * or will wait for us as soon as it wakes up, then just go
                     * on (instead of producing a deadlock in the former case).
                     */
                    if (op->waiting_for_op) {
                        continue;
                    }

                    self->waiting_for_op = op;
                }

                qemu_co_queue_wait(&op->waiting_requests, NULL);

                if (self) {
                    self->waiting_for_op = NULL;
                }

                break;
            }
        }
    }
}


f_slice(typval_T *argvars, typval_T *rettv)
{
    if (in_vim9script()
	    && ((argvars[0].v_type != VAR_STRING
		    && argvars[0].v_type != VAR_LIST
		    && argvars[0].v_type != VAR_BLOB
		    && check_for_list_arg(argvars, 0) == FAIL)
		|| check_for_number_arg(argvars, 1) == FAIL
		|| check_for_opt_number_arg(argvars, 2) == FAIL))
	return;

    if (check_can_index(argvars, TRUE, FALSE) == OK)
    {
	copy_tv(argvars, rettv);
	eval_index_inner(rettv, TRUE, argvars + 1,
		argvars[2].v_type == VAR_UNKNOWN ? NULL : argvars + 2,
		TRUE, NULL, 0, FALSE);
    }
}


call_func_rettv(
	char_u	    **arg,
	evalarg_T   *evalarg,
	typval_T    *rettv,
	int	    evaluate,
	dict_T	    *selfdict,
	typval_T    *basetv)
{
    partial_T	*pt = NULL;
    funcexe_T	funcexe;
    typval_T	functv;
    char_u	*s;
    int		ret;

    // need to copy the funcref so that we can clear rettv
    if (evaluate)
    {
	functv = *rettv;
	rettv->v_type = VAR_UNKNOWN;

	// Invoke the function.  Recursive!
	if (functv.v_type == VAR_PARTIAL)
	{
	    pt = functv.vval.v_partial;
	    s = partial_name(pt);
	}
	else
	{
	    s = functv.vval.v_string;
	    if (s == NULL || *s == NUL)
	    {
		emsg(_(e_empty_function_name));
		ret = FAIL;
		goto theend;
	    }
	}
    }
    else
	s = (char_u *)"";

    CLEAR_FIELD(funcexe);
    funcexe.fe_firstline = curwin->w_cursor.lnum;
    funcexe.fe_lastline = curwin->w_cursor.lnum;
    funcexe.fe_evaluate = evaluate;
    funcexe.fe_partial = pt;
    funcexe.fe_selfdict = selfdict;
    funcexe.fe_basetv = basetv;
    ret = get_func_tv(s, -1, rettv, arg, evalarg, &funcexe);

theend:
    // Clear the funcref afterwards, so that deleting it while
    // evaluating the arguments is possible (see test55).
    if (evaluate)
	clear_tv(&functv);

    return ret;
}


handle_subscript(
    char_u	**arg,
    typval_T	*rettv,
    evalarg_T	*evalarg,
    int		verbose)	// give error messages
{
    int		evaluate = evalarg != NULL
				      && (evalarg->eval_flags & EVAL_EVALUATE);
    int		ret = OK;
    dict_T	*selfdict = NULL;
    int		check_white = TRUE;
    int		getnext;
    char_u	*p;

    while (ret == OK)
    {
	// When at the end of the line and ".name" or "->{" or "->X" follows in
	// the next line then consume the line break.
	p = eval_next_non_blank(*arg, evalarg, &getnext);
	if (getnext
	    && ((rettv->v_type == VAR_DICT && *p == '.' && eval_isdictc(p[1]))
		|| (p[0] == '-' && p[1] == '>' && (p[2] == '{'
			|| ASCII_ISALPHA(in_vim9script() ? *skipwhite(p + 2)
								    : p[2])))))
	{
	    *arg = eval_next_line(evalarg);
	    p = *arg;
	    check_white = FALSE;
	}

	if (rettv->v_type == VAR_ANY)
	{
	    char_u	*exp_name;
	    int		cc;
	    int		idx;
	    ufunc_T	*ufunc;
	    type_T	*type;

	    // Found script from "import * as {name}", script item name must
	    // follow.
	    if (**arg != '.')
	    {
		if (verbose)
		    semsg(_(e_expected_str_but_got_str), "'.'", *arg);
		ret = FAIL;
		break;
	    }
	    ++*arg;
	    if (IS_WHITE_OR_NUL(**arg))
	    {
		if (verbose)
		    emsg(_(e_no_white_space_allowed_after_dot));
		ret = FAIL;
		break;
	    }

	    // isolate the name
	    exp_name = *arg;
	    while (eval_isnamec(**arg))
		++*arg;
	    cc = **arg;
	    **arg = NUL;

	    idx = find_exported(rettv->vval.v_number, exp_name, &ufunc, &type,
						  evalarg->eval_cctx, verbose);
	    **arg = cc;
	    *arg = skipwhite(*arg);

	    if (idx < 0 && ufunc == NULL)
	    {
		ret = FAIL;
		break;
	    }
	    if (idx >= 0)
	    {
		scriptitem_T    *si = SCRIPT_ITEM(rettv->vval.v_number);
		svar_T		*sv = ((svar_T *)si->sn_var_vals.ga_data) + idx;

		copy_tv(sv->sv_tv, rettv);
	    }
	    else
	    {
		rettv->v_type = VAR_FUNC;
		rettv->vval.v_string = vim_strsave(ufunc->uf_name);
	    }
	}

	if ((**arg == '(' && (!evaluate || rettv->v_type == VAR_FUNC
			    || rettv->v_type == VAR_PARTIAL))
		    && (!check_white || !VIM_ISWHITE(*(*arg - 1))))
	{
	    ret = call_func_rettv(arg, evalarg, rettv, evaluate,
							       selfdict, NULL);

	    // Stop the expression evaluation when immediately aborting on
	    // error, or when an interrupt occurred or an exception was thrown
	    // but not caught.
	    if (aborting())
	    {
		if (ret == OK)
		    clear_tv(rettv);
		ret = FAIL;
	    }
	    dict_unref(selfdict);
	    selfdict = NULL;
	}
	else if (p[0] == '-' && p[1] == '>')
	{
	    if (in_vim9script())
		*arg = skipwhite(p + 2);
	    else
		*arg = p + 2;
	    if (ret == OK)
	    {
		if (VIM_ISWHITE(**arg))
		{
		    emsg(_(e_nowhitespace));
		    ret = FAIL;
		}
		else if ((**arg == '{' && !in_vim9script()) || **arg == '(')
		    // expr->{lambda}() or expr->(lambda)()
		    ret = eval_lambda(arg, rettv, evalarg, verbose);
		else
		    // expr->name()
		    ret = eval_method(arg, rettv, evalarg, verbose);
	    }
	}
	// "." is ".name" lookup when we found a dict or when evaluating and
	// scriptversion is at least 2, where string concatenation is "..".
	else if (**arg == '['
		|| (**arg == '.' && (rettv->v_type == VAR_DICT
			|| (!evaluate
			    && (*arg)[1] != '.'
			    && !in_old_script(2)))))
	{
	    dict_unref(selfdict);
	    if (rettv->v_type == VAR_DICT)
	    {
		selfdict = rettv->vval.v_dict;
		if (selfdict != NULL)
		    ++selfdict->dv_refcount;
	    }
	    else
		selfdict = NULL;
	    if (eval_index(arg, rettv, evalarg, verbose) == FAIL)
	    {
		clear_tv(rettv);
		ret = FAIL;
	    }
	}
	else
	    break;
    }

    // Turn "dict.Func" into a partial for "Func" bound to "dict".
    // Don't do this when "Func" is already a partial that was bound
    // explicitly (pt_auto is FALSE).
    if (selfdict != NULL
	    && (rettv->v_type == VAR_FUNC
		|| (rettv->v_type == VAR_PARTIAL
		    && (rettv->vval.v_partial->pt_auto
			|| rettv->vval.v_partial->pt_dict == NULL))))
	selfdict = make_partial(selfdict, rettv);

    dict_unref(selfdict);
    return ret;
}


typval2string(typval_T *tv, int convert)
{
    garray_T	ga;
    char_u	*retval;
#ifdef FEAT_FLOAT
    char_u	numbuf[NUMBUFLEN];
#endif

    if (convert && tv->v_type == VAR_LIST)
    {
	ga_init2(&ga, (int)sizeof(char), 80);
	if (tv->vval.v_list != NULL)
	{
	    list_join(&ga, tv->vval.v_list, (char_u *)"\n", TRUE, FALSE, 0);
	    if (tv->vval.v_list->lv_len > 0)
		ga_append(&ga, NL);
	}
	ga_append(&ga, NUL);
	retval = (char_u *)ga.ga_data;
    }
#ifdef FEAT_FLOAT
    else if (convert && tv->v_type == VAR_FLOAT)
    {
	vim_snprintf((char *)numbuf, NUMBUFLEN, "%g", tv->vval.v_float);
	retval = vim_strsave(numbuf);
    }
#endif
    else
	retval = vim_strsave(tv_get_string(tv));
    return retval;
}