struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
				   unsigned long addr,
				   pmd_t *pmd,
				   unsigned int flags)
{
	struct mm_struct *mm = vma->vm_mm;
	struct page *page = NULL;

	assert_spin_locked(pmd_lockptr(mm, pmd));

	if (flags & FOLL_WRITE && !can_follow_write_pmd(*pmd, flags))
		goto out;

	/* Avoid dumping huge zero page */
	if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
		return ERR_PTR(-EFAULT);

	/* Full NUMA hinting faults to serialise migration in fault paths */
	if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
		goto out;

	page = pmd_page(*pmd);
	VM_BUG_ON_PAGE(!PageHead(page) && !is_zone_device_page(page), page);

	if (!try_grab_page(page, flags))
		return ERR_PTR(-ENOMEM);

	if (flags & FOLL_TOUCH)
		touch_pmd(vma, addr, pmd, flags);

	if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
		/*
		 * We don't mlock() pte-mapped THPs. This way we can avoid
		 * leaking mlocked pages into non-VM_LOCKED VMAs.
		 *
		 * For anon THP:
		 *
		 * In most cases the pmd is the only mapping of the page as we
		 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for
		 * writable private mappings in populate_vma_page_range().
		 *
		 * The only scenario when we have the page shared here is if we
		 * mlocking read-only mapping shared over fork(). We skip
		 * mlocking such pages.
		 *
		 * For file THP:
		 *
		 * We can expect PageDoubleMap() to be stable under page lock:
		 * for file pages we set it in page_add_file_rmap(), which
		 * requires page to be locked.
		 */

		if (PageAnon(page) && compound_mapcount(page) != 1)
			goto skip_mlock;
		if (PageDoubleMap(page) || !page->mapping)
			goto skip_mlock;
		if (!trylock_page(page))
			goto skip_mlock;
		lru_add_drain();
		if (page->mapping && !PageDoubleMap(page))
			mlock_vma_page(page);
		unlock_page(page);
	}
skip_mlock:
	page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
	VM_BUG_ON_PAGE(!PageCompound(page) && !is_zone_device_page(page), page);

out:
	return page;
}


int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
		     unsigned long address, unsigned int fault_flags,
		     bool *unlocked)
{
	struct vm_area_struct *vma;
	vm_fault_t ret, major = 0;

	address = untagged_addr(address);

	if (unlocked)
		fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;

retry:
	vma = find_extend_vma(mm, address);
	if (!vma || address < vma->vm_start)
		return -EFAULT;

	if (!vma_permits_fault(vma, fault_flags))
		return -EFAULT;

	if ((fault_flags & FAULT_FLAG_KILLABLE) &&
	    fatal_signal_pending(current))
		return -EINTR;

	ret = handle_mm_fault(vma, address, fault_flags);
	major |= ret & VM_FAULT_MAJOR;
	if (ret & VM_FAULT_ERROR) {
		int err = vm_fault_to_errno(ret, 0);

		if (err)
			return err;
		BUG();
	}

	if (ret & VM_FAULT_RETRY) {
		down_read(&mm->mmap_sem);
		*unlocked = true;
		fault_flags |= FAULT_FLAG_TRIED;
		goto retry;
	}

	if (tsk) {
		if (major)
			tsk->maj_flt++;
		else
			tsk->min_flt++;
	}
	return 0;
}


static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
				 unsigned long end, unsigned int flags,
				 struct page **pages, int *nr)
{
	unsigned long fault_pfn;
	int nr_start = *nr;

	fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
	if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
		return 0;

	if (unlikely(pud_val(orig) != pud_val(*pudp))) {
		undo_dev_pagemap(nr, nr_start, flags, pages);
		return 0;
	}
	return 1;
}


static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
				 unsigned long end, unsigned int flags,
				 struct page **pages, int *nr)
{
	BUILD_BUG();
	return 0;
}


static void add_object(struct i915_mmu_object *mo)
{
	GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb));
	interval_tree_insert(&mo->it, &mo->mn->objects);
}


static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
		       unsigned long end, unsigned int flags,
		       struct page **pages, int *nr)
{
	unsigned long pte_end;
	struct page *head, *page;
	pte_t pte;
	int refs;

	pte_end = (addr + sz) & ~(sz-1);
	if (pte_end < end)
		end = pte_end;

	pte = READ_ONCE(*ptep);

	if (!pte_access_permitted(pte, flags & FOLL_WRITE))
		return 0;

	/* hugepages are never "special" */
	VM_BUG_ON(!pfn_valid(pte_pfn(pte)));

	head = pte_page(pte);
	page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
	refs = record_subpages(page, addr, end, pages + *nr);

	head = try_grab_compound_head(head, refs, flags);
	if (!head)
		return 0;

	if (unlikely(pte_val(pte) != pte_val(*ptep))) {
		put_compound_head(head, refs, flags);
		return 0;
	}

	*nr += refs;
	SetPageReferenced(head);
	return 1;
}


static int get_gate_page(struct mm_struct *mm, unsigned long address,
		unsigned int gup_flags, struct vm_area_struct **vma,
		struct page **page)
{
	pgd_t *pgd;
	p4d_t *p4d;
	pud_t *pud;
	pmd_t *pmd;
	pte_t *pte;
	int ret = -EFAULT;

	/* user gate pages are read-only */
	if (gup_flags & FOLL_WRITE)
		return -EFAULT;
	if (address > TASK_SIZE)
		pgd = pgd_offset_k(address);
	else
		pgd = pgd_offset_gate(mm, address);
	if (pgd_none(*pgd))
		return -EFAULT;
	p4d = p4d_offset(pgd, address);
	if (p4d_none(*p4d))
		return -EFAULT;
	pud = pud_offset(p4d, address);
	if (pud_none(*pud))
		return -EFAULT;
	pmd = pmd_offset(pud, address);
	if (!pmd_present(*pmd))
		return -EFAULT;
	VM_BUG_ON(pmd_trans_huge(*pmd));
	pte = pte_offset_map(pmd, address);
	if (pte_none(*pte))
		goto unmap;
	*vma = get_gate_vma(mm);
	if (!page)
		goto out;
	*page = vm_normal_page(*vma, address, *pte);
	if (!*page) {
		if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
			goto unmap;
		*page = pte_page(*pte);
	}
	if (unlikely(!try_get_page(*page))) {
		ret = -ENOMEM;
		goto unmap;
	}
out:
	ret = 0;
unmap:
	pte_unmap(pte);
	return ret;
}


static bool check_dax_vmas(struct vm_area_struct **vmas, long nr_pages)
{
	long i;
	struct vm_area_struct *vma_prev = NULL;

	for (i = 0; i < nr_pages; i++) {
		struct vm_area_struct *vma = vmas[i];

		if (vma == vma_prev)
			continue;

		vma_prev = vma;

		if (vma_is_fsdax(vma))
			return true;
	}
	return false;
}


static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
			unsigned long end, unsigned int flags,
			struct page **pages, int *nr)
{
	int refs;
	struct page *head, *page;

	if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
		return 0;

	BUILD_BUG_ON(pgd_devmap(orig));

	page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
	refs = record_subpages(page, addr, end, pages + *nr);

	head = try_grab_compound_head(pgd_page(orig), refs, flags);
	if (!head)
		return 0;

	if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
		put_compound_head(head, refs, flags);
		return 0;
	}

	*nr += refs;
	SetPageReferenced(head);
	return 1;
}


static struct page *follow_p4d_mask(struct vm_area_struct *vma,
				    unsigned long address, pgd_t *pgdp,
				    unsigned int flags,
				    struct follow_page_context *ctx)
{
	p4d_t *p4d;
	struct page *page;

	p4d = p4d_offset(pgdp, address);
	if (p4d_none(*p4d))
		return no_page_table(vma, flags);
	BUILD_BUG_ON(p4d_huge(*p4d));
	if (unlikely(p4d_bad(*p4d)))
		return no_page_table(vma, flags);

	if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
		page = follow_huge_pd(vma, address,
				      __hugepd(p4d_val(*p4d)), flags,
				      P4D_SHIFT);
		if (page)
			return page;
		return no_page_table(vma, flags);
	}
	return follow_pud_mask(vma, address, p4d, flags, ctx);
}


int mm_take_all_locks(struct mm_struct *mm)
{
	struct vm_area_struct *vma;
	struct anon_vma_chain *avc;

	BUG_ON(mmap_read_trylock(mm));

	mutex_lock(&mm_all_locks_mutex);

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (signal_pending(current))
			goto out_unlock;
		if (vma->vm_file && vma->vm_file->f_mapping &&
				is_vm_hugetlb_page(vma))
			vm_lock_mapping(mm, vma->vm_file->f_mapping);
	}

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (signal_pending(current))
			goto out_unlock;
		if (vma->vm_file && vma->vm_file->f_mapping &&
				!is_vm_hugetlb_page(vma))
			vm_lock_mapping(mm, vma->vm_file->f_mapping);
	}

	for (vma = mm->mmap; vma; vma = vma->vm_next) {
		if (signal_pending(current))
			goto out_unlock;
		if (vma->anon_vma)
			list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
				vm_lock_anon_vma(mm, avc->anon_vma);
	}

	return 0;

out_unlock:
	mm_drop_all_locks(mm);
	return -EINTR;
}


static inline int init_cache_random_seq(struct kmem_cache *s)
{
	return 0;
}


void __kmemcg_cache_deactivate_after_rcu(struct kmem_cache *s)
{
	/*
	 * Called with all the locks held after a sched RCU grace period.
	 * Even if @s becomes empty after shrinking, we can't know that @s
	 * doesn't have allocations already in-flight and thus can't
	 * destroy @s until the associated memcg is released.
	 *
	 * However, let's remove the sysfs files for empty caches here.
	 * Each cache has a lot of interface files which aren't
	 * particularly useful for empty draining caches; otherwise, we can
	 * easily end up with millions of unnecessary sysfs files on
	 * systems which have a lot of memory and transient cgroups.
	 */
	if (!__kmem_cache_shrink(s))
		sysfs_slab_remove(s);
}


void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{
	void *ret = slab_alloc(s, gfpflags, _RET_IP_);

	trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
				s->size, gfpflags);

	return ret;
}


static void free_kmem_cache_nodes(struct kmem_cache *s)
{
	int node;
	struct kmem_cache_node *n;

	for_each_kmem_cache_node(s, node, n) {
		s->node[node] = NULL;
		kmem_cache_free(kmem_cache_node, n);
	}
}


static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
							{ return 0; }


static void clear_stat(struct kmem_cache *s, enum stat_item si)
{
	int cpu;

	for_each_online_cpu(cpu)
		per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
}


static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
{
	unsigned long freeptr_addr = (unsigned long)object + s->offset;

#ifdef CONFIG_SLAB_FREELIST_HARDENED
	BUG_ON(object == fp); /* naive detection of double free or corruption */
#endif

	*(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);
}


void __init kmem_cache_init_late(void)
{
}


static ssize_t red_zone_store(struct kmem_cache *s,
				const char *buf, size_t length)
{
	if (any_slab_objects(s))
		return -EBUSY;

	s->flags &= ~SLAB_RED_ZONE;
	if (buf[0] == '1') {
		s->flags |= SLAB_RED_ZONE;
	}
	calculate_sizes(s, -1);
	return length;
}


static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%u\n", oo_objects(s->oo));
}


static ssize_t shrink_store(struct kmem_cache *s,
			const char *buf, size_t length)
{
	if (buf[0] == '1')
		kmem_cache_shrink_all(s);
	else
		return -EINVAL;
	return length;
}


static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
{
	stat(s, CPUSLAB_FLUSH);
	deactivate_slab(s, c->page, c->freelist, c);

	c->tid = next_tid(c->tid);
}


static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
{
	return x.x & OO_MASK;
}


static bool has_cpu_slab(int cpu, void *info)
{
	struct kmem_cache *s = info;
	struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);

	return c->page || slub_percpu_partial(c);
}


static __always_inline void slab_unlock(struct page *page)
{
	VM_BUG_ON_PAGE(PageTail(page), page);
	__bit_spin_unlock(PG_locked, &page->flags);
}


static void rcu_free_slab(struct rcu_head *h)
{
	struct page *page = container_of(h, struct page, rcu_head);

	__free_slab(page->slab_cache, page);
}


void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
			 bool to_user)
{
	struct kmem_cache *s;
	unsigned int offset;
	size_t object_size;

	ptr = kasan_reset_tag(ptr);

	/* Find object and usable object size. */
	s = page->slab_cache;

	/* Reject impossible pointers. */
	if (ptr < page_address(page))
		usercopy_abort("SLUB object not in SLUB page?!", NULL,
			       to_user, 0, n);

	/* Find offset within object. */
	offset = (ptr - page_address(page)) % s->size;

	/* Adjust for redzone and reject if within the redzone. */
	if (kmem_cache_debug(s) && s->flags & SLAB_RED_ZONE) {
		if (offset < s->red_left_pad)
			usercopy_abort("SLUB object in left red zone",
				       s->name, to_user, offset, n);
		offset -= s->red_left_pad;
	}

	/* Allow address range falling entirely within usercopy region. */
	if (offset >= s->useroffset &&
	    offset - s->useroffset <= s->usersize &&
	    n <= s->useroffset - offset + s->usersize)
		return;

	/*
	 * If the copy is still within the allocated object, produce
	 * a warning instead of rejecting the copy. This is intended
	 * to be a temporary method to find any missing usercopy
	 * whitelists.
	 */
	object_size = slab_ksize(s);
	if (usercopy_fallback &&
	    offset <= object_size && n <= object_size - offset) {
		usercopy_warn("SLUB object", s->name, to_user, offset, n);
		return;
	}

	usercopy_abort("SLUB object", s->name, to_user, offset, n);
}


static inline int free_consistency_checks(struct kmem_cache *s,
		struct page *page, void *object, unsigned long addr)
{
	if (!check_valid_pointer(s, page, object)) {
		slab_err(s, page, "Invalid object pointer 0x%p", object);
		return 0;
	}

	if (on_freelist(s, page, object)) {
		object_err(s, page, object, "Object already free");
		return 0;
	}

	if (!check_object(s, page, object, SLUB_RED_ACTIVE))
		return 0;

	if (unlikely(s != page->slab_cache)) {
		if (!PageSlab(page)) {
			slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
				 object);
		} else if (!page->slab_cache) {
			pr_err("SLUB <none>: no slab for object 0x%p.\n",
			       object);
			dump_stack();
		} else
			object_err(s, page, object,
					"page slab pointer corrupt.");
		return 0;
	}
	return 1;
}


void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{
	void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);

	trace_kmem_cache_alloc_node(_RET_IP_, ret,
				    s->object_size, s->size, gfpflags, node);

	return ret;
}


static inline unsigned long tid_to_event(unsigned long tid)
{
	return tid / TID_STEP;
}


static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
{
	return sprintf(buf, "%u\n", s->remote_node_defrag_ratio / 10);
}


static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
		void *freelist_old, unsigned long counters_old,
		void *freelist_new, unsigned long counters_new,
		const char *n)
{
	VM_BUG_ON(!irqs_disabled());
#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
    defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
	if (s->flags & __CMPXCHG_DOUBLE) {
		if (cmpxchg_double(&page->freelist, &page->counters,
				   freelist_old, counters_old,
				   freelist_new, counters_new))
			return true;
	} else
#endif
	{
		slab_lock(page);
		if (page->freelist == freelist_old &&
					page->counters == counters_old) {
			page->freelist = freelist_new;
			page->counters = counters_new;
			slab_unlock(page);
			return true;
		}
		slab_unlock(page);
	}

	cpu_relax();
	stat(s, CMPXCHG_DOUBLE_FAIL);

#ifdef SLUB_DEBUG_CMPXCHG
	pr_info("%s %s: cmpxchg double redo ", n, s->name);
#endif

	return false;
}


static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
{
	BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
			KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));

	/*
	 * Must align to double word boundary for the double cmpxchg
	 * instructions to work; see __pcpu_double_call_return_bool().
	 */
	s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
				     2 * sizeof(void *));

	if (!s->cpu_slab)
		return 0;

	init_kmem_cache_cpus(s);

	return 1;
}


static int __init setup_slub_max_order(char *str)
{
	get_option(&str, (int *)&slub_max_order);
	slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1);

	return 1;
}


static int __init setup_slub_min_order(char *str)
{
	get_option(&str, (int *)&slub_min_order);

	return 1;
}


static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
			  unsigned long addr, struct kmem_cache_cpu *c)
{
	void *freelist;
	struct page *page;

	page = c->page;
	if (!page)
		goto new_slab;
redo:

	if (unlikely(!node_match(page, node))) {
		int searchnode = node;

		if (node != NUMA_NO_NODE && !node_present_pages(node))
			searchnode = node_to_mem_node(node);

		if (unlikely(!node_match(page, searchnode))) {
			stat(s, ALLOC_NODE_MISMATCH);
			deactivate_slab(s, page, c->freelist, c);
			goto new_slab;
		}
	}

	/*
	 * By rights, we should be searching for a slab page that was
	 * PFMEMALLOC but right now, we are losing the pfmemalloc
	 * information when the page leaves the per-cpu allocator
	 */
	if (unlikely(!pfmemalloc_match(page, gfpflags))) {
		deactivate_slab(s, page, c->freelist, c);
		goto new_slab;
	}

	/* must check again c->freelist in case of cpu migration or IRQ */
	freelist = c->freelist;
	if (freelist)
		goto load_freelist;

	freelist = get_freelist(s, page);

	if (!freelist) {
		c->page = NULL;
		stat(s, DEACTIVATE_BYPASS);
		goto new_slab;
	}

	stat(s, ALLOC_REFILL);

load_freelist:
	/*
	 * freelist is pointing to the list of objects to be used.
	 * page is pointing to the page from which the objects are obtained.
	 * That page must be frozen for per cpu allocations to work.
	 */
	VM_BUG_ON(!c->page->frozen);
	c->freelist = get_freepointer(s, freelist);
	c->tid = next_tid(c->tid);
	return freelist;

new_slab:

	if (slub_percpu_partial(c)) {
		page = c->page = slub_percpu_partial(c);
		slub_set_percpu_partial(c, page);
		stat(s, CPU_PARTIAL_ALLOC);
		goto redo;
	}

	freelist = new_slab_objects(s, gfpflags, node, &c);

	if (unlikely(!freelist)) {
		slab_out_of_memory(s, gfpflags, node);
		return NULL;
	}

	page = c->page;
	if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
		goto load_freelist;

	/* Only entered in the debug case */
	if (kmem_cache_debug(s) &&
			!alloc_debug_processing(s, page, freelist, addr))
		goto new_slab;	/* Slab failed checks. Next slab needed */

	deactivate_slab(s, page, get_freepointer(s, freelist), c);
	return freelist;
}


static inline void note_cmpxchg_failure(const char *n,
		const struct kmem_cache *s, unsigned long tid)
{
#ifdef SLUB_DEBUG_CMPXCHG
	unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);

	pr_info("%s %s: cmpxchg redo ", n, s->name);

#ifdef CONFIG_PREEMPTION
	if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
		pr_warn("due to cpu change %d -> %d\n",
			tid_to_cpu(tid), tid_to_cpu(actual_tid));
	else
#endif
	if (tid_to_event(tid) != tid_to_event(actual_tid))
		pr_warn("due to cpu running other code. Event %ld->%ld\n",
			tid_to_event(tid), tid_to_event(actual_tid));
	else
		pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
			actual_tid, tid, next_tid(tid));
#endif
	stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
}


static inline struct kset *cache_kset(struct kmem_cache *s)
{
#ifdef CONFIG_MEMCG
	if (!is_root_cache(s))
		return s->memcg_params.root_cache->memcg_kset;
#endif
	return slab_kset;


static void flush_all(struct kmem_cache *s)
{
	on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1);
}


int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
{
	int err;

	err = kmem_cache_open(s, flags);
	if (err)
		return err;

	/* Mutex is not taken during early boot */
	if (slab_state <= UP)
		return 0;

	memcg_propagate_slab_attrs(s);
	err = sysfs_slab_add(s);
	if (err)
		__kmem_cache_release(s);

	return err;
}


static long madvise_willneed(struct vm_area_struct *vma,
			     struct vm_area_struct **prev,
			     unsigned long start, unsigned long end)
{
	struct file *file = vma->vm_file;
	loff_t offset;

	*prev = vma;
#ifdef CONFIG_SWAP
	if (!file) {
		walk_page_range(vma->vm_mm, start, end, &swapin_walk_ops, vma);
		lru_add_drain(); /* Push any new pages onto the LRU now */
		return 0;
	}

	if (shmem_mapping(file->f_mapping)) {
		force_shm_swapin_readahead(vma, start, end,
					file->f_mapping);
		return 0;
	}
#else
	if (!file)
		return -EBADF;
#endif

	if (IS_DAX(file_inode(file))) {
		/* no bad return value, but ignore advice */
		return 0;
	}

	/*
	 * Filesystem's fadvise may need to take various locks.  We need to
	 * explicitly grab a reference because the vma (and hence the
	 * vma's reference to the file) can go away as soon as we drop
	 * mmap_sem.
	 */
	*prev = NULL;	/* tell sys_madvise we drop mmap_sem */
	get_file(file);
	up_read(&current->mm->mmap_sem);
	offset = (loff_t)(start - vma->vm_start)
			+ ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
	vfs_fadvise(file, offset, end - start, POSIX_FADV_WILLNEED);
	fput(file);
	down_read(&current->mm->mmap_sem);
	return 0;
}


static int madvise_need_mmap_write(int behavior)
{
	switch (behavior) {
	case MADV_REMOVE:
	case MADV_WILLNEED:
	case MADV_DONTNEED:
	case MADV_COLD:
	case MADV_PAGEOUT:
	case MADV_FREE:
		return 0;
	default:
		/* be safe, default to 1. list exceptions explicitly */
		return 1;
	}
}


SYSCALL_DEFINE3(madvise, unsigned long, start, size_t, len_in, int, behavior)
{
	return do_madvise(start, len_in, behavior);
}


static long madvise_dontneed_free(struct vm_area_struct *vma,
				  struct vm_area_struct **prev,
				  unsigned long start, unsigned long end,
				  int behavior)
{
	*prev = vma;
	if (!can_madv_lru_vma(vma))
		return -EINVAL;

	if (!userfaultfd_remove(vma, start, end)) {
		*prev = NULL; /* mmap_sem has been dropped, prev is stale */

		down_read(&current->mm->mmap_sem);
		vma = find_vma(current->mm, start);
		if (!vma)
			return -ENOMEM;
		if (start < vma->vm_start) {
			/*
			 * This "vma" under revalidation is the one
			 * with the lowest vma->vm_start where start
			 * is also < vma->vm_end. If start <
			 * vma->vm_start it means an hole materialized
			 * in the user address space within the
			 * virtual range passed to MADV_DONTNEED
			 * or MADV_FREE.
			 */
			return -ENOMEM;
		}
		if (!can_madv_lru_vma(vma))
			return -EINVAL;
		if (end > vma->vm_end) {
			/*
			 * Don't fail if end > vma->vm_end. If the old
			 * vma was splitted while the mmap_sem was
			 * released the effect of the concurrent
			 * operation may not cause madvise() to
			 * have an undefined result. There may be an
			 * adjacent next vma that we'll walk
			 * next. userfaultfd_remove() will generate an
			 * UFFD_EVENT_REMOVE repetition on the
			 * end-vma->vm_end range, but the manager can
			 * handle a repetition fine.
			 */
			end = vma->vm_end;
		}
		VM_WARN_ON(start >= end);
	}

	if (behavior == MADV_DONTNEED)
		return madvise_dontneed_single_vma(vma, start, end);
	else if (behavior == MADV_FREE)
		return madvise_free_single_vma(vma, start, end);
	else
		return -EINVAL;
}


static void io_kill_timeout(struct io_kiocb *req)
{
	int ret;

	ret = hrtimer_try_to_cancel(&req->io->timeout.timer);
	if (ret != -1) {
		atomic_inc(&req->ctx->cq_timeouts);
		list_del_init(&req->list);
		io_cqring_fill_event(req, 0);
		io_put_req(req);
	}
}


static int io_madvise(struct io_kiocb *req, struct io_kiocb **nxt,
		      bool force_nonblock)
{
#if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU)
	struct io_madvise *ma = &req->madvise;
	int ret;

	if (force_nonblock)
		return -EAGAIN;

	ret = do_madvise(ma->addr, ma->len, ma->advice);
	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);
	io_put_req_find_next(req, nxt);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}


static int io_openat(struct io_kiocb *req, struct io_kiocb **nxt,
		     bool force_nonblock)
{
	req->open.how = build_open_how(req->open.how.flags, req->open.how.mode);
	return io_openat2(req, nxt, force_nonblock);
}


static int io_close(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
{
	int ret;

	req->close.put_file = NULL;
	ret = __close_fd_get_file(req->close.fd, &req->close.put_file);
	if (ret < 0)
		return ret;

	/* if the file has a flush method, be safe and punt to async */
	if (req->close.put_file->f_op->flush && !io_wq_current_is_worker())
		goto eagain;

	/*
	 * No ->flush(), safely close from here and just punt the
	 * fput() to async context.
	 */
	ret = filp_close(req->close.put_file, current->files);

	if (ret < 0)
		req_set_fail_links(req);
	io_cqring_add_event(req, ret);

	if (io_wq_current_is_worker()) {
		struct io_wq_work *old_work, *work;

		old_work = work = &req->work;
		io_close_finish(&work);
		if (work && work != old_work)
			*nxt = container_of(work, struct io_kiocb, work);
		return 0;
	}

eagain:
	req->work.func = io_close_finish;
	/*
	 * Do manual async queue here to avoid grabbing files - we don't
	 * need the files, and it'll cause io_close_finish() to close
	 * the file again and cause a double CQE entry for this request
	 */
	io_queue_async_work(req);
	return 0;
}


static int io_write(struct io_kiocb *req, struct io_kiocb **nxt,
		    bool force_nonblock)
{
	struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs;
	struct kiocb *kiocb = &req->rw.kiocb;
	struct iov_iter iter;
	size_t iov_count;
	ssize_t ret, io_size;

	ret = io_import_iovec(WRITE, req, &iovec, &iter);
	if (ret < 0)
		return ret;

	/* Ensure we clear previously set non-block flag */
	if (!force_nonblock)
		req->rw.kiocb.ki_flags &= ~IOCB_NOWAIT;

	req->result = 0;
	io_size = ret;
	if (req->flags & REQ_F_LINK)
		req->result = io_size;

	/*
	 * If the file doesn't support async, mark it as REQ_F_MUST_PUNT so
	 * we know to async punt it even if it was opened O_NONBLOCK
	 */
	if (force_nonblock && !io_file_supports_async(req->file)) {
		req->flags |= REQ_F_MUST_PUNT;
		goto copy_iov;
	}

	/* file path doesn't support NOWAIT for non-direct_IO */
	if (force_nonblock && !(kiocb->ki_flags & IOCB_DIRECT) &&
	    (req->flags & REQ_F_ISREG))
		goto copy_iov;

	iov_count = iov_iter_count(&iter);
	ret = rw_verify_area(WRITE, req->file, &kiocb->ki_pos, iov_count);
	if (!ret) {
		ssize_t ret2;

		/*
		 * Open-code file_start_write here to grab freeze protection,
		 * which will be released by another thread in
		 * io_complete_rw().  Fool lockdep by telling it the lock got
		 * released so that it doesn't complain about the held lock when
		 * we return to userspace.
		 */
		if (req->flags & REQ_F_ISREG) {
			__sb_start_write(file_inode(req->file)->i_sb,
						SB_FREEZE_WRITE, true);
			__sb_writers_release(file_inode(req->file)->i_sb,
						SB_FREEZE_WRITE);
		}
		kiocb->ki_flags |= IOCB_WRITE;

		if (req->file->f_op->write_iter)
			ret2 = call_write_iter(req->file, kiocb, &iter);
		else
			ret2 = loop_rw_iter(WRITE, req->file, kiocb, &iter);
		/*
		 * Raw bdev writes will -EOPNOTSUPP for IOCB_NOWAIT. Just
		 * retry them without IOCB_NOWAIT.
		 */
		if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT))
			ret2 = -EAGAIN;
		if (!force_nonblock || ret2 != -EAGAIN) {
			kiocb_done(kiocb, ret2, nxt, req->in_async);
		} else {
copy_iov:
			ret = io_setup_async_rw(req, io_size, iovec,
						inline_vecs, &iter);
			if (ret)
				goto out_free;
			return -EAGAIN;
		}
	}
out_free:
	req->flags &= ~REQ_F_NEED_CLEANUP;
	kfree(iovec);
	return ret;
}


static void io_close_finish(struct io_wq_work **workptr)
{
	struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work);
	struct io_kiocb *nxt = NULL;

	/* Invoked with files, we need to do the close */
	if (req->work.files) {
		int ret;

		ret = filp_close(req->close.put_file, req->work.files);
		if (ret < 0)
			req_set_fail_links(req);
		io_cqring_add_event(req, ret);
	}

	fput(req->close.put_file);

	io_put_req_find_next(req, &nxt);
	if (nxt)
		io_wq_assign_next(workptr, nxt);
}


static void io_atomic_switch(struct percpu_ref *ref)
{
	struct fixed_file_data *data;

	data = container_of(ref, struct fixed_file_data, refs);
	clear_bit(FFD_F_ATOMIC, &data->state);
}


static inline void req_set_fail_links(struct io_kiocb *req)
{
	if ((req->flags & (REQ_F_LINK | REQ_F_HARDLINK)) == REQ_F_LINK)
		req->flags |= REQ_F_FAIL_LINK;
}


static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
{
	struct io_uring_probe *p;
	size_t size;
	int i, ret;

	size = struct_size(p, ops, nr_args);
	if (size == SIZE_MAX)
		return -EOVERFLOW;
	p = kzalloc(size, GFP_KERNEL);
	if (!p)
		return -ENOMEM;

	ret = -EFAULT;
	if (copy_from_user(p, arg, size))
		goto out;
	ret = -EINVAL;
	if (memchr_inv(p, 0, size))
		goto out;

	p->last_op = IORING_OP_LAST - 1;
	if (nr_args > IORING_OP_LAST)
		nr_args = IORING_OP_LAST;

	for (i = 0; i < nr_args; i++) {
		p->ops[i].op = i;
		if (!io_op_defs[i].not_supported)
			p->ops[i].flags = IO_URING_OP_SUPPORTED;
	}
	p->ops_len = i;

	ret = 0;
	if (copy_to_user(arg, p, size))
		ret = -EFAULT;
out:
	kfree(p);
	return ret;
}


static bool io_register_op_must_quiesce(int op)
{
	switch (op) {
	case IORING_UNREGISTER_FILES:
	case IORING_REGISTER_FILES_UPDATE:
	case IORING_REGISTER_PROBE:
	case IORING_REGISTER_PERSONALITY:
	case IORING_UNREGISTER_PERSONALITY:
		return false;
	default:
		return true;
	}
}


static int io_req_defer(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_ring_ctx *ctx = req->ctx;
	int ret;

	/* Still need defer if there is pending req in defer list. */
	if (!req_need_defer(req) && list_empty(&ctx->defer_list))
		return 0;

	if (!req->io && io_alloc_async_ctx(req))
		return -EAGAIN;

	ret = io_req_defer_prep(req, sqe);
	if (ret < 0)
		return ret;

	spin_lock_irq(&ctx->completion_lock);
	if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
		spin_unlock_irq(&ctx->completion_lock);
		return 0;
	}

	trace_io_uring_defer(ctx, req, req->user_data);
	list_add_tail(&req->list, &ctx->defer_list);
	spin_unlock_irq(&ctx->completion_lock);
	return -EIOCBQUEUED;
}


static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
{
	const struct cred *old_creds;

	old_creds = idr_remove(&ctx->personality_idr, id);
	if (old_creds) {
		put_cred(old_creds);
		return 0;
	}

	return -EINVAL;
}


static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
#if defined(CONFIG_NET)
	struct io_accept *accept = &req->accept;

	if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL)))
		return -EINVAL;
	if (sqe->ioprio || sqe->len || sqe->buf_index)
		return -EINVAL;

	accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
	accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
	accept->flags = READ_ONCE(sqe->accept_flags);
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}


static int io_accept(struct io_kiocb *req, struct io_kiocb **nxt,
		     bool force_nonblock)
{
#if defined(CONFIG_NET)
	int ret;

	ret = __io_accept(req, nxt, force_nonblock);
	if (ret == -EAGAIN && force_nonblock) {
		req->work.func = io_accept_finish;
		io_put_req(req);
		return -EAGAIN;
	}
	return 0;
#else
	return -EOPNOTSUPP;
#endif
}


p11_rpc_buffer_add_ulong_value (p11_buffer *buffer,
				const void *value,
				CK_ULONG value_length)
{
	CK_ULONG ulong_value = 0;

	/* Check if value can be converted to CK_ULONG. */
	if (value_length > sizeof (CK_ULONG)) {
		p11_buffer_fail (buffer);
		return;
	}
	if (value)
		memcpy (&ulong_value, value, value_length);

	/* Check if ulong_value can be converted to uint64_t. */
	if (ulong_value > UINT64_MAX) {
		p11_buffer_fail (buffer);
		return;
	}

	p11_rpc_buffer_add_uint64 (buffer, ulong_value);
}


p11_rpc_message_read_space_string (p11_rpc_message *msg,
                                   CK_UTF8CHAR *buffer,
                                   CK_ULONG length)
{
	const unsigned char *data;
	size_t n_data;

	assert (msg != NULL);
	assert (msg->input != NULL);
	assert (buffer != NULL);
	assert (length != 0);

	assert (!msg->signature || p11_rpc_message_verify_part (msg, "s"));

	if (!p11_rpc_buffer_get_byte_array (msg->input, &msg->parsed, &data, &n_data))
		return false;

	if (n_data != length) {
		p11_message ("invalid length space padded string received: %d != %d",
		             (int)length, (int)n_data);
		return false;
	}

	memcpy (buffer, data, length);
	return true;
}


p11_rpc_buffer_decode_uint32 (unsigned char* ptr)
{
	uint32_t val = (uint32_t) ptr[0] << 24 | ptr[1] << 16 | ptr[2] << 8 | ptr[3];
	return val;
}


rpc_C_VerifyRecoverInit (CK_X_FUNCTION_LIST *self,
                         p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_MECHANISM mechanism;
	CK_OBJECT_HANDLE key;

	BEGIN_CALL (VerifyRecoverInit);
		IN_ULONG (session);
		IN_MECHANISM (mechanism);
		IN_ULONG (key);
	PROCESS_CALL ((self, session, &mechanism, key));
	END_CALL;
}


merge_attrs (CK_ATTRIBUTE *output,
             CK_ULONG *noutput,
             CK_ATTRIBUTE *merge,
             CK_ULONG nmerge,
             p11_array *to_free)
{
	CK_ULONG i;

	for (i = 0; i < nmerge; i++) {
		/* Already have this attribute? */
		if (p11_attrs_findn (output, *noutput, merge[i].type)) {
			p11_array_push (to_free, merge[i].pValue);

		} else {
			memcpy (output + *noutput, merge + i, sizeof (CK_ATTRIBUTE));
			(*noutput)++;
		}
	}

	/* Freeing the array itself */
	p11_array_push (to_free, merge);
}


rpc_C_Sign (CK_X_FUNCTION_LIST *self,
            p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_BYTE_PTR part;
	CK_ULONG part_len;
	CK_BYTE_PTR signature;
	CK_ULONG signature_len;

	BEGIN_CALL (Sign);
		IN_ULONG (session);
		IN_BYTE_ARRAY (part, part_len);
		IN_BYTE_BUFFER (signature, signature_len);
	PROCESS_CALL ((self, session, part, part_len, signature, &signature_len));
		OUT_BYTE_ARRAY (signature, signature_len);
	END_CALL;

}


p11_index_set (p11_index *index,
               CK_OBJECT_HANDLE handle,
               CK_ATTRIBUTE *attrs,
               CK_ULONG count)
{
	CK_ATTRIBUTE *update;
	index_object *obj;

	return_val_if_fail (index != NULL, CKR_GENERAL_ERROR);

	obj = p11_dict_get (index->objects, &handle);
	if (obj == NULL)
		return CKR_OBJECT_HANDLE_INVALID;

	update = p11_attrs_buildn (NULL, attrs, count);
	return_val_if_fail (update != NULL, CKR_HOST_MEMORY);

	return p11_index_update (index, handle, update);
}


rpc_C_FindObjects (CK_X_FUNCTION_LIST *self,
                    p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_OBJECT_HANDLE_PTR objects;
	CK_ULONG max_object_count;
	CK_ULONG object_count;

	BEGIN_CALL (FindObjects);
		IN_ULONG (session);
		IN_ULONG_BUFFER (objects, max_object_count);
	PROCESS_CALL ((self, session, objects, max_object_count, &object_count));
		OUT_ULONG_ARRAY (objects, object_count);
	END_CALL;
}


proxy_C_VerifyInit (CK_X_FUNCTION_LIST *self,
                    CK_SESSION_HANDLE handle,
                    CK_MECHANISM_PTR mechanism,
                    CK_OBJECT_HANDLE key)
{
	State *state = (State *)self;
	Mapping map;
	CK_RV rv;

	rv = map_session_to_real (state->px, &handle, &map, NULL);
	if (rv != CKR_OK)
		return rv;
	return (map.funcs->C_VerifyInit) (handle, mechanism, key);
}


p11_index_new (p11_index_build_cb build,
               p11_index_store_cb store,
               p11_index_remove_cb remove,
               p11_index_notify_cb notify,
               void *data)
{
	p11_index *index;

	index = calloc (1, sizeof (p11_index));
	return_val_if_fail (index != NULL, NULL);

	if (build == NULL)
		build = default_build;
	if (store == NULL)
		store = default_store;
	if (notify == NULL)
		notify = default_notify;
	if (remove == NULL)
		remove = default_remove;

	index->build = build;
	index->store = store;
	index->notify = notify;
	index->remove = remove;
	index->data = data;

	index->objects = p11_dict_new (p11_dict_ulongptr_hash,
	                               p11_dict_ulongptr_equal,
	                               NULL, free_object);
	if (index->objects == NULL) {
		p11_index_free (index);
		return_val_if_reached (NULL);
	}

	index->buckets = calloc (NUM_BUCKETS, sizeof (index_bucket));
	if (index->buckets == NULL) {
		p11_index_free (index);
		return_val_if_reached (NULL);
	}

	return index;
}


proxy_free (Proxy *py, unsigned finalize)
{
	if (py) {
		if (finalize)
			p11_kit_modules_finalize (py->inited);
		free (py->inited);
		p11_dict_free (py->sessions);
		free (py->mappings);
		free (py);
	}
}


proxy_C_SignRecoverInit (CK_X_FUNCTION_LIST *self,
                         CK_SESSION_HANDLE handle,
                         CK_MECHANISM_PTR mechanism,
                         CK_OBJECT_HANDLE key)
{
	State *state = (State *)self;
	Mapping map;
	CK_RV rv;

	rv = map_session_to_real (state->px, &handle, &map, NULL);
	if (rv != CKR_OK)
		return rv;
	return (map.funcs->C_SignRecoverInit) (handle, mechanism, key);
}


proxy_C_SignUpdate (CK_X_FUNCTION_LIST *self,
                    CK_SESSION_HANDLE handle,
                    CK_BYTE_PTR part,
                    CK_ULONG part_len)
{
	State *state = (State *)self;
	Mapping map;
	CK_RV rv;

	rv = map_session_to_real (state->px, &handle, &map, NULL);
	if (rv != CKR_OK)
		return rv;
	return (map.funcs->C_SignUpdate) (handle, part, part_len);
}


finish_module (P11KitIter *iter)
{
	iter->num_slots = 0;
	iter->saw_slots = 0;
	iter->module = NULL;
}


p11_index_free (p11_index *index)
{
	int i;

	return_if_fail (index != NULL);

	p11_dict_free (index->objects);
	p11_dict_free (index->changes);
	if (index->buckets) {
		for (i = 0; i < NUM_BUCKETS; i++)
			free (index->buckets[i].elem);
		free (index->buckets);
	}
	free (index);
}


bucket_push (index_bucket *bucket,
             CK_OBJECT_HANDLE handle)
{
	unsigned int alloc;

	alloc = alloc_size (bucket->num);
	if (bucket->num + 1 > alloc) {
		CK_OBJECT_HANDLE *elem;

		alloc = alloc ? alloc * 2 : 1;
		return_val_if_fail (alloc != 0, false);
		elem = reallocarray (bucket->elem, alloc, sizeof (CK_OBJECT_HANDLE));
		return_val_if_fail (elem != NULL, false);
		bucket->elem = elem;
	}

	return_val_if_fail (bucket->elem != NULL, false);
	bucket->elem[bucket->num++] = handle;
	return true;
}


proxy_C_GenerateKey (CK_X_FUNCTION_LIST *self,
                     CK_SESSION_HANDLE handle,
                     CK_MECHANISM_PTR mechanism,
                     CK_ATTRIBUTE_PTR template,
                     CK_ULONG count,
                     CK_OBJECT_HANDLE_PTR key)
{
	State *state = (State *)self;
	Mapping map;
	CK_RV rv;

	rv = map_session_to_real (state->px, &handle, &map, NULL);
	if (rv != CKR_OK)
		return rv;
	return (map.funcs->C_GenerateKey) (handle, mechanism, template, count, key);
}


proxy_C_GenerateKeyPair (CK_X_FUNCTION_LIST *self,
                         CK_SESSION_HANDLE handle,
                         CK_MECHANISM_PTR mechanism,
                         CK_ATTRIBUTE_PTR pub_template,
                         CK_ULONG pub_count,
                         CK_ATTRIBUTE_PTR priv_template,
                         CK_ULONG priv_count,
                         CK_OBJECT_HANDLE_PTR pub_key,
                         CK_OBJECT_HANDLE_PTR priv_key)
{
	State *state = (State *)self;
	Mapping map;
	CK_RV rv;

	rv = map_session_to_real (state->px, &handle, &map, NULL);
	if (rv != CKR_OK)
		return rv;
	return (map.funcs->C_GenerateKeyPair) (handle, mechanism, pub_template, pub_count, priv_template, priv_count, pub_key, priv_key);
}


rpc_C_SignEncryptUpdate (CK_X_FUNCTION_LIST *self,
                         p11_rpc_message *msg)
{
	CK_SESSION_HANDLE session;
	CK_BYTE_PTR part;
	CK_ULONG part_len;
	CK_BYTE_PTR encrypted_part;
	CK_ULONG encrypted_part_len;

	BEGIN_CALL (SignEncryptUpdate);
		IN_ULONG (session);
		IN_BYTE_ARRAY (part, part_len);
		IN_BYTE_BUFFER (encrypted_part, encrypted_part_len);
	PROCESS_CALL ((self, session, part, part_len, encrypted_part, &encrypted_part_len));
		OUT_BYTE_ARRAY (encrypted_part, encrypted_part_len);
	END_CALL;
}


p11_kit_iter_get_slot_info (P11KitIter *iter)
{
	return_val_if_fail (iter != NULL, NULL);
	return &iter->slot_info;
}


finish_object (P11KitIter *iter)
{
	iter->object = 0;
}


p11_kit_iter_next (P11KitIter *iter)
{
	CK_ULONG batch;
	CK_ULONG count;
	CK_BBOOL matches;
	CK_RV rv;

	return_val_if_fail (iter->iterating, CKR_OPERATION_NOT_INITIALIZED);

	COROUTINE_BEGIN (iter_next);

	iter->object = 0;

	if (iter->match_nothing)
		return finish_iterating (iter, CKR_CANCEL);

	if (!(iter->with_modules || iter->with_slots || iter->with_tokens || iter->with_objects))
		return finish_iterating (iter, CKR_CANCEL);

	/*
	 * If we have outstanding objects, then iterate one through those
	 * Note that we pass each object through the filters, and only
	 * assume it's iterated if it matches
	 */
	while (iter->with_objects && iter->saw_objects < iter->num_objects) {
		iter->object = iter->objects[iter->saw_objects++];

		rv = call_all_filters (iter, &matches);
		if (rv != CKR_OK)
			return finish_iterating (iter, rv);

		if (matches && iter->with_objects) {
			iter->kind = P11_KIT_ITER_KIND_OBJECT;
			COROUTINE_RETURN (iter_next, 1, CKR_OK);
		}
	}

	/* Move to next session, if we have finished searching
	 * objects, or we are looking for modules/slots/tokens */
	if ((iter->with_objects && iter->searched) ||
	    (!iter->with_objects &&
	     (iter->with_modules || iter->with_slots || iter->with_tokens))) {
		/* Use iter->kind as the sentinel to detect the case where
		 * any match (except object) is successful in
		 * move_next_session() */
		do {
			iter->kind = P11_KIT_ITER_KIND_UNKNOWN;
			rv = move_next_session (iter);
			if (rv != CKR_OK)
				return finish_iterating (iter, rv);
			if (iter->kind != P11_KIT_ITER_KIND_UNKNOWN)
				COROUTINE_RETURN (iter_next, 2, CKR_OK);
		} while (iter->move_next_session_state > 0);
	}

	/* Ready to start searching */
	if (iter->with_objects && !iter->searching && !iter->searched) {
		count = p11_attrs_count (iter->match_attrs);
		rv = (iter->module->C_FindObjectsInit) (iter->session, iter->match_attrs, count);
		if (rv != CKR_OK)
			return finish_iterating (iter, rv);
		iter->searching = 1;
		iter->searched = 0;
	}

	/* If we have searched on this session then try to continue */
	if (iter->with_objects && iter->searching) {
		assert (iter->module != NULL);
		assert (iter->session != 0);
		iter->num_objects = 0;
		iter->saw_objects = 0;

		for (;;) {
			if (iter->max_objects - iter->num_objects == 0) {
				CK_OBJECT_HANDLE *objects;

				iter->max_objects = iter->max_objects ? iter->max_objects * 2 : 64;
				objects = reallocarray (iter->objects, iter->max_objects, sizeof (CK_ULONG));
				return_val_if_fail (objects != NULL, CKR_HOST_MEMORY);
				iter->objects = objects;
			}

			batch = iter->max_objects - iter->num_objects;
			rv = (iter->module->C_FindObjects) (iter->session,
			                                    iter->objects + iter->num_objects,
			                                    batch, &count);
			if (rv != CKR_OK)
				return finish_iterating (iter, rv);

			iter->num_objects += count;

			/*
			 * Done searching on this session, although there are still
			 * objects outstanding, which will be returned on next
			 * iterations.
			 */
			if (batch != count) {
				iter->searching = 0;
				iter->searched = 1;
				(iter->module->C_FindObjectsFinal) (iter->session);
				break;
			}

			if (!iter->preload_results)
				break;
		}
	}

	COROUTINE_END (iter_next);

	/* Try again */
	iter->iter_next_state = 0;
	iter->move_next_session_state = 0;
	iter->kind = P11_KIT_ITER_KIND_UNKNOWN;
	return p11_kit_iter_next (iter);
}


p11_kit_iter_get_attributes (P11KitIter *iter,
                             CK_ATTRIBUTE *template,
                             CK_ULONG count)
{
	return_val_if_fail (iter != NULL, CKR_GENERAL_ERROR);
	return_val_if_fail (iter->iterating, CKR_GENERAL_ERROR);
	return_val_if_fail (iter->module != NULL, CKR_GENERAL_ERROR);
	return_val_if_fail (iter->session != 0, CKR_GENERAL_ERROR);
	return_val_if_fail (iter->object != 0, CKR_GENERAL_ERROR);

	return (iter->module->C_GetAttributeValue) (iter->session, iter->object,
	                                            template, count);
}


call_all_filters (P11KitIter *iter,
                  CK_BBOOL *matches)
{
	Callback *cb;
	CK_RV rv;

	*matches = CK_TRUE;

	for (cb = iter->callbacks; cb != NULL; cb = cb->next) {
		rv = (cb->func) (iter, matches, cb->callback_data);
		if (rv != CKR_OK || !*matches)
			return rv;
	}

	return CKR_OK;
}


p11_mmap_open (const char *path,
               struct stat *sb,
               void **data,
               size_t *size)
{
	struct stat stb;
	p11_mmap *map;

	map = calloc (1, sizeof (p11_mmap));
	if (map == NULL)
		return NULL;

	map->fd = open (path, O_RDONLY | O_CLOEXEC);
	if (map->fd == -1) {
		free (map);
		return NULL;
	}

	if (sb == NULL) {
		sb = &stb;
		if (fstat (map->fd, &stb) < 0) {
			close (map->fd);
			free (map);
			return NULL;
		}
	}

	/* Workaround for broken ZFS on Linux */
	if (S_ISDIR (sb->st_mode)) {
		errno = EISDIR;
		close (map->fd);
		free (map);
		return NULL;
	}

	if (sb->st_size == 0) {
		*data = "";
		*size = 0;
		return map;
	}

	map->size = sb->st_size;
	map->data = mmap (NULL, map->size, PROT_READ, MAP_PRIVATE, map->fd, 0);
	if (map->data == MAP_FAILED) {
		close (map->fd);
		free (map);
		return NULL;
	}

	*data = map->data;
	*size = map->size;
	return map;
}


static int skiploop(void)
{
	int skip = evalskip;

	switch (skip) {
	case 0:
		break;

	case SKIPBREAK:
	case SKIPCONT:
		if (likely(--skipcount <= 0)) {
			evalskip = 0;
			break;
		}

		skip = SKIPBREAK;
		break;
	}

	return skip;
}


breakcmd(int argc, char **argv)
{
	int n = argc > 1 ? number(argv[1]) : 1;

	if (n <= 0)
		badnum(argv[1]);
	if (n > loopnest)
		n = loopnest;
	if (n > 0) {
		evalskip = (**argv == 'c')? SKIPCONT : SKIPBREAK;
		skipcount = n;
	}
	return 0;
}


static inline void rb_event_discard(struct ring_buffer_event *event)
{
	if (extended_time(event))
		event = skip_time_extend(event);

	/* array[0] holds the actual length for the discarded event */
	event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE;
	event->type_len = RINGBUF_TYPE_PADDING;
	/* time delta must be non zero */
	if (!event->time_delta)
		event->time_delta = 1;
}


void ring_buffer_nest_start(struct trace_buffer *buffer)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	int cpu;

	/* Enabled by ring_buffer_nest_end() */
	preempt_disable_notrace();
	cpu = raw_smp_processor_id();
	cpu_buffer = buffer->buffers[cpu];
	/* This is the shift value for the above recursive locking */
	cpu_buffer->nest += NESTED_BITS;
}


static inline u64 rb_time_stamp(struct trace_buffer *buffer)
{
	u64 ts;

	/* Skip retpolines :-( */
	if (IS_ENABLED(CONFIG_RETPOLINE) && likely(buffer->clock == trace_clock_local))
		ts = trace_clock_local();
	else
		ts = buffer->clock();

	/* shift to debug/test normalization and TIME_EXTENTS */
	return ts << DEBUG_SHIFT;
}


rb_event_data(struct ring_buffer_event *event)
{
	if (extended_time(event))
		event = skip_time_extend(event);
	WARN_ON_ONCE(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
	/* If length is in len field, then array[0] has the data */
	if (event->type_len)
		return (void *)&event->array[0];
	/* Otherwise length is in array[0] and array[1] has the data */
	return (void *)&event->array[1];
}


void ring_buffer_discard_commit(struct trace_buffer *buffer,
				struct ring_buffer_event *event)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	int cpu;

	/* The event is discarded regardless */
	rb_event_discard(event);

	cpu = smp_processor_id();
	cpu_buffer = buffer->buffers[cpu];

	/*
	 * This must only be called if the event has not been
	 * committed yet. Thus we can assume that preemption
	 * is still disabled.
	 */
	RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));

	rb_decrement_entry(cpu_buffer, event);
	if (rb_try_to_discard(cpu_buffer, event))
		goto out;

 out:
	rb_end_commit(cpu_buffer);

	trace_recursive_unlock(cpu_buffer);

	preempt_enable_notrace();

}


void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, void *data)
{
	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
	struct buffer_data_page *bpage = data;
	struct page *page = virt_to_page(bpage);
	unsigned long flags;

	/* If the page is still in use someplace else, we can't reuse it */
	if (page_ref_count(page) > 1)
		goto out;

	local_irq_save(flags);
	arch_spin_lock(&cpu_buffer->lock);

	if (!cpu_buffer->free_page) {
		cpu_buffer->free_page = bpage;
		bpage = NULL;
	}

	arch_spin_unlock(&cpu_buffer->lock);
	local_irq_restore(flags);

 out:
	free_page((unsigned long)bpage);
}


static void rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer,
			       struct buffer_page *tail_page,
			       struct buffer_page *next_page)
{
	unsigned long old_entries;
	unsigned long old_write;

	/*
	 * The tail page now needs to be moved forward.
	 *
	 * We need to reset the tail page, but without messing
	 * with possible erasing of data brought in by interrupts
	 * that have moved the tail page and are currently on it.
	 *
	 * We add a counter to the write field to denote this.
	 */
	old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
	old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);

	local_inc(&cpu_buffer->pages_touched);
	/*
	 * Just make sure we have seen our old_write and synchronize
	 * with any interrupts that come in.
	 */
	barrier();

	/*
	 * If the tail page is still the same as what we think
	 * it is, then it is up to us to update the tail
	 * pointer.
	 */
	if (tail_page == READ_ONCE(cpu_buffer->tail_page)) {
		/* Zero the write counter */
		unsigned long val = old_write & ~RB_WRITE_MASK;
		unsigned long eval = old_entries & ~RB_WRITE_MASK;

		/*
		 * This will only succeed if an interrupt did
		 * not come in and change it. In which case, we
		 * do not want to modify it.
		 *
		 * We add (void) to let the compiler know that we do not care
		 * about the return value of these functions. We use the
		 * cmpxchg to only update if an interrupt did not already
		 * do it for us. If the cmpxchg fails, we don't care.
		 */
		(void)local_cmpxchg(&next_page->write, old_write, val);
		(void)local_cmpxchg(&next_page->entries, old_entries, eval);

		/*
		 * No need to worry about races with clearing out the commit.
		 * it only can increment when a commit takes place. But that
		 * only happens in the outer most nested commit.
		 */
		local_set(&next_page->page->commit, 0);

		/* Again, either we update tail_page or an interrupt does */
		(void)cmpxchg(&cpu_buffer->tail_page, tail_page, next_page);
	}
}


static __always_inline unsigned rb_page_commit(struct buffer_page *bpage)
{
	return local_read(&bpage->page->commit);
}


size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu)
{
	size_t read;
	size_t cnt;

	read = local_read(&buffer->buffers[cpu]->pages_read);
	cnt = local_read(&buffer->buffers[cpu]->pages_touched);
	/* The reader can read an empty page, but not more than that */
	if (cnt < read) {
		WARN_ON_ONCE(read > cnt + 1);
		return 0;
	}

	return cnt - read;
}


unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned long ret;

	if (!cpumask_test_cpu(cpu, buffer->cpumask))
		return 0;

	cpu_buffer = buffer->buffers[cpu];
	ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes;

	return ret;
}


static void rb_time_set(rb_time_t *t, u64 val)
{
	local64_set(&t->time, val);
}


static inline void rb_time_val_set(local_t *t, unsigned long val, unsigned long cnt)
{
	val = rb_time_val_cnt(val, cnt);
	local_set(t, val);
}


static inline u64 rb_time_val(unsigned long top, unsigned long bottom)
{
	u64 val;

	val = top & RB_TIME_VAL_MASK;
	val <<= RB_TIME_SHIFT;
	val |= bottom & RB_TIME_VAL_MASK;

	return val;
}


static int rb_time_cmpxchg(rb_time_t *t, u64 expect, u64 set)
{
	unsigned long cnt, top, bottom;
	unsigned long cnt2, top2, bottom2;
	u64 val;

	/* The cmpxchg always fails if it interrupted an update */
	 if (!__rb_time_read(t, &val, &cnt2))
		 return false;

	 if (val != expect)
		 return false;

	 cnt = local_read(&t->cnt);
	 if ((cnt & 3) != cnt2)
		 return false;

	 cnt2 = cnt + 1;

	 rb_time_split(val, &top, &bottom);
	 top = rb_time_val_cnt(top, cnt);
	 bottom = rb_time_val_cnt(bottom, cnt);

	 rb_time_split(set, &top2, &bottom2);
	 top2 = rb_time_val_cnt(top2, cnt2);
	 bottom2 = rb_time_val_cnt(bottom2, cnt2);

	if (!rb_time_read_cmpxchg(&t->cnt, cnt, cnt2))
		return false;
	if (!rb_time_read_cmpxchg(&t->top, top, top2))
		return false;
	if (!rb_time_read_cmpxchg(&t->bottom, bottom, bottom2))
		return false;
	return true;
}


unsigned long ring_buffer_entries(struct trace_buffer *buffer)
{
	struct ring_buffer_per_cpu *cpu_buffer;
	unsigned long entries = 0;
	int cpu;

	/* if you care about this being correct, lock the buffer */
	for_each_buffer_cpu(buffer, cpu) {
		cpu_buffer = buffer->buffers[cpu];
		entries += rb_num_of_entries(cpu_buffer);
	}

	return entries;
}


rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
		  struct ring_buffer_event *event)
{
	unsigned long new_index, old_index;
	struct buffer_page *bpage;
	unsigned long index;
	unsigned long addr;
	u64 write_stamp;
	u64 delta;

	new_index = rb_event_index(event);
	old_index = new_index + rb_event_ts_length(event);
	addr = (unsigned long)event;
	addr &= PAGE_MASK;

	bpage = READ_ONCE(cpu_buffer->tail_page);

	delta = rb_time_delta(event);

	if (!rb_time_read(&cpu_buffer->write_stamp, &write_stamp))
		return 0;

	/* Make sure the write stamp is read before testing the location */
	barrier();

	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
		unsigned long write_mask =
			local_read(&bpage->write) & ~RB_WRITE_MASK;
		unsigned long event_length = rb_event_length(event);

		/* Something came in, can't discard */
		if (!rb_time_cmpxchg(&cpu_buffer->write_stamp,
				       write_stamp, write_stamp - delta))
			return 0;

		/*
		 * If an event were to come in now, it would see that the
		 * write_stamp and the before_stamp are different, and assume
		 * that this event just added itself before updating
		 * the write stamp. The interrupting event will fix the
		 * write stamp for us, and use the before stamp as its delta.
		 */

		/*
		 * This is on the tail page. It is possible that
		 * a write could come in and move the tail page
		 * and write to the next page. That is fine
		 * because we just shorten what is on this page.
		 */
		old_index += write_mask;
		new_index += write_mask;
		index = local_cmpxchg(&bpage->write, old_index, new_index);
		if (index == old_index) {
			/* update counters */
			local_sub(event_length, &cpu_buffer->entries_bytes);
			return 1;
		}
	}

	/* could not discard */
	return 0;
}


void ring_buffer_record_disable(struct trace_buffer *buffer)
{
	atomic_inc(&buffer->record_disabled);
}


rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
		     struct ring_buffer_event *event)
{
	u64 delta;

	switch (event->type_len) {
	case RINGBUF_TYPE_PADDING:
		return;

	case RINGBUF_TYPE_TIME_EXTEND:
		delta = ring_buffer_event_time_stamp(event);
		cpu_buffer->read_stamp += delta;
		return;

	case RINGBUF_TYPE_TIME_STAMP:
		delta = ring_buffer_event_time_stamp(event);
		cpu_buffer->read_stamp = delta;
		return;

	case RINGBUF_TYPE_DATA:
		cpu_buffer->read_stamp += event->time_delta;
		return;

	default:
		RB_WARN_ON(cpu_buffer, 1);
	}
	return;
}


bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer)
{
	return buffer->time_stamp_abs;
}


static int __rb_allocate_pages(long nr_pages, struct list_head *pages, int cpu)
{
	struct buffer_page *bpage, *tmp;
	bool user_thread = current->mm != NULL;
	gfp_t mflags;
	long i;

	/*
	 * Check if the available memory is there first.
	 * Note, si_mem_available() only gives us a rough estimate of available
	 * memory. It may not be accurate. But we don't care, we just want
	 * to prevent doing any allocation when it is obvious that it is
	 * not going to succeed.
	 */
	i = si_mem_available();
	if (i < nr_pages)
		return -ENOMEM;

	/*
	 * __GFP_RETRY_MAYFAIL flag makes sure that the allocation fails
	 * gracefully without invoking oom-killer and the system is not
	 * destabilized.
	 */
	mflags = GFP_KERNEL | __GFP_RETRY_MAYFAIL;

	/*
	 * If a user thread allocates too much, and si_mem_available()
	 * reports there's enough memory, even though there is not.
	 * Make sure the OOM killer kills this thread. This can happen
	 * even with RETRY_MAYFAIL because another task may be doing
	 * an allocation after this task has taken all memory.
	 * This is the task the OOM killer needs to take out during this
	 * loop, even if it was triggered by an allocation somewhere else.
	 */
	if (user_thread)
		set_current_oom_origin();
	for (i = 0; i < nr_pages; i++) {
		struct page *page;

		bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
				    mflags, cpu_to_node(cpu));
		if (!bpage)
			goto free_pages;

		list_add(&bpage->list, pages);

		page = alloc_pages_node(cpu_to_node(cpu), mflags, 0);
		if (!page)
			goto free_pages;
		bpage->page = page_address(page);
		rb_init_page(bpage->page);

		if (user_thread && fatal_signal_pending(current))
			goto free_pages;
	}
	if (user_thread)
		clear_current_oom_origin();

	return 0;

free_pages:
	list_for_each_entry_safe(bpage, tmp, pages, list) {
		list_del_init(&bpage->list);
		free_buffer_page(bpage);
	}
	if (user_thread)
		clear_current_oom_origin();

	return -ENOMEM;
}


static long io_uring_setup(u32 entries, struct io_uring_params __user *params)
{
	struct io_uring_params p;
	int i;

	if (copy_from_user(&p, params, sizeof(p)))
		return -EFAULT;
	for (i = 0; i < ARRAY_SIZE(p.resv); i++) {
		if (p.resv[i])
			return -EINVAL;
	}

	if (p.flags & ~(IORING_SETUP_IOPOLL | IORING_SETUP_SQPOLL |
			IORING_SETUP_SQ_AFF | IORING_SETUP_CQSIZE |
			IORING_SETUP_CLAMP | IORING_SETUP_ATTACH_WQ))
		return -EINVAL;

	return  io_uring_create(entries, &p, params);
}


static unsigned int sane_fdtable_size(struct fdtable *fdt, unsigned int max_fds)
{
	unsigned int count;

	count = count_open_files(fdt);
	if (max_fds < NR_OPEN_DEFAULT)
		max_fds = NR_OPEN_DEFAULT;
	return min(count, max_fds);
}


static int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	struct io_splice* sp = &req->splice;

	sp->off_in = READ_ONCE(sqe->splice_off_in);
	sp->off_out = READ_ONCE(sqe->off);
	return __io_splice_prep(req, sqe);
}


static void __io_req_task_cancel(struct io_kiocb *req, int error)
{
	struct io_ring_ctx *ctx = req->ctx;

	spin_lock_irq(&ctx->completion_lock);
	io_cqring_fill_event(req, error);
	io_commit_cqring(ctx);
	spin_unlock_irq(&ctx->completion_lock);

	io_cqring_ev_posted(ctx);
	req_set_fail_links(req);
	io_double_put_req(req);
}


static struct io_poll_iocb *io_poll_get_double(struct io_kiocb *req)
{
	/* pure poll stashes this in ->io, poll driven retry elsewhere */
	if (req->opcode == IORING_OP_POLL_ADD)
		return (struct io_poll_iocb *) req->io;
	return req->apoll->double_poll;
}


static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret)
{
	switch (ret) {
	case -EIOCBQUEUED:
		break;
	case -ERESTARTSYS:
	case -ERESTARTNOINTR:
	case -ERESTARTNOHAND:
	case -ERESTART_RESTARTBLOCK:
		/*
		 * We can't just restart the syscall, since previously
		 * submitted sqes may already be in progress. Just fail this
		 * IO with EINTR.
		 */
		ret = -EINTR;
		fallthrough;
	default:
		kiocb->ki_complete(kiocb, ret, 0);
	}
}


static int io_uring_get_fd(struct io_ring_ctx *ctx)
{
	struct file *file;
	int ret;

#if defined(CONFIG_UNIX)
	ret = sock_create_kern(&init_net, PF_UNIX, SOCK_RAW, IPPROTO_IP,
				&ctx->ring_sock);
	if (ret)
		return ret;
#endif

	ret = get_unused_fd_flags(O_RDWR | O_CLOEXEC);
	if (ret < 0)
		goto err;

	file = anon_inode_getfile("[io_uring]", &io_uring_fops, ctx,
					O_RDWR | O_CLOEXEC);
	if (IS_ERR(file)) {
err_fd:
		put_unused_fd(ret);
		ret = PTR_ERR(file);
		goto err;
	}

#if defined(CONFIG_UNIX)
	ctx->ring_sock->file = file;
#endif
	if (unlikely(io_uring_add_task_file(file))) {
		file = ERR_PTR(-ENOMEM);
		goto err_fd;
	}
	fd_install(ret, file);
	return ret;
err:
#if defined(CONFIG_UNIX)
	sock_release(ctx->ring_sock);
	ctx->ring_sock = NULL;
#endif
	return ret;
}


static int do_execveat_common(int fd, struct filename *filename,
			      struct user_arg_ptr argv,
			      struct user_arg_ptr envp,
			      int flags)
{
	struct linux_binprm *bprm;
	int retval;

	if (IS_ERR(filename))
		return PTR_ERR(filename);

	/*
	 * We move the actual failure in case of RLIMIT_NPROC excess from
	 * set*uid() to execve() because too many poorly written programs
	 * don't check setuid() return code.  Here we additionally recheck
	 * whether NPROC limit is still exceeded.
	 */
	if ((current->flags & PF_NPROC_EXCEEDED) &&
	    atomic_read(&current_user()->processes) > rlimit(RLIMIT_NPROC)) {
		retval = -EAGAIN;
		goto out_ret;
	}

	/* We're below the limit (still or again), so we don't want to make
	 * further execve() calls fail. */
	current->flags &= ~PF_NPROC_EXCEEDED;

	bprm = alloc_bprm(fd, filename);
	if (IS_ERR(bprm)) {
		retval = PTR_ERR(bprm);
		goto out_ret;
	}

	retval = count(argv, MAX_ARG_STRINGS);
	if (retval < 0)
		goto out_free;
	bprm->argc = retval;

	retval = count(envp, MAX_ARG_STRINGS);
	if (retval < 0)
		goto out_free;
	bprm->envc = retval;

	retval = bprm_stack_limits(bprm);
	if (retval < 0)
		goto out_free;

	retval = copy_string_kernel(bprm->filename, bprm);
	if (retval < 0)
		goto out_free;
	bprm->exec = bprm->p;

	retval = copy_strings(bprm->envc, envp, bprm);
	if (retval < 0)
		goto out_free;

	retval = copy_strings(bprm->argc, argv, bprm);
	if (retval < 0)
		goto out_free;

	retval = bprm_execve(bprm, fd, filename, flags);
out_free:
	free_bprm(bprm);

out_ret:
	putname(filename);
	return retval;
}


static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock)
{
	int ret;

	/* sync_file_range always requires a blocking context */
	if (force_nonblock)
		return -EAGAIN;

	ret = sync_file_range(req->file, req->sync.off, req->sync.len,
				req->sync.flags);
	if (ret < 0)
		req_set_fail_links(req);
	io_req_complete(req, ret);
	return 0;
}


static unsigned long rings_size(unsigned sq_entries, unsigned cq_entries,
				size_t *sq_offset)
{
	struct io_rings *rings;
	size_t off, sq_array_size;

	off = struct_size(rings, cqes, cq_entries);
	if (off == SIZE_MAX)
		return SIZE_MAX;

#ifdef CONFIG_SMP
	off = ALIGN(off, SMP_CACHE_BYTES);
	if (off == 0)
		return SIZE_MAX;
#endif

	if (sq_offset)
		*sq_offset = off;

	sq_array_size = array_size(sizeof(u32), sq_entries);
	if (sq_array_size == SIZE_MAX)
		return SIZE_MAX;

	if (check_add_overflow(off, sq_array_size, &off))
		return SIZE_MAX;

	return off;
}


static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}


static void io_req_map_rw(struct io_kiocb *req, const struct iovec *iovec,
			  const struct iovec *fast_iov, struct iov_iter *iter)
{
	struct io_async_rw *rw = &req->io->rw;

	memcpy(&rw->iter, iter, sizeof(*iter));
	rw->free_iovec = NULL;
	rw->bytes_done = 0;
	/* can only be fixed buffers, no need to do anything */
	if (iter->type == ITER_BVEC)
		return;
	if (!iovec) {
		unsigned iov_off = 0;

		rw->iter.iov = rw->fast_iov;
		if (iter->iov != fast_iov) {
			iov_off = iter->iov - fast_iov;
			rw->iter.iov += iov_off;
		}
		if (rw->fast_iov != fast_iov)
			memcpy(rw->fast_iov + iov_off, fast_iov + iov_off,
			       sizeof(struct iovec) * iter->nr_segs);
	} else {
		rw->free_iovec = iovec;
		req->flags |= REQ_F_NEED_CLEANUP;
	}
}


static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}


static inline void __set_open_fd(unsigned int fd, struct fdtable *fdt)
{
	__set_bit(fd, fdt->open_fds);
	fd /= BITS_PER_LONG;
	if (!~fdt->open_fds[fd])
		__set_bit(fd, fdt->full_fds_bits);
}


static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
			      wait_queue_func_t wake_func)
{
	poll->head = NULL;
	poll->done = false;
	poll->canceled = false;
	poll->events = events;
	INIT_LIST_HEAD(&poll->wait.entry);
	init_waitqueue_func_entry(&poll->wait, wake_func);
}


static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
{
	bool has_lock;
	int i;

	/*
	 * Avoid ABBA deadlock between the seq lock and the io_uring mutex,
	 * since fdinfo case grabs it in the opposite direction of normal use
	 * cases. If we fail to get the lock, we just don't iterate any
	 * structures that could be going away outside the io_uring mutex.
	 */
	has_lock = mutex_trylock(&ctx->uring_lock);

	seq_printf(m, "UserFiles:\t%u\n", ctx->nr_user_files);
	for (i = 0; has_lock && i < ctx->nr_user_files; i++) {
		struct fixed_file_table *table;
		struct file *f;

		table = &ctx->file_data->table[i >> IORING_FILE_TABLE_SHIFT];
		f = table->files[i & IORING_FILE_TABLE_MASK];
		if (f)
			seq_printf(m, "%5u: %s\n", i, file_dentry(f)->d_iname);
		else
			seq_printf(m, "%5u: <none>\n", i);
	}
	seq_printf(m, "UserBufs:\t%u\n", ctx->nr_user_bufs);
	for (i = 0; has_lock && i < ctx->nr_user_bufs; i++) {
		struct io_mapped_ubuf *buf = &ctx->user_bufs[i];

		seq_printf(m, "%5u: 0x%llx/%u\n", i, buf->ubuf,
						(unsigned int) buf->len);
	}
	if (has_lock && !idr_is_empty(&ctx->personality_idr)) {
		seq_printf(m, "Personalities:\n");
		idr_for_each(&ctx->personality_idr, io_uring_show_cred, m);
	}
	seq_printf(m, "PollList:\n");
	spin_lock_irq(&ctx->completion_lock);
	for (i = 0; i < (1U << ctx->cancel_hash_bits); i++) {
		struct hlist_head *list = &ctx->cancel_hash[i];
		struct io_kiocb *req;

		hlist_for_each_entry(req, list, hash_node)
			seq_printf(m, "  op=%d, task_works=%d\n", req->opcode,
					req->task->task_works != NULL);
	}
	spin_unlock_irq(&ctx->completion_lock);
	if (has_lock)
		mutex_unlock(&ctx->uring_lock);
}


static int __io_sq_thread_acquire_mm(struct io_ring_ctx *ctx)
{
	if (!current->mm) {
		if (unlikely(!(ctx->flags & IORING_SETUP_SQPOLL) ||
			     !ctx->sqo_task->mm ||
			     !mmget_not_zero(ctx->sqo_task->mm)))
			return -EFAULT;
		kthread_use_mm(ctx->sqo_task->mm);
	}

	return 0;
}


static inline void __set_close_on_exec(unsigned int fd, struct fdtable *fdt)
{
	__set_bit(fd, fdt->close_on_exec);
}


static int do_execveat(int fd, struct filename *filename,
		const char __user *const __user *__argv,
		const char __user *const __user *__envp,
		int flags)
{
	struct user_arg_ptr argv = { .ptr.native = __argv };
	struct user_arg_ptr envp = { .ptr.native = __envp };

	return do_execveat_common(fd, filename, argv, envp, flags);
}


static inline bool io_async_submit(struct io_ring_ctx *ctx)
{
	return ctx->flags & IORING_SETUP_SQPOLL;
}


static int io_provide_buffers_prep(struct io_kiocb *req,
				   const struct io_uring_sqe *sqe)
{
	struct io_provide_buf *p = &req->pbuf;
	u64 tmp;

	if (sqe->ioprio || sqe->rw_flags)
		return -EINVAL;

	tmp = READ_ONCE(sqe->fd);
	if (!tmp || tmp > USHRT_MAX)
		return -E2BIG;
	p->nbufs = tmp;
	p->addr = READ_ONCE(sqe->addr);
	p->len = READ_ONCE(sqe->len);

	if (!access_ok(u64_to_user_ptr(p->addr), (p->len * p->nbufs)))
		return -EFAULT;

	p->bgid = READ_ONCE(sqe->buf_group);
	tmp = READ_ONCE(sqe->off);
	if (tmp > USHRT_MAX)
		return -E2BIG;
	p->bid = tmp;
	return 0;
}


int __close_fd_get_file(unsigned int fd, struct file **res)
{
	struct files_struct *files = current->files;
	struct file *file;
	struct fdtable *fdt;

	spin_lock(&files->file_lock);
	fdt = files_fdtable(files);
	if (fd >= fdt->max_fds)
		goto out_unlock;
	file = fdt->fd[fd];
	if (!file)
		goto out_unlock;
	rcu_assign_pointer(fdt->fd[fd], NULL);
	__put_unused_fd(files, fd);
	spin_unlock(&files->file_lock);
	get_file(file);
	*res = file;
	return 0;

out_unlock:
	spin_unlock(&files->file_lock);
	*res = NULL;
	return -ENOENT;
}


static bool io_poll_remove_one(struct io_kiocb *req)
{
	bool do_complete;

	io_poll_remove_double(req);

	if (req->opcode == IORING_OP_POLL_ADD) {
		do_complete = __io_poll_remove_one(req, &req->poll);
	} else {
		struct async_poll *apoll = req->apoll;

		/* non-poll requests have submit ref still */
		do_complete = __io_poll_remove_one(req, &apoll->poll);
		if (do_complete) {
			io_put_req(req);
			kfree(apoll->double_poll);
			kfree(apoll);
		}
	}

	if (do_complete) {
		io_cqring_fill_event(req, -ECANCELED);
		io_commit_cqring(req->ctx);
		req->flags |= REQ_F_COMP_LOCKED;
		req_set_fail_links(req);
		io_put_req(req);
	}

	return do_complete;
}


static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
{
	return -EOPNOTSUPP;
}


static int copy_strings_kernel(int argc, const char *const *argv,
			       struct linux_binprm *bprm)
{
	while (argc-- > 0) {
		int ret = copy_string_kernel(argv[argc], bprm);
		if (ret < 0)
			return ret;
		if (fatal_signal_pending(current))
			return -ERESTARTNOHAND;
		cond_resched();
	}
	return 0;
}


static inline void io_uring_files_cancel(struct files_struct *files)
{
	if (current->io_uring && !xa_empty(&current->io_uring->xa))
		__io_uring_files_cancel(files);
}


static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
					      int index)
{
	struct fixed_file_table *table;

	table = &ctx->file_data->table[index >> IORING_FILE_TABLE_SHIFT];
	return table->files[index & IORING_FILE_TABLE_MASK];
}


static inline void io_consume_sqe(struct io_ring_ctx *ctx)
{
	ctx->cached_sq_head++;
}


static int exec_binprm(struct linux_binprm *bprm)
{
	pid_t old_pid, old_vpid;
	int ret, depth;

	/* Need to fetch pid before load_binary changes it */
	old_pid = current->pid;
	rcu_read_lock();
	old_vpid = task_pid_nr_ns(current, task_active_pid_ns(current->parent));
	rcu_read_unlock();

	/* This allows 4 levels of binfmt rewrites before failing hard. */
	for (depth = 0;; depth++) {
		struct file *exec;
		if (depth > 5)
			return -ELOOP;

		ret = search_binary_handler(bprm);
		if (ret < 0)
			return ret;
		if (!bprm->interpreter)
			break;

		exec = bprm->file;
		bprm->file = bprm->interpreter;
		bprm->interpreter = NULL;

		allow_write_access(exec);
		if (unlikely(bprm->have_execfd)) {
			if (bprm->executable) {
				fput(exec);
				return -ENOEXEC;
			}
			bprm->executable = exec;
		} else
			fput(exec);
	}

	audit_bprm(bprm);
	trace_sched_process_exec(current, old_pid, bprm);
	ptrace_event(PTRACE_EVENT_EXEC, old_vpid);
	proc_exec_connector(current);
	return 0;
}


static void io_destroy_buffers(struct io_ring_ctx *ctx)
{
	idr_for_each(&ctx->io_buffer_idr, __io_destroy_buffers, ctx);
	idr_destroy(&ctx->io_buffer_idr);
}


static int io_uring_alloc_task_context(struct task_struct *task)
{
	struct io_uring_task *tctx;

	tctx = kmalloc(sizeof(*tctx), GFP_KERNEL);
	if (unlikely(!tctx))
		return -ENOMEM;

	xa_init(&tctx->xa);
	init_waitqueue_head(&tctx->wait);
	tctx->last = NULL;
	tctx->in_idle = 0;
	atomic_long_set(&tctx->req_issue, 0);
	atomic_long_set(&tctx->req_complete, 0);
	task->io_uring = tctx;
	return 0;
}


static int io_openat(struct io_kiocb *req, bool force_nonblock)
{
	return io_openat2(req, force_nonblock);
}


static struct io_kiocb *io_req_link_next(struct io_kiocb *req)
{
	struct io_kiocb *nxt;

	/*
	 * The list should never be empty when we are called here. But could
	 * potentially happen if the chain is messed up, check to be on the
	 * safe side.
	 */
	if (unlikely(list_empty(&req->link_list)))
		return NULL;

	nxt = list_first_entry(&req->link_list, struct io_kiocb, link_list);
	list_del_init(&req->link_list);
	if (!list_empty(&nxt->link_list))
		nxt->flags |= REQ_F_LINK_HEAD;
	return nxt;
}


ldbm_config_modify_entry_callback(Slapi_PBlock *pb, Slapi_Entry *entryBefore, Slapi_Entry *e, int *returncode, char *returntext, void *arg)
{
    int i;
    char *attr_name;
    LDAPMod **mods;
    Slapi_Mods smods_moved;
    int rc = LDAP_SUCCESS;
    int apply_mod = 0;
    struct ldbminfo *li = (struct ldbminfo *)arg;
    int reapply_mods = 0;
    int move_mods = 0;
    int idx = 0;
    int internal_op = 0;
    Slapi_Operation *operation = NULL;

    /* This lock is probably way too conservative, but we don't expect much
     * contention for it. */
    PR_Lock(li->li_config_mutex);

    dblayer_private *priv = (dblayer_private *)li->li_dblayer_private;
    slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods);
    slapi_mods_init(&smods_moved, 0);

    slapi_pblock_get(pb, SLAPI_OPERATION, &operation);
    internal_op = operation_is_flag_set(operation, OP_FLAG_INTERNAL);

    returntext[0] = '\0';

    slapi_log_err(SLAPI_LOG_CONFIG, "ldbm_config_modify_entry_callback", "Executing for entry (%s) with flags (%d) operation is internal: %d\n",
                    slapi_entry_get_dn_const(e), li->li_flags, internal_op);
    /*
     * First pass: set apply mods to 0 so only input validation will be done;
     * 2nd pass: set apply mods to 1 to apply changes to internal storage
     */
    for (apply_mod = 0; apply_mod <= 1 && LDAP_SUCCESS == rc; apply_mod++) {
        for (i = 0; mods && mods[i] && LDAP_SUCCESS == rc; i++) {
            attr_name = mods[i]->mod_type;

            /* There are some attributes that we don't care about, like modifiersname. */
            if (ldbm_config_ignored_attr(attr_name)) {
                if (apply_mod) {
                    Slapi_Attr *origattr = NULL;
                    Slapi_ValueSet *origvalues = NULL;
                    mods[idx++] = mods[i];
                    /* we also need to restore the entryAfter e to its original
                       state, because the dse code will attempt to reapply
                       the mods again */
                    slapi_entry_attr_find(entryBefore, attr_name, &origattr);
                    if (NULL != origattr) {
                        slapi_attr_get_valueset(origattr, &origvalues);
                        if (NULL != origvalues) {
                            slapi_entry_add_valueset(e, attr_name, origvalues);
                            slapi_valueset_free(origvalues);
                        }
                    }
                    reapply_mods = 1; /* there is at least one mod we removed */
                }
                continue;
            }

            if (ldbm_config_moved_attr(attr_name) && !internal_op) {
                rc = priv->dblayer_config_set_fn(li, attr_name, apply_mod, mods[i]->mod_op,
                                            CONFIG_PHASE_RUNNING,
                                            (mods[i]->mod_bvalues == NULL) ? NULL
                                            : mods[i]->mod_bvalues[0]->bv_val);
                if (apply_mod) {
                    slapi_entry_attr_delete(e, attr_name);
                    slapi_mods_add_ldapmod(&smods_moved, mods[i]);
                    move_mods++;
                    reapply_mods = 1; /* there is at least one mod we removed */
                }
                continue;
            }
            /* when deleting a value, and this is the last or only value, set
               the config param to its default value
               when adding a value, if the value is set to its default value, replace
               it with the new value - otherwise, if it is single valued, reject the
               operation with TYPE_OR_VALUE_EXISTS */
            /* This assumes there is only one bval for this mod. */
            rc = ldbm_config_set((void *)li, attr_name, ldbm_config,
                                 (mods[i]->mod_bvalues == NULL) ? NULL
                                                                : mods[i]->mod_bvalues[0],
                                 returntext,
                                 ((li->li_flags & LI_FORCE_MOD_CONFIG) ? CONFIG_PHASE_INTERNAL : CONFIG_PHASE_RUNNING),
                                 apply_mod, mods[i]->mod_op);
            if (rc) {
                slapi_log_err(SLAPI_LOG_ERR, "ldbm_config_modify_entry_callback", "Modifying config attribute %s failed (err=%d)\n", attr_name, rc);
            }
            if (apply_mod) {
                mod_free(mods[i]);
                mods[i] = NULL;
            }
        }
    }

    PR_Unlock(li->li_config_mutex);

    if (reapply_mods) {
        mods[idx] = NULL;
        slapi_pblock_set(pb, SLAPI_DSE_REAPPLY_MODS, &reapply_mods);
    }

    if (move_mods) {
        char *dn = slapi_ch_smprintf("cn=bdb,%s",CONFIG_LDBM_DN);
        Slapi_PBlock *mod_pb = slapi_pblock_new();
        slapi_modify_internal_set_pb(mod_pb, dn,
                                     slapi_mods_get_ldapmods_byref(&smods_moved),
                                     NULL, NULL, li->li_identity, 0);
        slapi_modify_internal_pb(mod_pb);
        slapi_pblock_destroy(mod_pb);
        slapi_ch_free_string(&dn);
        slapi_mods_done(&smods_moved);
    }

    *returncode = rc;
    if (LDAP_SUCCESS == rc) {
        return SLAPI_DSE_CALLBACK_OK;
    } else {
        return SLAPI_DSE_CALLBACK_ERROR;
    }
}


ldbm_config_db_old_idl_maxids_set(void *arg,
                                  void *value,
                                  char *errorbuf,
                                  int phase __attribute__((unused)),
                                  int apply)
{
    struct ldbminfo *li = (struct ldbminfo *)arg;
    int retval = LDAP_SUCCESS;
    int val = (int)((uintptr_t)value);

    if (val < 0) {
        slapi_create_errormsg(errorbuf, SLAPI_DSE_RETURNTEXT_SIZE,
                              "Error: Invalid value for %s (%d). Value must be equal or greater than zero.",
                              CONFIG_DB_OLD_IDL_MAXIDS, val);
        return LDAP_UNWILLING_TO_PERFORM;
    }

    if (apply) {
        li->li_old_idl_maxids = val;
    }

    return retval;
}


send_nobackend_ldap_result(Slapi_PBlock *pb)
{
    struct berval **refurls;
    int err;

    refurls = g_get_default_referral();
    err = (refurls == NULL) ? LDAP_NO_SUCH_OBJECT : LDAP_REFERRAL;
    /* richm 20010831 - bug 556992 - the post op code needs to know what the
       ldap error sent to the client was - slapi_send_ldap_result sets the
       err in the pblock, so this function needs to also */
    slapi_pblock_set(pb, SLAPI_RESULT_CODE, &err);

    slapi_send_ldap_result(pb, err, NULL, NULL, 0, refurls);
}


ldbm_config_exclude_from_export_get(void *arg)
{
    struct ldbminfo *li = (struct ldbminfo *)arg;
    char *p, *retstr = NULL;
    size_t len = 0;

    if (NULL != li->li_attrs_to_exclude_from_export &&
        NULL != li->li_attrs_to_exclude_from_export[0]) {
        int i;

        for (i = 0; li->li_attrs_to_exclude_from_export[i] != NULL; ++i) {
            len += strlen(li->li_attrs_to_exclude_from_export[i]) + 1;
        }
        p = retstr = slapi_ch_malloc(len);
        for (i = 0; li->li_attrs_to_exclude_from_export[i] != NULL; ++i) {
            if (i > 0) {
                *p++ = ' ';
            }
            strcpy(p, li->li_attrs_to_exclude_from_export[i]);
            p += strlen(p);
        }
        *p = '\0';
    } else {
        retstr = slapi_ch_strdup("");
    }

    return (void *)retstr;
}


g_set_default_referral(struct berval **ldap_url)
{
    struct berval **default_referral;
    slapdFrontendConfig_t *slapdFrontendConfig = getFrontendConfig();
    int nReferrals;

    /* check to see if we want to delete all referrals */
    if (ldap_url && ldap_url[0] &&
        PL_strncasecmp((char *)ldap_url[0]->bv_val, REFERRAL_REMOVE_CMD, ldap_url[0]->bv_len) == 0) {
        delete_default_referral(slapdFrontendConfig->defaultreferral);
        slapdFrontendConfig->defaultreferral = NULL;
        return;
    }

    /* count the number of referrals */
    for (nReferrals = 0; ldap_url && ldap_url[nReferrals]; nReferrals++)
        ;

    default_referral = (struct berval **)
        slapi_ch_malloc((nReferrals + 1) * sizeof(struct berval *));

    /* terminate the end, and add the referrals backwards */
    default_referral[nReferrals--] = NULL;

    while (nReferrals >= 0) {
        default_referral[nReferrals] = ber_bvdup(ldap_url[nReferrals]);
        nReferrals--;
    }

    delete_default_referral(slapdFrontendConfig->defaultreferral);
    slapdFrontendConfig->defaultreferral = default_referral;
}


send_ldap_result(
    Slapi_PBlock *pb,
    int err,
    char *matched,
    char *text,
    int nentries,
    struct berval **urls)
{
    send_ldap_result_ext(pb, err, matched, text, nentries, urls, NULL);
}


ldbm_config_read_instance_entries(struct ldbminfo *li, const char *backend_type)
{
    Slapi_PBlock *tmp_pb;
    Slapi_Entry **entries = NULL;
    char *basedn = NULL;
    int rc = 0;

    /* Construct the base dn of the subtree that holds the instance entries. */
    basedn = slapi_create_dn_string("cn=%s,cn=plugins,cn=config", backend_type);
    if (NULL == basedn) {
        slapi_log_err(SLAPI_LOG_ERR,
                      "ldbm_config_read_instance_entries",
                      "failed to create backend dn for %s\n", backend_type);
        return 1;
    }

    /* Do a search of the subtree containing the instance entries */
    tmp_pb = slapi_pblock_new();
    slapi_search_internal_set_pb(tmp_pb, basedn, LDAP_SCOPE_SUBTREE, "(objectclass=nsBackendInstance)", NULL, 0, NULL, NULL, li->li_identity, 0);
    slapi_search_internal_pb(tmp_pb);
    slapi_pblock_get(tmp_pb, SLAPI_PLUGIN_INTOP_SEARCH_ENTRIES, &entries);
    if (entries != NULL) {
        int i;
        for (i = 0; entries[i] != NULL; i++) {
            rc = ldbm_instance_add_instance_entry_callback(NULL,
                                                           entries[i], NULL, NULL, NULL, li);
            if (SLAPI_DSE_CALLBACK_ERROR == rc) {
                slapi_log_err(SLAPI_LOG_ERR,
                              "ldbm_config_read_instance_entries",
                              "Failed to add instance entry %s\n",
                              slapi_entry_get_dn_const(entries[i]));
                break;
            }
            rc = 0;
        }
    }

    slapi_free_search_results_internal(tmp_pb);
    slapi_pblock_destroy(tmp_pb);
    slapi_ch_free_string(&basedn);

    return rc;
}


dse_callback_delete(struct dse_callback **pp)
{
    if (pp != NULL) {
        slapi_sdn_free(&((*pp)->base));
        slapi_ch_free((void **)&((*pp)->filter));
        slapi_filter_free((*pp)->slapifilter, 1);
        slapi_ch_free((void **)pp);
    }
}


slapi_config_remove_callback(int operation, int flags, const char *base, int scope, const char *filter, dseCallbackFn fn)
{
    int rc = 0;
    Slapi_Backend *be = slapi_be_select_by_instance_name(DSE_BACKEND);
    if (be != NULL) {
        struct dse *pdse = (struct dse *)be->be_database->plg_private;
        if (pdse != NULL) {
            Slapi_DN dn;
            slapi_sdn_init_dn_byref(&dn, base);
            dse_remove_callback(pdse, operation, flags, &dn, scope, filter, fn);
            slapi_sdn_done(&dn);
            rc = 1;
        }
    }
    return rc;
}


dse_node_delete(struct dse_node **pp)
{
    slapi_entry_free((*pp)->entry);
    slapi_ch_free((void **)&(*pp));
    PR_DECREMENT_COUNTER(dse_entries_exist);
}


dse_write_file_nolock(struct dse *pdse)
{
    FPWrapper fpw;
    int rc = 0;

    if (dont_ever_write_dse_files) {
        return rc;
    }

    fpw.fpw_rc = 0;
    fpw.fpw_prfd = NULL;

    if (NULL != pdse->dse_filename) {
        if ((fpw.fpw_prfd = PR_Open(pdse->dse_tmpfile, PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, SLAPD_DEFAULT_FILE_MODE)) == NULL) {
            rc = PR_GetOSError();
            slapi_log_err(SLAPI_LOG_ERR, "dse_write_file_nolock", "Cannot open "
                                                                  "temporary DSE file \"%s\" for update: OS error %d (%s)\n",
                          pdse->dse_tmpfile, rc, slapd_system_strerror(rc));
        } else {
            fpw.fpw_pdse = pdse;
            if (avl_apply(pdse->dse_tree, dse_write_entry, &fpw, STOP_TRAVERSAL, AVL_INORDER) == STOP_TRAVERSAL) {
                rc = fpw.fpw_rc;
                slapi_log_err(SLAPI_LOG_ERR, "dse_write_file_nolock", "Cannot write "
                                                                      " temporary DSE file \"%s\": OS error %d (%s)\n",
                              pdse->dse_tmpfile, rc, slapd_system_strerror(rc));
                (void)PR_Close(fpw.fpw_prfd);
                fpw.fpw_prfd = NULL;
            } else {
                (void)PR_Close(fpw.fpw_prfd);
                fpw.fpw_prfd = NULL;
                if (pdse->dse_fileback != NULL) {
                    rc = slapi_destructive_rename(pdse->dse_filename, pdse->dse_fileback);
                    if (rc != 0) {
                        slapi_log_err(SLAPI_LOG_ERR, "dse_write_file_nolock", "Cannot backup"
                                                                              " DSE file \"%s\" to \"%s\": OS error %d (%s)\n",
                                      pdse->dse_filename, pdse->dse_fileback,
                                      rc, slapd_system_strerror(rc));
                    }
                }
                rc = slapi_destructive_rename(pdse->dse_tmpfile, pdse->dse_filename);
                if (rc != 0) {
                    slapi_log_err(SLAPI_LOG_ERR, "dse_write_file_nolock", "Cannot rename"
                                                                          " temporary DSE file \"%s\" to \"%s\":"
                                                                          " OS error %d (%s)\n",
                                  pdse->dse_tmpfile, pdse->dse_filename,
                                  rc, slapd_system_strerror(rc));
                }
                /*
                 * We have now written to the tmp location, and renamed it
                 * we need to open and fsync the dir to make the rename stick.
                 */
                int fp_configdir =
#ifdef O_PATH
                    open(pdse->dse_configdir, O_PATH | O_DIRECTORY)
#else
                    open(pdse->dse_configdir, O_RDONLY | O_DIRECTORY)
#endif
                    ;
                if (fp_configdir != -1) {
                    fsync(fp_configdir);
                    close(fp_configdir);
                }
            }
        }
        if (fpw.fpw_prfd)
            (void)PR_Close(fpw.fpw_prfd);
    }

    return rc;
}


dse_modify(Slapi_PBlock *pb) /* JCM There should only be one exit point from this function! */
{
    int err;             /*House keeping stuff*/
    LDAPMod **mods;      /*Used to apply the modifications*/
    LDAPMod **original_mods = NULL; /* some mods can be removed by callback, save them for later logging */
    char *errbuf = NULL; /* To get error back */
    struct dse *pdse;
    Slapi_Entry *ec = NULL;
    Slapi_Entry *ecc = NULL;
    int returncode = LDAP_SUCCESS;
    char returntext[SLAPI_DSE_RETURNTEXT_SIZE] = "";
    Slapi_DN *sdn = NULL;
    int dont_write_file = 0; /* default */
    int rc = SLAPI_DSE_CALLBACK_DO_NOT_APPLY;
    int retval = -1;
    int need_be_postop = 0;
    int plugin_started = 0;
    int internal_op = 0;
    PRBool global_lock_owned = PR_FALSE;
    Operation *pb_op = NULL;

    PR_ASSERT(pb);
    if (slapi_pblock_get(pb, SLAPI_PLUGIN_PRIVATE, &pdse) < 0 ||
        /* slapi_pblock_get( pb, SLAPI_MODIFY_TARGET, &dn ) < 0 || */
        slapi_pblock_get(pb, SLAPI_MODIFY_TARGET_SDN, &sdn) < 0 ||
        slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods) < 0 || (NULL == pdse)) {
        returncode = LDAP_OPERATIONS_ERROR;
        goto done;
    }

    slapi_pblock_get(pb, SLAPI_DSE_DONT_WRITE_WHEN_ADDING, &dont_write_file);
    if (!dont_write_file && dse_check_for_readonly_error(pb, pdse)) {
        /* already returned result */
        return retval;
    }

    slapi_pblock_get(pb, SLAPI_OPERATION, &pb_op);
    if (pb_op){
        internal_op = operation_is_flag_set(pb_op, OP_FLAG_INTERNAL);
    }
    /* Find the entry we are about to modify. */
    ec = dse_get_entry_copy(pdse, sdn, DSE_USE_LOCK);
    if (ec == NULL) {
        returncode = LDAP_NO_SUCH_OBJECT;
        goto done;
    }

    /* Check acl */
    err = plugin_call_acl_mods_access(pb, ec, mods, &errbuf);
    if (err != LDAP_SUCCESS) {
        returncode = err;
        if (errbuf) {
            PL_strncpyz(returntext, errbuf, sizeof(returntext));
            slapi_ch_free_string(&errbuf);
        }
        goto done;
    }

    /* Save away a copy of the entry, before modifications */
    slapi_pblock_set(pb, SLAPI_ENTRY_PRE_OP, slapi_entry_dup(ec)); /* JCM - When does this get free'd? */
    /* richm - it is freed in modify.c */

    /* Modify a copy of the entry*/
    ecc = slapi_entry_dup(ec);
    err = entry_apply_mods(ecc, mods);

    /* Possibly acquire the global backend lock */
    if (global_backend_lock_requested()) {
        global_backend_lock_lock();
        global_lock_owned = PR_TRUE;
    }
    original_mods = copy_mods(mods);

    /* XXXmcs: should we expand objectclass values here?? */
    /* give the dse callbacks the first crack at the modify */
    rc = dse_call_callback(pdse, pb, SLAPI_OPERATION_MODIFY, DSE_FLAG_PREOP, ec, ecc, &returncode, returntext);
    if (SLAPI_DSE_CALLBACK_OK == rc) {
        int plugin_rc;

        /* next, give the be plugins a crack at it */
        slapi_pblock_set(pb, SLAPI_RESULT_CODE, &returncode);
        slapi_pblock_set(pb, SLAPI_MODIFY_EXISTING_ENTRY, ecc);
        plugin_rc = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_PRE_MODIFY_FN);
        need_be_postop = 1; /* if the be preops were called, have to call the be postops too */
        if (!returncode) {
            slapi_pblock_get(pb, SLAPI_RESULT_CODE, &returncode);
        }
        if (!plugin_rc && !returncode) {
            /* finally, give the betxn plugins a crack at it */
            plugin_rc = plugin_call_plugins(pb, SLAPI_PLUGIN_BE_TXN_PRE_MODIFY_FN);
            if (!returncode) {
                slapi_pblock_get(pb, SLAPI_RESULT_CODE, &returncode);
            }
            if (plugin_rc || returncode) {
                slapi_log_err(SLAPI_DSE_TRACELEVEL,
                              "dse_modify", "SLAPI_PLUGIN_BE_TXN_PRE_MODIFY_FN failed - rc %d LDAP error %d:%s\n",
                              plugin_rc, returncode, ldap_err2string(returncode));
            }
        } else {
            slapi_log_err(SLAPI_DSE_TRACELEVEL,
                          "dse_modify", "SLAPI_PLUGIN_BE_PRE_MODIFY_FN failed - rc %d LDAP error %d:%s\n",
                          rc, returncode, ldap_err2string(returncode));
        }
        if (plugin_rc || returncode) {
            char *ldap_result_message = NULL;
            rc = SLAPI_DSE_CALLBACK_ERROR;
            if (!returncode) {
                slapi_log_err(SLAPI_DSE_TRACELEVEL,
                              "dse_modify", "PRE_MODIFY plugin returned non-zero but did not set an LDAP error\n");
                returncode = LDAP_OPERATIONS_ERROR;
            }
            if (!returntext[0]) {
                slapi_pblock_get(pb, SLAPI_PB_RESULT_TEXT, &ldap_result_message);
                if (ldap_result_message && ldap_result_message[0]) {
                    PL_strncpyz(returntext, ldap_result_message, sizeof(returntext));
                }
            }
        } else {
            /*
             * If we are using dynamic plugins, and we are modifying a plugin
             * we need to do some additional checks.  First, check if we are
             * enabling/disabling a plugin.  Then make sure the plugin still
             * starts after applying the plugin changes.
             */
            rc = SLAPI_DSE_CALLBACK_OK;
            if (slapi_entry_attr_hasvalue(ec, SLAPI_ATTR_OBJECTCLASS, "nsSlapdPlugin")) {
                if (config_get_dynamic_plugins()) {
                    if ((plugin_started = dse_modify_plugin(ec, ecc, returntext)) == -1) {
                        returncode = LDAP_UNWILLING_TO_PERFORM;
                        rc = SLAPI_DSE_CALLBACK_ERROR;
                        retval = -1;
                        goto done;
                    }
                    /*
                     * If this is not a internal operation, make sure the plugin
                     * can be restarted.
                     */
                    if (!internal_op) {
                        if (dse_pre_modify_plugin(ec, ecc, mods)) {
                            char *errtext;
                            slapi_pblock_get(pb, SLAPI_PB_RESULT_TEXT, &errtext);
                            if (errtext) {
                                PL_strncpyz(returntext,
                                            "Failed to apply plugin config change, "
                                            "check the errors log for more info.",
                                            sizeof(returntext));
                            }
                            returncode = LDAP_UNWILLING_TO_PERFORM;
                            rc = SLAPI_DSE_CALLBACK_ERROR;
                            retval = -1;
                            goto done;
                        }
                    }
                } else {
                    slapi_log_err(SLAPI_LOG_NOTICE, "dse_modify", "A plugin has been enabled or disabled, but nsslapd-dynamic-plugins is off. A server restart is required to change this plugin state.\n");
                } /* end config_get_dynamic_plugins */
            } /* end has nsSlapdPlugin */
        }
    }

    switch (rc) {
    case SLAPI_DSE_CALLBACK_ERROR:
        /* Error occured in the callback -- return error code from callback */
        goto done;
        break;
    case SLAPI_DSE_CALLBACK_DO_NOT_APPLY:
        /* Callback says don't apply the changes -- return Success */
        returncode = LDAP_SUCCESS;
        returntext[0] = '\0';
        retval = 0;
        goto done;
        break;
    case SLAPI_DSE_CALLBACK_OK: {
        /* The callback may alter the mods in the pblock.  This happens
           for example in the schema code.  Since the schema attributes
           are managed exclusively by the schema code, we should not
           apply those mods.  However, for reasons unknown to me, we
           must in the general case call entry_apply_mods before calling
           the modify callback above.  In the case of schema, the schema
           code will remove the schema attributes from the mods.  So, we
           reapply the mods to the entry for the attributes we manage in
           the dse code (e.g. aci)
        */
        int reapply_mods = 0; /* default is to not reapply entry_apply_mods */
        slapi_pblock_get(pb, SLAPI_DSE_REAPPLY_MODS, &reapply_mods);
        /* Callback says apply the changes */
        if (reapply_mods) {
            LDAPMod **modsagain = NULL; /*Used to apply the modifications*/
            slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &modsagain);
            if (NULL != modsagain) {
                /* the dse modify callback must have modified ecc back to it's
                   original state, before the earlier apply_mods, but without the
                   attributes it did not want us to apply mods to */
                err = entry_apply_mods(ecc, modsagain);
            }
        }

        if (err != 0) {
            returncode = err;
            returntext[0] = '\0';
            retval = -1;
            goto done;
        }
        break;
    }
    }

    /* We're applying the mods... check that the entry still obeys the schema */
    if (slapi_entry_schema_check(pb, ecc) != 0) {
        char *errtext;

        slapi_pblock_get(pb, SLAPI_PB_RESULT_TEXT, &errtext);
        if (errtext) {
            PL_strncpyz(returntext, errtext, sizeof(returntext));
        }
        returncode = LDAP_OBJECT_CLASS_VIOLATION;
        retval = -1;
        goto done;
    }

    /* Check if the attribute values in the mods obey the syntaxes */
    if (slapi_mods_syntax_check(pb, mods, 0) != 0) {
        char *errtext;

        slapi_pblock_get(pb, SLAPI_PB_RESULT_TEXT, &errtext);
        if (errtext) {
            PL_strncpyz(returntext, errtext, sizeof(returntext));
        }
        returncode = LDAP_INVALID_SYNTAX;
        retval = -1;
        goto done;
    }

    /* Change the entry itself both on disk and in the AVL tree */
    /* dse_replace_entry free's the existing entry. */
    if (dse_replace_entry(pdse, ecc, !dont_write_file, DSE_USE_LOCK) != 0) {
        returncode = LDAP_OPERATIONS_ERROR;
        retval = -1;
        goto done;
    }
    retval = 0;                                                      /* so far, so good */
    slapi_pblock_set(pb, SLAPI_ENTRY_POST_OP, slapi_entry_dup(ecc)); /* JCM - When does this get free'd? */
    /* richm - it is freed in modify.c */
    /* give the dse callbacks the first crack at the modify */
    rc = dse_call_callback(pdse, pb, SLAPI_OPERATION_MODIFY, DSE_FLAG_POSTOP, ec, ecc, &returncode, returntext);
done:
    if (rc != SLAPI_DSE_CALLBACK_DO_NOT_APPLY) {
        /* make sure OPRETURN is set */
        slapi_pblock_get(pb, SLAPI_PLUGIN_OPRETURN, &err);
        if ((retval || returncode) && !err) {
            slapi_pblock_set(pb, SLAPI_PLUGIN_OPRETURN, retval ? &retval : &returncode);
        }
        /* next, give the betxn plugins a crack at it */
        slapi_pblock_set(pb, SLAPI_RESULT_CODE, &returncode);
        slapi_pblock_set(pb, SLAPI_MODIFY_EXISTING_ENTRY, ecc);
        if (need_be_postop) {
            plugin_call_plugins(pb, SLAPI_PLUGIN_BE_TXN_POST_MODIFY_FN);
            if (!returncode) {
                slapi_pblock_get(pb, SLAPI_RESULT_CODE, &returncode);
            }
            if (returncode && !returntext[0]) {
                char *ldap_result_message = NULL;
                slapi_pblock_get(pb, SLAPI_PB_RESULT_TEXT, &ldap_result_message);
                if (ldap_result_message && ldap_result_message[0]) {
                    PL_strncpyz(returntext, ldap_result_message, sizeof(returntext));
                }
            }

            plugin_call_plugins(pb, SLAPI_PLUGIN_BE_POST_MODIFY_FN);
            if (!returncode) {
                slapi_pblock_get(pb, SLAPI_RESULT_CODE, &returncode);
            }
        }
    } else {
        /* It should not happen but just be paranoiac, do not
         * forget to call the postop if needed
         */
        if (need_be_postop) {
            plugin_call_plugins(pb, SLAPI_PLUGIN_BE_TXN_POST_MODIFY_FN);
            if (!returncode) {
                slapi_pblock_get(pb, SLAPI_RESULT_CODE, &returncode);
            }

            plugin_call_plugins(pb, SLAPI_PLUGIN_BE_POST_MODIFY_FN);
            if (!returncode) {
                slapi_pblock_get(pb, SLAPI_RESULT_CODE, &returncode);
            }
        }
    }

    /* time to restore original mods */
    if (original_mods) {
        LDAPMod **mods_from_callback;
        slapi_pblock_get(pb, SLAPI_MODIFY_MODS, &mods_from_callback);
        ldap_mods_free(mods_from_callback, 1 /* Free the Array and the Elements */);
        slapi_pblock_set(pb, SLAPI_MODIFY_MODS, original_mods);
    }
    if (global_lock_owned) {
        global_backend_lock_unlock();
    }
    slapi_send_ldap_result(pb, returncode, NULL, returntext[0] ? returntext : NULL, 0, NULL);

    return dse_modify_return(retval, ec, ecc);
}


dse_numsubordinates(Slapi_Entry *entry)
{
    int ret = 0;
    Slapi_Attr *read_attr = NULL;
    size_t current_sub_count = 0;

    /* Get the present value of the subcount attr, or 0 if not present */
    ret = slapi_entry_attr_find(entry, subordinatecount, &read_attr);
    if (0 == ret) {
        /* decode the value */
        Slapi_Value *sval;
        slapi_attr_first_value(read_attr, &sval);
        if (sval != NULL) {
            const struct berval *bval = slapi_value_get_berval(sval);
            if (NULL != bval)
                current_sub_count = atol(bval->bv_val);
        }
    }
    return current_sub_count;
}