Commit message: mm: make page ref count overflow check tighter and more explicit

We have a VM_BUG_ON() to check that the page reference count doesn't
underflow (or get close to overflow) by checking the sign of the count.

That's all fine, but we actually want to allow people to use a "get page
ref unless it's already very high" helper function, and we want that one
to use the sign of the page ref (without triggering this VM_BUG_ON).

Change the VM_BUG_ON to only check for small underflows (or _very_ close
to overflowing), and ignore overflows which have strayed into negative
territory.

Acked-by: Matthew Wilcox <willy@infradead.org>
Cc: Jann Horn <jannh@google.com>
Cc: stable@kernel.org
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
CVE: CVE-2019-11487
CWE: CWE-416
Vulnerability description: The Linux kernel before 5.1-rc5 allows page->_refcount reference count overflow, with resultant use-after-free issues, if about 140 GiB of RAM exists. This is related to fs/fuse/dev.c, fs/pipe.c, fs/splice.c, include/linux/mm.h, include/linux/pipe_fs_i.h, kernel/trace/trace.c, mm/gup.c, and mm/hugetlb.c. It can occur with FUSE requests.

Project: linux
File: mm.h

static inline void update_hiwater_rss(struct mm_struct *mm)
{
	unsigned long _rss = get_mm_rss(mm);

	if ((mm)->hiwater_rss < _rss)
		(mm)->hiwater_rss = _rss;
}

Project: linux
File: mm.h

static inline struct mem_cgroup *page_memcg(struct page *page)
{
	return NULL;
}

Project: linux
File: mm.h

static inline struct zone *page_zone(const struct page *page)
{
	return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
}

Project: linux
File: mm.h

static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
						unsigned long address)
{
	return 0;
}

Project: linux
File: mm.h

static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
{
	return atomic_long_read(&mm->pgtables_bytes);
}

Project: linux
File: mm.h

static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
{
	return NULL;
}

Project: linux
File: mm.h

static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
{
	return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
		NULL: pmd_offset(pud, address);
}

Project: linux
File: mm.h

static inline void clear_page_pfmemalloc(struct page *page)
{
	page->index = 0;
}

Project: linux
File: mm.h

static inline spinlock_t *pud_lockptr(struct mm_struct *mm, pud_t *pud)
{
	return &mm->page_table_lock;
}

Project: linux
File: mm.h

static inline bool page_is_pfmemalloc(struct page *page)
{
	/*
	 * Page index cannot be this large so this must be
	 * a pfmemalloc page.
	 */
	return page->index == -1UL;
}

Project: linux
File: mm.h

static inline void set_page_links(struct page *page, enum zone_type zone,
	unsigned long node, unsigned long pfn)
{
	set_page_zone(page, zone);
	set_page_node(page, node);
#ifdef SECTION_IN_PAGE_FLAGS
	set_page_section(page, pfn_to_section_nr(pfn));
#endif
}

Project: linux
File: mm.h

static inline bool is_pci_p2pdma_page(const struct page *page)
{
	return false;
}

Project: linux
File: mm.h

static __always_inline void *lowmem_page_address(const struct page *page)
{
	return page_to_virt(page);
}

Project: linux
File: mm.h

static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
					 struct mm_struct *mm)
{
	unsigned long hiwater_rss = get_mm_hiwater_rss(mm);

	if (*maxrss < hiwater_rss)
		*maxrss = hiwater_rss;
}

Project: linux
File: mm.h

static inline spinlock_t *ptlock_ptr(struct page *page)
{
	return page->ptl;
}

Project: linux
File: mm.h

static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
{
	return 0;
}

Project: linux
File: mm.h

static inline void setup_nr_node_ids(void) {}

Project: linux
File: mm.h

static inline void __free_reserved_page(struct page *page)
{
	ClearPageReserved(page);
	init_page_count(page);
	__free_page(page);
}

Project: linux
File: mm.h

static inline void mm_inc_nr_puds(struct mm_struct *mm)
{
	if (mm_pud_folded(mm))
		return;
	atomic_long_add(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
}

Project: linux
File: mm.h

static inline int cpupid_to_nid(int cpupid)
{
	return cpu_to_node(cpupid_to_cpu(cpupid));
}

Project: linux
File: mm.h

static inline bool vma_is_anonymous(struct vm_area_struct *vma)
{
	return !vma->vm_ops;
}

Project: linux
File: mm.h

static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
{
	if (likely(vma->vm_flags & VM_WRITE))
		pte = pte_mkwrite(pte);
	return pte;
}

Project: linux
File: mm.h

static inline int cpu_pid_to_cpupid(int nid, int pid)
{
	return -1;
}

Project: linux
File: mm.h

static inline unsigned long totalram_pages(void)
{
	return (unsigned long)atomic_long_read(&_totalram_pages);
}

Project: linux
File: mm.h

static inline int page_cpupid_last(struct page *page)
{
	return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
}

Project: linux
File: mm.h

static inline int page_trans_huge_mapcount(struct page *page,
					   int *total_mapcount)
{
	int mapcount = page_mapcount(page);
	if (total_mapcount)
		*total_mapcount = mapcount;
	return mapcount;
}

Project: linux
File: mm.h

static inline int cpu_pid_to_cpupid(int cpu, int pid)
{
	return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
}

Project: linux
File: mm.h

static inline bool cpupid_cpu_unset(int cpupid)
{
	return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
}

Project: linux
File: mm.h

static inline u8 page_kasan_tag(const struct page *page)
{
	return (page->flags >> KASAN_TAG_PGSHIFT) & KASAN_TAG_MASK;
}

Project: linux
File: mm.h

static inline int mm_counter(struct page *page)
{
	if (PageAnon(page))
		return MM_ANONPAGES;
	return mm_counter_file(page);
}

Project: linux
File: mm.h

static inline void mm_dec_nr_ptes(struct mm_struct *mm)
{
	atomic_long_sub(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
}

Project: linux
File: mm.h

static inline void ptlock_free(struct page *page) {}

Project: linux
File: mm.h

static inline void *kvzalloc_node(size_t size, gfp_t flags, int node)
{
	return kvmalloc_node(size, flags | __GFP_ZERO, node);
}

Project: linux
File: mm.h

static inline bool pgtable_page_ctor(struct page *page)
{
	if (!ptlock_init(page))
		return false;
	__SetPageTable(page);
	inc_zone_page_state(page, NR_PAGETABLE);
	return true;
}

Project: linux
File: mm.h

static inline unsigned int debug_guardpage_minorder(void) { return 0; }

Project: linux
File: mm.h

static inline void get_page(struct page *page)
{
	page = compound_head(page);
	/*
	 * Getting a normal page or the head of a compound page
	 * requires to already have an elevated page->_refcount.
	 */
	VM_BUG_ON_PAGE(page_ref_zero_or_close_to_overflow(page), page);
	page_ref_inc(page);
}

Project: linux
File: mm.h

static inline void pgtable_init(void)
{
	ptlock_cache_init();
	pgtable_cache_init();
}

Project: linux
File: mm.h

static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
{
	VM_BUG_ON_PAGE(page[1].compound_dtor >= NR_COMPOUND_DTORS, page);
	return compound_page_dtors[page[1].compound_dtor];
}

Project: linux
File: mm.h

vm_unmapped_area(struct vm_unmapped_area_info *info)
{
	if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
		return unmapped_area_topdown(info);
	else
		return unmapped_area(info);
}

Project: linux
File: mm.h

static inline bool debug_pagealloc_enabled(void)
{
	return _debug_pagealloc_enabled;
}

Project: linux
File: mm.h

static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }

Project: linux
File: mm.h

static inline unsigned long vma_pages(struct vm_area_struct *vma)
{
	return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
}

Project: linux
File: mm.h

static inline bool put_devmap_managed_page(struct page *page)
{
	return false;
}

Project: linux
File: mm.h

static inline void mm_inc_nr_pmds(struct mm_struct *mm)
{
	if (mm_pmd_folded(mm))
		return;
	atomic_long_add(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
}

Project: linux
File: mm.h

static inline void page_kasan_tag_reset(struct page *page)
{
	page_kasan_tag_set(page, 0xff);
}

Project: linux
File: mm.h

static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}

Project: linux
File: mm.h

static inline int put_page_testzero(struct page *page)
{
	VM_BUG_ON_PAGE(page_ref_count(page) == 0, page);
	return page_ref_dec_and_test(page);
}

Project: linux
File: mm.h

static inline unsigned long vm_end_gap(struct vm_area_struct *vma)
{
	unsigned long vm_end = vma->vm_end;

	if (vma->vm_flags & VM_GROWSUP) {
		vm_end += stack_guard_gap;
		if (vm_end < vma->vm_end)
			vm_end = -PAGE_SIZE;
	}
	return vm_end;
}

Project: linux
File: mm.h

static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags)
{
	if (vm_fault & VM_FAULT_OOM)
		return -ENOMEM;
	if (vm_fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE))
		return (foll_flags & FOLL_HWPOISON) ? -EHWPOISON : -EFAULT;
	if (vm_fault & (VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV))
		return -EFAULT;
	return 0;
}

Project: linux
File: mm.h

static inline bool ptlock_alloc(struct page *page)
{
	return true;
}

Project: linux
File: mm.h

static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
{
	atomic_long_add(value, &mm->rss_stat.count[member]);
}

Project: linux
File: mm.h

static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
				unsigned long vm_start, unsigned long vm_end)
{
	struct vm_area_struct *vma = find_vma(mm, vm_start);

	if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
		vma = NULL;

	return vma;
}

Project: linux
File: mm.h

static inline bool page_is_guard(struct page *page)
{
	struct page_ext *page_ext;

	if (!debug_guardpage_enabled())
		return false;

	page_ext = lookup_page_ext(page);
	if (unlikely(!page_ext))
		return false;

	return test_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
}

Project: linux
File: mm.h

static inline void page_cpupid_reset_last(struct page *page)
{
	page->_last_cpupid = -1 & LAST_CPUPID_MASK;
}

Project: linux
File: mm.h

static inline long get_user_pages_longterm(unsigned long start,
		unsigned long nr_pages, unsigned int gup_flags,
		struct page **pages, struct vm_area_struct **vmas)
{
	return get_user_pages(start, nr_pages, gup_flags, pages, vmas);
}

Project: linux
File: mm.h

do_mmap_pgoff(struct file *file, unsigned long addr,
	unsigned long len, unsigned long prot, unsigned long flags,
	unsigned long pgoff, unsigned long *populate,
	struct list_head *uf)
{
	return do_mmap(file, addr, len, prot, flags, 0, pgoff, populate, uf);
}

Project: linux
File: mm.h

static inline int is_vmalloc_or_module_addr(const void *x)
{
	return 0;
}

Project: linux
File: mm.h

static inline int check_data_rlimit(unsigned long rlim,
				    unsigned long new,
				    unsigned long start,
				    unsigned long end_data,
				    unsigned long start_data)
{
	if (rlim < RLIM_INFINITY) {
		if (((new - start) + (end_data - start_data)) > rlim)
			return -ENOSPC;
	}

	return 0;
}

Project: linux
File: mm.h

static inline bool debug_pagealloc_enabled(void)
{
	return false;
}

Project: linux
File: mm.h

static inline void set_page_zone(struct page *page, enum zone_type zone)
{
	page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
	page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
}

Project: linux
File: mm.h

static inline int page_cpupid_last(struct page *page)
{
	return page_to_nid(page); /* XXX */
}

Project: linux
File: mm.h

static inline void vma_set_page_prot(struct vm_area_struct *vma)
{
	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
}

Project: linux
File: mm.h

static inline bool ptlock_init(struct page *page)
{
	/*
	 * prep_new_page() initialize page->private (and therefore page->ptl)
	 * with 0. Make sure nobody took it in use in between.
	 *
	 * It can happen if arch try to use slab for page table allocation:
	 * slab code uses page->slab_cache, which share storage with page->ptl.
	 */
	VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
	if (!ptlock_alloc(page))
		return false;
	spin_lock_init(ptlock_ptr(page));
	return true;
}

Project: linux
File: mm.h

static inline unsigned long *frame_vector_pfns(struct frame_vector *vec)
{
	if (!vec->is_pfns)
		frame_vector_to_pfns(vec);
	return (unsigned long *)(vec->ptrs);
}

Project: linux
File: mm.h

static inline int cpupid_to_pid(int cpupid)
{
	return -1;
}

Project: linux
File: mm.h

static inline void mm_inc_nr_ptes(struct mm_struct *mm)
{
	atomic_long_add(PTRS_PER_PTE * sizeof(pte_t), &mm->pgtables_bytes);
}

Project: linux
File: mm.h

struct address_space *page_file_mapping(struct page *page)
{
	if (unlikely(PageSwapCache(page)))
		return __page_file_mapping(page);

	return page->mapping;
}

Project: linux
File: mm.h

static inline int pgd_devmap(pgd_t pgd)
{
	return 0;
}

Project: linux
File: mm.h

static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
{
	mm->hiwater_rss = get_mm_rss(mm);
}

Project: linux
File: mm.h

static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
{
	return page_to_nid(page); /* XXX */
}

Project: linux
File: mm.h

static inline struct page *virt_to_head_page(const void *x)
{
	struct page *page = virt_to_page(x);

	return compound_head(page);
}

Project: linux
File: mm.h

static inline vm_fault_t vmf_insert_page(struct vm_area_struct *vma,
				unsigned long addr, struct page *page)
{
	int err = vm_insert_page(vma, addr, page);

	if (err == -ENOMEM)
		return VM_FAULT_OOM;
	if (err < 0 && err != -EBUSY)
		return VM_FAULT_SIGBUS;

	return VM_FAULT_NOPAGE;
}

Project: linux
File: mm.h

static inline void put_page(struct page *page)
{
	page = compound_head(page);

	/*
	 * For devmap managed pages we need to catch refcount transition from
	 * 2 to 1, when refcount reach one it means the page is free and we
	 * need to inform the device driver through callback. See
	 * include/linux/memremap.h and HMM for details.
	 */
	if (put_devmap_managed_page(page))
		return;

	if (put_page_testzero(page))
		__put_page(page);
}

Project: linux
File: mm.h

static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}

Project: linux
File: mm.h

static inline void mm_dec_nr_puds(struct mm_struct *mm)
{
	if (mm_pud_folded(mm))
		return;
	atomic_long_sub(PTRS_PER_PUD * sizeof(pud_t), &mm->pgtables_bytes);
}

Project: linux
File: mm.h

static inline bool is_vmalloc_addr(const void *x)
{
#ifdef CONFIG_MMU
	unsigned long addr = (unsigned long)x;

	return addr >= VMALLOC_START && addr < VMALLOC_END;
#else
	return false;
#endif
}

Project: linux
File: mm.h

static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
	return ptlock_ptr(pmd_page(*pmd));
}

Project: linux
File: mm.h

static inline void vma_init(struct vm_area_struct *vma, struct mm_struct *mm)
{
	static const struct vm_operations_struct dummy_vm_ops = {};

	memset(vma, 0, sizeof(*vma));
	vma->vm_mm = mm;
	vma->vm_ops = &dummy_vm_ops;
	INIT_LIST_HEAD(&vma->anon_vma_chain);
}

Project: linux
File: mm.h

static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
{
	struct vm_area_struct * vma = find_vma(mm,start_addr);

	if (vma && end_addr <= vma->vm_start)
		vma = NULL;
	return vma;
}

Project: linux
File: mm.h

static inline void kernel_poison_pages(struct page *page, int numpages,
					int enable) { }

Project: linux
File: mm.h

static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
{
	spinlock_t *ptl = pmd_lockptr(mm, pmd);
	spin_lock(ptl);
	return ptl;
}

Project: linux
File: mm.h

static inline void page_mapcount_reset(struct page *page)
{
	atomic_set(&(page)->_mapcount, -1);
}

Project: linux
File: mm.h

static inline int pte_devmap(pte_t pte)
{
	return 0;
}

Project: linux
File: mm.h

static inline void *kvcalloc(size_t n, size_t size, gfp_t flags)
{
	return kvmalloc_array(n, size, flags | __GFP_ZERO);
}

Project: linux
File: mm.h

static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
{
	return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
}

Project: linux
File: mm.h

static inline void mm_pgtables_bytes_init(struct mm_struct *mm) {}

Project: linux
File: mm.h

static inline struct mem_cgroup *page_memcg(struct page *page)
{
	return page->mem_cgroup;
}

Project: linux
File: mm.h

static inline void page_kasan_tag_set(struct page *page, u8 tag) { }

Project: linux
File: mm.h

static inline void totalram_pages_dec(void)
{
	atomic_long_dec(&_totalram_pages);
}

Project: linux
File: mm.h

static inline pgoff_t page_index(struct page *page)
{
	if (unlikely(PageSwapCache(page)))
		return __page_file_index(page);
	return page->index;
}

Project: linux
File: mm.h

static inline void page_cpupid_reset_last(struct page *page)
{
	page->flags |= LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT;
}

Project: linux
File: mm.h

static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}

Project: linux
File: mm.h

static inline unsigned long free_initmem_default(int poison)
{
	extern char __init_begin[], __init_end[];

	return free_reserved_area(&__init_begin, &__init_end,
				  poison, "unused kernel");
}

Project: linux
File: mm.h

static inline bool pgtable_pmd_page_ctor(struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	page->pmd_huge_pte = NULL;
#endif
	return ptlock_init(page);
}

Project: linux
File: mm.h

static inline int cpupid_to_cpu(int cpupid)
{
	return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
}

Project: linux
File: mm.h

static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
	return ptlock_ptr(pmd_to_page(pmd));
}

Project: linux
File: mm.h

static inline unsigned long get_num_physpages(void)
{
	int nid;
	unsigned long phys_pages = 0;

	for_each_online_node(nid)
		phys_pages += node_present_pages(nid);

	return phys_pages;
}

Project: linux
File: mm.h

static inline void set_compound_order(struct page *page, unsigned int order)
{
	page[1].compound_order = order;
}

Project: linux
File: mm.h

static inline bool is_device_public_page(const struct page *page)
{
	return is_zone_device_page(page) &&
		page->pgmap->type == MEMORY_DEVICE_PUBLIC;
}

Project: linux
File: mm.h

static inline void dec_mm_counter(struct mm_struct *mm, int member)
{
	atomic_long_dec(&mm->rss_stat.count[member]);
}

Project: linux
File: mm.h

static inline bool page_is_guard(struct page *page) { return false; }

Project: linux
File: mm.h

static inline void unmap_mapping_range(struct address_space *mapping,
		loff_t const holebegin, loff_t const holelen, int even_cows) { }

Project: linux
File: mm.h

static inline void set_max_mapnr(unsigned long limit) { }

Project: linux
File: mm.h

static inline int fixup_user_fault(struct task_struct *tsk,
		struct mm_struct *mm, unsigned long address,
		unsigned int fault_flags, bool *unlocked)
{
	/* should never happen if there's no MMU */
	BUG();
	return -EFAULT;
}

Project: linux
File: mm.h

static inline void totalram_pages_add(long count)
{
	atomic_long_add(count, &_totalram_pages);
}

Project: linux
File: mm.h

static inline void totalram_pages_set(long val)
{
	atomic_long_set(&_totalram_pages, val);
}

Project: linux
File: mm.h

static inline spinlock_t *pud_lock(struct mm_struct *mm, pud_t *pud)
{
	spinlock_t *ptl = pud_lockptr(mm, pud);

	spin_lock(ptl);
	return ptl;
}

Project: linux
File: mm.h

static inline void mm_dec_nr_puds(struct mm_struct *mm) {}

Project: linux
File: mm.h

static inline bool kernel_page_present(struct page *page) { return true; }

Project: linux
File: mm.h

static inline void vma_set_anonymous(struct vm_area_struct *vma)
{
	vma->vm_ops = NULL;
}

Project: linux
File: mm.h

static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
						unsigned long address)
{
	return 0;
}

Project: linux
File: mm.h

static inline bool is_zone_device_page(const struct page *page)
{
	return false;
}

Project: linux
File: mm.h

static inline void *kvmalloc_array(size_t n, size_t size, gfp_t flags)
{
	size_t bytes;

	if (unlikely(check_mul_overflow(n, size, &bytes)))
		return NULL;

	return kvmalloc(bytes, flags);
}

Project: linux
File: mm.h

static inline int page_to_nid(const struct page *page)
{
	struct page *p = (struct page *)page;

	return (PF_POISONED_CHECK(p)->flags >> NODES_PGSHIFT) & NODES_MASK;
}

Project: linux
File: mm.h

static inline void mm_inc_nr_puds(struct mm_struct *mm) {}

Project: linux
File: mm.h

static inline void dev_pagemap_put_ops(void)
{
}

Project: linux
File: mm.h

static inline unsigned int frame_vector_count(struct frame_vector *vec)
{
	return vec->nr_frames;
}

Project: linux
File: mm.h

static inline int get_page_unless_zero(struct page *page)
{
	return page_ref_add_unless(page, 1, 0);
}

Project: linux
File: mm.h

static inline int mm_counter_file(struct page *page)
{
	if (PageSwapBacked(page))
		return MM_SHMEMPAGES;
	return MM_FILEPAGES;
}

Project: linux
File: mm.h

static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
{
	WARN_ON_ONCE(!rcu_read_lock_held());
	return READ_ONCE(page->mem_cgroup);
}

Project: linux
File: mm.h

static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
{
	return false;
}

Project: linux
File: mm.h

static inline bool vma_is_shmem(struct vm_area_struct *vma) { return false; }

Project: linux
File: mm.h

static inline void totalram_pages_inc(void)
{
	atomic_long_inc(&_totalram_pages);
}

Project: linux
File: mm.h

static inline int total_mapcount(struct page *page)
{
	return page_mapcount(page);
}

Project: linux
File: mm.h

static inline int __pud_alloc(struct mm_struct *mm, p4d_t *p4d,
						unsigned long address)
{
	return 0;
}

Project: linux
File: mm.h

static inline void ptlock_cache_init(void)
{
}

Project: linux
File: mm.h

static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
	return &mm->page_table_lock;
}

Project: linux
File: mm.h

static inline bool is_device_public_page(const struct page *page)
{
	return false;
}

Project: linux
File: mm.h

static inline unsigned int compound_order(struct page *page)
{
	if (!PageHead(page))
		return 0;
	return page[1].compound_order;
}

Project: linux
File: mm.h

static inline bool cpupid_pid_unset(int cpupid)
{
	return 1;
}

Project: linux
File: mm.h

static inline void mm_dec_nr_pmds(struct mm_struct *mm)
{
	if (mm_pmd_folded(mm))
		return;
	atomic_long_sub(PTRS_PER_PMD * sizeof(pmd_t), &mm->pgtables_bytes);
}

Project: linux
File: mm.h

static inline unsigned int debug_guardpage_minorder(void)
{
	return _debug_guardpage_minorder;
}

Project: linux
File: mm.h

static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
{
	return __pgprot(0);
}

Project: linux
File: mm.h

static inline bool page_poisoning_enabled(void) { return false; }

Project: linux
File: mm.h

static inline void unmap_shared_mapping_range(struct address_space *mapping,
		loff_t const holebegin, loff_t const holelen)
{
	unmap_mapping_range(mapping, holebegin, holelen, 0);
}

Project: linux
File: mm.h

static inline unsigned long page_to_section(const struct page *page)
{
	return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
}

Project: linux
File: mm.h

static inline void page_kasan_tag_set(struct page *page, u8 tag)
{
	page->flags &= ~(KASAN_TAG_MASK << KASAN_TAG_PGSHIFT);
	page->flags |= (tag & KASAN_TAG_MASK) << KASAN_TAG_PGSHIFT;
}

Project: linux
File: mm.h

static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
{
	return max(mm->hiwater_rss, get_mm_rss(mm));
}

Project: linux
File: mm.h

static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
{
	return max(mm->hiwater_vm, mm->total_vm);
}

Project: linux
File: mm.h

static inline bool is_device_private_page(const struct page *page)
{
	return false;
}

Project: linux
File: mm.h

static inline void set_page_node(struct page *page, unsigned long node)
{
	page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
	page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
}

Project: linux
File: mm.h

static inline unsigned long mm_pgtables_bytes(const struct mm_struct *mm)
{
	return 0;
}

Project: linux
File: mm.h

static inline pg_data_t *page_pgdat(const struct page *page)
{
	return NODE_DATA(page_to_nid(page));
}

Project: linux
File: mm.h

static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
				    spinlock_t **ptl)
{
	pte_t *ptep;
	__cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
	return ptep;
}

Project: linux
File: mm.h

static inline void mark_page_reserved(struct page *page)
{
	SetPageReserved(page);
	adjust_managed_page_count(page, -1);
}

Project: linux
File: mm.h

static inline int page_mapcount(struct page *page)
{
	VM_BUG_ON_PAGE(PageSlab(page), page);

	if (unlikely(PageCompound(page)))
		return __page_mapcount(page);
	return atomic_read(&page->_mapcount) + 1;
}

Project: linux
File: mm.h

static inline void mm_populate(unsigned long addr, unsigned long len)
{
	/* Ignore errors */
	(void) __mm_populate(addr, len, 1);
}

Project: linux
File: mm.h

static inline void set_page_section(struct page *page, unsigned long section)
{
	page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
	page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
}

Project: linux
File: mm.h

static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
{
	long val = atomic_long_read(&mm->rss_stat.count[member]);

#ifdef SPLIT_RSS_COUNTING
	/*
	 * counter is updated in asynchronous manner and may go to minus.
	 * But it's never be expected number for users.
	 */
	if (val < 0)
		val = 0;
#endif
	return (unsigned long)val;
}

Project: linux
File: mm.h

static inline void dev_pagemap_get_ops(void)
{
}

Project: linux
File: mm.h

static inline void *kvmalloc(size_t size, gfp_t flags)
{
	return kvmalloc_node(size, flags, NUMA_NO_NODE);
}

Project: linux
File: mm.h

static inline int cpupid_to_pid(int cpupid)
{
	return cpupid & LAST__PID_MASK;
}

Project: linux
File: mm.h

static inline u8 page_kasan_tag(const struct page *page)
{
	return 0xff;
}

Project: linux
File: mm.h

static inline vm_fault_t vmf_error(int err)
{
	if (err == -ENOMEM)
		return VM_FAULT_OOM;
	return VM_FAULT_SIGBUS;
}

Project: linux
File: mm.h

static inline void inc_mm_counter(struct mm_struct *mm, int member)
{
	atomic_long_inc(&mm->rss_stat.count[member]);
}

Project: linux
File: mm.h

static inline void pgtable_page_dtor(struct page *page)
{
	ptlock_free(page);
	__ClearPageTable(page);
	dec_zone_page_state(page, NR_PAGETABLE);
}

Project: linux
File: mm.h

static inline void zero_resv_unavail(void) {}

Project: linux
File: mm.h

kernel_map_pages(struct page *page, int numpages, int enable) {}

Project: linux
File: mm.h

static inline bool is_pci_p2pdma_page(const struct page *page)
{
	return is_zone_device_page(page) &&
		page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
}

Project: linux
File: mm.h

static inline unsigned long get_mm_rss(struct mm_struct *mm)
{
	return get_mm_counter(mm, MM_FILEPAGES) +
		get_mm_counter(mm, MM_ANONPAGES) +
		get_mm_counter(mm, MM_SHMEMPAGES);
}

Project: linux
File: mm.h

static inline void free_reserved_page(struct page *page)
{
	__free_reserved_page(page);
	adjust_managed_page_count(page, 1);
}

Project: linux
File: mm.h

static inline int page_cpupid_last(struct page *page)
{
	return page->_last_cpupid;
}

Project: linux
File: mm.h

static inline void mm_inc_nr_ptes(struct mm_struct *mm) {}

Project: linux
File: mm.h

static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
		unsigned long address, unsigned int flags)
{
	/* should never happen if there's no MMU */
	BUG();
	return VM_FAULT_SIGBUS;
}

Project: linux
File: mm.h

static inline struct page **frame_vector_pages(struct frame_vector *vec)
{
	if (vec->is_pfns) {
		int err = frame_vector_to_pages(vec);

		if (err)
			return ERR_PTR(err);
	}
	return (struct page **)(vec->ptrs);
}

Project: linux
File: mm.h

static inline void pgtable_pmd_page_dtor(struct page *page)
{
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
	VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
#endif
	ptlock_free(page);
}

Project: linux
File: mm.h

static inline int page_zone_id(struct page *page)
{
	return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
}

Project: linux
File: mm.h

static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
{
	return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
}

Project: linux
File: mm.h

kernel_map_pages(struct page *page, int numpages, int enable)
{
	if (!debug_pagealloc_enabled())
		return;

	__kernel_map_pages(page, numpages, enable);
}

Project: linux
File: mm.h

static inline int __early_pfn_to_nid(unsigned long pfn,
					struct mminit_pfnnid_cache *state)
{
	return 0;
}

Project: linux
File: mm.h

static inline void pgtable_pmd_page_dtor(struct page *page) {}

Project: linux
File: mm.h

static inline int vma_adjust(struct vm_area_struct *vma, unsigned long start,
	unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
{
	return __vma_adjust(vma, start, end, pgoff, insert, NULL);
}

Project: linux
File: mm.h

static inline void set_compound_page_dtor(struct page *page,
		enum compound_dtor_id compound_dtor)
{
	VM_BUG_ON_PAGE(compound_dtor >= NR_COMPOUND_DTORS, page);
	page[1].compound_dtor = compound_dtor;
}

Project: linux
File: mm.h

static inline void set_page_address(struct page *page, void *address)
{
	page->virtual = address;
}

Project: linux
File: mm.h

static inline bool put_devmap_managed_page(struct page *page)
{
	if (!static_branch_unlikely(&devmap_managed_key))
		return false;
	if (!is_zone_device_page(page))
		return false;
	switch (page->pgmap->type) {
	case MEMORY_DEVICE_PRIVATE:
	case MEMORY_DEVICE_PUBLIC:
	case MEMORY_DEVICE_FS_DAX:
		__put_devmap_managed_page(page);
		return true;
	default:
		break;
	}
	return false;
}

Project: linux
File: mm.h

static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }

Project: linux
File: mm.h

static inline bool is_zone_device_page(const struct page *page)
{
	return page_zonenum(page) == ZONE_DEVICE;
}

Project: linux
File: mm.h

static inline bool debug_guardpage_enabled(void)
{
	return _debug_guardpage_enabled;
}

Project: linux
File: mm.h

static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d,
		unsigned long address)
{
	return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ?
		NULL : pud_offset(p4d, address);
}

Project: linux
File: mm.h

static inline atomic_t *compound_mapcount_ptr(struct page *page)
{
	return &page[1].compound_mapcount;
}

Project: linux
File: mm.h

static inline int cpupid_to_nid(int cpupid)
{
	return -1;
}

Project: linux
File: mm.h

static inline void set_max_mapnr(unsigned long limit)
{
	max_mapnr = limit;
}

Project: linux
File: mm.h

static inline enum zone_type page_zonenum(const struct page *page)
{
	return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
}

Project: linux
File: mm.h

static inline bool range_in_vma(struct vm_area_struct *vma,
				unsigned long start, unsigned long end)
{
	return (vma && vma->vm_start <= start && end <= vma->vm_end);
}

Project: linux
File: mm.h

static inline void *page_address(const struct page *page)
{
	return page->virtual;
}

Project: linux
File: mm.h

static inline bool debug_guardpage_enabled(void) { return false; }

Project: linux
File: mm.h

static inline int cpupid_to_cpu(int cpupid)
{
	return -1;
}

Project: linux
File: mm.h

static inline bool is_device_private_page(const struct page *page)
{
	return is_zone_device_page(page) &&
		page->pgmap->type == MEMORY_DEVICE_PRIVATE;
}

Project: linux
File: mm.h

static inline void page_cpupid_reset_last(struct page *page)
{
}

Project: linux
File: mm.h

static inline int pud_devmap(pud_t pud)
{
	return 0;
}

Project: linux
File: mm.h

static inline void cancel_dirty_page(struct page *page)
{
	/* Avoid atomic ops, locking, etc. when not actually needed. */
	if (PageDirty(page))
		__cancel_dirty_page(page);
}

Project: linux
File: mm.h

static inline void sync_mm_rss(struct mm_struct *mm)
{
}

Project: linux
File: mm.h

static inline void update_hiwater_vm(struct mm_struct *mm)
{
	if (mm->hiwater_vm < mm->total_vm)
		mm->hiwater_vm = mm->total_vm;
}

Project: linux
File: mm.h

static inline bool cpupid_pid_unset(int cpupid)
{
	return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
}

Project: linux
File: mm.h

static struct page *pmd_to_page(pmd_t *pmd)
{
	unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
	return virt_to_page((void *)((unsigned long) pmd & mask));
}

Project: linux
File: mm.h

static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
	return &mm->page_table_lock;
}

Project: linux
File: mm.h

static inline void mm_populate(unsigned long addr, unsigned long len) {}

Project: linux
File: mm.h

static inline void unmap_mapping_pages(struct address_space *mapping,
		pgoff_t start, pgoff_t nr, bool even_cows) { }

Project: linux
File: mm.h

static inline struct mem_cgroup *page_memcg_rcu(struct page *page)
{
	WARN_ON_ONCE(!rcu_read_lock_held());
	return NULL;
}

Project: linux
File: mm.h

static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd,
		unsigned long address)
{
	return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ?
		NULL : p4d_offset(pgd, address);
}

Project: linux
File: mm.h

static inline spinlock_t *ptlock_ptr(struct page *page)
{
	return &page->ptl;
}

Project: linux
File: mm.h

static inline unsigned long vm_start_gap(struct vm_area_struct *vma)
{
	unsigned long vm_start = vma->vm_start;

	if (vma->vm_flags & VM_GROWSDOWN) {
		vm_start -= stack_guard_gap;
		if (vm_start > vma->vm_start)
			vm_start = 0;
	}
	return vm_start;
}

Project: linux
File: mm.h

static inline void set_page_pfmemalloc(struct page *page)
{
	page->index = -1UL;
}

Project: linux
File: mm.h

static inline void page_kasan_tag_reset(struct page *page) { }

Project: linux
File: mm.h

static inline void mm_pgtables_bytes_init(struct mm_struct *mm)
{
	atomic_long_set(&mm->pgtables_bytes, 0);
}

Project: linux
File: mm.h

static inline int compound_mapcount(struct page *page)
{
	VM_BUG_ON_PAGE(!PageCompound(page), page);
	page = compound_head(page);
	return atomic_read(compound_mapcount_ptr(page)) + 1;
}

Project: linux
File: mm.h

static inline bool ptlock_init(struct page *page) { return true; }

Project: linux
File: mm.h

static inline void *kvzalloc(size_t size, gfp_t flags)
{
	return kvmalloc(size, flags | __GFP_ZERO);
}

Project: linux
File: mm.h

static inline int pmd_devmap(pmd_t pmd)
{
	return 0;
}