mm/gup.c

Source file repositories/reference/linux-study-clean/mm/gup.c

File Facts

System
Linux kernel
Corpus path
mm/gup.c
Extension
.c
Size
103284 bytes
Lines
3558
Domain
Core OS
Bucket
Memory Management
Inferred role
Core OS: exported/initcall integration point
Status
integration implementation candidate

Why This File Exists

Core operating-system implementation surface: boot, tasks, memory, VFS, syscall-facing interfaces, synchronization, credentials, and isolation.

Dependency Surface

Detected Declarations

Annotated Snippet

* page tables.  Return error instead of NULL to skip handle_mm_fault,
	 * then get_dump_page() will return NULL to leave a hole in the dump.
	 * But we can only make this optimization where a hole would surely
	 * be zero-filled if handle_mm_fault() actually did handle it.
	 */
	if (is_vm_hugetlb_page(vma)) {
		struct hstate *h = hstate_vma(vma);

		if (!hugetlbfs_pagecache_present(h, vma, address))
			return ERR_PTR(-EFAULT);
	} else if ((vma_is_anonymous(vma) || !vma->vm_ops->fault)) {
		return ERR_PTR(-EFAULT);
	}

	return NULL;
}

#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES
/* FOLL_FORCE can write to even unwritable PUDs in COW mappings. */
static inline bool can_follow_write_pud(pud_t pud, struct page *page,
					struct vm_area_struct *vma,
					unsigned int flags)
{
	/* If the pud is writable, we can write to the page. */
	if (pud_write(pud))
		return true;

	return can_follow_write_common(page, vma, flags);
}

static struct page *follow_huge_pud(struct vm_area_struct *vma,
				    unsigned long addr, pud_t *pudp,
				    int flags, unsigned long *page_mask)
{
	struct mm_struct *mm = vma->vm_mm;
	struct page *page;
	pud_t pud = *pudp;
	unsigned long pfn = pud_pfn(pud);
	int ret;

	assert_spin_locked(pud_lockptr(mm, pudp));

	if (!pud_present(pud))
		return NULL;

	if ((flags & FOLL_WRITE) &&
	    !can_follow_write_pud(pud, pfn_to_page(pfn), vma, flags))
		return NULL;

	pfn += (addr & ~PUD_MASK) >> PAGE_SHIFT;
	page = pfn_to_page(pfn);

	if (!pud_write(pud) && gup_must_unshare(vma, flags, page))
		return ERR_PTR(-EMLINK);

	ret = try_grab_folio(page_folio(page), 1, flags);
	if (ret)
		page = ERR_PTR(ret);
	else
		*page_mask = HPAGE_PUD_NR - 1;

	return page;
}

/* FOLL_FORCE can write to even unwritable PMDs in COW mappings. */
static inline bool can_follow_write_pmd(pmd_t pmd, struct page *page,
					struct vm_area_struct *vma,
					unsigned int flags)
{
	/* If the pmd is writable, we can write to the page. */
	if (pmd_write(pmd))
		return true;

	if (!can_follow_write_common(page, vma, flags))
		return false;

	/* ... and a write-fault isn't required for other reasons. */
	if (pmd_needs_soft_dirty_wp(vma, pmd))
		return false;
	return !userfaultfd_huge_pmd_wp(vma, pmd);
}

static struct page *follow_huge_pmd(struct vm_area_struct *vma,
				    unsigned long addr, pmd_t *pmd,
				    unsigned int flags,
				    unsigned long *page_mask)
{
	struct mm_struct *mm = vma->vm_mm;
	pmd_t pmdval = *pmd;
	struct page *page;

Annotation

Implementation Notes