io_uring/memmap.c

Source file repositories/reference/linux-study-clean/io_uring/memmap.c

File Facts

System
Linux kernel
Corpus path
io_uring/memmap.c
Extension
.c
Size
11243 bytes
Lines
440
Domain
Kernel Services
Bucket
io_uring
Inferred role
Kernel Services: implementation source
Status
source implementation candidate

Why This File Exists

Shared kernel service surface used by multiple subsystems, including helpers, cryptography, virtualization support, and async I/O infrastructure.

Dependency Surface

Detected Declarations

Annotated Snippet

if (ifd.nr_folios == 1 && !PageHighMem(mr->pages[0])) {
			mr->ptr = page_address(mr->pages[0]);
			return 0;
		}
	}
	ptr = vmap(mr->pages, mr->nr_pages, VM_MAP, PAGE_KERNEL);
	if (!ptr)
		return -ENOMEM;

	mr->ptr = ptr;
	mr->flags |= IO_REGION_F_VMAP;
	return 0;
}

static int io_region_pin_pages(struct io_mapped_region *mr,
			       struct io_uring_region_desc *reg)
{
	size_t size = io_region_size(mr);
	struct page **pages;
	int nr_pages;

	pages = io_pin_pages(reg->user_addr, size, &nr_pages);
	if (IS_ERR(pages))
		return PTR_ERR(pages);
	if (WARN_ON_ONCE(nr_pages != mr->nr_pages))
		return -EFAULT;

	mr->pages = pages;
	mr->flags |= IO_REGION_F_USER_PROVIDED;
	return 0;
}

static int io_region_allocate_pages(struct io_mapped_region *mr,
				    struct io_uring_region_desc *reg,
				    unsigned long mmap_offset)
{
	gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN;
	size_t size = io_region_size(mr);
	unsigned long nr_allocated;
	struct page **pages;

	pages = kvmalloc_objs(*pages, mr->nr_pages, gfp);
	if (!pages)
		return -ENOMEM;

	if (io_mem_alloc_compound(pages, mr->nr_pages, size, gfp)) {
		mr->flags |= IO_REGION_F_SINGLE_REF;
		goto done;
	}

	nr_allocated = alloc_pages_bulk_node(gfp, NUMA_NO_NODE,
					     mr->nr_pages, pages);
	if (nr_allocated != mr->nr_pages) {
		if (nr_allocated)
			release_pages(pages, nr_allocated);
		kvfree(pages);
		return -ENOMEM;
	}
done:
	reg->mmap_offset = mmap_offset;
	mr->pages = pages;
	return 0;
}

int io_create_region(struct io_ring_ctx *ctx, struct io_mapped_region *mr,
		     struct io_uring_region_desc *reg,
		     unsigned long mmap_offset)
{
	int nr_pages, ret;
	u64 end;

	if (WARN_ON_ONCE(mr->pages || mr->ptr || mr->nr_pages))
		return -EFAULT;
	if (memchr_inv(&reg->__resv, 0, sizeof(reg->__resv)))
		return -EINVAL;
	if (reg->flags & ~IORING_MEM_REGION_TYPE_USER)
		return -EINVAL;
	/* user_addr should be set IFF it's a user memory backed region */
	if ((reg->flags & IORING_MEM_REGION_TYPE_USER) != !!reg->user_addr)
		return -EFAULT;
	if (!reg->size || reg->mmap_offset || reg->id)
		return -EINVAL;
	if ((reg->size >> PAGE_SHIFT) > INT_MAX)
		return -E2BIG;
	if ((reg->user_addr | reg->size) & ~PAGE_MASK)
		return -EINVAL;
	if (check_add_overflow(reg->user_addr, reg->size, &end))
		return -EOVERFLOW;

	nr_pages = reg->size >> PAGE_SHIFT;

Annotation

Implementation Notes