arch/x86/mm/init_64.c

Source file repositories/reference/linux-study-clean/arch/x86/mm/init_64.c

File Facts

System
Linux kernel
Corpus path
arch/x86/mm/init_64.c
Extension
.c
Size
43714 bytes
Lines
1670
Domain
Architecture Layer
Bucket
arch/x86
Inferred role
Architecture Layer: exported/initcall integration point
Status
integration implementation candidate

Why This File Exists

CPU and platform-specific kernel glue: boot entry, traps, syscall entry, interrupts, page tables, context switch, and low-level barriers.

Dependency Surface

Detected Declarations

Annotated Snippet

list_for_each_entry(page, &pgd_list, lru) {
			pgd_t *pgd;
			spinlock_t *pgt_lock;

			pgd = (pgd_t *)page_address(page) + pgd_index(addr);
			/* the pgt_lock only for Xen */
			pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
			spin_lock(pgt_lock);

			if (!pgd_none(*pgd_ref) && !pgd_none(*pgd))
				BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));

			if (pgd_none(*pgd))
				set_pgd(pgd, *pgd_ref);

			spin_unlock(pgt_lock);
		}
		spin_unlock(&pgd_lock);
	}
}

static void sync_global_pgds_l4(unsigned long start, unsigned long end)
{
	unsigned long addr;

	for (addr = start; addr <= end; addr = ALIGN(addr + 1, PGDIR_SIZE)) {
		pgd_t *pgd_ref = pgd_offset_k(addr);
		const p4d_t *p4d_ref;
		struct page *page;

		/*
		 * With folded p4d, pgd_none() is always false, we need to
		 * handle synchronization on p4d level.
		 */
		MAYBE_BUILD_BUG_ON(pgd_none(*pgd_ref));
		p4d_ref = p4d_offset(pgd_ref, addr);

		if (p4d_none(*p4d_ref))
			continue;

		spin_lock(&pgd_lock);
		list_for_each_entry(page, &pgd_list, lru) {
			pgd_t *pgd;
			p4d_t *p4d;
			spinlock_t *pgt_lock;

			pgd = (pgd_t *)page_address(page) + pgd_index(addr);
			p4d = p4d_offset(pgd, addr);
			/* the pgt_lock only for Xen */
			pgt_lock = &pgd_page_get_mm(page)->page_table_lock;
			spin_lock(pgt_lock);

			if (!p4d_none(*p4d_ref) && !p4d_none(*p4d))
				BUG_ON(p4d_pgtable(*p4d)
				       != p4d_pgtable(*p4d_ref));

			if (p4d_none(*p4d))
				set_p4d(p4d, *p4d_ref);

			spin_unlock(pgt_lock);
		}
		spin_unlock(&pgd_lock);
	}
}

/*
 * When memory was added make sure all the processes MM have
 * suitable PGD entries in the local PGD level page.
 */
static void sync_global_pgds(unsigned long start, unsigned long end)
{
	if (pgtable_l5_enabled())
		sync_global_pgds_l5(start, end);
	else
		sync_global_pgds_l4(start, end);
}

/*
 * Make kernel mappings visible in all page tables in the system.
 * This is necessary except when the init task populates kernel mappings
 * during the boot process. In that case, all processes originating from
 * the init task copies the kernel mappings, so there is no issue.
 * Otherwise, missing synchronization could lead to kernel crashes due
 * to missing page table entries for certain kernel mappings.
 *
 * Synchronization is performed at the top level, which is the PGD in
 * 5-level paging systems. But in 4-level paging systems, however,
 * pgd_populate() is a no-op, so synchronization is done at the P4D level.
 * sync_global_pgds() handles this difference between paging levels.
 */

Annotation

Implementation Notes