arch/arm64/include/asm/tlbflush.h

Source file repositories/reference/linux-study-clean/arch/arm64/include/asm/tlbflush.h

File Facts

System
Linux kernel
Corpus path
arch/arm64/include/asm/tlbflush.h
Extension
.h
Size
21395 bytes
Lines
759
Domain
Architecture Layer
Bucket
arch/arm64
Inferred role
Architecture Layer: implementation source
Status
source implementation candidate

Why This File Exists

CPU and platform-specific kernel glue: boot entry, traps, syscall entry, interrupts, page tables, context switch, and low-level barriers.

Dependency Surface

Detected Declarations

Annotated Snippet

if (num >= 0) {
			__tlbi_range(rop, addr, asid, scale, num, level, lpa2);
			addr += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT;
		}

		scale--;
		continue;
invalidate_one:
		__tlbi_level_asid(lop, addr, level, asid);
		addr += stride;
	}
}

#define __flush_s1_tlb_range_op(op, start, pages, stride, asid, tlb_level) \
	__flush_tlb_range_op(op, r##op, start, pages, stride, asid, tlb_level, lpa2_is_enabled())

#define __flush_s2_tlb_range_op(op, start, pages, stride, tlb_level) \
	__flush_tlb_range_op(op, r##op, start, pages, stride, 0, tlb_level, kvm_lpa2_is_enabled())

static inline bool __flush_tlb_range_limit_excess(unsigned long pages,
						  unsigned long stride)
{
	/*
	 * Assume that the worst case number of DVM ops required to flush a
	 * given range on a system that supports tlb-range is 20 (4 scales, 1
	 * final page, 15 for alignment on LPA2 systems), which is much smaller
	 * than MAX_DVM_OPS.
	 */
	if (system_supports_tlb_range())
		return pages > MAX_TLBI_RANGE_PAGES;

	return pages >= (MAX_DVM_OPS * stride) >> PAGE_SHIFT;
}

typedef unsigned __bitwise tlbf_t;

/* No special behaviour. */
#define TLBF_NONE		((__force tlbf_t)0)

/* Invalidate tlb entries only, leaving the page table walk cache intact. */
#define TLBF_NOWALKCACHE	((__force tlbf_t)BIT(0))

/* Skip the trailing dsb after issuing tlbi. */
#define TLBF_NOSYNC		((__force tlbf_t)BIT(1))

/* Suppress tlb notifier callbacks for this flush operation. */
#define TLBF_NONOTIFY		((__force tlbf_t)BIT(2))

/* Perform the tlbi locally without broadcasting to other CPUs. */
#define TLBF_NOBROADCAST	((__force tlbf_t)BIT(3))

static __always_inline void __do_flush_tlb_range(struct vm_area_struct *vma,
					unsigned long start, unsigned long end,
					unsigned long stride, int tlb_level,
					tlbf_t flags)
{
	struct mm_struct *mm = vma->vm_mm;
	unsigned long asid, pages;

	pages = (end - start) >> PAGE_SHIFT;

	if (__flush_tlb_range_limit_excess(pages, stride)) {
		flush_tlb_mm(mm);
		return;
	}

	if (!(flags & TLBF_NOBROADCAST))
		dsb(ishst);
	else
		dsb(nshst);

	asid = ASID(mm);

	switch (flags & (TLBF_NOWALKCACHE | TLBF_NOBROADCAST)) {
	case TLBF_NONE:
		__flush_s1_tlb_range_op(vae1is, start, pages, stride,
					asid, tlb_level);
		break;
	case TLBF_NOWALKCACHE:
		__flush_s1_tlb_range_op(vale1is, start, pages, stride,
					asid, tlb_level);
		break;
	case TLBF_NOBROADCAST:
		/* Combination unused */
		BUG();
		break;
	case TLBF_NOWALKCACHE | TLBF_NOBROADCAST:
		__flush_s1_tlb_range_op(vale1, start, pages, stride,
					asid, tlb_level);
		break;

Annotation

Implementation Notes