lib/crc/x86/crc32c-3way.S

Source file repositories/reference/linux-study-clean/lib/crc/x86/crc32c-3way.S

File Facts

System
Linux kernel
Corpus path
lib/crc/x86/crc32c-3way.S
Extension
.S
Size
10957 bytes
Lines
361
Domain
Kernel Services
Bucket
lib
Inferred role
Kernel Services: lib
Status
atlas-only

Why This File Exists

Shared kernel service surface used by multiple subsystems, including helpers, cryptography, virtualization support, and async I/O infrastructure.

Dependency Surface

Detected Declarations

Annotated Snippet

#include <linux/linkage.h>

## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction

# Define threshold below which buffers are considered "small" and routed to
# regular CRC code that does not interleave the CRC instructions.
#define SMALL_SIZE 200

# u32 crc32c_x86_3way(u32 crc, const u8 *buffer, size_t len);

.text
SYM_FUNC_START(crc32c_x86_3way)
#define    crc0		  %edi
#define    crc0_q	  %rdi
#define    bufp		  %rsi
#define    bufp_d	  %esi
#define    len		  %rdx
#define    len_dw	  %edx
#define    n_misaligned	  %ecx /* overlaps chunk_bytes! */
#define    n_misaligned_q %rcx
#define    chunk_bytes	  %ecx /* overlaps n_misaligned! */
#define    chunk_bytes_q  %rcx
#define    crc1		  %r8
#define    crc2		  %r9

	cmp	$SMALL_SIZE, len
	jb	.Lsmall

	################################################################
	## 1) ALIGN:
	################################################################
	mov	bufp_d, n_misaligned
	neg	n_misaligned
	and	$7, n_misaligned	# calculate the misalignment amount of
					# the address
	je	.Laligned		# Skip if aligned

	# Process 1 <= n_misaligned <= 7 bytes individually in order to align
	# the remaining data to an 8-byte boundary.
.Ldo_align:
	movq	(bufp), %rax
	add	n_misaligned_q, bufp
	sub	n_misaligned_q, len
.Lalign_loop:
	crc32b	%al, crc0		# compute crc32 of 1-byte
	shr	$8, %rax		# get next byte
	dec	n_misaligned
	jne     .Lalign_loop
.Laligned:

	################################################################
	## 2) PROCESS BLOCK:
	################################################################

	cmp	$128*24, len
	jae     .Lfull_block

.Lpartial_block:
	# Compute floor(len / 24) to get num qwords to process from each lane.
	imul	$2731, len_dw, %eax	# 2731 = ceil(2^16 / 24)
	shr	$16, %eax
	jmp	.Lcrc_3lanes

.Lfull_block:
	# Processing 128 qwords from each lane.
	mov	$128, %eax

	################################################################
	## 3) CRC each of three lanes:
	################################################################

Annotation

Implementation Notes