lib/raid/raid6/x86/avx512.c

Source file repositories/reference/linux-study-clean/lib/raid/raid6/x86/avx512.c

File Facts

System
Linux kernel
Corpus path
lib/raid/raid6/x86/avx512.c
Extension
.c
Size
17623 bytes
Lines
540
Domain
Kernel Services
Bucket
lib
Inferred role
Kernel Services: implementation source
Status
source implementation candidate

Why This File Exists

Shared kernel service surface used by multiple subsystems, including helpers, cryptography, virtualization support, and async I/O infrastructure.

Dependency Surface

Detected Declarations

Annotated Snippet

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * Copyright (C) 2016 Intel Corporation
 *
 * Author: Gayatri Kammela <gayatri.kammela@intel.com>
 * Author: Megha Dey <megha.dey@linux.intel.com>
 *
 * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved
 * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved
 *
 * AVX512 implementation of RAID-6 syndrome functions
 */

#include <asm/cpufeature.h>
#include <asm/fpu/api.h>
#include "algos.h"

static const struct raid6_avx512_constants {
	u64 x1d[8];
} raid6_avx512_constants __aligned(512/8) = {
	{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,
	  0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
};

static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
{
	u8 **dptr = (u8 **)ptrs;
	u8 *p, *q;
	int d, z, z0;

	z0 = disks - 3;         /* Highest data disk */
	p = dptr[z0+1];         /* XOR parity */
	q = dptr[z0+2];         /* RS syndrome */

	kernel_fpu_begin();

	asm volatile("vmovdqa64 %0,%%zmm0\n\t"
		     "vpxorq %%zmm1,%%zmm1,%%zmm1" /* Zero temp */
		     :
		     : "m" (raid6_avx512_constants.x1d[0]));

	for (d = 0; d < bytes; d += 64) {
		asm volatile("prefetchnta %0\n\t"
			     "vmovdqa64 %0,%%zmm2\n\t"     /* P[0] */
			     "prefetchnta %1\n\t"
			     "vmovdqa64 %%zmm2,%%zmm4\n\t" /* Q[0] */
			     "vmovdqa64 %1,%%zmm6"
			     :
			     : "m" (dptr[z0][d]), "m" (dptr[z0-1][d]));
		for (z = z0-2; z >= 0; z--) {
			asm volatile("prefetchnta %0\n\t"
				     "vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
				     "vpmovm2b %%k1,%%zmm5\n\t"
				     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
				     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
				     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
				     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
				     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
				     "vmovdqa64 %0,%%zmm6"
				     :
				     : "m" (dptr[z][d]));
		}
		asm volatile("vpcmpgtb %%zmm4,%%zmm1,%%k1\n\t"
			     "vpmovm2b %%k1,%%zmm5\n\t"
			     "vpaddb %%zmm4,%%zmm4,%%zmm4\n\t"
			     "vpandq %%zmm0,%%zmm5,%%zmm5\n\t"
			     "vpxorq %%zmm5,%%zmm4,%%zmm4\n\t"
			     "vpxorq %%zmm6,%%zmm2,%%zmm2\n\t"
			     "vpxorq %%zmm6,%%zmm4,%%zmm4\n\t"
			     "vmovntdq %%zmm2,%0\n\t"
			     "vpxorq %%zmm2,%%zmm2,%%zmm2\n\t"
			     "vmovntdq %%zmm4,%1\n\t"
			     "vpxorq %%zmm4,%%zmm4,%%zmm4"
			     :
			     : "m" (p[d]), "m" (q[d]));
	}

	asm volatile("sfence" : : : "memory");
	kernel_fpu_end();
}

static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
				       size_t bytes, void **ptrs)
{
	u8 **dptr = (u8 **)ptrs;
	u8 *p, *q;
	int d, z, z0;

Annotation

Implementation Notes