67988 lines
1.8 MiB
67988 lines
1.8 MiB
// Code generated by command: go run gen.go -out ../galois_gen_nopshufb_amd64.s -stubs ../galois_gen_nopshufb_amd64.go -pkg=reedsolomon. DO NOT EDIT.
|
|
|
|
//go:build !appengine && !noasm && !nogen && nopshufb && gc
|
|
|
|
#include "textflag.h"
|
|
|
|
// func _dummy_()
|
|
TEXT ·_dummy_(SB), $0
|
|
#ifdef GOAMD64_v4
|
|
#define XOR3WAY(ignore, a, b, dst) \
|
|
VPTERNLOGD $0x96, a, b, dst
|
|
|
|
#else
|
|
#define XOR3WAY(ignore, a, b, dst) \
|
|
VPXOR a, dst, dst \
|
|
VPXOR b, dst, dst
|
|
|
|
#endif
|
|
RET
|
|
|
|
// sSE2XorSlice will XOR in with out and store in out.
|
|
// Processes 16 bytes/loop.
|
|
|
|
// func sSE2XorSlice(in []byte, out []byte)
|
|
// Requires: SSE2
|
|
TEXT ·sSE2XorSlice(SB), $0-48
|
|
MOVQ in_base+0(FP), AX
|
|
MOVQ out_base+24(FP), CX
|
|
MOVQ in_len+8(FP), DX
|
|
SHRQ $0x04, DX
|
|
JZ end
|
|
|
|
loop:
|
|
MOVOU (AX), X0
|
|
MOVOU (CX), X1
|
|
PXOR X0, X1
|
|
MOVOU X1, (CX)
|
|
ADDQ $0x10, AX
|
|
ADDQ $0x10, CX
|
|
DECQ DX
|
|
JNZ loop
|
|
|
|
end:
|
|
RET
|
|
|
|
// sSE2XorSlice_64 will XOR in with out and store in out.
|
|
// Processes 64 bytes/loop.
|
|
|
|
// func sSE2XorSlice_64(in []byte, out []byte)
|
|
// Requires: SSE2
|
|
TEXT ·sSE2XorSlice_64(SB), $0-48
|
|
MOVQ in_base+0(FP), AX
|
|
MOVQ out_base+24(FP), CX
|
|
MOVQ in_len+8(FP), DX
|
|
SHRQ $0x06, DX
|
|
JZ end
|
|
|
|
loop:
|
|
MOVOU (AX), X0
|
|
MOVOU 16(AX), X2
|
|
MOVOU 32(AX), X4
|
|
MOVOU 48(AX), X6
|
|
MOVOU (CX), X1
|
|
MOVOU 16(CX), X3
|
|
MOVOU 32(CX), X5
|
|
MOVOU 48(CX), X7
|
|
PXOR X0, X1
|
|
PXOR X2, X3
|
|
PXOR X4, X5
|
|
PXOR X6, X7
|
|
MOVOU X1, (CX)
|
|
MOVOU X3, 16(CX)
|
|
MOVOU X5, 32(CX)
|
|
MOVOU X7, 48(CX)
|
|
ADDQ $0x40, AX
|
|
ADDQ $0x40, CX
|
|
DECQ DX
|
|
JNZ loop
|
|
|
|
end:
|
|
RET
|
|
|
|
// avx2XorSlice_64 will XOR in with out and store in out.
|
|
// Processes 64 bytes/loop.
|
|
|
|
// func avx2XorSlice_64(in []byte, out []byte)
|
|
// Requires: AVX, AVX2
|
|
TEXT ·avx2XorSlice_64(SB), $0-48
|
|
MOVQ in_base+0(FP), AX
|
|
MOVQ out_base+24(FP), CX
|
|
MOVQ in_len+8(FP), DX
|
|
SHRQ $0x06, DX
|
|
JZ end
|
|
|
|
loop:
|
|
VMOVDQU (AX), Y0
|
|
VMOVDQU 32(AX), Y2
|
|
VMOVDQU (CX), Y1
|
|
VMOVDQU 32(CX), Y3
|
|
VPXOR Y0, Y1, Y1
|
|
VPXOR Y2, Y3, Y3
|
|
VMOVDQU Y1, (CX)
|
|
VMOVDQU Y3, 32(CX)
|
|
ADDQ $0x40, AX
|
|
ADDQ $0x40, CX
|
|
DECQ DX
|
|
JNZ loop
|
|
|
|
end:
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func mulGFNI_1x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 4 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ start+72(FP), BX
|
|
|
|
// Add start offset to output
|
|
ADDQ BX, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ BX, CX
|
|
|
|
mulGFNI_1x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (CX), Z1
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z1, Z1
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z1, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 4 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ start+72(FP), BX
|
|
|
|
// Add start offset to output
|
|
ADDQ BX, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ BX, CX
|
|
|
|
mulAvxGFNI_1x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (CX), Y1
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y1, Y1
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y1, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 4 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ start+72(FP), BX
|
|
|
|
// Add start offset to output
|
|
ADDQ BX, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ BX, CX
|
|
|
|
mulGFNI_1x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (DX), Z1
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (CX), Z2
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z2, Z2
|
|
VXORPD Z1, Z2, Z1
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z1, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 4 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ start+72(FP), BX
|
|
|
|
// Add start offset to output
|
|
ADDQ BX, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ BX, CX
|
|
|
|
mulAvxGFNI_1x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (DX), Y1
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (CX), Y2
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y2, Y2
|
|
VXORPD Y1, Y2, Y1
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y1, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 6 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ start+72(FP), SI
|
|
|
|
// Add start offset to output
|
|
ADDQ SI, BX
|
|
ADDQ SI, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ SI, CX
|
|
|
|
mulGFNI_1x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (CX), Z3
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z3, Z2
|
|
VGF2P8AFFINEQB $0x00, Z1, Z3, Z3
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z2, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z3, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x2(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 6 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ start+72(FP), SI
|
|
|
|
// Add start offset to output
|
|
ADDQ SI, BX
|
|
ADDQ SI, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ SI, CX
|
|
|
|
mulAvxGFNI_1x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (CX), Y3
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y3, Y2
|
|
VGF2P8AFFINEQB $0x00, Y1, Y3, Y3
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y2, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y3, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 6 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ start+72(FP), SI
|
|
|
|
// Add start offset to output
|
|
ADDQ SI, BX
|
|
ADDQ SI, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ SI, CX
|
|
|
|
mulGFNI_1x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (BX), Z2
|
|
VMOVDQU64 (DX), Z3
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (CX), Z4
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
|
|
VXORPD Z2, Z5, Z2
|
|
VGF2P8AFFINEQB $0x00, Z1, Z4, Z5
|
|
VXORPD Z3, Z5, Z3
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z2, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z3, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x2Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 6 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ start+72(FP), SI
|
|
|
|
// Add start offset to output
|
|
ADDQ SI, BX
|
|
ADDQ SI, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ SI, CX
|
|
|
|
mulAvxGFNI_1x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (BX), Y2
|
|
VMOVDQU (DX), Y3
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (CX), Y4
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y4, Y5
|
|
VXORPD Y2, Y5, Y2
|
|
VGF2P8AFFINEQB $0x00, Y1, Y4, Y5
|
|
VXORPD Y3, Y5, Y3
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y2, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y3, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x3_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, BX
|
|
ADDQ DI, SI
|
|
ADDQ DI, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, CX
|
|
|
|
mulGFNI_1x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (CX), Z5
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z5, Z3
|
|
VGF2P8AFFINEQB $0x00, Z1, Z5, Z4
|
|
VGF2P8AFFINEQB $0x00, Z2, Z5, Z5
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z3, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z4, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z5, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x3(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, BX
|
|
ADDQ DI, SI
|
|
ADDQ DI, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, CX
|
|
|
|
mulAvxGFNI_1x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (CX), Y5
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y5, Y3
|
|
VGF2P8AFFINEQB $0x00, Y1, Y5, Y4
|
|
VGF2P8AFFINEQB $0x00, Y2, Y5, Y5
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y3, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y4, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y5, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x3_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, BX
|
|
ADDQ DI, SI
|
|
ADDQ DI, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, CX
|
|
|
|
mulGFNI_1x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (BX), Z3
|
|
VMOVDQU64 (SI), Z4
|
|
VMOVDQU64 (DX), Z5
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (CX), Z6
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z6, Z7
|
|
VXORPD Z3, Z7, Z3
|
|
VGF2P8AFFINEQB $0x00, Z1, Z6, Z7
|
|
VXORPD Z4, Z7, Z4
|
|
VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
|
|
VXORPD Z5, Z7, Z5
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z3, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z4, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z5, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x3Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, BX
|
|
ADDQ DI, SI
|
|
ADDQ DI, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, CX
|
|
|
|
mulAvxGFNI_1x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (BX), Y3
|
|
VMOVDQU (SI), Y4
|
|
VMOVDQU (DX), Y5
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (CX), Y6
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y6, Y7
|
|
VXORPD Y3, Y7, Y3
|
|
VGF2P8AFFINEQB $0x00, Y1, Y6, Y7
|
|
VXORPD Y4, Y7, Y4
|
|
VGF2P8AFFINEQB $0x00, Y2, Y6, Y7
|
|
VXORPD Y5, Y7, Y5
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y3, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y4, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y5, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x4_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, BX
|
|
ADDQ R8, SI
|
|
ADDQ R8, DI
|
|
ADDQ R8, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, CX
|
|
|
|
mulGFNI_1x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (CX), Z7
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z7, Z4
|
|
VGF2P8AFFINEQB $0x00, Z1, Z7, Z5
|
|
VGF2P8AFFINEQB $0x00, Z2, Z7, Z6
|
|
VGF2P8AFFINEQB $0x00, Z3, Z7, Z7
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z4, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z5, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z6, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z7, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x4(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, BX
|
|
ADDQ R8, SI
|
|
ADDQ R8, DI
|
|
ADDQ R8, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, CX
|
|
|
|
mulAvxGFNI_1x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (CX), Y7
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y7, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y7, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y7, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y7, Y7
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y4, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y5, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y6, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y7, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x4_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, BX
|
|
ADDQ R8, SI
|
|
ADDQ R8, DI
|
|
ADDQ R8, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, CX
|
|
|
|
mulGFNI_1x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU64 (BX), Z4
|
|
VMOVDQU64 (SI), Z5
|
|
VMOVDQU64 (DI), Z6
|
|
VMOVDQU64 (DX), Z7
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (CX), Z8
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z8, Z9
|
|
VXORPD Z4, Z9, Z4
|
|
VGF2P8AFFINEQB $0x00, Z1, Z8, Z9
|
|
VXORPD Z5, Z9, Z5
|
|
VGF2P8AFFINEQB $0x00, Z2, Z8, Z9
|
|
VXORPD Z6, Z9, Z6
|
|
VGF2P8AFFINEQB $0x00, Z3, Z8, Z9
|
|
VXORPD Z7, Z9, Z7
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z4, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z5, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z6, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z7, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x4Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, BX
|
|
ADDQ R8, SI
|
|
ADDQ R8, DI
|
|
ADDQ R8, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, CX
|
|
|
|
mulAvxGFNI_1x4Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU (BX), Y4
|
|
VMOVDQU (SI), Y5
|
|
VMOVDQU (DI), Y6
|
|
VMOVDQU (DX), Y7
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (CX), Y8
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y8, Y9
|
|
VXORPD Y4, Y9, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y8, Y9
|
|
VXORPD Y5, Y9, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y8, Y9
|
|
VXORPD Y6, Y9, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y8, Y9
|
|
VXORPD Y7, Y9, Y7
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y4, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y5, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y6, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y7, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x5_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_1x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (CX), Z9
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z9, Z5
|
|
VGF2P8AFFINEQB $0x00, Z1, Z9, Z6
|
|
VGF2P8AFFINEQB $0x00, Z2, Z9, Z7
|
|
VGF2P8AFFINEQB $0x00, Z3, Z9, Z8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z9, Z9
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z5, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z6, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z7, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z8, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z9, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x5(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_1x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (CX), Y9
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y9, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y9, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y9, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y9, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y9, Y9
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y5, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y6, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y7, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x5_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_1x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU64 (BX), Z5
|
|
VMOVDQU64 (SI), Z6
|
|
VMOVDQU64 (DI), Z7
|
|
VMOVDQU64 (R8), Z8
|
|
VMOVDQU64 (DX), Z9
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (CX), Z10
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z10, Z11
|
|
VXORPD Z5, Z11, Z5
|
|
VGF2P8AFFINEQB $0x00, Z1, Z10, Z11
|
|
VXORPD Z6, Z11, Z6
|
|
VGF2P8AFFINEQB $0x00, Z2, Z10, Z11
|
|
VXORPD Z7, Z11, Z7
|
|
VGF2P8AFFINEQB $0x00, Z3, Z10, Z11
|
|
VXORPD Z8, Z11, Z8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z10, Z11
|
|
VXORPD Z9, Z11, Z9
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z5, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z6, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z7, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z8, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z9, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x5Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_1x5Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU (BX), Y5
|
|
VMOVDQU (SI), Y6
|
|
VMOVDQU (DI), Y7
|
|
VMOVDQU (R8), Y8
|
|
VMOVDQU (DX), Y9
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (CX), Y10
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y10, Y11
|
|
VXORPD Y5, Y11, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y10, Y11
|
|
VXORPD Y6, Y11, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y10, Y11
|
|
VXORPD Y7, Y11, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y10, Y11
|
|
VXORPD Y8, Y11, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y10, Y11
|
|
VXORPD Y9, Y11, Y9
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y5, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y6, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y7, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x6_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_1x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (CX), Z11
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z11, Z6
|
|
VGF2P8AFFINEQB $0x00, Z1, Z11, Z7
|
|
VGF2P8AFFINEQB $0x00, Z2, Z11, Z8
|
|
VGF2P8AFFINEQB $0x00, Z3, Z11, Z9
|
|
VGF2P8AFFINEQB $0x00, Z4, Z11, Z10
|
|
VGF2P8AFFINEQB $0x00, Z5, Z11, Z11
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z6, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z7, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z8, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z9, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z10, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z11, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x6(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, CX
|
|
|
|
mulAvxGFNI_1x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (CX), Y11
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y11, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y11, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y11, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y11, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y11, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y11, Y11
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y6, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y7, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y8, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y9, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x6_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_1x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU64 (BX), Z6
|
|
VMOVDQU64 (SI), Z7
|
|
VMOVDQU64 (DI), Z8
|
|
VMOVDQU64 (R8), Z9
|
|
VMOVDQU64 (R9), Z10
|
|
VMOVDQU64 (DX), Z11
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (CX), Z12
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z12, Z13
|
|
VXORPD Z6, Z13, Z6
|
|
VGF2P8AFFINEQB $0x00, Z1, Z12, Z13
|
|
VXORPD Z7, Z13, Z7
|
|
VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
|
|
VXORPD Z8, Z13, Z8
|
|
VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
|
|
VXORPD Z9, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z6, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z7, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z8, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z9, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z10, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z11, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x6Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, CX
|
|
|
|
mulAvxGFNI_1x6Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU (BX), Y6
|
|
VMOVDQU (SI), Y7
|
|
VMOVDQU (DI), Y8
|
|
VMOVDQU (R8), Y9
|
|
VMOVDQU (R9), Y10
|
|
VMOVDQU (DX), Y11
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (CX), Y12
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y12, Y13
|
|
VXORPD Y6, Y13, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y12, Y13
|
|
VXORPD Y7, Y13, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
|
|
VXORPD Y8, Y13, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
|
|
VXORPD Y9, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y6, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y7, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y8, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y9, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x7_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 16 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_1x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (CX), Z13
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z13, Z7
|
|
VGF2P8AFFINEQB $0x00, Z1, Z13, Z8
|
|
VGF2P8AFFINEQB $0x00, Z2, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z3, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z4, Z13, Z11
|
|
VGF2P8AFFINEQB $0x00, Z5, Z13, Z12
|
|
VGF2P8AFFINEQB $0x00, Z6, Z13, Z13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z7, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z8, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z9, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z10, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z11, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z12, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z13, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x7(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 16 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, CX
|
|
|
|
mulAvxGFNI_1x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (CX), Y13
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y13, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y13, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y13, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y13, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y13, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y8, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x7_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 16 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_1x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU64 (BX), Z7
|
|
VMOVDQU64 (SI), Z8
|
|
VMOVDQU64 (DI), Z9
|
|
VMOVDQU64 (R8), Z10
|
|
VMOVDQU64 (R9), Z11
|
|
VMOVDQU64 (R10), Z12
|
|
VMOVDQU64 (DX), Z13
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (CX), Z14
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z14, Z15
|
|
VXORPD Z7, Z15, Z7
|
|
VGF2P8AFFINEQB $0x00, Z1, Z14, Z15
|
|
VXORPD Z8, Z15, Z8
|
|
VGF2P8AFFINEQB $0x00, Z2, Z14, Z15
|
|
VXORPD Z9, Z15, Z9
|
|
VGF2P8AFFINEQB $0x00, Z3, Z14, Z15
|
|
VXORPD Z10, Z15, Z10
|
|
VGF2P8AFFINEQB $0x00, Z4, Z14, Z15
|
|
VXORPD Z11, Z15, Z11
|
|
VGF2P8AFFINEQB $0x00, Z5, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z6, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z7, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z8, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z9, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z10, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z11, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z12, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z13, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x7Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 16 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, CX
|
|
|
|
mulAvxGFNI_1x7Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU (BX), Y7
|
|
VMOVDQU (SI), Y8
|
|
VMOVDQU (DI), Y9
|
|
VMOVDQU (R8), Y10
|
|
VMOVDQU (R9), Y11
|
|
VMOVDQU (R10), Y12
|
|
VMOVDQU (DX), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (CX), Y14
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (BX)
|
|
ADDQ $0x20, BX
|
|
VMOVDQU Y8, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (DX)
|
|
ADDQ $0x20, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x8_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_1x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (CX), Z15
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z15, Z8
|
|
VGF2P8AFFINEQB $0x00, Z1, Z15, Z9
|
|
VGF2P8AFFINEQB $0x00, Z2, Z15, Z10
|
|
VGF2P8AFFINEQB $0x00, Z3, Z15, Z11
|
|
VGF2P8AFFINEQB $0x00, Z4, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z5, Z15, Z13
|
|
VGF2P8AFFINEQB $0x00, Z6, Z15, Z14
|
|
VGF2P8AFFINEQB $0x00, Z7, Z15, Z15
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z8, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z9, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z10, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z11, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z12, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z13, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z14, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z15, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x8(SB), $0-88
|
|
// Loading 6 of 8 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), BX
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_1x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (DX), Y13
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y13, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y13, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y13, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y13, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y13, Y12
|
|
VBROADCASTSD 56(CX), Y14
|
|
VGF2P8AFFINEQB $0x00, Y14, Y13, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y7, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x8_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_1x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU64 (BX), Z8
|
|
VMOVDQU64 (SI), Z9
|
|
VMOVDQU64 (DI), Z10
|
|
VMOVDQU64 (R8), Z11
|
|
VMOVDQU64 (R9), Z12
|
|
VMOVDQU64 (R10), Z13
|
|
VMOVDQU64 (R11), Z14
|
|
VMOVDQU64 (DX), Z15
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (CX), Z16
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z16, Z17
|
|
VXORPD Z8, Z17, Z8
|
|
VGF2P8AFFINEQB $0x00, Z1, Z16, Z17
|
|
VXORPD Z9, Z17, Z9
|
|
VGF2P8AFFINEQB $0x00, Z2, Z16, Z17
|
|
VXORPD Z10, Z17, Z10
|
|
VGF2P8AFFINEQB $0x00, Z3, Z16, Z17
|
|
VXORPD Z11, Z17, Z11
|
|
VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
|
|
VXORPD Z12, Z17, Z12
|
|
VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
|
|
VXORPD Z13, Z17, Z13
|
|
VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z8, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z9, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z10, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z11, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z12, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z13, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z14, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z15, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x8Xor(SB), $0-88
|
|
// Loading 6 of 8 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), BX
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_1x8Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU (SI), Y6
|
|
VMOVDQU (DI), Y7
|
|
VMOVDQU (R8), Y8
|
|
VMOVDQU (R9), Y9
|
|
VMOVDQU (R10), Y10
|
|
VMOVDQU (R11), Y11
|
|
VMOVDQU (R12), Y12
|
|
VMOVDQU (BX), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y7, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x9_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_1x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (CX), Z17
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z17, Z9
|
|
VGF2P8AFFINEQB $0x00, Z1, Z17, Z10
|
|
VGF2P8AFFINEQB $0x00, Z2, Z17, Z11
|
|
VGF2P8AFFINEQB $0x00, Z3, Z17, Z12
|
|
VGF2P8AFFINEQB $0x00, Z4, Z17, Z13
|
|
VGF2P8AFFINEQB $0x00, Z5, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z6, Z17, Z15
|
|
VGF2P8AFFINEQB $0x00, Z7, Z17, Z16
|
|
VGF2P8AFFINEQB $0x00, Z8, Z17, Z17
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU64 Z9, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z10, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z11, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z12, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z13, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z14, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z15, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z16, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z17, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x9(SB), $0-88
|
|
// Loading 5 of 9 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), R13
|
|
MOVQ 192(BX), BX
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_1x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (DX), Y13
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y13, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y13, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y13, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y13, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y13, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y13, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y13, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y13, Y12
|
|
VBROADCASTSD 64(CX), Y14
|
|
VGF2P8AFFINEQB $0x00, Y14, Y13, Y13
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU Y5, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y6, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y7, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y8, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x9_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_1x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
VMOVDQU64 (BX), Z9
|
|
VMOVDQU64 (SI), Z10
|
|
VMOVDQU64 (DI), Z11
|
|
VMOVDQU64 (R8), Z12
|
|
VMOVDQU64 (R9), Z13
|
|
VMOVDQU64 (R10), Z14
|
|
VMOVDQU64 (R11), Z15
|
|
VMOVDQU64 (R12), Z16
|
|
VMOVDQU64 (DX), Z17
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (CX), Z18
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z18, Z19
|
|
VXORPD Z9, Z19, Z9
|
|
VGF2P8AFFINEQB $0x00, Z1, Z18, Z19
|
|
VXORPD Z10, Z19, Z10
|
|
VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
|
|
VXORPD Z11, Z19, Z11
|
|
VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
|
|
VXORPD Z12, Z19, Z12
|
|
VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
|
|
VXORPD Z13, Z19, Z13
|
|
VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
|
|
VXORPD Z14, Z19, Z14
|
|
VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU64 Z9, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z10, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z11, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z12, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z13, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z14, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z15, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z16, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z17, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x9Xor(SB), $0-88
|
|
// Loading 5 of 9 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), R13
|
|
MOVQ 192(BX), BX
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_1x9Xor_loop:
|
|
// Load 9 outputs
|
|
VMOVDQU (SI), Y5
|
|
VMOVDQU (DI), Y6
|
|
VMOVDQU (R8), Y7
|
|
VMOVDQU (R9), Y8
|
|
VMOVDQU (R10), Y9
|
|
VMOVDQU (R11), Y10
|
|
VMOVDQU (R12), Y11
|
|
VMOVDQU (R13), Y12
|
|
VMOVDQU (BX), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU Y5, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y6, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y7, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y8, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x10_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_1x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (CX), Z19
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z19, Z10
|
|
VGF2P8AFFINEQB $0x00, Z1, Z19, Z11
|
|
VGF2P8AFFINEQB $0x00, Z2, Z19, Z12
|
|
VGF2P8AFFINEQB $0x00, Z3, Z19, Z13
|
|
VGF2P8AFFINEQB $0x00, Z4, Z19, Z14
|
|
VGF2P8AFFINEQB $0x00, Z5, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z6, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z7, Z19, Z17
|
|
VGF2P8AFFINEQB $0x00, Z8, Z19, Z18
|
|
VGF2P8AFFINEQB $0x00, Z9, Z19, Z19
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU64 Z10, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z11, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z12, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z13, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z14, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z15, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z16, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z17, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z18, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z19, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x10(SB), $0-88
|
|
// Loading 4 of 10 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), R13
|
|
MOVQ 192(BX), R14
|
|
MOVQ 216(BX), BX
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_1x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (DX), Y13
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y13, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y13, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y13, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y13, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y13, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y13, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y13, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y13, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y13, Y12
|
|
VBROADCASTSD 72(CX), Y14
|
|
VGF2P8AFFINEQB $0x00, Y14, Y13, Y13
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU Y4, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y5, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y6, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y7, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_1x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_1x10_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_1x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), CX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ out_base+48(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, DX
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_1x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
VMOVDQU64 (BX), Z10
|
|
VMOVDQU64 (SI), Z11
|
|
VMOVDQU64 (DI), Z12
|
|
VMOVDQU64 (R8), Z13
|
|
VMOVDQU64 (R9), Z14
|
|
VMOVDQU64 (R10), Z15
|
|
VMOVDQU64 (R11), Z16
|
|
VMOVDQU64 (R12), Z17
|
|
VMOVDQU64 (R13), Z18
|
|
VMOVDQU64 (DX), Z19
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (CX), Z20
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z20, Z21
|
|
VXORPD Z10, Z21, Z10
|
|
VGF2P8AFFINEQB $0x00, Z1, Z20, Z21
|
|
VXORPD Z11, Z21, Z11
|
|
VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
|
|
VXORPD Z12, Z21, Z12
|
|
VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
|
|
VXORPD Z13, Z21, Z13
|
|
VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
|
|
VXORPD Z14, Z21, Z14
|
|
VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
|
|
VXORPD Z15, Z21, Z15
|
|
VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU64 Z10, (BX)
|
|
ADDQ $0x40, BX
|
|
VMOVDQU64 Z11, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z12, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z13, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z14, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z15, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z16, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z17, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z18, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z19, (DX)
|
|
ADDQ $0x40, DX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_1x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_1x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_1x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_1x10Xor(SB), $0-88
|
|
// Loading 4 of 10 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_1x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), DX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), R13
|
|
MOVQ 192(BX), R14
|
|
MOVQ 216(BX), BX
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_1x10Xor_loop:
|
|
// Load 10 outputs
|
|
VMOVDQU (SI), Y4
|
|
VMOVDQU (DI), Y5
|
|
VMOVDQU (R8), Y6
|
|
VMOVDQU (R9), Y7
|
|
VMOVDQU (R10), Y8
|
|
VMOVDQU (R11), Y9
|
|
VMOVDQU (R12), Y10
|
|
VMOVDQU (R13), Y11
|
|
VMOVDQU (R14), Y12
|
|
VMOVDQU (BX), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU Y4, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y5, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y6, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y7, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_1x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_1x10Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 5 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), BX
|
|
MOVQ start+72(FP), SI
|
|
|
|
// Add start offset to output
|
|
ADDQ SI, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ SI, DX
|
|
ADDQ SI, CX
|
|
|
|
mulGFNI_2x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z3
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z3, Z2
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (CX), Z3
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z3, Z3
|
|
VXORPD Z2, Z3, Z2
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z2, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 5 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), BX
|
|
MOVQ start+72(FP), SI
|
|
|
|
// Add start offset to output
|
|
ADDQ SI, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ SI, DX
|
|
ADDQ SI, CX
|
|
|
|
mulAvxGFNI_2x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y3
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y3, Y2
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (CX), Y3
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y3, Y3
|
|
VXORPD Y2, Y3, Y2
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y2, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 5 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), BX
|
|
MOVQ start+72(FP), SI
|
|
|
|
// Add start offset to output
|
|
ADDQ SI, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ SI, DX
|
|
ADDQ SI, CX
|
|
|
|
mulGFNI_2x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (BX), Z2
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z3
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z3, Z3
|
|
VXORPD Z2, Z3, Z2
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (CX), Z3
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z3, Z3
|
|
VXORPD Z2, Z3, Z2
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z2, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 5 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), BX
|
|
MOVQ start+72(FP), SI
|
|
|
|
// Add start offset to output
|
|
ADDQ SI, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ SI, DX
|
|
ADDQ SI, CX
|
|
|
|
mulAvxGFNI_2x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (BX), Y2
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y3
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y3, Y3
|
|
VXORPD Y2, Y3, Y2
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (CX), Y3
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y3, Y3
|
|
VXORPD Y2, Y3, Y2
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y2, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), BX
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, SI
|
|
ADDQ DI, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, DX
|
|
ADDQ DI, CX
|
|
|
|
mulGFNI_2x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z6
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z6, Z4
|
|
VGF2P8AFFINEQB $0x00, Z1, Z6, Z5
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (CX), Z6
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
|
|
VXORPD Z4, Z7, Z4
|
|
VGF2P8AFFINEQB $0x00, Z3, Z6, Z7
|
|
VXORPD Z5, Z7, Z5
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z4, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z5, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x2(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), BX
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, SI
|
|
ADDQ DI, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, DX
|
|
ADDQ DI, CX
|
|
|
|
mulAvxGFNI_2x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y6
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y6, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y6, Y5
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (CX), Y6
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y6, Y7
|
|
VXORPD Y4, Y7, Y4
|
|
VGF2P8AFFINEQB $0x00, Y3, Y6, Y7
|
|
VXORPD Y5, Y7, Y5
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y4, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y5, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), BX
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, SI
|
|
ADDQ DI, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, DX
|
|
ADDQ DI, CX
|
|
|
|
mulGFNI_2x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (SI), Z4
|
|
VMOVDQU64 (BX), Z5
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z6
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z6, Z7
|
|
VXORPD Z4, Z7, Z4
|
|
VGF2P8AFFINEQB $0x00, Z1, Z6, Z7
|
|
VXORPD Z5, Z7, Z5
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (CX), Z6
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
|
|
VXORPD Z4, Z7, Z4
|
|
VGF2P8AFFINEQB $0x00, Z3, Z6, Z7
|
|
VXORPD Z5, Z7, Z5
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z4, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z5, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x2Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), BX
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, SI
|
|
ADDQ DI, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, DX
|
|
ADDQ DI, CX
|
|
|
|
mulAvxGFNI_2x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (SI), Y4
|
|
VMOVDQU (BX), Y5
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y6
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y6, Y7
|
|
VXORPD Y4, Y7, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y6, Y7
|
|
VXORPD Y5, Y7, Y5
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (CX), Y6
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y6, Y7
|
|
VXORPD Y4, Y7, Y4
|
|
VGF2P8AFFINEQB $0x00, Y3, Y6, Y7
|
|
VXORPD Y5, Y7, Y5
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y4, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y5, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x3_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 11 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), BX
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, SI
|
|
ADDQ R8, DI
|
|
ADDQ R8, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, CX
|
|
|
|
mulGFNI_2x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z9
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z9, Z6
|
|
VGF2P8AFFINEQB $0x00, Z1, Z9, Z7
|
|
VGF2P8AFFINEQB $0x00, Z2, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (CX), Z9
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z9, Z10
|
|
VXORPD Z6, Z10, Z6
|
|
VGF2P8AFFINEQB $0x00, Z4, Z9, Z10
|
|
VXORPD Z7, Z10, Z7
|
|
VGF2P8AFFINEQB $0x00, Z5, Z9, Z10
|
|
VXORPD Z8, Z10, Z8
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z6, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z7, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z8, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x3(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 11 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), BX
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, SI
|
|
ADDQ R8, DI
|
|
ADDQ R8, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, CX
|
|
|
|
mulAvxGFNI_2x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (DX), Y9
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y9, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y9, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (CX), Y9
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y3, Y9, Y10
|
|
VXORPD Y6, Y10, Y6
|
|
VGF2P8AFFINEQB $0x00, Y4, Y9, Y10
|
|
VXORPD Y7, Y10, Y7
|
|
VGF2P8AFFINEQB $0x00, Y5, Y9, Y10
|
|
VXORPD Y8, Y10, Y8
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y6, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y7, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y8, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x3_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 11 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), BX
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, SI
|
|
ADDQ R8, DI
|
|
ADDQ R8, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, CX
|
|
|
|
mulGFNI_2x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (SI), Z6
|
|
VMOVDQU64 (DI), Z7
|
|
VMOVDQU64 (BX), Z8
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z9
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z9, Z10
|
|
VXORPD Z6, Z10, Z6
|
|
VGF2P8AFFINEQB $0x00, Z1, Z9, Z10
|
|
VXORPD Z7, Z10, Z7
|
|
VGF2P8AFFINEQB $0x00, Z2, Z9, Z10
|
|
VXORPD Z8, Z10, Z8
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (CX), Z9
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z9, Z10
|
|
VXORPD Z6, Z10, Z6
|
|
VGF2P8AFFINEQB $0x00, Z4, Z9, Z10
|
|
VXORPD Z7, Z10, Z7
|
|
VGF2P8AFFINEQB $0x00, Z5, Z9, Z10
|
|
VXORPD Z8, Z10, Z8
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z6, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z7, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z8, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x3Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 11 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), BX
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, SI
|
|
ADDQ R8, DI
|
|
ADDQ R8, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, CX
|
|
|
|
mulAvxGFNI_2x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (SI), Y6
|
|
VMOVDQU (DI), Y7
|
|
VMOVDQU (BX), Y8
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (DX), Y9
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y9, Y10
|
|
VXORPD Y6, Y10, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y9, Y10
|
|
VXORPD Y7, Y10, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y9, Y10
|
|
VXORPD Y8, Y10, Y8
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (CX), Y9
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y3, Y9, Y10
|
|
VXORPD Y6, Y10, Y6
|
|
VGF2P8AFFINEQB $0x00, Y4, Y9, Y10
|
|
VXORPD Y7, Y10, Y7
|
|
VGF2P8AFFINEQB $0x00, Y5, Y9, Y10
|
|
VXORPD Y8, Y10, Y8
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y6, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y7, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y8, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x4_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), BX
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_2x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z12
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z12, Z8
|
|
VGF2P8AFFINEQB $0x00, Z1, Z12, Z9
|
|
VGF2P8AFFINEQB $0x00, Z2, Z12, Z10
|
|
VGF2P8AFFINEQB $0x00, Z3, Z12, Z11
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (CX), Z12
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
|
|
VXORPD Z8, Z13, Z8
|
|
VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
|
|
VXORPD Z9, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z8, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z9, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z10, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z11, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x4(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), BX
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_2x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (DX), Y12
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y12, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y12, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y12, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y12, Y11
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (CX), Y12
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
|
|
VXORPD Y8, Y13, Y8
|
|
VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
|
|
VXORPD Y9, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y8, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x4_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), BX
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_2x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU64 (SI), Z8
|
|
VMOVDQU64 (DI), Z9
|
|
VMOVDQU64 (R8), Z10
|
|
VMOVDQU64 (BX), Z11
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z12
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z12, Z13
|
|
VXORPD Z8, Z13, Z8
|
|
VGF2P8AFFINEQB $0x00, Z1, Z12, Z13
|
|
VXORPD Z9, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (CX), Z12
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
|
|
VXORPD Z8, Z13, Z8
|
|
VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
|
|
VXORPD Z9, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z8, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z9, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z10, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z11, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x4Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), BX
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_2x4Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU (SI), Y8
|
|
VMOVDQU (DI), Y9
|
|
VMOVDQU (R8), Y10
|
|
VMOVDQU (BX), Y11
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (DX), Y12
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y12, Y13
|
|
VXORPD Y8, Y13, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y12, Y13
|
|
VXORPD Y9, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (CX), Y12
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
|
|
VXORPD Y8, Y13, Y8
|
|
VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
|
|
VXORPD Y9, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y8, (SI)
|
|
ADDQ $0x20, SI
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (BX)
|
|
ADDQ $0x20, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x5_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 17 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), BX
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_2x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z15
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z15, Z10
|
|
VGF2P8AFFINEQB $0x00, Z1, Z15, Z11
|
|
VGF2P8AFFINEQB $0x00, Z2, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z3, Z15, Z13
|
|
VGF2P8AFFINEQB $0x00, Z4, Z15, Z14
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (CX), Z15
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z15, Z16
|
|
VXORPD Z10, Z16, Z10
|
|
VGF2P8AFFINEQB $0x00, Z6, Z15, Z16
|
|
VXORPD Z11, Z16, Z11
|
|
VGF2P8AFFINEQB $0x00, Z7, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z8, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z9, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z10, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z11, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z12, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z13, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z14, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x5(SB), $0-88
|
|
// Loading 9 of 10 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 17 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), SI
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_2x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x5_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 17 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), BX
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_2x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU64 (SI), Z10
|
|
VMOVDQU64 (DI), Z11
|
|
VMOVDQU64 (R8), Z12
|
|
VMOVDQU64 (R9), Z13
|
|
VMOVDQU64 (BX), Z14
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z15
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z15, Z16
|
|
VXORPD Z10, Z16, Z10
|
|
VGF2P8AFFINEQB $0x00, Z1, Z15, Z16
|
|
VXORPD Z11, Z16, Z11
|
|
VGF2P8AFFINEQB $0x00, Z2, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z3, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z4, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (CX), Z15
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z15, Z16
|
|
VXORPD Z10, Z16, Z10
|
|
VGF2P8AFFINEQB $0x00, Z6, Z15, Z16
|
|
VXORPD Z11, Z16, Z11
|
|
VGF2P8AFFINEQB $0x00, Z7, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z8, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z9, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z10, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z11, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z12, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z13, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z14, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x5Xor(SB), $0-88
|
|
// Loading 9 of 10 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 17 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), SI
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_2x5Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU (DI), Y9
|
|
VMOVDQU (R8), Y10
|
|
VMOVDQU (R9), Y11
|
|
VMOVDQU (R10), Y12
|
|
VMOVDQU (SI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x6_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), BX
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_2x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (DX), Z18
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z18, Z12
|
|
VGF2P8AFFINEQB $0x00, Z1, Z18, Z13
|
|
VGF2P8AFFINEQB $0x00, Z2, Z18, Z14
|
|
VGF2P8AFFINEQB $0x00, Z3, Z18, Z15
|
|
VGF2P8AFFINEQB $0x00, Z4, Z18, Z16
|
|
VGF2P8AFFINEQB $0x00, Z5, Z18, Z17
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (CX), Z18
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
|
|
VXORPD Z12, Z19, Z12
|
|
VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
|
|
VXORPD Z13, Z19, Z13
|
|
VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
|
|
VXORPD Z14, Z19, Z14
|
|
VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z12, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z13, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z14, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z15, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z16, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z17, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x6(SB), $0-88
|
|
// Loading 8 of 12 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), SI
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_2x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y9, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y12, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x6_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), BX
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_2x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU64 (SI), Z12
|
|
VMOVDQU64 (DI), Z13
|
|
VMOVDQU64 (R8), Z14
|
|
VMOVDQU64 (R9), Z15
|
|
VMOVDQU64 (R10), Z16
|
|
VMOVDQU64 (BX), Z17
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (DX), Z18
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z18, Z19
|
|
VXORPD Z12, Z19, Z12
|
|
VGF2P8AFFINEQB $0x00, Z1, Z18, Z19
|
|
VXORPD Z13, Z19, Z13
|
|
VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
|
|
VXORPD Z14, Z19, Z14
|
|
VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (CX), Z18
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
|
|
VXORPD Z12, Z19, Z12
|
|
VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
|
|
VXORPD Z13, Z19, Z13
|
|
VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
|
|
VXORPD Z14, Z19, Z14
|
|
VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z12, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z13, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z14, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z15, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z16, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z17, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x6Xor(SB), $0-88
|
|
// Loading 8 of 12 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), SI
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_2x6Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU (DI), Y8
|
|
VMOVDQU (R8), Y9
|
|
VMOVDQU (R9), Y10
|
|
VMOVDQU (R10), Y11
|
|
VMOVDQU (R11), Y12
|
|
VMOVDQU (SI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y9, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y12, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x7_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 23 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), BX
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_2x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (DX), Z21
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z21, Z14
|
|
VGF2P8AFFINEQB $0x00, Z1, Z21, Z15
|
|
VGF2P8AFFINEQB $0x00, Z2, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z3, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z4, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z5, Z21, Z19
|
|
VGF2P8AFFINEQB $0x00, Z6, Z21, Z20
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (CX), Z21
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z7, Z21, Z22
|
|
VXORPD Z14, Z22, Z14
|
|
VGF2P8AFFINEQB $0x00, Z8, Z21, Z22
|
|
VXORPD Z15, Z22, Z15
|
|
VGF2P8AFFINEQB $0x00, Z9, Z21, Z22
|
|
VXORPD Z16, Z22, Z16
|
|
VGF2P8AFFINEQB $0x00, Z10, Z21, Z22
|
|
VXORPD Z17, Z22, Z17
|
|
VGF2P8AFFINEQB $0x00, Z11, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z12, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z13, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z14, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z15, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z16, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z17, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z18, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z19, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z20, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x7(SB), $0-88
|
|
// Loading 7 of 14 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 23 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), SI
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_2x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x7_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 23 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), BX
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_2x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU64 (SI), Z14
|
|
VMOVDQU64 (DI), Z15
|
|
VMOVDQU64 (R8), Z16
|
|
VMOVDQU64 (R9), Z17
|
|
VMOVDQU64 (R10), Z18
|
|
VMOVDQU64 (R11), Z19
|
|
VMOVDQU64 (BX), Z20
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (DX), Z21
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z21, Z22
|
|
VXORPD Z14, Z22, Z14
|
|
VGF2P8AFFINEQB $0x00, Z1, Z21, Z22
|
|
VXORPD Z15, Z22, Z15
|
|
VGF2P8AFFINEQB $0x00, Z2, Z21, Z22
|
|
VXORPD Z16, Z22, Z16
|
|
VGF2P8AFFINEQB $0x00, Z3, Z21, Z22
|
|
VXORPD Z17, Z22, Z17
|
|
VGF2P8AFFINEQB $0x00, Z4, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z5, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z6, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (CX), Z21
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z7, Z21, Z22
|
|
VXORPD Z14, Z22, Z14
|
|
VGF2P8AFFINEQB $0x00, Z8, Z21, Z22
|
|
VXORPD Z15, Z22, Z15
|
|
VGF2P8AFFINEQB $0x00, Z9, Z21, Z22
|
|
VXORPD Z16, Z22, Z16
|
|
VGF2P8AFFINEQB $0x00, Z10, Z21, Z22
|
|
VXORPD Z17, Z22, Z17
|
|
VGF2P8AFFINEQB $0x00, Z11, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z12, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z13, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z14, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z15, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z16, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z17, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z18, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z19, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z20, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x7Xor(SB), $0-88
|
|
// Loading 7 of 14 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 23 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), SI
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_2x7Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU (DI), Y7
|
|
VMOVDQU (R8), Y8
|
|
VMOVDQU (R9), Y9
|
|
VMOVDQU (R10), Y10
|
|
VMOVDQU (R11), Y11
|
|
VMOVDQU (R12), Y12
|
|
VMOVDQU (SI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x8_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), BX
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_2x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (DX), Z24
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z24, Z16
|
|
VGF2P8AFFINEQB $0x00, Z1, Z24, Z17
|
|
VGF2P8AFFINEQB $0x00, Z2, Z24, Z18
|
|
VGF2P8AFFINEQB $0x00, Z3, Z24, Z19
|
|
VGF2P8AFFINEQB $0x00, Z4, Z24, Z20
|
|
VGF2P8AFFINEQB $0x00, Z5, Z24, Z21
|
|
VGF2P8AFFINEQB $0x00, Z6, Z24, Z22
|
|
VGF2P8AFFINEQB $0x00, Z7, Z24, Z23
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (CX), Z24
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
|
|
VXORPD Z16, Z25, Z16
|
|
VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
|
|
VXORPD Z17, Z25, Z17
|
|
VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
|
|
VXORPD Z18, Z25, Z18
|
|
VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
|
|
VXORPD Z19, Z25, Z19
|
|
VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z16, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z17, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z18, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z19, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z20, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z21, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z22, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z23, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x8(SB), $0-88
|
|
// Loading 6 of 16 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), SI
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_2x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 56(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y7, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y8, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x8_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), BX
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_2x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU64 (SI), Z16
|
|
VMOVDQU64 (DI), Z17
|
|
VMOVDQU64 (R8), Z18
|
|
VMOVDQU64 (R9), Z19
|
|
VMOVDQU64 (R10), Z20
|
|
VMOVDQU64 (R11), Z21
|
|
VMOVDQU64 (R12), Z22
|
|
VMOVDQU64 (BX), Z23
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (DX), Z24
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z24, Z25
|
|
VXORPD Z16, Z25, Z16
|
|
VGF2P8AFFINEQB $0x00, Z1, Z24, Z25
|
|
VXORPD Z17, Z25, Z17
|
|
VGF2P8AFFINEQB $0x00, Z2, Z24, Z25
|
|
VXORPD Z18, Z25, Z18
|
|
VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
|
|
VXORPD Z19, Z25, Z19
|
|
VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (CX), Z24
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
|
|
VXORPD Z16, Z25, Z16
|
|
VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
|
|
VXORPD Z17, Z25, Z17
|
|
VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
|
|
VXORPD Z18, Z25, Z18
|
|
VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
|
|
VXORPD Z19, Z25, Z19
|
|
VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z16, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z17, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z18, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z19, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z20, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z21, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z22, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z23, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x8Xor(SB), $0-88
|
|
// Loading 6 of 16 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), SI
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_2x8Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU (DI), Y6
|
|
VMOVDQU (R8), Y7
|
|
VMOVDQU (R9), Y8
|
|
VMOVDQU (R10), Y9
|
|
VMOVDQU (R11), Y10
|
|
VMOVDQU (R12), Y11
|
|
VMOVDQU (R13), Y12
|
|
VMOVDQU (SI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y7, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y8, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x9_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 29 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), R13
|
|
MOVQ 192(BX), BX
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_2x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (DX), Z27
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z27, Z18
|
|
VGF2P8AFFINEQB $0x00, Z1, Z27, Z19
|
|
VGF2P8AFFINEQB $0x00, Z2, Z27, Z20
|
|
VGF2P8AFFINEQB $0x00, Z3, Z27, Z21
|
|
VGF2P8AFFINEQB $0x00, Z4, Z27, Z22
|
|
VGF2P8AFFINEQB $0x00, Z5, Z27, Z23
|
|
VGF2P8AFFINEQB $0x00, Z6, Z27, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z27, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z27, Z26
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (CX), Z27
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z9, Z27, Z28
|
|
VXORPD Z18, Z28, Z18
|
|
VGF2P8AFFINEQB $0x00, Z10, Z27, Z28
|
|
VXORPD Z19, Z28, Z19
|
|
VGF2P8AFFINEQB $0x00, Z11, Z27, Z28
|
|
VXORPD Z20, Z28, Z20
|
|
VGF2P8AFFINEQB $0x00, Z12, Z27, Z28
|
|
VXORPD Z21, Z28, Z21
|
|
VGF2P8AFFINEQB $0x00, Z13, Z27, Z28
|
|
VXORPD Z22, Z28, Z22
|
|
VGF2P8AFFINEQB $0x00, Z14, Z27, Z28
|
|
VXORPD Z23, Z28, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU64 Z18, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z19, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z20, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z21, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z22, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z23, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z24, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z25, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z26, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x9(SB), $0-88
|
|
// Loading 5 of 18 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 29 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), R14
|
|
MOVQ 192(SI), SI
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_2x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 64(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU Y5, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y6, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y7, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x9_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 29 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), R13
|
|
MOVQ 192(BX), BX
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_2x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
VMOVDQU64 (SI), Z18
|
|
VMOVDQU64 (DI), Z19
|
|
VMOVDQU64 (R8), Z20
|
|
VMOVDQU64 (R9), Z21
|
|
VMOVDQU64 (R10), Z22
|
|
VMOVDQU64 (R11), Z23
|
|
VMOVDQU64 (R12), Z24
|
|
VMOVDQU64 (R13), Z25
|
|
VMOVDQU64 (BX), Z26
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (DX), Z27
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z27, Z28
|
|
VXORPD Z18, Z28, Z18
|
|
VGF2P8AFFINEQB $0x00, Z1, Z27, Z28
|
|
VXORPD Z19, Z28, Z19
|
|
VGF2P8AFFINEQB $0x00, Z2, Z27, Z28
|
|
VXORPD Z20, Z28, Z20
|
|
VGF2P8AFFINEQB $0x00, Z3, Z27, Z28
|
|
VXORPD Z21, Z28, Z21
|
|
VGF2P8AFFINEQB $0x00, Z4, Z27, Z28
|
|
VXORPD Z22, Z28, Z22
|
|
VGF2P8AFFINEQB $0x00, Z5, Z27, Z28
|
|
VXORPD Z23, Z28, Z23
|
|
VGF2P8AFFINEQB $0x00, Z6, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (CX), Z27
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z9, Z27, Z28
|
|
VXORPD Z18, Z28, Z18
|
|
VGF2P8AFFINEQB $0x00, Z10, Z27, Z28
|
|
VXORPD Z19, Z28, Z19
|
|
VGF2P8AFFINEQB $0x00, Z11, Z27, Z28
|
|
VXORPD Z20, Z28, Z20
|
|
VGF2P8AFFINEQB $0x00, Z12, Z27, Z28
|
|
VXORPD Z21, Z28, Z21
|
|
VGF2P8AFFINEQB $0x00, Z13, Z27, Z28
|
|
VXORPD Z22, Z28, Z22
|
|
VGF2P8AFFINEQB $0x00, Z14, Z27, Z28
|
|
VXORPD Z23, Z28, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU64 Z18, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z19, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z20, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z21, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z22, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z23, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z24, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z25, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z26, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x9Xor(SB), $0-88
|
|
// Loading 5 of 18 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 29 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), R14
|
|
MOVQ 192(SI), SI
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_2x9Xor_loop:
|
|
// Load 9 outputs
|
|
VMOVDQU (DI), Y5
|
|
VMOVDQU (R8), Y6
|
|
VMOVDQU (R9), Y7
|
|
VMOVDQU (R10), Y8
|
|
VMOVDQU (R11), Y9
|
|
VMOVDQU (R12), Y10
|
|
VMOVDQU (R13), Y11
|
|
VMOVDQU (R14), Y12
|
|
VMOVDQU (SI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU Y5, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y6, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y7, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x10_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), R13
|
|
MOVQ 192(BX), R14
|
|
MOVQ 216(BX), BX
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, DX
|
|
ADDQ R15, CX
|
|
|
|
mulGFNI_2x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (CX), Z30
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU64 Z20, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z21, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z22, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z23, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x10(SB), $8-88
|
|
// Loading 4 of 20 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), R14
|
|
MOVQ 192(SI), R15
|
|
MOVQ 216(SI), SI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_2x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 72(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU Y4, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_2x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_2x10_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_2x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), CX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ out_base+48(FP), BX
|
|
MOVQ (BX), SI
|
|
MOVQ 24(BX), DI
|
|
MOVQ 48(BX), R8
|
|
MOVQ 72(BX), R9
|
|
MOVQ 96(BX), R10
|
|
MOVQ 120(BX), R11
|
|
MOVQ 144(BX), R12
|
|
MOVQ 168(BX), R13
|
|
MOVQ 192(BX), R14
|
|
MOVQ 216(BX), BX
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, BX
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, DX
|
|
ADDQ R15, CX
|
|
|
|
mulGFNI_2x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
VMOVDQU64 (SI), Z20
|
|
VMOVDQU64 (DI), Z21
|
|
VMOVDQU64 (R8), Z22
|
|
VMOVDQU64 (R9), Z23
|
|
VMOVDQU64 (R10), Z24
|
|
VMOVDQU64 (R11), Z25
|
|
VMOVDQU64 (R12), Z26
|
|
VMOVDQU64 (R13), Z27
|
|
VMOVDQU64 (R14), Z28
|
|
VMOVDQU64 (BX), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (CX), Z30
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU64 Z20, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z21, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z22, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z23, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (BX)
|
|
ADDQ $0x40, BX
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_2x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_2x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_2x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_2x10Xor(SB), $8-88
|
|
// Loading 4 of 20 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_2x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), DX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), R14
|
|
MOVQ 192(SI), R15
|
|
MOVQ 216(SI), SI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_2x10Xor_loop:
|
|
// Load 10 outputs
|
|
VMOVDQU (DI), Y4
|
|
VMOVDQU (R8), Y5
|
|
VMOVDQU (R9), Y6
|
|
VMOVDQU (R10), Y7
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (SI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU Y4, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_2x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_2x10Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 6 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), SI
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, DX
|
|
ADDQ DI, BX
|
|
ADDQ DI, CX
|
|
|
|
mulGFNI_3x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z4
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z4, Z3
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z4
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z4, Z4
|
|
VXORPD Z3, Z4, Z3
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (CX), Z4
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z4, Z4
|
|
VXORPD Z3, Z4, Z3
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z3, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 6 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), SI
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, DX
|
|
ADDQ DI, BX
|
|
ADDQ DI, CX
|
|
|
|
mulAvxGFNI_3x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y4
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y4, Y3
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y4
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y4, Y4
|
|
VXORPD Y3, Y4, Y3
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (CX), Y4
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y4, Y4
|
|
VXORPD Y3, Y4, Y3
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y3, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 6 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), SI
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, DX
|
|
ADDQ DI, BX
|
|
ADDQ DI, CX
|
|
|
|
mulGFNI_3x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (SI), Z3
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z4
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z4, Z4
|
|
VXORPD Z3, Z4, Z3
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z4
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z4, Z4
|
|
VXORPD Z3, Z4, Z3
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (CX), Z4
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z4, Z4
|
|
VXORPD Z3, Z4, Z3
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z3, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 6 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), SI
|
|
MOVQ start+72(FP), DI
|
|
|
|
// Add start offset to output
|
|
ADDQ DI, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ DI, DX
|
|
ADDQ DI, BX
|
|
ADDQ DI, CX
|
|
|
|
mulAvxGFNI_3x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (SI), Y3
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y4
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y4, Y4
|
|
VXORPD Y3, Y4, Y3
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y4
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y4, Y4
|
|
VXORPD Y3, Y4, Y3
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (CX), Y4
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y4, Y4
|
|
VXORPD Y3, Y4, Y3
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y3, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), SI
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, DI
|
|
ADDQ R8, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, BX
|
|
ADDQ R8, CX
|
|
|
|
mulGFNI_3x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z8
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z8, Z6
|
|
VGF2P8AFFINEQB $0x00, Z1, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z8
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z8, Z9
|
|
VXORPD Z6, Z9, Z6
|
|
VGF2P8AFFINEQB $0x00, Z3, Z8, Z9
|
|
VXORPD Z7, Z9, Z7
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (CX), Z8
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z8, Z9
|
|
VXORPD Z6, Z9, Z6
|
|
VGF2P8AFFINEQB $0x00, Z5, Z8, Z9
|
|
VXORPD Z7, Z9, Z7
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z6, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z7, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x2(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), SI
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, DI
|
|
ADDQ R8, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, BX
|
|
ADDQ R8, CX
|
|
|
|
mulAvxGFNI_3x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y8
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y8, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (BX), Y8
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y8, Y9
|
|
VXORPD Y6, Y9, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y8, Y9
|
|
VXORPD Y7, Y9, Y7
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (CX), Y8
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y4, Y8, Y9
|
|
VXORPD Y6, Y9, Y6
|
|
VGF2P8AFFINEQB $0x00, Y5, Y8, Y9
|
|
VXORPD Y7, Y9, Y7
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y6, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y7, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), SI
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, DI
|
|
ADDQ R8, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, BX
|
|
ADDQ R8, CX
|
|
|
|
mulGFNI_3x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (DI), Z6
|
|
VMOVDQU64 (SI), Z7
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z8
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z8, Z9
|
|
VXORPD Z6, Z9, Z6
|
|
VGF2P8AFFINEQB $0x00, Z1, Z8, Z9
|
|
VXORPD Z7, Z9, Z7
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z8
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z8, Z9
|
|
VXORPD Z6, Z9, Z6
|
|
VGF2P8AFFINEQB $0x00, Z3, Z8, Z9
|
|
VXORPD Z7, Z9, Z7
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (CX), Z8
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z8, Z9
|
|
VXORPD Z6, Z9, Z6
|
|
VGF2P8AFFINEQB $0x00, Z5, Z8, Z9
|
|
VXORPD Z7, Z9, Z7
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z6, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z7, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x2Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), SI
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, DI
|
|
ADDQ R8, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, BX
|
|
ADDQ R8, CX
|
|
|
|
mulAvxGFNI_3x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (DI), Y6
|
|
VMOVDQU (SI), Y7
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y8
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y8, Y9
|
|
VXORPD Y6, Y9, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y8, Y9
|
|
VXORPD Y7, Y9, Y7
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (BX), Y8
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y8, Y9
|
|
VXORPD Y6, Y9, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y8, Y9
|
|
VXORPD Y7, Y9, Y7
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (CX), Y8
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y4, Y8, Y9
|
|
VXORPD Y6, Y9, Y6
|
|
VGF2P8AFFINEQB $0x00, Y5, Y8, Y9
|
|
VXORPD Y7, Y9, Y7
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y6, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y7, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x3_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), SI
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_3x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z12
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z12, Z9
|
|
VGF2P8AFFINEQB $0x00, Z1, Z12, Z10
|
|
VGF2P8AFFINEQB $0x00, Z2, Z12, Z11
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z12
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
|
|
VXORPD Z9, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (CX), Z12
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
|
|
VXORPD Z9, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z8, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z9, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z10, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z11, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x3(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), SI
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_3x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (DX), Y12
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y12, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y12, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y12, Y11
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (BX), Y12
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
|
|
VXORPD Y9, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (CX), Y12
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
|
|
VXORPD Y9, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y8, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x3_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), SI
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_3x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (DI), Z9
|
|
VMOVDQU64 (R8), Z10
|
|
VMOVDQU64 (SI), Z11
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z12
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z12, Z13
|
|
VXORPD Z9, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z1, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z12
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
|
|
VXORPD Z9, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (CX), Z12
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
|
|
VXORPD Z9, Z13, Z9
|
|
VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z8, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z9, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z10, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z11, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x3Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), SI
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, DI
|
|
ADDQ R9, R8
|
|
ADDQ R9, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_3x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (DI), Y9
|
|
VMOVDQU (R8), Y10
|
|
VMOVDQU (SI), Y11
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (DX), Y12
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y12, Y13
|
|
VXORPD Y9, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (BX), Y12
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
|
|
VXORPD Y9, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (CX), Y12
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
|
|
VXORPD Y9, Y13, Y9
|
|
VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y8, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x4_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), SI
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_3x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z16
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z1, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z2, Z16, Z14
|
|
VGF2P8AFFINEQB $0x00, Z3, Z16, Z15
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z16
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
|
|
VXORPD Z12, Z17, Z12
|
|
VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
|
|
VXORPD Z13, Z17, Z13
|
|
VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (CX), Z16
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z16, Z17
|
|
VXORPD Z12, Z17, Z12
|
|
VGF2P8AFFINEQB $0x00, Z9, Z16, Z17
|
|
VXORPD Z13, Z17, Z13
|
|
VGF2P8AFFINEQB $0x00, Z10, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z11, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z12, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z13, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z14, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z15, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x4(SB), $0-88
|
|
// Loading 10 of 12 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), DI
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_3x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x4_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), SI
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_3x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU64 (DI), Z12
|
|
VMOVDQU64 (R8), Z13
|
|
VMOVDQU64 (R9), Z14
|
|
VMOVDQU64 (SI), Z15
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z16
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z16, Z17
|
|
VXORPD Z12, Z17, Z12
|
|
VGF2P8AFFINEQB $0x00, Z1, Z16, Z17
|
|
VXORPD Z13, Z17, Z13
|
|
VGF2P8AFFINEQB $0x00, Z2, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z3, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z16
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
|
|
VXORPD Z12, Z17, Z12
|
|
VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
|
|
VXORPD Z13, Z17, Z13
|
|
VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (CX), Z16
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z16, Z17
|
|
VXORPD Z12, Z17, Z12
|
|
VGF2P8AFFINEQB $0x00, Z9, Z16, Z17
|
|
VXORPD Z13, Z17, Z13
|
|
VGF2P8AFFINEQB $0x00, Z10, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z11, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z12, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z13, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z14, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z15, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x4Xor(SB), $0-88
|
|
// Loading 10 of 12 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), DI
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_3x4Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU (R8), Y10
|
|
VMOVDQU (R9), Y11
|
|
VMOVDQU (R10), Y12
|
|
VMOVDQU (DI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y11, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x5_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), SI
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_3x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z20
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z20, Z15
|
|
VGF2P8AFFINEQB $0x00, Z1, Z20, Z16
|
|
VGF2P8AFFINEQB $0x00, Z2, Z20, Z17
|
|
VGF2P8AFFINEQB $0x00, Z3, Z20, Z18
|
|
VGF2P8AFFINEQB $0x00, Z4, Z20, Z19
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (BX), Z20
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
|
|
VXORPD Z15, Z21, Z15
|
|
VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (CX), Z20
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
|
|
VXORPD Z15, Z21, Z15
|
|
VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z15, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z16, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z17, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z18, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z19, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x5(SB), $0-88
|
|
// Loading 9 of 15 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), DI
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_3x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y12, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x5_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), SI
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_3x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU64 (DI), Z15
|
|
VMOVDQU64 (R8), Z16
|
|
VMOVDQU64 (R9), Z17
|
|
VMOVDQU64 (R10), Z18
|
|
VMOVDQU64 (SI), Z19
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z20
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z20, Z21
|
|
VXORPD Z15, Z21, Z15
|
|
VGF2P8AFFINEQB $0x00, Z1, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (BX), Z20
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
|
|
VXORPD Z15, Z21, Z15
|
|
VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (CX), Z20
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
|
|
VXORPD Z15, Z21, Z15
|
|
VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z15, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z16, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z17, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z18, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z19, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x5Xor(SB), $0-88
|
|
// Loading 9 of 15 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), DI
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_3x5Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU (R8), Y9
|
|
VMOVDQU (R9), Y10
|
|
VMOVDQU (R10), Y11
|
|
VMOVDQU (R11), Y12
|
|
VMOVDQU (DI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y12, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x6_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), SI
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_3x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (DX), Z24
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z24, Z18
|
|
VGF2P8AFFINEQB $0x00, Z1, Z24, Z19
|
|
VGF2P8AFFINEQB $0x00, Z2, Z24, Z20
|
|
VGF2P8AFFINEQB $0x00, Z3, Z24, Z21
|
|
VGF2P8AFFINEQB $0x00, Z4, Z24, Z22
|
|
VGF2P8AFFINEQB $0x00, Z5, Z24, Z23
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (BX), Z24
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
|
|
VXORPD Z18, Z25, Z18
|
|
VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
|
|
VXORPD Z19, Z25, Z19
|
|
VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (CX), Z24
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
|
|
VXORPD Z18, Z25, Z18
|
|
VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
|
|
VXORPD Z19, Z25, Z19
|
|
VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z18, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z19, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z20, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z21, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z22, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z23, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x6(SB), $0-88
|
|
// Loading 8 of 18 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), DI
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_3x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x6_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), SI
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_3x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU64 (DI), Z18
|
|
VMOVDQU64 (R8), Z19
|
|
VMOVDQU64 (R9), Z20
|
|
VMOVDQU64 (R10), Z21
|
|
VMOVDQU64 (R11), Z22
|
|
VMOVDQU64 (SI), Z23
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (DX), Z24
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z24, Z25
|
|
VXORPD Z18, Z25, Z18
|
|
VGF2P8AFFINEQB $0x00, Z1, Z24, Z25
|
|
VXORPD Z19, Z25, Z19
|
|
VGF2P8AFFINEQB $0x00, Z2, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (BX), Z24
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
|
|
VXORPD Z18, Z25, Z18
|
|
VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
|
|
VXORPD Z19, Z25, Z19
|
|
VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (CX), Z24
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
|
|
VXORPD Z18, Z25, Z18
|
|
VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
|
|
VXORPD Z19, Z25, Z19
|
|
VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z18, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z19, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z20, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z21, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z22, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z23, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x6Xor(SB), $0-88
|
|
// Loading 8 of 18 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), DI
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_3x6Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU (R8), Y8
|
|
VMOVDQU (R9), Y9
|
|
VMOVDQU (R10), Y10
|
|
VMOVDQU (R11), Y11
|
|
VMOVDQU (R12), Y12
|
|
VMOVDQU (DI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x7_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 30 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), SI
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_3x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (DX), Z28
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z28, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z28, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z28, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z28, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z28, Z27
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (BX), Z28
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z7, Z28, Z29
|
|
VXORPD Z21, Z29, Z21
|
|
VGF2P8AFFINEQB $0x00, Z8, Z28, Z29
|
|
VXORPD Z22, Z29, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z28, Z29
|
|
VXORPD Z23, Z29, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (CX), Z28
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z14, Z28, Z29
|
|
VXORPD Z21, Z29, Z21
|
|
VGF2P8AFFINEQB $0x00, Z15, Z28, Z29
|
|
VXORPD Z22, Z29, Z22
|
|
VGF2P8AFFINEQB $0x00, Z16, Z28, Z29
|
|
VXORPD Z23, Z29, Z23
|
|
VGF2P8AFFINEQB $0x00, Z17, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z18, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z19, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z20, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z21, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z22, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z23, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x7(SB), $0-88
|
|
// Loading 7 of 21 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 30 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), DI
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_3x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y8, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x7_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 30 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), CX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), SI
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_3x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU64 (DI), Z21
|
|
VMOVDQU64 (R8), Z22
|
|
VMOVDQU64 (R9), Z23
|
|
VMOVDQU64 (R10), Z24
|
|
VMOVDQU64 (R11), Z25
|
|
VMOVDQU64 (R12), Z26
|
|
VMOVDQU64 (SI), Z27
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (DX), Z28
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z28, Z29
|
|
VXORPD Z21, Z29, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z28, Z29
|
|
VXORPD Z22, Z29, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z28, Z29
|
|
VXORPD Z23, Z29, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (BX), Z28
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z7, Z28, Z29
|
|
VXORPD Z21, Z29, Z21
|
|
VGF2P8AFFINEQB $0x00, Z8, Z28, Z29
|
|
VXORPD Z22, Z29, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z28, Z29
|
|
VXORPD Z23, Z29, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (CX), Z28
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z14, Z28, Z29
|
|
VXORPD Z21, Z29, Z21
|
|
VGF2P8AFFINEQB $0x00, Z15, Z28, Z29
|
|
VXORPD Z22, Z29, Z22
|
|
VGF2P8AFFINEQB $0x00, Z16, Z28, Z29
|
|
VXORPD Z23, Z29, Z23
|
|
VGF2P8AFFINEQB $0x00, Z17, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z18, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z19, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z20, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z21, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z22, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z23, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x7Xor(SB), $0-88
|
|
// Loading 7 of 21 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 30 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), DI
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_3x7Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU (R8), Y7
|
|
VMOVDQU (R9), Y8
|
|
VMOVDQU (R10), Y9
|
|
VMOVDQU (R11), Y10
|
|
VMOVDQU (R12), Y11
|
|
VMOVDQU (R13), Y12
|
|
VMOVDQU (DI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y8, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x8_64(SB), $0-88
|
|
// Loading 22 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 34 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), DI
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_3x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z22, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z23, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x8(SB), $0-88
|
|
// Loading 6 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 34 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), DI
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_3x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 56(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y7, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x8_64Xor(SB), $0-88
|
|
// Loading 22 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 34 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), DI
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_3x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU64 (R8), Z22
|
|
VMOVDQU64 (R9), Z23
|
|
VMOVDQU64 (R10), Z24
|
|
VMOVDQU64 (R11), Z25
|
|
VMOVDQU64 (R12), Z26
|
|
VMOVDQU64 (R13), Z27
|
|
VMOVDQU64 (R14), Z28
|
|
VMOVDQU64 (DI), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z22, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z23, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x8Xor(SB), $0-88
|
|
// Loading 6 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 34 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), DI
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_3x8Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU (R8), Y6
|
|
VMOVDQU (R9), Y7
|
|
VMOVDQU (R10), Y8
|
|
VMOVDQU (R11), Y9
|
|
VMOVDQU (R12), Y10
|
|
VMOVDQU (R13), Y11
|
|
VMOVDQU (R14), Y12
|
|
VMOVDQU (DI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y7, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x9_64(SB), $8-88
|
|
// Loading 21 of 27 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), R15
|
|
MOVQ 192(DI), DI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_3x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU64 Z21, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x9(SB), $8-88
|
|
// Loading 5 of 27 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), R15
|
|
MOVQ 192(DI), DI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_3x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 64(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x9_64Xor(SB), $8-88
|
|
// Loading 21 of 27 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), R15
|
|
MOVQ 192(DI), DI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_3x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
VMOVDQU64 (R8), Z21
|
|
VMOVDQU64 (R9), Z22
|
|
VMOVDQU64 (R10), Z23
|
|
VMOVDQU64 (R11), Z24
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (DI), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU64 Z21, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_3x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x9Xor(SB), $8-88
|
|
// Loading 5 of 27 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), R15
|
|
MOVQ 192(DI), DI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_3x9Xor_loop:
|
|
// Load 9 outputs
|
|
VMOVDQU (R8), Y5
|
|
VMOVDQU (R9), Y6
|
|
VMOVDQU (R10), Y7
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (DI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_3x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x10_64(SB), $8-88
|
|
// Loading 20 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), AX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), R14
|
|
MOVQ 192(SI), R15
|
|
MOVQ 216(SI), SI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_3x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU64 Z20, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z21, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_3x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x10(SB), $8-88
|
|
// Loading 4 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), AX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), R14
|
|
MOVQ 192(SI), R15
|
|
MOVQ 216(SI), SI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_3x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 72(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU Y4, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_3x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_3x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_3x10_64Xor(SB), $8-88
|
|
// Loading 20 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_3x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), AX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), R14
|
|
MOVQ 192(SI), R15
|
|
MOVQ 216(SI), SI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_3x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
VMOVDQU64 (DI), Z20
|
|
VMOVDQU64 (R8), Z21
|
|
VMOVDQU64 (R9), Z22
|
|
VMOVDQU64 (R10), Z23
|
|
VMOVDQU64 (R11), Z24
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (SI), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU64 Z20, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z21, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (SI)
|
|
ADDQ $0x40, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_3x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_3x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_3x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_3x10Xor(SB), $8-88
|
|
// Loading 4 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_3x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), AX
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ out_base+48(FP), SI
|
|
MOVQ (SI), DI
|
|
MOVQ 24(SI), R8
|
|
MOVQ 48(SI), R9
|
|
MOVQ 72(SI), R10
|
|
MOVQ 96(SI), R11
|
|
MOVQ 120(SI), R12
|
|
MOVQ 144(SI), R13
|
|
MOVQ 168(SI), R14
|
|
MOVQ 192(SI), R15
|
|
MOVQ 216(SI), SI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, SI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_3x10Xor_loop:
|
|
// Load 10 outputs
|
|
VMOVDQU (DI), Y4
|
|
VMOVDQU (R8), Y5
|
|
VMOVDQU (R9), Y6
|
|
VMOVDQU (R10), Y7
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (SI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
VMOVDQU Y4, (DI)
|
|
ADDQ $0x20, DI
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (SI)
|
|
ADDQ $0x20, SI
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_3x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_3x10Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 7 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), DI
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, BX
|
|
ADDQ R8, SI
|
|
ADDQ R8, CX
|
|
|
|
mulGFNI_4x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z5
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z5, Z4
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z5
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z5, Z5
|
|
VXORPD Z4, Z5, Z4
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z5
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z5, Z5
|
|
VXORPD Z4, Z5, Z4
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (CX), Z5
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z5, Z5
|
|
VXORPD Z4, Z5, Z4
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z4, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 7 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), DI
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, BX
|
|
ADDQ R8, SI
|
|
ADDQ R8, CX
|
|
|
|
mulAvxGFNI_4x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y5
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y5, Y4
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y5
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y5, Y5
|
|
VXORPD Y4, Y5, Y4
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y5
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y5, Y5
|
|
VXORPD Y4, Y5, Y4
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (CX), Y5
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y3, Y5, Y5
|
|
VXORPD Y4, Y5, Y4
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y4, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 7 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), DI
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, BX
|
|
ADDQ R8, SI
|
|
ADDQ R8, CX
|
|
|
|
mulGFNI_4x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (DI), Z4
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z5
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z5, Z5
|
|
VXORPD Z4, Z5, Z4
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z5
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z5, Z5
|
|
VXORPD Z4, Z5, Z4
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z5
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z5, Z5
|
|
VXORPD Z4, Z5, Z4
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (CX), Z5
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z5, Z5
|
|
VXORPD Z4, Z5, Z4
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z4, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 7 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), DI
|
|
MOVQ start+72(FP), R8
|
|
|
|
// Add start offset to output
|
|
ADDQ R8, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R8, DX
|
|
ADDQ R8, BX
|
|
ADDQ R8, SI
|
|
ADDQ R8, CX
|
|
|
|
mulAvxGFNI_4x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (DI), Y4
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y5
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y5, Y5
|
|
VXORPD Y4, Y5, Y4
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y5
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y5, Y5
|
|
VXORPD Y4, Y5, Y4
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y5
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y5, Y5
|
|
VXORPD Y4, Y5, Y4
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (CX), Y5
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y3, Y5, Y5
|
|
VXORPD Y4, Y5, Y4
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y4, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), DI
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, R8
|
|
ADDQ R9, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_4x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z10
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z10, Z8
|
|
VGF2P8AFFINEQB $0x00, Z1, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z10
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z10, Z11
|
|
VXORPD Z8, Z11, Z8
|
|
VGF2P8AFFINEQB $0x00, Z3, Z10, Z11
|
|
VXORPD Z9, Z11, Z9
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z10
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z10, Z11
|
|
VXORPD Z8, Z11, Z8
|
|
VGF2P8AFFINEQB $0x00, Z5, Z10, Z11
|
|
VXORPD Z9, Z11, Z9
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (CX), Z10
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z10, Z11
|
|
VXORPD Z8, Z11, Z8
|
|
VGF2P8AFFINEQB $0x00, Z7, Z10, Z11
|
|
VXORPD Z9, Z11, Z9
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z8, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z9, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x2(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), DI
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, R8
|
|
ADDQ R9, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_4x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y10
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y10, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (BX), Y10
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y10, Y11
|
|
VXORPD Y8, Y11, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y10, Y11
|
|
VXORPD Y9, Y11, Y9
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (SI), Y10
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y10, Y11
|
|
VXORPD Y8, Y11, Y8
|
|
VGF2P8AFFINEQB $0x00, Y5, Y10, Y11
|
|
VXORPD Y9, Y11, Y9
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (CX), Y10
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y10, Y11
|
|
VXORPD Y8, Y11, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y10, Y11
|
|
VXORPD Y9, Y11, Y9
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), DI
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, R8
|
|
ADDQ R9, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_4x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (R8), Z8
|
|
VMOVDQU64 (DI), Z9
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z10
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z10, Z11
|
|
VXORPD Z8, Z11, Z8
|
|
VGF2P8AFFINEQB $0x00, Z1, Z10, Z11
|
|
VXORPD Z9, Z11, Z9
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z10
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z10, Z11
|
|
VXORPD Z8, Z11, Z8
|
|
VGF2P8AFFINEQB $0x00, Z3, Z10, Z11
|
|
VXORPD Z9, Z11, Z9
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z10
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z10, Z11
|
|
VXORPD Z8, Z11, Z8
|
|
VGF2P8AFFINEQB $0x00, Z5, Z10, Z11
|
|
VXORPD Z9, Z11, Z9
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (CX), Z10
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z10, Z11
|
|
VXORPD Z8, Z11, Z8
|
|
VGF2P8AFFINEQB $0x00, Z7, Z10, Z11
|
|
VXORPD Z9, Z11, Z9
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z8, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z9, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x2Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), DI
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, R8
|
|
ADDQ R9, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_4x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (R8), Y8
|
|
VMOVDQU (DI), Y9
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y10
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y10, Y11
|
|
VXORPD Y8, Y11, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y10, Y11
|
|
VXORPD Y9, Y11, Y9
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (BX), Y10
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y10, Y11
|
|
VXORPD Y8, Y11, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y10, Y11
|
|
VXORPD Y9, Y11, Y9
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (SI), Y10
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y10, Y11
|
|
VXORPD Y8, Y11, Y8
|
|
VGF2P8AFFINEQB $0x00, Y5, Y10, Y11
|
|
VXORPD Y9, Y11, Y9
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (CX), Y10
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y10, Y11
|
|
VXORPD Y8, Y11, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y10, Y11
|
|
VXORPD Y9, Y11, Y9
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y8, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y9, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x3_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 17 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), DI
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_4x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z15
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z1, Z15, Z13
|
|
VGF2P8AFFINEQB $0x00, Z2, Z15, Z14
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z15
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z4, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z5, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z15
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z7, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z8, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (CX), Z15
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z9, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z10, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z11, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z12, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z13, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z14, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x3(SB), $0-88
|
|
// Loading 11 of 12 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 17 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R8
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_4x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x3_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 17 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), DI
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R8
|
|
ADDQ R10, R9
|
|
ADDQ R10, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_4x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (R8), Z12
|
|
VMOVDQU64 (R9), Z13
|
|
VMOVDQU64 (DI), Z14
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z15
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z1, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z2, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z15
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z4, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z5, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z15
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z7, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z8, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (CX), Z15
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z9, Z15, Z16
|
|
VXORPD Z12, Z16, Z12
|
|
VGF2P8AFFINEQB $0x00, Z10, Z15, Z16
|
|
VXORPD Z13, Z16, Z13
|
|
VGF2P8AFFINEQB $0x00, Z11, Z15, Z16
|
|
VXORPD Z14, Z16, Z14
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z12, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z13, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z14, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x3Xor(SB), $0-88
|
|
// Loading 11 of 12 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 17 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R8
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_4x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (R9), Y11
|
|
VMOVDQU (R10), Y12
|
|
VMOVDQU (R8), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x4_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), DI
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_4x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z20
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z20, Z16
|
|
VGF2P8AFFINEQB $0x00, Z1, Z20, Z17
|
|
VGF2P8AFFINEQB $0x00, Z2, Z20, Z18
|
|
VGF2P8AFFINEQB $0x00, Z3, Z20, Z19
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z20
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (SI), Z20
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (CX), Z20
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z15, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z16, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z17, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z18, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z19, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x4(SB), $0-88
|
|
// Loading 10 of 16 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R8
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_4x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y12, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x4_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), DI
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_4x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU64 (R8), Z16
|
|
VMOVDQU64 (R9), Z17
|
|
VMOVDQU64 (R10), Z18
|
|
VMOVDQU64 (DI), Z19
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z20
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z1, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z20
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (SI), Z20
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (CX), Z20
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
|
|
VXORPD Z16, Z21, Z16
|
|
VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
|
|
VXORPD Z17, Z21, Z17
|
|
VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z15, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z16, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z17, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z18, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z19, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x4Xor(SB), $0-88
|
|
// Loading 10 of 16 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R8
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_4x4Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU (R9), Y10
|
|
VMOVDQU (R10), Y11
|
|
VMOVDQU (R11), Y12
|
|
VMOVDQU (R8), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y12, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x5_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 27 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), DI
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_4x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z25
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z25, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z25, Z24
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (BX), Z25
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z25, Z26
|
|
VXORPD Z20, Z26, Z20
|
|
VGF2P8AFFINEQB $0x00, Z6, Z25, Z26
|
|
VXORPD Z21, Z26, Z21
|
|
VGF2P8AFFINEQB $0x00, Z7, Z25, Z26
|
|
VXORPD Z22, Z26, Z22
|
|
VGF2P8AFFINEQB $0x00, Z8, Z25, Z26
|
|
VXORPD Z23, Z26, Z23
|
|
VGF2P8AFFINEQB $0x00, Z9, Z25, Z26
|
|
VXORPD Z24, Z26, Z24
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (SI), Z25
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z25, Z26
|
|
VXORPD Z20, Z26, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z25, Z26
|
|
VXORPD Z21, Z26, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z25, Z26
|
|
VXORPD Z22, Z26, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z25, Z26
|
|
VXORPD Z23, Z26, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z25, Z26
|
|
VXORPD Z24, Z26, Z24
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (CX), Z25
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z15, Z25, Z26
|
|
VXORPD Z20, Z26, Z20
|
|
VGF2P8AFFINEQB $0x00, Z16, Z25, Z26
|
|
VXORPD Z21, Z26, Z21
|
|
VGF2P8AFFINEQB $0x00, Z17, Z25, Z26
|
|
VXORPD Z22, Z26, Z22
|
|
VGF2P8AFFINEQB $0x00, Z18, Z25, Z26
|
|
VXORPD Z23, Z26, Z23
|
|
VGF2P8AFFINEQB $0x00, Z19, Z25, Z26
|
|
VXORPD Z24, Z26, Z24
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z20, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z21, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z22, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z23, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z24, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x5(SB), $0-88
|
|
// Loading 9 of 20 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 27 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R8
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_4x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x5_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 27 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), DI
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_4x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU64 (R8), Z20
|
|
VMOVDQU64 (R9), Z21
|
|
VMOVDQU64 (R10), Z22
|
|
VMOVDQU64 (R11), Z23
|
|
VMOVDQU64 (DI), Z24
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z25
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z25, Z26
|
|
VXORPD Z20, Z26, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z25, Z26
|
|
VXORPD Z21, Z26, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z25, Z26
|
|
VXORPD Z22, Z26, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z25, Z26
|
|
VXORPD Z23, Z26, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z25, Z26
|
|
VXORPD Z24, Z26, Z24
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (BX), Z25
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z25, Z26
|
|
VXORPD Z20, Z26, Z20
|
|
VGF2P8AFFINEQB $0x00, Z6, Z25, Z26
|
|
VXORPD Z21, Z26, Z21
|
|
VGF2P8AFFINEQB $0x00, Z7, Z25, Z26
|
|
VXORPD Z22, Z26, Z22
|
|
VGF2P8AFFINEQB $0x00, Z8, Z25, Z26
|
|
VXORPD Z23, Z26, Z23
|
|
VGF2P8AFFINEQB $0x00, Z9, Z25, Z26
|
|
VXORPD Z24, Z26, Z24
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (SI), Z25
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z25, Z26
|
|
VXORPD Z20, Z26, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z25, Z26
|
|
VXORPD Z21, Z26, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z25, Z26
|
|
VXORPD Z22, Z26, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z25, Z26
|
|
VXORPD Z23, Z26, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z25, Z26
|
|
VXORPD Z24, Z26, Z24
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (CX), Z25
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z15, Z25, Z26
|
|
VXORPD Z20, Z26, Z20
|
|
VGF2P8AFFINEQB $0x00, Z16, Z25, Z26
|
|
VXORPD Z21, Z26, Z21
|
|
VGF2P8AFFINEQB $0x00, Z17, Z25, Z26
|
|
VXORPD Z22, Z26, Z22
|
|
VGF2P8AFFINEQB $0x00, Z18, Z25, Z26
|
|
VXORPD Z23, Z26, Z23
|
|
VGF2P8AFFINEQB $0x00, Z19, Z25, Z26
|
|
VXORPD Z24, Z26, Z24
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z20, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z21, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z22, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z23, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z24, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x5Xor(SB), $0-88
|
|
// Loading 9 of 20 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 27 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R8
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_4x5Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU (R9), Y9
|
|
VMOVDQU (R10), Y10
|
|
VMOVDQU (R11), Y11
|
|
VMOVDQU (R12), Y12
|
|
VMOVDQU (R8), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x6_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), DI
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_4x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (CX), Z30
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z24, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z25, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z26, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z27, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z28, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z29, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x6(SB), $0-88
|
|
// Loading 8 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R8
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_4x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x6_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), CX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), DI
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_4x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU64 (R8), Z24
|
|
VMOVDQU64 (R9), Z25
|
|
VMOVDQU64 (R10), Z26
|
|
VMOVDQU64 (R11), Z27
|
|
VMOVDQU64 (R12), Z28
|
|
VMOVDQU64 (DI), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (CX), Z30
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z24, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z25, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z26, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z27, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z28, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z29, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x6Xor(SB), $0-88
|
|
// Loading 8 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R8
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_4x6Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU (R9), Y8
|
|
VMOVDQU (R10), Y9
|
|
VMOVDQU (R11), Y10
|
|
VMOVDQU (R12), Y11
|
|
VMOVDQU (R13), Y12
|
|
VMOVDQU (R8), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x7_64(SB), $0-88
|
|
// Loading 23 of 28 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 37 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R8
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_4x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z23, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x7(SB), $0-88
|
|
// Loading 7 of 28 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 37 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R8
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_4x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x7_64Xor(SB), $0-88
|
|
// Loading 23 of 28 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 37 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R8
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_4x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU64 (R9), Z23
|
|
VMOVDQU64 (R10), Z24
|
|
VMOVDQU64 (R11), Z25
|
|
VMOVDQU64 (R12), Z26
|
|
VMOVDQU64 (R13), Z27
|
|
VMOVDQU64 (R14), Z28
|
|
VMOVDQU64 (R8), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z23, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x7Xor(SB), $0-88
|
|
// Loading 7 of 28 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 37 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R8
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_4x7Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU (R9), Y7
|
|
VMOVDQU (R10), Y8
|
|
VMOVDQU (R11), Y9
|
|
VMOVDQU (R12), Y10
|
|
VMOVDQU (R13), Y11
|
|
VMOVDQU (R14), Y12
|
|
VMOVDQU (R8), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x8_64(SB), $8-88
|
|
// Loading 22 of 32 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R15
|
|
MOVQ 168(R8), R8
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_4x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x8(SB), $8-88
|
|
// Loading 6 of 32 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R15
|
|
MOVQ 168(R8), R8
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_4x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 56(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x8_64Xor(SB), $8-88
|
|
// Loading 22 of 32 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R15
|
|
MOVQ 168(R8), R8
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_4x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU64 (R9), Z22
|
|
VMOVDQU64 (R10), Z23
|
|
VMOVDQU64 (R11), Z24
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R8), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_4x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x8Xor(SB), $8-88
|
|
// Loading 6 of 32 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R15
|
|
MOVQ 168(R8), R8
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_4x8Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU (R9), Y6
|
|
VMOVDQU (R10), Y7
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R8), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x9_64(SB), $8-88
|
|
// Loading 21 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 47 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), AX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), R15
|
|
MOVQ 192(DI), DI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_4x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU64 Z21, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_4x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x9(SB), $8-88
|
|
// Loading 5 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 47 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), AX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), R15
|
|
MOVQ 192(DI), DI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_4x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 64(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_4x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x9_64Xor(SB), $8-88
|
|
// Loading 21 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 47 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), AX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), R15
|
|
MOVQ 192(DI), DI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_4x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
VMOVDQU64 (R8), Z21
|
|
VMOVDQU64 (R9), Z22
|
|
VMOVDQU64 (R10), Z23
|
|
VMOVDQU64 (R11), Z24
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (DI), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU64 Z21, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (DI)
|
|
ADDQ $0x40, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_4x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x9Xor(SB), $8-88
|
|
// Loading 5 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 47 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), AX
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ out_base+48(FP), DI
|
|
MOVQ (DI), R8
|
|
MOVQ 24(DI), R9
|
|
MOVQ 48(DI), R10
|
|
MOVQ 72(DI), R11
|
|
MOVQ 96(DI), R12
|
|
MOVQ 120(DI), R13
|
|
MOVQ 144(DI), R14
|
|
MOVQ 168(DI), R15
|
|
MOVQ 192(DI), DI
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, DI
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_4x9Xor_loop:
|
|
// Load 9 outputs
|
|
VMOVDQU (R8), Y5
|
|
VMOVDQU (R9), Y6
|
|
VMOVDQU (R10), Y7
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (DI), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (DI)
|
|
ADDQ $0x20, DI
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_4x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x10_64(SB), $0-88
|
|
// Loading 20 of 40 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 52 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, DX
|
|
|
|
mulGFNI_4x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R8), R10
|
|
VMOVDQU64 Z20, (R10)(R9*1)
|
|
MOVQ 24(R8), R10
|
|
VMOVDQU64 Z21, (R10)(R9*1)
|
|
MOVQ 48(R8), R10
|
|
VMOVDQU64 Z22, (R10)(R9*1)
|
|
MOVQ 72(R8), R10
|
|
VMOVDQU64 Z23, (R10)(R9*1)
|
|
MOVQ 96(R8), R10
|
|
VMOVDQU64 Z24, (R10)(R9*1)
|
|
MOVQ 120(R8), R10
|
|
VMOVDQU64 Z25, (R10)(R9*1)
|
|
MOVQ 144(R8), R10
|
|
VMOVDQU64 Z26, (R10)(R9*1)
|
|
MOVQ 168(R8), R10
|
|
VMOVDQU64 Z27, (R10)(R9*1)
|
|
MOVQ 192(R8), R10
|
|
VMOVDQU64 Z28, (R10)(R9*1)
|
|
MOVQ 216(R8), R10
|
|
VMOVDQU64 Z29, (R10)(R9*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R9
|
|
DECQ AX
|
|
JNZ mulGFNI_4x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x10(SB), $0-88
|
|
// Loading 4 of 40 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 52 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, DX
|
|
|
|
mulAvxGFNI_4x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 72(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R8), R10
|
|
VMOVDQU Y4, (R10)(R9*1)
|
|
MOVQ 24(R8), R10
|
|
VMOVDQU Y5, (R10)(R9*1)
|
|
MOVQ 48(R8), R10
|
|
VMOVDQU Y6, (R10)(R9*1)
|
|
MOVQ 72(R8), R10
|
|
VMOVDQU Y7, (R10)(R9*1)
|
|
MOVQ 96(R8), R10
|
|
VMOVDQU Y8, (R10)(R9*1)
|
|
MOVQ 120(R8), R10
|
|
VMOVDQU Y9, (R10)(R9*1)
|
|
MOVQ 144(R8), R10
|
|
VMOVDQU Y10, (R10)(R9*1)
|
|
MOVQ 168(R8), R10
|
|
VMOVDQU Y11, (R10)(R9*1)
|
|
MOVQ 192(R8), R10
|
|
VMOVDQU Y12, (R10)(R9*1)
|
|
MOVQ 216(R8), R10
|
|
VMOVDQU Y13, (R10)(R9*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R9
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_4x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_4x10_64Xor(SB), $0-88
|
|
// Loading 20 of 40 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 52 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_4x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, DX
|
|
|
|
mulGFNI_4x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z20
|
|
MOVQ 24(R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z21
|
|
MOVQ 48(R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z22
|
|
MOVQ 72(R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z23
|
|
MOVQ 96(R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z24
|
|
MOVQ 120(R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z25
|
|
MOVQ 144(R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z26
|
|
MOVQ 168(R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z27
|
|
MOVQ 192(R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z28
|
|
MOVQ 216(R8), R10
|
|
VMOVDQU64 (R10)(R9*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R8), R10
|
|
VMOVDQU64 Z20, (R10)(R9*1)
|
|
MOVQ 24(R8), R10
|
|
VMOVDQU64 Z21, (R10)(R9*1)
|
|
MOVQ 48(R8), R10
|
|
VMOVDQU64 Z22, (R10)(R9*1)
|
|
MOVQ 72(R8), R10
|
|
VMOVDQU64 Z23, (R10)(R9*1)
|
|
MOVQ 96(R8), R10
|
|
VMOVDQU64 Z24, (R10)(R9*1)
|
|
MOVQ 120(R8), R10
|
|
VMOVDQU64 Z25, (R10)(R9*1)
|
|
MOVQ 144(R8), R10
|
|
VMOVDQU64 Z26, (R10)(R9*1)
|
|
MOVQ 168(R8), R10
|
|
VMOVDQU64 Z27, (R10)(R9*1)
|
|
MOVQ 192(R8), R10
|
|
VMOVDQU64 Z28, (R10)(R9*1)
|
|
MOVQ 216(R8), R10
|
|
VMOVDQU64 Z29, (R10)(R9*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R9
|
|
DECQ AX
|
|
JNZ mulGFNI_4x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_4x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_4x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_4x10Xor(SB), $0-88
|
|
// Loading 4 of 40 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 52 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_4x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), DX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, DX
|
|
|
|
mulAvxGFNI_4x10Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R8), R10
|
|
VMOVDQU (R10)(R9*1), Y4
|
|
MOVQ 24(R8), R10
|
|
VMOVDQU (R10)(R9*1), Y5
|
|
MOVQ 48(R8), R10
|
|
VMOVDQU (R10)(R9*1), Y6
|
|
MOVQ 72(R8), R10
|
|
VMOVDQU (R10)(R9*1), Y7
|
|
MOVQ 96(R8), R10
|
|
VMOVDQU (R10)(R9*1), Y8
|
|
MOVQ 120(R8), R10
|
|
VMOVDQU (R10)(R9*1), Y9
|
|
MOVQ 144(R8), R10
|
|
VMOVDQU (R10)(R9*1), Y10
|
|
MOVQ 168(R8), R10
|
|
VMOVDQU (R10)(R9*1), Y11
|
|
MOVQ 192(R8), R10
|
|
VMOVDQU (R10)(R9*1), Y12
|
|
MOVQ 216(R8), R10
|
|
VMOVDQU (R10)(R9*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R8), R10
|
|
VMOVDQU Y4, (R10)(R9*1)
|
|
MOVQ 24(R8), R10
|
|
VMOVDQU Y5, (R10)(R9*1)
|
|
MOVQ 48(R8), R10
|
|
VMOVDQU Y6, (R10)(R9*1)
|
|
MOVQ 72(R8), R10
|
|
VMOVDQU Y7, (R10)(R9*1)
|
|
MOVQ 96(R8), R10
|
|
VMOVDQU Y8, (R10)(R9*1)
|
|
MOVQ 120(R8), R10
|
|
VMOVDQU Y9, (R10)(R9*1)
|
|
MOVQ 144(R8), R10
|
|
VMOVDQU Y10, (R10)(R9*1)
|
|
MOVQ 168(R8), R10
|
|
VMOVDQU Y11, (R10)(R9*1)
|
|
MOVQ 192(R8), R10
|
|
VMOVDQU Y12, (R10)(R9*1)
|
|
MOVQ 216(R8), R10
|
|
VMOVDQU Y13, (R10)(R9*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R9
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_4x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_4x10Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R8
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_5x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z6
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z6, Z5
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z6
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z6, Z6
|
|
VXORPD Z5, Z6, Z5
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z6
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z6, Z6
|
|
VXORPD Z5, Z6, Z5
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z6
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z6, Z6
|
|
VXORPD Z5, Z6, Z5
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (CX), Z6
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z6, Z6
|
|
VXORPD Z5, Z6, Z5
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z5, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R8
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_5x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y6
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y6, Y5
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y6
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y6, Y6
|
|
VXORPD Y5, Y6, Y5
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y6
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y6, Y6
|
|
VXORPD Y5, Y6, Y5
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y6
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y6, Y6
|
|
VXORPD Y5, Y6, Y5
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (CX), Y6
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y4, Y6, Y6
|
|
VXORPD Y5, Y6, Y5
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R8
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, CX
|
|
|
|
mulGFNI_5x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (R8), Z5
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z6
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z6, Z6
|
|
VXORPD Z5, Z6, Z5
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z6
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z6, Z6
|
|
VXORPD Z5, Z6, Z5
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z6
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z6, Z6
|
|
VXORPD Z5, Z6, Z5
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z6
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z6, Z6
|
|
VXORPD Z5, Z6, Z5
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (CX), Z6
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z6, Z6
|
|
VXORPD Z5, Z6, Z5
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z5, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 8 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R8
|
|
MOVQ start+72(FP), R9
|
|
|
|
// Add start offset to output
|
|
ADDQ R9, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R9, DX
|
|
ADDQ R9, BX
|
|
ADDQ R9, SI
|
|
ADDQ R9, DI
|
|
ADDQ R9, CX
|
|
|
|
mulAvxGFNI_5x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (R8), Y5
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y6
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y6, Y6
|
|
VXORPD Y5, Y6, Y5
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y6
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y6, Y6
|
|
VXORPD Y5, Y6, Y5
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y6
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y6, Y6
|
|
VXORPD Y5, Y6, Y5
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y6
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y6, Y6
|
|
VXORPD Y5, Y6, Y5
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (CX), Y6
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y4, Y6, Y6
|
|
VXORPD Y5, Y6, Y5
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y5, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R8
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R9
|
|
ADDQ R10, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_5x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z12
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z12, Z10
|
|
VGF2P8AFFINEQB $0x00, Z1, Z12, Z11
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z12
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z12
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z12
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (CX), Z12
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z9, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z10, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z11, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x2(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R8
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R9
|
|
ADDQ R10, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, CX
|
|
|
|
mulAvxGFNI_5x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y12
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y12, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y12, Y11
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (BX), Y12
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (SI), Y12
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (DI), Y12
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (CX), Y12
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y8, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R8
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R9
|
|
ADDQ R10, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_5x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (R9), Z10
|
|
VMOVDQU64 (R8), Z11
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z12
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z1, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z12
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z3, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z12
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z5, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z12
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z7, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (CX), Z12
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z12, Z13
|
|
VXORPD Z10, Z13, Z10
|
|
VGF2P8AFFINEQB $0x00, Z9, Z12, Z13
|
|
VXORPD Z11, Z13, Z11
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z10, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z11, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x2Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 14 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R8
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R9
|
|
ADDQ R10, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, CX
|
|
|
|
mulAvxGFNI_5x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (R9), Y10
|
|
VMOVDQU (R8), Y11
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y12
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (BX), Y12
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (SI), Y12
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (DI), Y12
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (CX), Y12
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y8, Y12, Y13
|
|
VXORPD Y10, Y13, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y12, Y13
|
|
VXORPD Y11, Y13, Y11
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y10, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y11, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x3_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R8
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_5x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z18
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z18, Z15
|
|
VGF2P8AFFINEQB $0x00, Z1, Z18, Z16
|
|
VGF2P8AFFINEQB $0x00, Z2, Z18, Z17
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z18
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z18
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z18
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (CX), Z18
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z12, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z13, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z14, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z15, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z16, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z17, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x3(SB), $0-88
|
|
// Loading 11 of 15 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R9
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_5x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y12, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x3_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R8
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R9
|
|
ADDQ R11, R10
|
|
ADDQ R11, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_5x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (R9), Z15
|
|
VMOVDQU64 (R10), Z16
|
|
VMOVDQU64 (R8), Z17
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z18
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z1, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z18
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z18
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z18
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (CX), Z18
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z12, Z18, Z19
|
|
VXORPD Z15, Z19, Z15
|
|
VGF2P8AFFINEQB $0x00, Z13, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z14, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z15, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z16, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z17, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x3Xor(SB), $0-88
|
|
// Loading 11 of 15 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R9
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_5x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (R10), Y11
|
|
VMOVDQU (R11), Y12
|
|
VMOVDQU (R9), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y12, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x4_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R8
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_5x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z24
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z24, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z24, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z24, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z24, Z23
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z24
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (SI), Z24
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (DI), Z24
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (CX), Z24
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z18, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z19, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z20, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z21, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z22, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z23, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x4(SB), $0-88
|
|
// Loading 10 of 20 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R9
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_5x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x4_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R8
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_5x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU64 (R9), Z20
|
|
VMOVDQU64 (R10), Z21
|
|
VMOVDQU64 (R11), Z22
|
|
VMOVDQU64 (R8), Z23
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z24
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z24
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (SI), Z24
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (DI), Z24
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (CX), Z24
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
|
|
VXORPD Z20, Z25, Z20
|
|
VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z18, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z19, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z20, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z21, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z22, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z23, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x4Xor(SB), $0-88
|
|
// Loading 10 of 20 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R9
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_5x4Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU (R10), Y10
|
|
VMOVDQU (R11), Y11
|
|
VMOVDQU (R12), Y12
|
|
VMOVDQU (R9), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x5_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R8
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_5x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (CX), Z30
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z25, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z26, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z27, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z28, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z29, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x5(SB), $0-88
|
|
// Loading 9 of 25 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R9
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_5x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x5_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), CX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R8
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_5x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU64 (R9), Z25
|
|
VMOVDQU64 (R10), Z26
|
|
VMOVDQU64 (R11), Z27
|
|
VMOVDQU64 (R12), Z28
|
|
VMOVDQU64 (R8), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (CX), Z30
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z25, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z26, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z27, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z28, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z29, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x5Xor(SB), $0-88
|
|
// Loading 9 of 25 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R9
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_5x5Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU (R10), Y9
|
|
VMOVDQU (R11), Y10
|
|
VMOVDQU (R12), Y11
|
|
VMOVDQU (R13), Y12
|
|
VMOVDQU (R9), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x6_64(SB), $0-88
|
|
// Loading 24 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R9
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_5x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x6(SB), $0-88
|
|
// Loading 8 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R9
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_5x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x6_64Xor(SB), $0-88
|
|
// Loading 24 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R9
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_5x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU64 (R10), Z24
|
|
VMOVDQU64 (R11), Z25
|
|
VMOVDQU64 (R12), Z26
|
|
VMOVDQU64 (R13), Z27
|
|
VMOVDQU64 (R14), Z28
|
|
VMOVDQU64 (R9), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x6Xor(SB), $0-88
|
|
// Loading 8 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R9
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_5x6Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU (R10), Y8
|
|
VMOVDQU (R11), Y9
|
|
VMOVDQU (R12), Y10
|
|
VMOVDQU (R13), Y11
|
|
VMOVDQU (R14), Y12
|
|
VMOVDQU (R9), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x7_64(SB), $8-88
|
|
// Loading 23 of 35 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 44 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R15
|
|
MOVQ 144(R9), R9
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_5x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x7(SB), $8-88
|
|
// Loading 7 of 35 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 44 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R15
|
|
MOVQ 144(R9), R9
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_5x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x7_64Xor(SB), $8-88
|
|
// Loading 23 of 35 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 44 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R15
|
|
MOVQ 144(R9), R9
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_5x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU64 (R10), Z23
|
|
VMOVDQU64 (R11), Z24
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R9), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_5x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x7Xor(SB), $8-88
|
|
// Loading 7 of 35 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 44 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R15
|
|
MOVQ 144(R9), R9
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_5x7Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU (R10), Y7
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R9), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x8_64(SB), $8-88
|
|
// Loading 22 of 40 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 50 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), AX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R15
|
|
MOVQ 168(R8), R8
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_5x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_5x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x8(SB), $8-88
|
|
// Loading 6 of 40 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 50 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), AX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R15
|
|
MOVQ 168(R8), R8
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_5x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 56(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_5x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x8_64Xor(SB), $8-88
|
|
// Loading 22 of 40 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 50 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), AX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R15
|
|
MOVQ 168(R8), R8
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_5x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU64 (R9), Z22
|
|
VMOVDQU64 (R10), Z23
|
|
VMOVDQU64 (R11), Z24
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R8), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU64 Z22, (R9)
|
|
ADDQ $0x40, R9
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R8)
|
|
ADDQ $0x40, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_5x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x8Xor(SB), $8-88
|
|
// Loading 6 of 40 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 50 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), AX
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ out_base+48(FP), R8
|
|
MOVQ (R8), R9
|
|
MOVQ 24(R8), R10
|
|
MOVQ 48(R8), R11
|
|
MOVQ 72(R8), R12
|
|
MOVQ 96(R8), R13
|
|
MOVQ 120(R8), R14
|
|
MOVQ 144(R8), R15
|
|
MOVQ 168(R8), R8
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R8
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_5x8Xor_loop:
|
|
// Load 8 outputs
|
|
VMOVDQU (R9), Y6
|
|
VMOVDQU (R10), Y7
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R8), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R8)
|
|
ADDQ $0x20, R8
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_5x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x9_64(SB), $0-88
|
|
// Loading 21 of 45 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 56 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, DX
|
|
|
|
mulGFNI_5x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU64 Z21, (R11)(R10*1)
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU64 Z22, (R11)(R10*1)
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU64 Z23, (R11)(R10*1)
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU64 Z24, (R11)(R10*1)
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU64 Z25, (R11)(R10*1)
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU64 Z26, (R11)(R10*1)
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU64 Z27, (R11)(R10*1)
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU64 Z28, (R11)(R10*1)
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU64 Z29, (R11)(R10*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R10
|
|
DECQ AX
|
|
JNZ mulGFNI_5x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x9(SB), $0-88
|
|
// Loading 5 of 45 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 56 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, DX
|
|
|
|
mulAvxGFNI_5x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 64(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU Y5, (R11)(R10*1)
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU Y6, (R11)(R10*1)
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU Y7, (R11)(R10*1)
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU Y8, (R11)(R10*1)
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU Y9, (R11)(R10*1)
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU Y10, (R11)(R10*1)
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU Y11, (R11)(R10*1)
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU Y12, (R11)(R10*1)
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU Y13, (R11)(R10*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R10
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x9_64Xor(SB), $0-88
|
|
// Loading 21 of 45 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 56 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, DX
|
|
|
|
mulGFNI_5x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z21
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z22
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z23
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z24
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z25
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z26
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z27
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z28
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU64 Z21, (R11)(R10*1)
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU64 Z22, (R11)(R10*1)
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU64 Z23, (R11)(R10*1)
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU64 Z24, (R11)(R10*1)
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU64 Z25, (R11)(R10*1)
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU64 Z26, (R11)(R10*1)
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU64 Z27, (R11)(R10*1)
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU64 Z28, (R11)(R10*1)
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU64 Z29, (R11)(R10*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R10
|
|
DECQ AX
|
|
JNZ mulGFNI_5x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x9Xor(SB), $0-88
|
|
// Loading 5 of 45 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 56 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, DX
|
|
|
|
mulAvxGFNI_5x9Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU (R11)(R10*1), Y5
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y6
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y7
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y8
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y9
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y10
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y11
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y12
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU Y5, (R11)(R10*1)
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU Y6, (R11)(R10*1)
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU Y7, (R11)(R10*1)
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU Y8, (R11)(R10*1)
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU Y9, (R11)(R10*1)
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU Y10, (R11)(R10*1)
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU Y11, (R11)(R10*1)
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU Y12, (R11)(R10*1)
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU Y13, (R11)(R10*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R10
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x10_64(SB), $0-88
|
|
// Loading 20 of 50 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 62 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, DX
|
|
|
|
mulGFNI_5x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU64 Z20, (R11)(R10*1)
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU64 Z21, (R11)(R10*1)
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU64 Z22, (R11)(R10*1)
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU64 Z23, (R11)(R10*1)
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU64 Z24, (R11)(R10*1)
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU64 Z25, (R11)(R10*1)
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU64 Z26, (R11)(R10*1)
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU64 Z27, (R11)(R10*1)
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU64 Z28, (R11)(R10*1)
|
|
MOVQ 216(R9), R11
|
|
VMOVDQU64 Z29, (R11)(R10*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R10
|
|
DECQ AX
|
|
JNZ mulGFNI_5x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x10(SB), $0-88
|
|
// Loading 4 of 50 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 62 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, DX
|
|
|
|
mulAvxGFNI_5x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 72(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU Y4, (R11)(R10*1)
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU Y5, (R11)(R10*1)
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU Y6, (R11)(R10*1)
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU Y7, (R11)(R10*1)
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU Y8, (R11)(R10*1)
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU Y9, (R11)(R10*1)
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU Y10, (R11)(R10*1)
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU Y11, (R11)(R10*1)
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU Y12, (R11)(R10*1)
|
|
MOVQ 216(R9), R11
|
|
VMOVDQU Y13, (R11)(R10*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R10
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_5x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_5x10_64Xor(SB), $0-88
|
|
// Loading 20 of 50 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 62 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_5x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, DX
|
|
|
|
mulGFNI_5x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z20
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z21
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z22
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z23
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z24
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z25
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z26
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z27
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z28
|
|
MOVQ 216(R9), R11
|
|
VMOVDQU64 (R11)(R10*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU64 Z20, (R11)(R10*1)
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU64 Z21, (R11)(R10*1)
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU64 Z22, (R11)(R10*1)
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU64 Z23, (R11)(R10*1)
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU64 Z24, (R11)(R10*1)
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU64 Z25, (R11)(R10*1)
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU64 Z26, (R11)(R10*1)
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU64 Z27, (R11)(R10*1)
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU64 Z28, (R11)(R10*1)
|
|
MOVQ 216(R9), R11
|
|
VMOVDQU64 Z29, (R11)(R10*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R10
|
|
DECQ AX
|
|
JNZ mulGFNI_5x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_5x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_5x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_5x10Xor(SB), $0-88
|
|
// Loading 4 of 50 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 62 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_5x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), DX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, DX
|
|
|
|
mulAvxGFNI_5x10Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU (R11)(R10*1), Y4
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y5
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y6
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y7
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y8
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y9
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y10
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y11
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y12
|
|
MOVQ 216(R9), R11
|
|
VMOVDQU (R11)(R10*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R9), R11
|
|
VMOVDQU Y4, (R11)(R10*1)
|
|
MOVQ 24(R9), R11
|
|
VMOVDQU Y5, (R11)(R10*1)
|
|
MOVQ 48(R9), R11
|
|
VMOVDQU Y6, (R11)(R10*1)
|
|
MOVQ 72(R9), R11
|
|
VMOVDQU Y7, (R11)(R10*1)
|
|
MOVQ 96(R9), R11
|
|
VMOVDQU Y8, (R11)(R10*1)
|
|
MOVQ 120(R9), R11
|
|
VMOVDQU Y9, (R11)(R10*1)
|
|
MOVQ 144(R9), R11
|
|
VMOVDQU Y10, (R11)(R10*1)
|
|
MOVQ 168(R9), R11
|
|
VMOVDQU Y11, (R11)(R10*1)
|
|
MOVQ 192(R9), R11
|
|
VMOVDQU Y12, (R11)(R10*1)
|
|
MOVQ 216(R9), R11
|
|
VMOVDQU Y13, (R11)(R10*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R10
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_5x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_5x10Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 9 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_6x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z7
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z7
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z7
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z7
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z7
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (CX), Z7
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z6, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 9 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, CX
|
|
|
|
mulAvxGFNI_6x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y7
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y7
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y7
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y7
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y7
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (CX), Y7
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y5, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 9 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, CX
|
|
|
|
mulGFNI_6x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (R9), Z6
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z7
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z7
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z7
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z7
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z7
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (CX), Z7
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z7, Z7
|
|
VXORPD Z6, Z7, Z6
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z6, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 9 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R9
|
|
MOVQ start+72(FP), R10
|
|
|
|
// Add start offset to output
|
|
ADDQ R10, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R10, DX
|
|
ADDQ R10, BX
|
|
ADDQ R10, SI
|
|
ADDQ R10, DI
|
|
ADDQ R10, R8
|
|
ADDQ R10, CX
|
|
|
|
mulAvxGFNI_6x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (R9), Y6
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y7
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y7
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y7
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y7
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y7
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (CX), Y7
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y5, Y7, Y7
|
|
VXORPD Y6, Y7, Y6
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y6, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 16 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R9
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R10
|
|
ADDQ R11, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_6x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z14
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z14, Z12
|
|
VGF2P8AFFINEQB $0x00, Z1, Z14, Z13
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z14
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z3, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z14
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z5, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z14
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z7, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z14
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z9, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (CX), Z14
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z10, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z11, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z12, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z13, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x2(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 16 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R9
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R10
|
|
ADDQ R11, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, CX
|
|
|
|
mulAvxGFNI_6x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (CX), Y14
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 16 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R9
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R10
|
|
ADDQ R11, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_6x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (R10), Z12
|
|
VMOVDQU64 (R9), Z13
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z14
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z1, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z14
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z3, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z14
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z5, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z14
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z7, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z14
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z9, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (CX), Z14
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z10, Z14, Z15
|
|
VXORPD Z12, Z15, Z12
|
|
VGF2P8AFFINEQB $0x00, Z11, Z14, Z15
|
|
VXORPD Z13, Z15, Z13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z12, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z13, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x2Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 16 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R9
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R10
|
|
ADDQ R11, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, CX
|
|
|
|
mulAvxGFNI_6x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (R10), Y12
|
|
VMOVDQU (R9), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (CX), Y14
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x3_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 23 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R9
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_6x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z21
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z1, Z21, Z19
|
|
VGF2P8AFFINEQB $0x00, Z2, Z21, Z20
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z21
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z4, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z5, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z21
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z7, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z8, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z21
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z10, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z11, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z21
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z13, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z14, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (CX), Z21
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z15, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z16, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z17, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z18, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z19, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z20, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x3(SB), $0-88
|
|
// Loading 11 of 18 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 23 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R10
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_6x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x3_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 23 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R9
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R10
|
|
ADDQ R12, R11
|
|
ADDQ R12, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_6x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (R10), Z18
|
|
VMOVDQU64 (R11), Z19
|
|
VMOVDQU64 (R9), Z20
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z21
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z1, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z2, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z21
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z4, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z5, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z21
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z7, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z8, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z21
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z10, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z11, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z21
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z13, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z14, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (CX), Z21
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z15, Z21, Z22
|
|
VXORPD Z18, Z22, Z18
|
|
VGF2P8AFFINEQB $0x00, Z16, Z21, Z22
|
|
VXORPD Z19, Z22, Z19
|
|
VGF2P8AFFINEQB $0x00, Z17, Z21, Z22
|
|
VXORPD Z20, Z22, Z20
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z18, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z19, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z20, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x3Xor(SB), $0-88
|
|
// Loading 11 of 18 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 23 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R10
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_6x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (R11), Y11
|
|
VMOVDQU (R12), Y12
|
|
VMOVDQU (R10), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x4_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 30 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R9
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_6x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z28
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z28, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z28, Z27
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z28
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (SI), Z28
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (DI), Z28
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R8), Z28
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z16, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z17, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z18, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z19, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (CX), Z28
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z20, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z21, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z22, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z23, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x4(SB), $0-88
|
|
// Loading 10 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 30 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R10
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_6x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x4_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 30 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), CX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R9
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_6x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU64 (R10), Z24
|
|
VMOVDQU64 (R11), Z25
|
|
VMOVDQU64 (R12), Z26
|
|
VMOVDQU64 (R9), Z27
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z28
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z28
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (SI), Z28
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (DI), Z28
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R8), Z28
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z16, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z17, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z18, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z19, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (CX), Z28
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z20, Z28, Z29
|
|
VXORPD Z24, Z29, Z24
|
|
VGF2P8AFFINEQB $0x00, Z21, Z28, Z29
|
|
VXORPD Z25, Z29, Z25
|
|
VGF2P8AFFINEQB $0x00, Z22, Z28, Z29
|
|
VXORPD Z26, Z29, Z26
|
|
VGF2P8AFFINEQB $0x00, Z23, Z28, Z29
|
|
VXORPD Z27, Z29, Z27
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z24, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x4Xor(SB), $0-88
|
|
// Loading 10 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 30 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R10
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_6x4Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU (R11), Y10
|
|
VMOVDQU (R12), Y11
|
|
VMOVDQU (R13), Y12
|
|
VMOVDQU (R10), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x5_64(SB), $0-88
|
|
// Loading 25 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 37 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R10
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_6x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x5(SB), $0-88
|
|
// Loading 9 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 37 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R10
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_6x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x5_64Xor(SB), $0-88
|
|
// Loading 25 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 37 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R10
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_6x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU64 (R11), Z25
|
|
VMOVDQU64 (R12), Z26
|
|
VMOVDQU64 (R13), Z27
|
|
VMOVDQU64 (R14), Z28
|
|
VMOVDQU64 (R10), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z25, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x5Xor(SB), $0-88
|
|
// Loading 9 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 37 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R10
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_6x5Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU (R11), Y9
|
|
VMOVDQU (R12), Y10
|
|
VMOVDQU (R13), Y11
|
|
VMOVDQU (R14), Y12
|
|
VMOVDQU (R10), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x6_64(SB), $8-88
|
|
// Loading 24 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 44 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R15
|
|
MOVQ 120(R10), R10
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_6x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x6(SB), $8-88
|
|
// Loading 8 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 44 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R15
|
|
MOVQ 120(R10), R10
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_6x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x6_64Xor(SB), $8-88
|
|
// Loading 24 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 44 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R15
|
|
MOVQ 120(R10), R10
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_6x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU64 (R11), Z24
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R10), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_6x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x6Xor(SB), $8-88
|
|
// Loading 8 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 44 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R15
|
|
MOVQ 120(R10), R10
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_6x6Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R10), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x7_64(SB), $8-88
|
|
// Loading 23 of 42 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 51 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), AX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R15
|
|
MOVQ 144(R9), R9
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_6x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_6x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x7(SB), $8-88
|
|
// Loading 7 of 42 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 51 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), AX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R15
|
|
MOVQ 144(R9), R9
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_6x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_6x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x7_64Xor(SB), $8-88
|
|
// Loading 23 of 42 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 51 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), AX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R15
|
|
MOVQ 144(R9), R9
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_6x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU64 (R10), Z23
|
|
VMOVDQU64 (R11), Z24
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R9), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R9)
|
|
ADDQ $0x40, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_6x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x7Xor(SB), $8-88
|
|
// Loading 7 of 42 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 51 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), AX
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ out_base+48(FP), R9
|
|
MOVQ (R9), R10
|
|
MOVQ 24(R9), R11
|
|
MOVQ 48(R9), R12
|
|
MOVQ 72(R9), R13
|
|
MOVQ 96(R9), R14
|
|
MOVQ 120(R9), R15
|
|
MOVQ 144(R9), R9
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R9
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_6x7Xor_loop:
|
|
// Load 7 outputs
|
|
VMOVDQU (R10), Y7
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R9), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R9)
|
|
ADDQ $0x20, R9
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_6x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x8_64(SB), $0-88
|
|
// Loading 22 of 48 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 58 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulGFNI_6x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU64 Z22, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU64 Z23, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU64 Z24, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU64 Z25, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU64 Z26, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU64 Z27, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU64 Z28, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU64 Z29, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R11
|
|
DECQ AX
|
|
JNZ mulGFNI_6x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x8(SB), $0-88
|
|
// Loading 6 of 48 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 58 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_6x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 56(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU Y6, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU Y7, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU Y8, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU Y9, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU Y10, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU Y11, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU Y12, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU Y13, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R11
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x8_64Xor(SB), $0-88
|
|
// Loading 22 of 48 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 58 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulGFNI_6x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z22
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z23
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z24
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z25
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z26
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z27
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z28
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU64 Z22, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU64 Z23, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU64 Z24, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU64 Z25, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU64 Z26, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU64 Z27, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU64 Z28, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU64 Z29, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R11
|
|
DECQ AX
|
|
JNZ mulGFNI_6x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x8Xor(SB), $0-88
|
|
// Loading 6 of 48 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 58 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_6x8Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU (R12)(R11*1), Y6
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y7
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y8
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y9
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y10
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y11
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y12
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU Y6, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU Y7, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU Y8, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU Y9, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU Y10, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU Y11, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU Y12, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU Y13, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R11
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x9_64(SB), $0-88
|
|
// Loading 21 of 54 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 65 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulGFNI_6x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU64 Z21, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU64 Z22, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU64 Z23, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU64 Z24, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU64 Z25, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU64 Z26, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU64 Z27, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU64 Z28, (R12)(R11*1)
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU64 Z29, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R11
|
|
DECQ AX
|
|
JNZ mulGFNI_6x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x9(SB), $0-88
|
|
// Loading 5 of 54 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 65 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_6x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 64(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU Y5, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU Y6, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU Y7, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU Y8, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU Y9, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU Y10, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU Y11, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU Y12, (R12)(R11*1)
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU Y13, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R11
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x9_64Xor(SB), $0-88
|
|
// Loading 21 of 54 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 65 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulGFNI_6x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z21
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z22
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z23
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z24
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z25
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z26
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z27
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z28
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU64 Z21, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU64 Z22, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU64 Z23, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU64 Z24, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU64 Z25, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU64 Z26, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU64 Z27, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU64 Z28, (R12)(R11*1)
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU64 Z29, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R11
|
|
DECQ AX
|
|
JNZ mulGFNI_6x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x9Xor(SB), $0-88
|
|
// Loading 5 of 54 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 65 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_6x9Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU (R12)(R11*1), Y5
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y6
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y7
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y8
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y9
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y10
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y11
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y12
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU Y5, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU Y6, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU Y7, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU Y8, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU Y9, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU Y10, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU Y11, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU Y12, (R12)(R11*1)
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU Y13, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R11
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x10_64(SB), $0-88
|
|
// Loading 20 of 60 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 72 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulGFNI_6x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU64 Z20, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU64 Z21, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU64 Z22, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU64 Z23, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU64 Z24, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU64 Z25, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU64 Z26, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU64 Z27, (R12)(R11*1)
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU64 Z28, (R12)(R11*1)
|
|
MOVQ 216(R10), R12
|
|
VMOVDQU64 Z29, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R11
|
|
DECQ AX
|
|
JNZ mulGFNI_6x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x10(SB), $0-88
|
|
// Loading 4 of 60 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 72 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_6x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 72(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU Y4, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU Y5, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU Y6, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU Y7, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU Y8, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU Y9, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU Y10, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU Y11, (R12)(R11*1)
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU Y12, (R12)(R11*1)
|
|
MOVQ 216(R10), R12
|
|
VMOVDQU Y13, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R11
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_6x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_6x10_64Xor(SB), $0-88
|
|
// Loading 20 of 60 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 72 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_6x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulGFNI_6x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z20
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z21
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z22
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z23
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z24
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z25
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z26
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z27
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z28
|
|
MOVQ 216(R10), R12
|
|
VMOVDQU64 (R12)(R11*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU64 Z20, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU64 Z21, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU64 Z22, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU64 Z23, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU64 Z24, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU64 Z25, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU64 Z26, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU64 Z27, (R12)(R11*1)
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU64 Z28, (R12)(R11*1)
|
|
MOVQ 216(R10), R12
|
|
VMOVDQU64 Z29, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R11
|
|
DECQ AX
|
|
JNZ mulGFNI_6x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_6x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_6x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_6x10Xor(SB), $0-88
|
|
// Loading 4 of 60 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 72 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_6x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), DX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, DX
|
|
|
|
mulAvxGFNI_6x10Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU (R12)(R11*1), Y4
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y5
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y6
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y7
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y8
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y9
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y10
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y11
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y12
|
|
MOVQ 216(R10), R12
|
|
VMOVDQU (R12)(R11*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R10), R12
|
|
VMOVDQU Y4, (R12)(R11*1)
|
|
MOVQ 24(R10), R12
|
|
VMOVDQU Y5, (R12)(R11*1)
|
|
MOVQ 48(R10), R12
|
|
VMOVDQU Y6, (R12)(R11*1)
|
|
MOVQ 72(R10), R12
|
|
VMOVDQU Y7, (R12)(R11*1)
|
|
MOVQ 96(R10), R12
|
|
VMOVDQU Y8, (R12)(R11*1)
|
|
MOVQ 120(R10), R12
|
|
VMOVDQU Y9, (R12)(R11*1)
|
|
MOVQ 144(R10), R12
|
|
VMOVDQU Y10, (R12)(R11*1)
|
|
MOVQ 168(R10), R12
|
|
VMOVDQU Y11, (R12)(R11*1)
|
|
MOVQ 192(R10), R12
|
|
VMOVDQU Y12, (R12)(R11*1)
|
|
MOVQ 216(R10), R12
|
|
VMOVDQU Y13, (R12)(R11*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R11
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_6x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_6x10Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), CX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_7x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z8
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z8
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z8
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z8
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z8
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (R9), Z8
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z5, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 6 to 1 outputs
|
|
VMOVDQU64 (CX), Z8
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z7, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), CX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, CX
|
|
|
|
mulAvxGFNI_7x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y8
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y8
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y8
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y8
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y8
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (R9), Y8
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y5, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 6 to 1 outputs
|
|
VMOVDQU (CX), Y8
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), CX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, CX
|
|
|
|
mulGFNI_7x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (R10), Z7
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z8
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z8
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z8
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z8
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z8
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (R9), Z8
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z5, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Load and process 64 bytes from input 6 to 1 outputs
|
|
VMOVDQU64 (CX), Z8
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z8, Z8
|
|
VXORPD Z7, Z8, Z7
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z7, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 10 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), CX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R10
|
|
MOVQ start+72(FP), R11
|
|
|
|
// Add start offset to output
|
|
ADDQ R11, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R11, DX
|
|
ADDQ R11, BX
|
|
ADDQ R11, SI
|
|
ADDQ R11, DI
|
|
ADDQ R11, R8
|
|
ADDQ R11, R9
|
|
ADDQ R11, CX
|
|
|
|
mulAvxGFNI_7x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (R10), Y7
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y8
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y8
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y8
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y8
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y8
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (R9), Y8
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y5, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Load and process 32 bytes from input 6 to 1 outputs
|
|
VMOVDQU (CX), Y8
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y8, Y8
|
|
VXORPD Y7, Y8, Y7
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y7, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), CX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R10
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R11
|
|
ADDQ R12, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_7x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z16
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z16, Z14
|
|
VGF2P8AFFINEQB $0x00, Z1, Z16, Z15
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z16
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z3, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z16
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z16
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z16
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z9, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (R9), Z16
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z10, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z11, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 6 to 2 outputs
|
|
VMOVDQU64 (CX), Z16
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z12, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z13, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z14, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z15, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x2(SB), $0-88
|
|
// Loading 12 of 14 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R11
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R12
|
|
ADDQ R13, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_7x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), CX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R10
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R11
|
|
ADDQ R12, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_7x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (R11), Z14
|
|
VMOVDQU64 (R10), Z15
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z16
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z1, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z16
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z3, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z16
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z5, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z16
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z7, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z16
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z9, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (R9), Z16
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z10, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z11, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Load and process 64 bytes from input 6 to 2 outputs
|
|
VMOVDQU64 (CX), Z16
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z12, Z16, Z17
|
|
VXORPD Z14, Z17, Z14
|
|
VGF2P8AFFINEQB $0x00, Z13, Z16, Z17
|
|
VXORPD Z15, Z17, Z15
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z14, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z15, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x2Xor(SB), $0-88
|
|
// Loading 12 of 14 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 18 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R11
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R12
|
|
ADDQ R13, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_7x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (R12), Y12
|
|
VMOVDQU (R11), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x3_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), CX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R10
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_7x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z24
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z24, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z24, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z24, Z23
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z24
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z24
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z24
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z24
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (R9), Z24
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 6 to 3 outputs
|
|
VMOVDQU64 (CX), Z24
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z18, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z21, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z22, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x3(SB), $0-88
|
|
// Loading 11 of 21 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R11
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_7x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x3_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), CX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R10
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R11
|
|
ADDQ R13, R12
|
|
ADDQ R13, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_7x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (R11), Z21
|
|
VMOVDQU64 (R12), Z22
|
|
VMOVDQU64 (R10), Z23
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z24
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z24
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z4, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z5, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z24
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z7, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z8, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z24
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z24
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z13, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z14, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (R9), Z24
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z15, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z16, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Load and process 64 bytes from input 6 to 3 outputs
|
|
VMOVDQU64 (CX), Z24
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z18, Z24, Z25
|
|
VXORPD Z21, Z25, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z24, Z25
|
|
VXORPD Z22, Z25, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z24, Z25
|
|
VXORPD Z23, Z25, Z23
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z21, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z22, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z23, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x3Xor(SB), $0-88
|
|
// Loading 11 of 21 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 26 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R11
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_7x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (R12), Y11
|
|
VMOVDQU (R13), Y12
|
|
VMOVDQU (R11), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x4_64(SB), $0-88
|
|
// Loading 26 of 28 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 34 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R11
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_7x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 4 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x4(SB), $0-88
|
|
// Loading 10 of 28 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 34 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R11
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_7x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x4_64Xor(SB), $0-88
|
|
// Loading 26 of 28 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 34 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R11
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_7x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU64 (R12), Z26
|
|
VMOVDQU64 (R13), Z27
|
|
VMOVDQU64 (R14), Z28
|
|
VMOVDQU64 (R11), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 4 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z26, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x4Xor(SB), $0-88
|
|
// Loading 10 of 28 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 34 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R11
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_7x4Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU (R12), Y10
|
|
VMOVDQU (R13), Y11
|
|
VMOVDQU (R14), Y12
|
|
VMOVDQU (R11), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x5_64(SB), $8-88
|
|
// Loading 25 of 35 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R15
|
|
MOVQ 96(R11), R11
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_7x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x5(SB), $8-88
|
|
// Loading 9 of 35 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R15
|
|
MOVQ 96(R11), R11
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_7x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x5_64Xor(SB), $8-88
|
|
// Loading 25 of 35 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R15
|
|
MOVQ 96(R11), R11
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_7x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R11), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_7x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x5Xor(SB), $8-88
|
|
// Loading 9 of 35 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R15
|
|
MOVQ 96(R11), R11
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_7x5Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R11), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x6_64(SB), $8-88
|
|
// Loading 24 of 42 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 50 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), AX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R15
|
|
MOVQ 120(R10), R10
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_7x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 6 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_7x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x6(SB), $8-88
|
|
// Loading 8 of 42 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 50 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), AX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R15
|
|
MOVQ 120(R10), R10
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_7x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 6 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_7x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x6_64Xor(SB), $8-88
|
|
// Loading 24 of 42 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 50 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), AX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R15
|
|
MOVQ 120(R10), R10
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_7x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU64 (R11), Z24
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R10), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 6 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU64 Z24, (R11)
|
|
ADDQ $0x40, R11
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R10)
|
|
ADDQ $0x40, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_7x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x6Xor(SB), $8-88
|
|
// Loading 8 of 42 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 50 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), AX
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ out_base+48(FP), R10
|
|
MOVQ (R10), R11
|
|
MOVQ 24(R10), R12
|
|
MOVQ 48(R10), R13
|
|
MOVQ 72(R10), R14
|
|
MOVQ 96(R10), R15
|
|
MOVQ 120(R10), R10
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R10
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_7x6Xor_loop:
|
|
// Load 6 outputs
|
|
VMOVDQU (R11), Y8
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R10), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 6 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R10)
|
|
ADDQ $0x20, R10
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_7x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x7_64(SB), $0-88
|
|
// Loading 23 of 49 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 58 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulGFNI_7x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 Z23, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 Z24, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 Z25, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 Z26, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 Z27, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 Z28, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 Z29, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R12
|
|
DECQ AX
|
|
JNZ mulGFNI_7x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x7(SB), $0-88
|
|
// Loading 7 of 49 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 58 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_7x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU Y7, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU Y8, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU Y9, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU Y10, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU Y11, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU Y12, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU Y13, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R12
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x7_64Xor(SB), $0-88
|
|
// Loading 23 of 49 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 58 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulGFNI_7x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z23
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z24
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z25
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z26
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z27
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z28
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 Z23, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 Z24, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 Z25, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 Z26, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 Z27, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 Z28, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 Z29, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R12
|
|
DECQ AX
|
|
JNZ mulGFNI_7x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x7Xor(SB), $0-88
|
|
// Loading 7 of 49 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 58 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_7x7Xor_loop:
|
|
// Load 7 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU (R13)(R12*1), Y7
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y8
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y9
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y10
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y11
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y12
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU Y7, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU Y8, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU Y9, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU Y10, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU Y11, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU Y12, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU Y13, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R12
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x8_64(SB), $0-88
|
|
// Loading 22 of 56 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 66 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulGFNI_7x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 Z22, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 Z23, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 Z24, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 Z25, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 Z26, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 Z27, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 Z28, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU64 Z29, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R12
|
|
DECQ AX
|
|
JNZ mulGFNI_7x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x8(SB), $0-88
|
|
// Loading 6 of 56 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 66 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_7x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 56(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU Y6, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU Y7, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU Y8, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU Y9, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU Y10, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU Y11, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU Y12, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU Y13, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R12
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x8_64Xor(SB), $0-88
|
|
// Loading 22 of 56 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 66 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulGFNI_7x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z22
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z23
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z24
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z25
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z26
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z27
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z28
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 Z22, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 Z23, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 Z24, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 Z25, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 Z26, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 Z27, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 Z28, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU64 Z29, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R12
|
|
DECQ AX
|
|
JNZ mulGFNI_7x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x8Xor(SB), $0-88
|
|
// Loading 6 of 56 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 66 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_7x8Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU (R13)(R12*1), Y6
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y7
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y8
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y9
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y10
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y11
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y12
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU Y6, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU Y7, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU Y8, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU Y9, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU Y10, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU Y11, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU Y12, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU Y13, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R12
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x9_64(SB), $0-88
|
|
// Loading 21 of 63 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 74 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulGFNI_7x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 Z21, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 Z22, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 Z23, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 Z24, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 Z25, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 Z26, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 Z27, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU64 Z28, (R13)(R12*1)
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU64 Z29, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R12
|
|
DECQ AX
|
|
JNZ mulGFNI_7x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x9(SB), $0-88
|
|
// Loading 5 of 63 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 74 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_7x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 64(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU Y5, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU Y6, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU Y7, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU Y8, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU Y9, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU Y10, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU Y11, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU Y12, (R13)(R12*1)
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU Y13, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R12
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x9_64Xor(SB), $0-88
|
|
// Loading 21 of 63 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 74 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulGFNI_7x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z21
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z22
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z23
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z24
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z25
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z26
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z27
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z28
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 Z21, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 Z22, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 Z23, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 Z24, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 Z25, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 Z26, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 Z27, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU64 Z28, (R13)(R12*1)
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU64 Z29, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R12
|
|
DECQ AX
|
|
JNZ mulGFNI_7x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x9Xor(SB), $0-88
|
|
// Loading 5 of 63 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 74 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_7x9Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU (R13)(R12*1), Y5
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y6
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y7
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y8
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y9
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y10
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y11
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y12
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU Y5, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU Y6, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU Y7, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU Y8, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU Y9, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU Y10, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU Y11, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU Y12, (R13)(R12*1)
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU Y13, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R12
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x10_64(SB), $0-88
|
|
// Loading 20 of 70 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 82 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulGFNI_7x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 Z20, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 Z21, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 Z22, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 Z23, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 Z24, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 Z25, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 Z26, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU64 Z27, (R13)(R12*1)
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU64 Z28, (R13)(R12*1)
|
|
MOVQ 216(R11), R13
|
|
VMOVDQU64 Z29, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R12
|
|
DECQ AX
|
|
JNZ mulGFNI_7x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x10(SB), $0-88
|
|
// Loading 4 of 70 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 82 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_7x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 72(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU Y4, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU Y5, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU Y6, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU Y7, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU Y8, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU Y9, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU Y10, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU Y11, (R13)(R12*1)
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU Y12, (R13)(R12*1)
|
|
MOVQ 216(R11), R13
|
|
VMOVDQU Y13, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R12
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_7x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_7x10_64Xor(SB), $0-88
|
|
// Loading 20 of 70 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 82 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_7x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulGFNI_7x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z20
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z21
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z22
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z23
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z24
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z25
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z26
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z27
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z28
|
|
MOVQ 216(R11), R13
|
|
VMOVDQU64 (R13)(R12*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU64 Z20, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU64 Z21, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU64 Z22, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU64 Z23, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU64 Z24, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU64 Z25, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU64 Z26, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU64 Z27, (R13)(R12*1)
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU64 Z28, (R13)(R12*1)
|
|
MOVQ 216(R11), R13
|
|
VMOVDQU64 Z29, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R12
|
|
DECQ AX
|
|
JNZ mulGFNI_7x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_7x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_7x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_7x10Xor(SB), $0-88
|
|
// Loading 4 of 70 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 82 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_7x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), DX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, DX
|
|
|
|
mulAvxGFNI_7x10Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU (R13)(R12*1), Y4
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y5
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y6
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y7
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y8
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y9
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y10
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y11
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y12
|
|
MOVQ 216(R11), R13
|
|
VMOVDQU (R13)(R12*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R11), R13
|
|
VMOVDQU Y4, (R13)(R12*1)
|
|
MOVQ 24(R11), R13
|
|
VMOVDQU Y5, (R13)(R12*1)
|
|
MOVQ 48(R11), R13
|
|
VMOVDQU Y6, (R13)(R12*1)
|
|
MOVQ 72(R11), R13
|
|
VMOVDQU Y7, (R13)(R12*1)
|
|
MOVQ 96(R11), R13
|
|
VMOVDQU Y8, (R13)(R12*1)
|
|
MOVQ 120(R11), R13
|
|
VMOVDQU Y9, (R13)(R12*1)
|
|
MOVQ 144(R11), R13
|
|
VMOVDQU Y10, (R13)(R12*1)
|
|
MOVQ 168(R11), R13
|
|
VMOVDQU Y11, (R13)(R12*1)
|
|
MOVQ 192(R11), R13
|
|
VMOVDQU Y12, (R13)(R12*1)
|
|
MOVQ 216(R11), R13
|
|
VMOVDQU Y13, (R13)(R12*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R12
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_7x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_7x10Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 11 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), CX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_8x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z9
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z9
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z9
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z9
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z9
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (R9), Z9
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z5, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 6 to 1 outputs
|
|
VMOVDQU64 (R10), Z9
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z6, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 7 to 1 outputs
|
|
VMOVDQU64 (CX), Z9
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z7, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z8, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_8x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 11 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), CX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, CX
|
|
|
|
mulAvxGFNI_8x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y9
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y9
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y9
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y9
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y9
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (R9), Y9
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y5, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 6 to 1 outputs
|
|
VMOVDQU (R10), Y9
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y6, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 7 to 1 outputs
|
|
VMOVDQU (CX), Y9
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y7, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 11 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), CX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, CX
|
|
|
|
mulGFNI_8x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (R11), Z8
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z9
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z9
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z9
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z9
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z9
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (R9), Z9
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z5, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 6 to 1 outputs
|
|
VMOVDQU64 (R10), Z9
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z6, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Load and process 64 bytes from input 7 to 1 outputs
|
|
VMOVDQU64 (CX), Z9
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z7, Z9, Z9
|
|
VXORPD Z8, Z9, Z8
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z8, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_8x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 11 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), CX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R11
|
|
MOVQ start+72(FP), R12
|
|
|
|
// Add start offset to output
|
|
ADDQ R12, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R12, DX
|
|
ADDQ R12, BX
|
|
ADDQ R12, SI
|
|
ADDQ R12, DI
|
|
ADDQ R12, R8
|
|
ADDQ R12, R9
|
|
ADDQ R12, R10
|
|
ADDQ R12, CX
|
|
|
|
mulAvxGFNI_8x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (R11), Y8
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y9
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y9
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y9
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y9
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y9
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (R9), Y9
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y5, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 6 to 1 outputs
|
|
VMOVDQU (R10), Y9
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y6, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Load and process 32 bytes from input 7 to 1 outputs
|
|
VMOVDQU (CX), Y9
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y7, Y9, Y9
|
|
VXORPD Y8, Y9, Y8
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y8, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), CX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R11
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R12
|
|
ADDQ R13, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_8x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z18
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z18, Z16
|
|
VGF2P8AFFINEQB $0x00, Z1, Z18, Z17
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z18
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z18
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z18
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z18
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (R9), Z18
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 6 to 2 outputs
|
|
VMOVDQU64 (R10), Z18
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z12, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z13, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 7 to 2 outputs
|
|
VMOVDQU64 (CX), Z18
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z14, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z15, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z16, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z17, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_8x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x2(SB), $0-88
|
|
// Loading 12 of 16 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R12
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R13
|
|
ADDQ R14, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_8x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 2 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), CX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R11
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R12
|
|
ADDQ R13, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_8x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (R12), Z16
|
|
VMOVDQU64 (R11), Z17
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z18
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z1, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z18
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z3, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z18
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z5, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z18
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z7, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z18
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z9, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (R9), Z18
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z10, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z11, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 6 to 2 outputs
|
|
VMOVDQU64 (R10), Z18
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z12, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z13, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Load and process 64 bytes from input 7 to 2 outputs
|
|
VMOVDQU64 (CX), Z18
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z14, Z18, Z19
|
|
VXORPD Z16, Z19, Z16
|
|
VGF2P8AFFINEQB $0x00, Z15, Z18, Z19
|
|
VXORPD Z17, Z19, Z17
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z16, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z17, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_8x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x2Xor(SB), $0-88
|
|
// Loading 12 of 16 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 20 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R12
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R13
|
|
ADDQ R14, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_8x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (R13), Y12
|
|
VMOVDQU (R12), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 2 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y13, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x3_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 29 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), CX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R11
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_8x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z27
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z27, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z27, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z27, Z26
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z27
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z27
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z27
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z10, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z27
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (R9), Z27
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z15, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 6 to 3 outputs
|
|
VMOVDQU64 (R10), Z27
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z18, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 7 to 3 outputs
|
|
VMOVDQU64 (CX), Z27
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z21, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z22, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z23, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z24, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z25, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z26, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_8x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x3(SB), $0-88
|
|
// Loading 11 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 29 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R12
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_8x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 3 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x3_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 29 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), CX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R11
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R12
|
|
ADDQ R14, R13
|
|
ADDQ R14, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_8x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (R12), Z24
|
|
VMOVDQU64 (R13), Z25
|
|
VMOVDQU64 (R11), Z26
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z27
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z27
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z27
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z27
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z10, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z27
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (R9), Z27
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z15, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 6 to 3 outputs
|
|
VMOVDQU64 (R10), Z27
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z18, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Load and process 64 bytes from input 7 to 3 outputs
|
|
VMOVDQU64 (CX), Z27
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z21, Z27, Z28
|
|
VXORPD Z24, Z28, Z24
|
|
VGF2P8AFFINEQB $0x00, Z22, Z27, Z28
|
|
VXORPD Z25, Z28, Z25
|
|
VGF2P8AFFINEQB $0x00, Z23, Z27, Z28
|
|
VXORPD Z26, Z28, Z26
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z24, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z25, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z26, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_8x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x3Xor(SB), $0-88
|
|
// Loading 11 of 24 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 29 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R12
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_8x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (R13), Y11
|
|
VMOVDQU (R14), Y12
|
|
VMOVDQU (R12), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 3 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x4_64(SB), $8-88
|
|
// Loading 26 of 32 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R15
|
|
MOVQ 72(R12), R12
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_8x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 4 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 4 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_8x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x4(SB), $8-88
|
|
// Loading 10 of 32 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R15
|
|
MOVQ 72(R12), R12
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_8x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 4 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x4_64Xor(SB), $8-88
|
|
// Loading 26 of 32 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R15
|
|
MOVQ 72(R12), R12
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, DX
|
|
|
|
mulGFNI_8x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R12), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 4 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 4 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_8x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x4Xor(SB), $8-88
|
|
// Loading 10 of 32 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 38 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R15
|
|
MOVQ 72(R12), R12
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_8x4Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R12), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 4 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x5_64(SB), $8-88
|
|
// Loading 25 of 40 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 47 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), AX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R15
|
|
MOVQ 96(R11), R11
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_8x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 5 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 5 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_8x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x5(SB), $8-88
|
|
// Loading 9 of 40 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 47 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), AX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R15
|
|
MOVQ 96(R11), R11
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_8x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 5 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 5 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_8x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x5_64Xor(SB), $8-88
|
|
// Loading 25 of 40 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 47 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), AX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R15
|
|
MOVQ 96(R11), R11
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_8x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU64 (R12), Z25
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R11), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 5 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 5 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU64 Z25, (R12)
|
|
ADDQ $0x40, R12
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R11)
|
|
ADDQ $0x40, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_8x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x5Xor(SB), $8-88
|
|
// Loading 9 of 40 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 47 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), AX
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ out_base+48(FP), R11
|
|
MOVQ (R11), R12
|
|
MOVQ 24(R11), R13
|
|
MOVQ 48(R11), R14
|
|
MOVQ 72(R11), R15
|
|
MOVQ 96(R11), R11
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R11
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_8x5Xor_loop:
|
|
// Load 5 outputs
|
|
VMOVDQU (R12), Y9
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R11), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 5 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 5 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R11)
|
|
ADDQ $0x20, R11
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_8x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x6_64(SB), $0-88
|
|
// Loading 24 of 48 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 56 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 6 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x6(SB), $0-88
|
|
// Loading 8 of 48 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 56 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 6 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x6_64Xor(SB), $0-88
|
|
// Loading 24 of 48 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 56 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z24
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z25
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z26
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z27
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z28
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 6 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x6Xor(SB), $0-88
|
|
// Loading 8 of 48 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 56 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x6Xor_loop:
|
|
// Load 6 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU (R14)(R13*1), Y8
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y9
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y10
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y11
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y12
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 6 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x7_64(SB), $0-88
|
|
// Loading 23 of 56 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 65 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 7 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z23, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x7(SB), $0-88
|
|
// Loading 7 of 56 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 65 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 7 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y7, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x7_64Xor(SB), $0-88
|
|
// Loading 23 of 56 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 65 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z23
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z24
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z25
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z26
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z27
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z28
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 7 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z23, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x7Xor(SB), $0-88
|
|
// Loading 7 of 56 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 65 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x7Xor_loop:
|
|
// Load 7 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU (R14)(R13*1), Y7
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y8
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y9
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y10
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y11
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y12
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 7 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y7, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x8_64(SB), $0-88
|
|
// Loading 22 of 64 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 74 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 8 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z22, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z23, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x8(SB), $0-88
|
|
// Loading 6 of 64 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 74 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 56(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 8 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y6, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y7, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x8_64Xor(SB), $0-88
|
|
// Loading 22 of 64 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 74 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z22
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z23
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z24
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z25
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z26
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z27
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z28
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 8 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z22, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z23, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x8Xor(SB), $0-88
|
|
// Loading 6 of 64 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 74 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x8Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU (R14)(R13*1), Y6
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y7
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y8
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y9
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y10
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y11
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y12
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 8 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y6, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y7, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x9_64(SB), $0-88
|
|
// Loading 21 of 72 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 83 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 9 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z21, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z22, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z23, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x9(SB), $0-88
|
|
// Loading 5 of 72 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 83 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 64(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 9 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y5, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y6, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y7, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x9_64Xor(SB), $0-88
|
|
// Loading 21 of 72 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 83 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z21
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z22
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z23
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z24
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z25
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z26
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z27
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z28
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 9 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z21, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z22, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z23, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x9Xor(SB), $0-88
|
|
// Loading 5 of 72 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 83 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x9Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU (R14)(R13*1), Y5
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y6
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y7
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y8
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y9
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y10
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y11
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y12
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 9 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y5, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y6, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y7, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x10_64(SB), $0-88
|
|
// Loading 20 of 80 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 92 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 10 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z20, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z21, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z22, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z23, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 216(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x10(SB), $0-88
|
|
// Loading 4 of 80 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 92 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 72(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 10 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y4, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y5, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y6, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y7, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 216(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_8x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_8x10_64Xor(SB), $0-88
|
|
// Loading 20 of 80 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 92 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_8x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulGFNI_8x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z20
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z21
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z22
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z23
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z24
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z25
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z26
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z27
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z28
|
|
MOVQ 216(R12), R14
|
|
VMOVDQU64 (R14)(R13*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 10 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU64 Z20, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU64 Z21, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU64 Z22, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU64 Z23, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU64 Z24, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU64 Z25, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU64 Z26, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU64 Z27, (R14)(R13*1)
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU64 Z28, (R14)(R13*1)
|
|
MOVQ 216(R12), R14
|
|
VMOVDQU64 Z29, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R13
|
|
DECQ AX
|
|
JNZ mulGFNI_8x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_8x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_8x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_8x10Xor(SB), $0-88
|
|
// Loading 4 of 80 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 92 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_8x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), DX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, DX
|
|
|
|
mulAvxGFNI_8x10Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU (R14)(R13*1), Y4
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y5
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y6
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y7
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y8
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y9
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y10
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y11
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y12
|
|
MOVQ 216(R12), R14
|
|
VMOVDQU (R14)(R13*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 10 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R12), R14
|
|
VMOVDQU Y4, (R14)(R13*1)
|
|
MOVQ 24(R12), R14
|
|
VMOVDQU Y5, (R14)(R13*1)
|
|
MOVQ 48(R12), R14
|
|
VMOVDQU Y6, (R14)(R13*1)
|
|
MOVQ 72(R12), R14
|
|
VMOVDQU Y7, (R14)(R13*1)
|
|
MOVQ 96(R12), R14
|
|
VMOVDQU Y8, (R14)(R13*1)
|
|
MOVQ 120(R12), R14
|
|
VMOVDQU Y9, (R14)(R13*1)
|
|
MOVQ 144(R12), R14
|
|
VMOVDQU Y10, (R14)(R13*1)
|
|
MOVQ 168(R12), R14
|
|
VMOVDQU Y11, (R14)(R13*1)
|
|
MOVQ 192(R12), R14
|
|
VMOVDQU Y12, (R14)(R13*1)
|
|
MOVQ 216(R12), R14
|
|
VMOVDQU Y13, (R14)(R13*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R13
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_8x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_8x10Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), CX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_9x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z10
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z10
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z10
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z10
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z10
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (R9), Z10
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z5, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 6 to 1 outputs
|
|
VMOVDQU64 (R10), Z10
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z6, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 7 to 1 outputs
|
|
VMOVDQU64 (R11), Z10
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z7, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 8 to 1 outputs
|
|
VMOVDQU64 (CX), Z10
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z9, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_9x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), CX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, CX
|
|
|
|
mulAvxGFNI_9x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y10
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y10
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y10
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y10
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y10
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (R9), Y10
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y5, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 6 to 1 outputs
|
|
VMOVDQU (R10), Y10
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y6, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 7 to 1 outputs
|
|
VMOVDQU (R11), Y10
|
|
ADDQ $0x20, R11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 8 to 1 outputs
|
|
VMOVDQU (CX), Y10
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y8, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), CX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, CX
|
|
|
|
mulGFNI_9x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (R12), Z9
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z10
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z10
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z10
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z10
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z10
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (R9), Z10
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z5, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 6 to 1 outputs
|
|
VMOVDQU64 (R10), Z10
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z6, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 7 to 1 outputs
|
|
VMOVDQU64 (R11), Z10
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z7, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Load and process 64 bytes from input 8 to 1 outputs
|
|
VMOVDQU64 (CX), Z10
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z8, Z10, Z10
|
|
VXORPD Z9, Z10, Z9
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z9, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_9x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 12 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), CX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R12
|
|
MOVQ start+72(FP), R13
|
|
|
|
// Add start offset to output
|
|
ADDQ R13, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R13, DX
|
|
ADDQ R13, BX
|
|
ADDQ R13, SI
|
|
ADDQ R13, DI
|
|
ADDQ R13, R8
|
|
ADDQ R13, R9
|
|
ADDQ R13, R10
|
|
ADDQ R13, R11
|
|
ADDQ R13, CX
|
|
|
|
mulAvxGFNI_9x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (R12), Y9
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y10
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y10
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y10
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y10
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y10
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (R9), Y10
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y5, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 6 to 1 outputs
|
|
VMOVDQU (R10), Y10
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y6, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 7 to 1 outputs
|
|
VMOVDQU (R11), Y10
|
|
ADDQ $0x20, R11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Load and process 32 bytes from input 8 to 1 outputs
|
|
VMOVDQU (CX), Y10
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y8, Y10, Y10
|
|
VXORPD Y9, Y10, Y9
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y9, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), CX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R12
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R13
|
|
ADDQ R14, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_9x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z20
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z20, Z18
|
|
VGF2P8AFFINEQB $0x00, Z1, Z20, Z19
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z20
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z20
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z20
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z20
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (R9), Z20
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 6 to 2 outputs
|
|
VMOVDQU64 (R10), Z20
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 7 to 2 outputs
|
|
VMOVDQU64 (R11), Z20
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z15, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 8 to 2 outputs
|
|
VMOVDQU64 (CX), Z20
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z16, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z17, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z18, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z19, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_9x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x2(SB), $0-88
|
|
// Loading 12 of 18 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R13
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R14
|
|
ADDQ R15, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_9x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 2 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 2 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R13)
|
|
ADDQ $0x20, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), CX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R12
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R13
|
|
ADDQ R14, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_9x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (R13), Z18
|
|
VMOVDQU64 (R12), Z19
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z20
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z1, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z20
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z3, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z20
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z5, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z20
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z7, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z20
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z9, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (R9), Z20
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z10, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z11, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 6 to 2 outputs
|
|
VMOVDQU64 (R10), Z20
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z12, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z13, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 7 to 2 outputs
|
|
VMOVDQU64 (R11), Z20
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z14, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z15, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Load and process 64 bytes from input 8 to 2 outputs
|
|
VMOVDQU64 (CX), Z20
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z16, Z20, Z21
|
|
VXORPD Z18, Z21, Z18
|
|
VGF2P8AFFINEQB $0x00, Z17, Z20, Z21
|
|
VXORPD Z19, Z21, Z19
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z18, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z19, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_9x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x2Xor(SB), $0-88
|
|
// Loading 12 of 18 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 22 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R13
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R14
|
|
ADDQ R15, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_9x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (R14), Y12
|
|
VMOVDQU (R13), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 2 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 2 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y13, (R13)
|
|
ADDQ $0x20, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x3_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
VBROADCASTF32X2 208(CX), Z26
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), CX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R12
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, DX
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, CX
|
|
|
|
mulGFNI_9x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 3 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 3 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 3 outputs
|
|
VMOVDQU64 (CX), Z30
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z26, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_9x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x3(SB), $8-88
|
|
// Loading 11 of 27 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R15
|
|
MOVQ 48(R13), R13
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_9x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 3 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 3 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R13)
|
|
ADDQ $0x20, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x3_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
VBROADCASTF32X2 208(CX), Z26
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), CX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R12
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R13
|
|
ADDQ R15, R14
|
|
ADDQ R15, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, DX
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, CX
|
|
|
|
mulGFNI_9x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (R13), Z27
|
|
VMOVDQU64 (R14), Z28
|
|
VMOVDQU64 (R12), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 3 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 3 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 3 outputs
|
|
VMOVDQU64 (CX), Z30
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z26, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z27, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z28, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z29, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_9x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x3Xor(SB), $8-88
|
|
// Loading 11 of 27 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 32 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R15
|
|
MOVQ 48(R13), R13
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_9x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R13), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 3 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 3 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R13)
|
|
ADDQ $0x20, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x4_64(SB), $8-88
|
|
// Loading 26 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), R11
|
|
MOVQ 192(AX), AX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R15
|
|
MOVQ 72(R12), R12
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_9x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 4 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 4 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 4 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_9x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x4(SB), $8-88
|
|
// Loading 10 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), R11
|
|
MOVQ 192(AX), AX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R15
|
|
MOVQ 72(R12), R12
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_9x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 4 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 4 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 4 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_9x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x4_64Xor(SB), $8-88
|
|
// Loading 26 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), R11
|
|
MOVQ 192(AX), AX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R15
|
|
MOVQ 72(R12), R12
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_9x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU64 (R13), Z26
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R12), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 4 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 4 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 4 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU64 Z26, (R13)
|
|
ADDQ $0x40, R13
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R12)
|
|
ADDQ $0x40, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_9x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x4Xor(SB), $8-88
|
|
// Loading 10 of 36 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 42 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), R11
|
|
MOVQ 192(AX), AX
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ out_base+48(FP), R12
|
|
MOVQ (R12), R13
|
|
MOVQ 24(R12), R14
|
|
MOVQ 48(R12), R15
|
|
MOVQ 72(R12), R12
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R13
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R12
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_9x4Xor_loop:
|
|
// Load 4 outputs
|
|
VMOVDQU (R13), Y10
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R12), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 4 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 4 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 4 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R12)
|
|
ADDQ $0x20, R12
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_9x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x5_64(SB), $0-88
|
|
// Loading 25 of 45 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 52 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 5 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 5 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x5(SB), $0-88
|
|
// Loading 9 of 45 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 52 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 5 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 5 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x5_64Xor(SB), $0-88
|
|
// Loading 25 of 45 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 52 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z25
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z26
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z27
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z28
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 5 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 5 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x5Xor(SB), $0-88
|
|
// Loading 9 of 45 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 52 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x5Xor_loop:
|
|
// Load 5 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU (R15)(R14*1), Y9
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y10
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y11
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y12
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 5 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 5 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x6_64(SB), $0-88
|
|
// Loading 24 of 54 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 62 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 6 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 6 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x6(SB), $0-88
|
|
// Loading 8 of 54 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 62 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 6 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 6 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x6_64Xor(SB), $0-88
|
|
// Loading 24 of 54 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 62 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z24
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z25
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z26
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z27
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z28
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 6 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 6 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x6Xor(SB), $0-88
|
|
// Loading 8 of 54 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 62 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x6Xor_loop:
|
|
// Load 6 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU (R15)(R14*1), Y8
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y9
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y10
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y11
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y12
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 6 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 6 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x7_64(SB), $0-88
|
|
// Loading 23 of 63 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 72 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 7 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 7 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z23, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x7(SB), $0-88
|
|
// Loading 7 of 63 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 72 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 7 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 7 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y7, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x7_64Xor(SB), $0-88
|
|
// Loading 23 of 63 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 72 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z23
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z24
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z25
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z26
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z27
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z28
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 7 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 7 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z23, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x7Xor(SB), $0-88
|
|
// Loading 7 of 63 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 72 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x7Xor_loop:
|
|
// Load 7 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU (R15)(R14*1), Y7
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y8
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y9
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y10
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y11
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y12
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 7 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 7 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y7, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x8_64(SB), $0-88
|
|
// Loading 22 of 72 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 82 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 8 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 8 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z22, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z23, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x8(SB), $0-88
|
|
// Loading 6 of 72 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 82 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 56(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 8 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 8 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y6, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y7, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x8_64Xor(SB), $0-88
|
|
// Loading 22 of 72 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 82 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z22
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z23
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z24
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z25
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z26
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z27
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z28
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 8 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 8 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z22, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z23, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x8Xor(SB), $0-88
|
|
// Loading 6 of 72 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 82 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x8Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU (R15)(R14*1), Y6
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y7
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y8
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y9
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y10
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y11
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y12
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 8 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 8 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y6, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y7, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x9_64(SB), $0-88
|
|
// Loading 21 of 81 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 92 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 9 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 9 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z21, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z22, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z23, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x9(SB), $0-88
|
|
// Loading 5 of 81 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 92 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 64(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 9 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 9 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 640(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y5, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y6, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y7, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x9_64Xor(SB), $0-88
|
|
// Loading 21 of 81 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 92 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z21
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z22
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z23
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z24
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z25
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z26
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z27
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z28
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 9 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 9 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z21, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z22, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z23, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x9Xor(SB), $0-88
|
|
// Loading 5 of 81 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 92 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x9Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU (R15)(R14*1), Y5
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y6
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y7
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y8
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y9
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y10
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y11
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y12
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 9 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 9 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 640(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y5, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y6, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y7, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x10_64(SB), $0-88
|
|
// Loading 20 of 90 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 102 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 10 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 10 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z20, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z21, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z22, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z23, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 216(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x10(SB), $0-88
|
|
// Loading 4 of 90 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 102 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 72(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 10 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 10 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 640(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 648(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 656(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 664(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 672(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 680(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 688(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 696(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 704(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 712(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y4, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y5, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y6, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y7, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 216(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_9x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_9x10_64Xor(SB), $0-88
|
|
// Loading 20 of 90 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 102 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_9x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulGFNI_9x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z20
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z21
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z22
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z23
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z24
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z25
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z26
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z27
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z28
|
|
MOVQ 216(R13), R15
|
|
VMOVDQU64 (R15)(R14*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 10 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 10 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU64 Z20, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU64 Z21, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU64 Z22, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU64 Z23, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU64 Z24, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU64 Z25, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU64 Z26, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU64 Z27, (R15)(R14*1)
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU64 Z28, (R15)(R14*1)
|
|
MOVQ 216(R13), R15
|
|
VMOVDQU64 Z29, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R14
|
|
DECQ AX
|
|
JNZ mulGFNI_9x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_9x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_9x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_9x10Xor(SB), $0-88
|
|
// Loading 4 of 90 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 102 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_9x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), DX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, DX
|
|
|
|
mulAvxGFNI_9x10Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU (R15)(R14*1), Y4
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y5
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y6
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y7
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y8
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y9
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y10
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y11
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y12
|
|
MOVQ 216(R13), R15
|
|
VMOVDQU (R15)(R14*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 10 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 10 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 640(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 648(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 656(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 664(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 672(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 680(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 688(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 696(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 704(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 712(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R13), R15
|
|
VMOVDQU Y4, (R15)(R14*1)
|
|
MOVQ 24(R13), R15
|
|
VMOVDQU Y5, (R15)(R14*1)
|
|
MOVQ 48(R13), R15
|
|
VMOVDQU Y6, (R15)(R14*1)
|
|
MOVQ 72(R13), R15
|
|
VMOVDQU Y7, (R15)(R14*1)
|
|
MOVQ 96(R13), R15
|
|
VMOVDQU Y8, (R15)(R14*1)
|
|
MOVQ 120(R13), R15
|
|
VMOVDQU Y9, (R15)(R14*1)
|
|
MOVQ 144(R13), R15
|
|
VMOVDQU Y10, (R15)(R14*1)
|
|
MOVQ 168(R13), R15
|
|
VMOVDQU Y11, (R15)(R14*1)
|
|
MOVQ 192(R13), R15
|
|
VMOVDQU Y12, (R15)(R14*1)
|
|
MOVQ 216(R13), R15
|
|
VMOVDQU Y13, (R15)(R14*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R14
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_9x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_9x10Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x1_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x1_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 13 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x1_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), R12
|
|
MOVQ 216(CX), CX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_10x1_64_loop:
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z11
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z11
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z11
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z11
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z11
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (R9), Z11
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z5, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 6 to 1 outputs
|
|
VMOVDQU64 (R10), Z11
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z6, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 7 to 1 outputs
|
|
VMOVDQU64 (R11), Z11
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z7, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 8 to 1 outputs
|
|
VMOVDQU64 (R12), Z11
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB $0x00, Z8, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 9 to 1 outputs
|
|
VMOVDQU64 (CX), Z11
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z9, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z10, (R13)
|
|
ADDQ $0x40, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_10x1_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x1_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x1(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x1(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 13 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x1_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), R12
|
|
MOVQ 216(CX), CX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, CX
|
|
|
|
mulAvxGFNI_10x1_loop:
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y11
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y11
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y11
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y11
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y11
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (R9), Y11
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y5, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 6 to 1 outputs
|
|
VMOVDQU (R10), Y11
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y6, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 7 to 1 outputs
|
|
VMOVDQU (R11), Y11
|
|
ADDQ $0x20, R11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 8 to 1 outputs
|
|
VMOVDQU (R12), Y11
|
|
ADDQ $0x20, R12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 9 to 1 outputs
|
|
VMOVDQU (CX), Y11
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y9, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x1_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x1_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x1_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x1_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 13 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x1_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), R12
|
|
MOVQ 216(CX), CX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, CX
|
|
|
|
mulGFNI_10x1_64Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU64 (R13), Z10
|
|
|
|
// Load and process 64 bytes from input 0 to 1 outputs
|
|
VMOVDQU64 (DX), Z11
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 1 to 1 outputs
|
|
VMOVDQU64 (BX), Z11
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z1, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 2 to 1 outputs
|
|
VMOVDQU64 (SI), Z11
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z2, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 3 to 1 outputs
|
|
VMOVDQU64 (DI), Z11
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z3, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 4 to 1 outputs
|
|
VMOVDQU64 (R8), Z11
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z4, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 5 to 1 outputs
|
|
VMOVDQU64 (R9), Z11
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z5, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 6 to 1 outputs
|
|
VMOVDQU64 (R10), Z11
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z6, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 7 to 1 outputs
|
|
VMOVDQU64 (R11), Z11
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z7, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 8 to 1 outputs
|
|
VMOVDQU64 (R12), Z11
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB $0x00, Z8, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Load and process 64 bytes from input 9 to 1 outputs
|
|
VMOVDQU64 (CX), Z11
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z9, Z11, Z11
|
|
VXORPD Z10, Z11, Z10
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU64 Z10, (R13)
|
|
ADDQ $0x40, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_10x1_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x1_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x1Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x1Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 13 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x1Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), R12
|
|
MOVQ 216(CX), CX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R13
|
|
MOVQ start+72(FP), R14
|
|
|
|
// Add start offset to output
|
|
ADDQ R14, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R14, DX
|
|
ADDQ R14, BX
|
|
ADDQ R14, SI
|
|
ADDQ R14, DI
|
|
ADDQ R14, R8
|
|
ADDQ R14, R9
|
|
ADDQ R14, R10
|
|
ADDQ R14, R11
|
|
ADDQ R14, R12
|
|
ADDQ R14, CX
|
|
|
|
mulAvxGFNI_10x1Xor_loop:
|
|
// Load 1 outputs
|
|
VMOVDQU (R13), Y10
|
|
|
|
// Load and process 32 bytes from input 0 to 1 outputs
|
|
VMOVDQU (DX), Y11
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 1 to 1 outputs
|
|
VMOVDQU (BX), Y11
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y1, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 2 to 1 outputs
|
|
VMOVDQU (SI), Y11
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 3 to 1 outputs
|
|
VMOVDQU (DI), Y11
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y3, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 4 to 1 outputs
|
|
VMOVDQU (R8), Y11
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 5 to 1 outputs
|
|
VMOVDQU (R9), Y11
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y5, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 6 to 1 outputs
|
|
VMOVDQU (R10), Y11
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y6, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 7 to 1 outputs
|
|
VMOVDQU (R11), Y11
|
|
ADDQ $0x20, R11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 8 to 1 outputs
|
|
VMOVDQU (R12), Y11
|
|
ADDQ $0x20, R12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Load and process 32 bytes from input 9 to 1 outputs
|
|
VMOVDQU (CX), Y11
|
|
ADDQ $0x20, CX
|
|
VGF2P8AFFINEQB $0x00, Y9, Y11, Y11
|
|
VXORPD Y10, Y11, Y10
|
|
|
|
// Store 1 outputs
|
|
VMOVDQU Y10, (R13)
|
|
ADDQ $0x20, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x1Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x1Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x2_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x2_64(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 24 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x2_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), R12
|
|
MOVQ 216(CX), CX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R13
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R14
|
|
ADDQ R15, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, DX
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, CX
|
|
|
|
mulGFNI_10x2_64_loop:
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z22
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z22, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z22, Z21
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z22
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z3, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z22
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z5, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z22
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z7, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z22
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z9, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (R9), Z22
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z10, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 6 to 2 outputs
|
|
VMOVDQU64 (R10), Z22
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z12, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z13, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 7 to 2 outputs
|
|
VMOVDQU64 (R11), Z22
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z14, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z15, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 8 to 2 outputs
|
|
VMOVDQU64 (R12), Z22
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB $0x00, Z16, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z17, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 9 to 2 outputs
|
|
VMOVDQU64 (CX), Z22
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z18, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z19, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z20, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z21, (R13)
|
|
ADDQ $0x40, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_10x2_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x2_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x2(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x2(SB), $8-88
|
|
// Loading 12 of 20 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 24 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x2_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ (R14), R15
|
|
MOVQ 24(R14), R14
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R15
|
|
ADDQ BP, R14
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_10x2_loop:
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 2 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 2 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 2 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R14)
|
|
ADDQ $0x20, R14
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x2_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x2_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x2_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x2_64Xor(SB), $0-88
|
|
// Loading all tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 24 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x2_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), CX
|
|
MOVQ (CX), DX
|
|
MOVQ 24(CX), BX
|
|
MOVQ 48(CX), SI
|
|
MOVQ 72(CX), DI
|
|
MOVQ 96(CX), R8
|
|
MOVQ 120(CX), R9
|
|
MOVQ 144(CX), R10
|
|
MOVQ 168(CX), R11
|
|
MOVQ 192(CX), R12
|
|
MOVQ 216(CX), CX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R13
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to output
|
|
ADDQ R15, R14
|
|
ADDQ R15, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, DX
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, CX
|
|
|
|
mulGFNI_10x2_64Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU64 (R14), Z20
|
|
VMOVDQU64 (R13), Z21
|
|
|
|
// Load and process 64 bytes from input 0 to 2 outputs
|
|
VMOVDQU64 (DX), Z22
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 1 to 2 outputs
|
|
VMOVDQU64 (BX), Z22
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z2, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z3, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 2 to 2 outputs
|
|
VMOVDQU64 (SI), Z22
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z5, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 3 to 2 outputs
|
|
VMOVDQU64 (DI), Z22
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z7, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 4 to 2 outputs
|
|
VMOVDQU64 (R8), Z22
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z8, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z9, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 5 to 2 outputs
|
|
VMOVDQU64 (R9), Z22
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z10, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 6 to 2 outputs
|
|
VMOVDQU64 (R10), Z22
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z12, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z13, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 7 to 2 outputs
|
|
VMOVDQU64 (R11), Z22
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z14, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z15, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 8 to 2 outputs
|
|
VMOVDQU64 (R12), Z22
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB $0x00, Z16, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z17, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Load and process 64 bytes from input 9 to 2 outputs
|
|
VMOVDQU64 (CX), Z22
|
|
ADDQ $0x40, CX
|
|
VGF2P8AFFINEQB $0x00, Z18, Z22, Z23
|
|
VXORPD Z20, Z23, Z20
|
|
VGF2P8AFFINEQB $0x00, Z19, Z22, Z23
|
|
VXORPD Z21, Z23, Z21
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU64 Z20, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z21, (R13)
|
|
ADDQ $0x40, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulGFNI_10x2_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x2_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x2Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x2Xor(SB), $8-88
|
|
// Loading 12 of 20 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 24 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x2Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
VBROADCASTSD 88(CX), Y11
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ (R14), R15
|
|
MOVQ 24(R14), R14
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R15
|
|
ADDQ BP, R14
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, R13
|
|
ADDQ BP, DX
|
|
|
|
mulAvxGFNI_10x2Xor_loop:
|
|
// Load 2 outputs
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R14), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 2 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 2 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 2 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 2 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 2 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 2 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 2 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 2 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 2 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 2 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 2 outputs
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R14)
|
|
ADDQ $0x20, R14
|
|
|
|
// Prepare for next loop
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x2Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x2Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x3_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x3_64(SB), $8-88
|
|
// Loading 27 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 35 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x3_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
VBROADCASTF32X2 208(CX), Z26
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), R11
|
|
MOVQ 192(AX), R12
|
|
MOVQ 216(AX), AX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R15
|
|
MOVQ 48(R13), R13
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_10x3_64_loop:
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 3 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 3 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 3 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z26, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 3 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R13)
|
|
ADDQ $0x40, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_10x3_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x3_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x3(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x3(SB), $8-88
|
|
// Loading 11 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 35 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x3_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), R11
|
|
MOVQ 192(AX), R12
|
|
MOVQ 216(AX), AX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R15
|
|
MOVQ 48(R13), R13
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_10x3_loop:
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 3 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 3 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 3 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 3 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R13)
|
|
ADDQ $0x20, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_10x3_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x3_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x3_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x3_64Xor(SB), $8-88
|
|
// Loading 27 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 35 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x3_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
VBROADCASTF32X2 208(CX), Z26
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), R11
|
|
MOVQ 192(AX), R12
|
|
MOVQ 216(AX), AX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R15
|
|
MOVQ 48(R13), R13
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x06, BP
|
|
|
|
mulGFNI_10x3_64Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU64 (R14), Z27
|
|
VMOVDQU64 (R15), Z28
|
|
VMOVDQU64 (R13), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 3 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 3 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 3 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 3 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 3 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 3 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 3 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 3 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 3 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z26, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 3 outputs
|
|
VMOVDQU64 (AX), Z30
|
|
ADDQ $0x40, AX
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU64 Z27, (R14)
|
|
ADDQ $0x40, R14
|
|
VMOVDQU64 Z28, (R15)
|
|
ADDQ $0x40, R15
|
|
VMOVDQU64 Z29, (R13)
|
|
ADDQ $0x40, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulGFNI_10x3_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x3_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x3Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x3Xor(SB), $8-88
|
|
// Loading 11 of 30 tables to registers
|
|
// Destination kept in GP registers
|
|
// Full registers estimated 35 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x3Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
VBROADCASTSD 80(CX), Y10
|
|
MOVQ in_base+24(FP), AX
|
|
MOVQ (AX), DX
|
|
MOVQ 24(AX), BX
|
|
MOVQ 48(AX), SI
|
|
MOVQ 72(AX), DI
|
|
MOVQ 96(AX), R8
|
|
MOVQ 120(AX), R9
|
|
MOVQ 144(AX), R10
|
|
MOVQ 168(AX), R11
|
|
MOVQ 192(AX), R12
|
|
MOVQ 216(AX), AX
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ out_base+48(FP), R13
|
|
MOVQ (R13), R14
|
|
MOVQ 24(R13), R15
|
|
MOVQ 48(R13), R13
|
|
MOVQ start+72(FP), BP
|
|
|
|
// Add start offset to output
|
|
ADDQ BP, R14
|
|
ADDQ BP, R15
|
|
ADDQ BP, R13
|
|
|
|
// Add start offset to input
|
|
ADDQ BP, DX
|
|
ADDQ BP, BX
|
|
ADDQ BP, SI
|
|
ADDQ BP, DI
|
|
ADDQ BP, R8
|
|
ADDQ BP, R9
|
|
ADDQ BP, R10
|
|
ADDQ BP, R11
|
|
ADDQ BP, R12
|
|
ADDQ BP, AX
|
|
|
|
// Reload length to save a register
|
|
MOVQ n+80(FP), BP
|
|
SHRQ $0x05, BP
|
|
|
|
mulAvxGFNI_10x3Xor_loop:
|
|
// Load 3 outputs
|
|
VMOVDQU (R14), Y11
|
|
VMOVDQU (R15), Y12
|
|
VMOVDQU (R13), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 3 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 3 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 3 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 3 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 3 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 3 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 3 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 3 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 3 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 3 outputs
|
|
VMOVDQU (AX), Y14
|
|
ADDQ $0x20, AX
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 3 outputs
|
|
VMOVDQU Y11, (R14)
|
|
ADDQ $0x20, R14
|
|
VMOVDQU Y12, (R15)
|
|
ADDQ $0x20, R15
|
|
VMOVDQU Y13, (R13)
|
|
ADDQ $0x20, R13
|
|
|
|
// Prepare for next loop
|
|
DECQ BP
|
|
JNZ mulAvxGFNI_10x3Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x3Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x4_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x4_64(SB), $8-88
|
|
// Loading 26 of 40 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 46 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x4_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x4_64_loop:
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 4 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 4 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 4 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 4 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 4 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x4_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x4_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x4(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x4(SB), $8-88
|
|
// Loading 10 of 40 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 46 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x4_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x4_loop:
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 4 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 4 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 4 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x4_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x4_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x4_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x4_64Xor(SB), $8-88
|
|
// Loading 26 of 40 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 46 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x4_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
VBROADCASTF32X2 200(CX), Z25
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x4_64Xor_loop:
|
|
// Load 4 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z26
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z27
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z28
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 4 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 4 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 4 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 4 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 4 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 4 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 4 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z25, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 4 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 4 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 4 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 4 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x4_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x4_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x4Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x4Xor(SB), $8-88
|
|
// Loading 10 of 40 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 46 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x4Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
VBROADCASTSD 72(CX), Y9
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x4Xor_loop:
|
|
// Load 4 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU (BP)(R15*1), Y10
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y11
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y12
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 4 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 4 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 4 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 4 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 4 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 4 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 4 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 4 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 4 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 4 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 4 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x4Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x4Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x5_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x5_64(SB), $8-88
|
|
// Loading 25 of 50 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 57 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x5_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x5_64_loop:
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 5 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 5 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 5 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x5_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x5_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x5(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x5(SB), $8-88
|
|
// Loading 9 of 50 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 57 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x5_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x5_loop:
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 5 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 5 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 5 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x5_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x5_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x5_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x5_64Xor(SB), $8-88
|
|
// Loading 25 of 50 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 57 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x5_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
VBROADCASTF32X2 192(CX), Z24
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x5_64Xor_loop:
|
|
// Load 5 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z25
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z26
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z27
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z28
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 5 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 5 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 5 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 5 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 5 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z24, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 5 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 5 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 5 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 5 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 5 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 5 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x5_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x5_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x5Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x5Xor(SB), $8-88
|
|
// Loading 9 of 50 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 57 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x5Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
VBROADCASTSD 64(CX), Y8
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x5Xor_loop:
|
|
// Load 5 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU (BP)(R15*1), Y9
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y10
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y11
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y12
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 5 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 5 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 5 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 5 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 5 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 5 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 5 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 5 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 5 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 5 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 5 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x5Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x5Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x6_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x6_64(SB), $8-88
|
|
// Loading 24 of 60 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 68 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x6_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x6_64_loop:
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 6 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 6 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 6 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x6_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x6_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x6(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x6(SB), $8-88
|
|
// Loading 8 of 60 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 68 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x6_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x6_loop:
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 6 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 6 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 6 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x6_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x6_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x6_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x6_64Xor(SB), $8-88
|
|
// Loading 24 of 60 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 68 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x6_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
VBROADCASTF32X2 184(CX), Z23
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x6_64Xor_loop:
|
|
// Load 6 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z24
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z25
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z26
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z27
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z28
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 6 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 6 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 6 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 6 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z23, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 6 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 6 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 6 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 6 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 6 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 6 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x6_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x6_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x6Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x6Xor(SB), $8-88
|
|
// Loading 8 of 60 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 68 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x6Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
VBROADCASTSD 56(CX), Y7
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x6Xor_loop:
|
|
// Load 6 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU (BP)(R15*1), Y8
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y9
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y10
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y11
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y12
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 6 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 6 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y7, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 6 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 6 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 6 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 6 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 6 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 6 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 6 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 6 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 6 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x6Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x6Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x7_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x7_64(SB), $8-88
|
|
// Loading 23 of 70 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 79 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x7_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x7_64_loop:
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 7 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 7 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 7 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z23, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x7_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x7_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x7(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x7(SB), $8-88
|
|
// Loading 7 of 70 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 79 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x7_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x7_loop:
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 7 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 7 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 7 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y7, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x7_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x7_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x7_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x7_64Xor(SB), $8-88
|
|
// Loading 23 of 70 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 79 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x7_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
VBROADCASTF32X2 176(CX), Z22
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x7_64Xor_loop:
|
|
// Load 7 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z23
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z24
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z25
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z26
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z27
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z28
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 7 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 7 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 7 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 7 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z22, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 7 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 7 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 7 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 7 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 7 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 7 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z23, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x7_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x7_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x7Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x7Xor(SB), $8-88
|
|
// Loading 7 of 70 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 79 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x7Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
VBROADCASTSD 48(CX), Y6
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x7Xor_loop:
|
|
// Load 7 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU (BP)(R15*1), Y7
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y8
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y9
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y10
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y11
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y12
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 7 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VGF2P8AFFINEQB $0x00, Y6, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 7 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 7 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 7 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 7 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 7 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 7 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 7 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 7 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 7 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 7 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y7, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x7Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x7Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x8_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x8_64(SB), $8-88
|
|
// Loading 22 of 80 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 90 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x8_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x8_64_loop:
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 8 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 8 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 8 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z22, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z23, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x8_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x8_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x8(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x8(SB), $8-88
|
|
// Loading 6 of 80 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 90 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x8_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x8_loop:
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y11
|
|
VBROADCASTSD 48(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 56(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 8 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 8 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 8 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y6, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y7, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x8_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x8_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x8_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x8_64Xor(SB), $8-88
|
|
// Loading 22 of 80 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 90 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x8_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
VBROADCASTF32X2 168(CX), Z21
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x8_64Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z22
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z23
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z24
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z25
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z26
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z27
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z28
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 8 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 8 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 8 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z21, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 8 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 8 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 8 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 8 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 8 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 8 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 8 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z22, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z23, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x8_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x8_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x8Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x8Xor(SB), $8-88
|
|
// Loading 6 of 80 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 90 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x8Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
VBROADCASTSD 40(CX), Y5
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x8Xor_loop:
|
|
// Load 8 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU (BP)(R15*1), Y6
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y7
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y8
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y9
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y10
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y11
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y12
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 8 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VGF2P8AFFINEQB $0x00, Y5, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 8 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 8 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 8 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 8 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 8 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 8 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 8 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 8 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 8 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 8 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y6, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y7, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x8Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x8Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x9_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x9_64(SB), $8-88
|
|
// Loading 21 of 90 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 101 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x9_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x9_64_loop:
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 9 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 9 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 9 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z21, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z22, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z23, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x9_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x9_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x9(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x9(SB), $8-88
|
|
// Loading 5 of 90 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 101 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x9_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x9_loop:
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y9
|
|
VBROADCASTSD 40(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 48(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 56(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 64(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 9 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 9 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 9 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 640(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 648(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 656(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 664(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 672(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 680(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 688(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 696(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 704(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 712(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y5, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y6, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y7, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x9_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x9_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x9_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x9_64Xor(SB), $8-88
|
|
// Loading 21 of 90 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 101 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x9_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
VBROADCASTF32X2 160(CX), Z20
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x9_64Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z21
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z22
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z23
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z24
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z25
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z26
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z27
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z28
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 9 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 9 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 9 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z20, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 9 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 9 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 9 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 9 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 9 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 9 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 9 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z21, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z22, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z23, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x9_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x9_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x9Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x9Xor(SB), $8-88
|
|
// Loading 5 of 90 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 101 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x9Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
VBROADCASTSD 32(CX), Y4
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x9Xor_loop:
|
|
// Load 9 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU (BP)(R15*1), Y5
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y6
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y7
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y8
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y9
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y10
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y11
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y12
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 9 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VGF2P8AFFINEQB $0x00, Y4, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 9 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 9 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 9 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 9 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 9 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 9 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 9 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 9 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 640(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 9 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 648(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 656(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 664(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 672(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 680(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 688(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 696(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 704(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 712(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 9 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y5, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y6, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y7, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x9Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x9Xor_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x10_64(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x10_64(SB), $8-88
|
|
// Loading 20 of 100 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 112 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x10_64_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x10_64_loop:
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 10 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 10 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 10 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 720(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 728(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 736(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 744(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 752(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 760(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 768(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 776(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 784(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 792(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z20, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z21, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z22, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z23, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 216(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x10_64_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x10_64_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x10(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x10(SB), $8-88
|
|
// Loading 4 of 100 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 112 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x10_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x10_loop:
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y7
|
|
VBROADCASTSD 32(CX), Y8
|
|
VGF2P8AFFINEQB $0x00, Y8, Y14, Y8
|
|
VBROADCASTSD 40(CX), Y9
|
|
VGF2P8AFFINEQB $0x00, Y9, Y14, Y9
|
|
VBROADCASTSD 48(CX), Y10
|
|
VGF2P8AFFINEQB $0x00, Y10, Y14, Y10
|
|
VBROADCASTSD 56(CX), Y11
|
|
VGF2P8AFFINEQB $0x00, Y11, Y14, Y11
|
|
VBROADCASTSD 64(CX), Y12
|
|
VGF2P8AFFINEQB $0x00, Y12, Y14, Y12
|
|
VBROADCASTSD 72(CX), Y13
|
|
VGF2P8AFFINEQB $0x00, Y13, Y14, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 10 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 10 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 10 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 640(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 648(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 656(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 664(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 672(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 680(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 688(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 696(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 704(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 712(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 720(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 728(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 736(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 744(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 752(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 760(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 768(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 776(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 784(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 792(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y4, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y5, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y6, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y7, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 216(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x10_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x10_end:
|
|
RET
|
|
|
|
// func mulGFNI_10x10_64Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·mulGFNI_10x10_64Xor(SB), $8-88
|
|
// Loading 20 of 100 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 112 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x06, AX
|
|
TESTQ AX, AX
|
|
JZ mulGFNI_10x10_64Xor_end
|
|
VBROADCASTF32X2 (CX), Z0
|
|
VBROADCASTF32X2 8(CX), Z1
|
|
VBROADCASTF32X2 16(CX), Z2
|
|
VBROADCASTF32X2 24(CX), Z3
|
|
VBROADCASTF32X2 32(CX), Z4
|
|
VBROADCASTF32X2 40(CX), Z5
|
|
VBROADCASTF32X2 48(CX), Z6
|
|
VBROADCASTF32X2 56(CX), Z7
|
|
VBROADCASTF32X2 64(CX), Z8
|
|
VBROADCASTF32X2 72(CX), Z9
|
|
VBROADCASTF32X2 80(CX), Z10
|
|
VBROADCASTF32X2 88(CX), Z11
|
|
VBROADCASTF32X2 96(CX), Z12
|
|
VBROADCASTF32X2 104(CX), Z13
|
|
VBROADCASTF32X2 112(CX), Z14
|
|
VBROADCASTF32X2 120(CX), Z15
|
|
VBROADCASTF32X2 128(CX), Z16
|
|
VBROADCASTF32X2 136(CX), Z17
|
|
VBROADCASTF32X2 144(CX), Z18
|
|
VBROADCASTF32X2 152(CX), Z19
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulGFNI_10x10_64Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z20
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z21
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z22
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z23
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z24
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z25
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z26
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z27
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z28
|
|
MOVQ 216(R14), BP
|
|
VMOVDQU64 (BP)(R15*1), Z29
|
|
|
|
// Load and process 64 bytes from input 0 to 10 outputs
|
|
VMOVDQU64 (BX), Z30
|
|
ADDQ $0x40, BX
|
|
VGF2P8AFFINEQB $0x00, Z0, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z1, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z2, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z3, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z4, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z5, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z6, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z7, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z8, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z9, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 1 to 10 outputs
|
|
VMOVDQU64 (SI), Z30
|
|
ADDQ $0x40, SI
|
|
VGF2P8AFFINEQB $0x00, Z10, Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB $0x00, Z11, Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB $0x00, Z12, Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB $0x00, Z13, Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB $0x00, Z14, Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB $0x00, Z15, Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB $0x00, Z16, Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB $0x00, Z17, Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB $0x00, Z18, Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB $0x00, Z19, Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 2 to 10 outputs
|
|
VMOVDQU64 (DI), Z30
|
|
ADDQ $0x40, DI
|
|
VGF2P8AFFINEQB.BCST $0x00, 160(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 168(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 176(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 184(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 192(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 200(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 208(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 216(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 224(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 232(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 3 to 10 outputs
|
|
VMOVDQU64 (R8), Z30
|
|
ADDQ $0x40, R8
|
|
VGF2P8AFFINEQB.BCST $0x00, 240(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 248(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 256(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 264(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 272(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 280(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 288(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 296(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 304(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 312(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 4 to 10 outputs
|
|
VMOVDQU64 (R9), Z30
|
|
ADDQ $0x40, R9
|
|
VGF2P8AFFINEQB.BCST $0x00, 320(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 328(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 336(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 344(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 352(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 360(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 368(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 376(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 384(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 392(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 5 to 10 outputs
|
|
VMOVDQU64 (R10), Z30
|
|
ADDQ $0x40, R10
|
|
VGF2P8AFFINEQB.BCST $0x00, 400(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 408(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 416(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 424(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 432(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 440(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 448(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 456(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 464(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 472(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 6 to 10 outputs
|
|
VMOVDQU64 (R11), Z30
|
|
ADDQ $0x40, R11
|
|
VGF2P8AFFINEQB.BCST $0x00, 480(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 488(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 496(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 504(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 512(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 520(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 528(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 536(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 544(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 552(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 7 to 10 outputs
|
|
VMOVDQU64 (R12), Z30
|
|
ADDQ $0x40, R12
|
|
VGF2P8AFFINEQB.BCST $0x00, 560(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 568(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 576(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 584(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 592(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 600(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 608(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 616(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 624(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 632(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 8 to 10 outputs
|
|
VMOVDQU64 (R13), Z30
|
|
ADDQ $0x40, R13
|
|
VGF2P8AFFINEQB.BCST $0x00, 640(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 648(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 656(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 664(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 672(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 680(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 688(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 696(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 704(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 712(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Load and process 64 bytes from input 9 to 10 outputs
|
|
VMOVDQU64 (DX), Z30
|
|
ADDQ $0x40, DX
|
|
VGF2P8AFFINEQB.BCST $0x00, 720(CX), Z30, Z31
|
|
VXORPD Z20, Z31, Z20
|
|
VGF2P8AFFINEQB.BCST $0x00, 728(CX), Z30, Z31
|
|
VXORPD Z21, Z31, Z21
|
|
VGF2P8AFFINEQB.BCST $0x00, 736(CX), Z30, Z31
|
|
VXORPD Z22, Z31, Z22
|
|
VGF2P8AFFINEQB.BCST $0x00, 744(CX), Z30, Z31
|
|
VXORPD Z23, Z31, Z23
|
|
VGF2P8AFFINEQB.BCST $0x00, 752(CX), Z30, Z31
|
|
VXORPD Z24, Z31, Z24
|
|
VGF2P8AFFINEQB.BCST $0x00, 760(CX), Z30, Z31
|
|
VXORPD Z25, Z31, Z25
|
|
VGF2P8AFFINEQB.BCST $0x00, 768(CX), Z30, Z31
|
|
VXORPD Z26, Z31, Z26
|
|
VGF2P8AFFINEQB.BCST $0x00, 776(CX), Z30, Z31
|
|
VXORPD Z27, Z31, Z27
|
|
VGF2P8AFFINEQB.BCST $0x00, 784(CX), Z30, Z31
|
|
VXORPD Z28, Z31, Z28
|
|
VGF2P8AFFINEQB.BCST $0x00, 792(CX), Z30, Z31
|
|
VXORPD Z29, Z31, Z29
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU64 Z20, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU64 Z21, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU64 Z22, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU64 Z23, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU64 Z24, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU64 Z25, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU64 Z26, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU64 Z27, (BP)(R15*1)
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU64 Z28, (BP)(R15*1)
|
|
MOVQ 216(R14), BP
|
|
VMOVDQU64 Z29, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x40, R15
|
|
DECQ AX
|
|
JNZ mulGFNI_10x10_64Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulGFNI_10x10_64Xor_end:
|
|
RET
|
|
|
|
// func mulAvxGFNI_10x10Xor(matrix []uint64, in [][]byte, out [][]byte, start int, n int)
|
|
// Requires: AVX, GFNI
|
|
TEXT ·mulAvxGFNI_10x10Xor(SB), $8-88
|
|
// Loading 4 of 100 tables to registers
|
|
// Destination kept on stack
|
|
// Full registers estimated 112 YMM used
|
|
MOVQ n+80(FP), AX
|
|
MOVQ matrix_base+0(FP), CX
|
|
SHRQ $0x05, AX
|
|
TESTQ AX, AX
|
|
JZ mulAvxGFNI_10x10Xor_end
|
|
VBROADCASTSD (CX), Y0
|
|
VBROADCASTSD 8(CX), Y1
|
|
VBROADCASTSD 16(CX), Y2
|
|
VBROADCASTSD 24(CX), Y3
|
|
MOVQ in_base+24(FP), DX
|
|
MOVQ (DX), BX
|
|
MOVQ 24(DX), SI
|
|
MOVQ 48(DX), DI
|
|
MOVQ 72(DX), R8
|
|
MOVQ 96(DX), R9
|
|
MOVQ 120(DX), R10
|
|
MOVQ 144(DX), R11
|
|
MOVQ 168(DX), R12
|
|
MOVQ 192(DX), R13
|
|
MOVQ 216(DX), DX
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ out_base+48(FP), R14
|
|
MOVQ start+72(FP), R15
|
|
|
|
// Add start offset to input
|
|
ADDQ R15, BX
|
|
ADDQ R15, SI
|
|
ADDQ R15, DI
|
|
ADDQ R15, R8
|
|
ADDQ R15, R9
|
|
ADDQ R15, R10
|
|
ADDQ R15, R11
|
|
ADDQ R15, R12
|
|
ADDQ R15, R13
|
|
ADDQ R15, DX
|
|
|
|
mulAvxGFNI_10x10Xor_loop:
|
|
// Load 10 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU (BP)(R15*1), Y4
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y5
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y6
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y7
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y8
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y9
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y10
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y11
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y12
|
|
MOVQ 216(R14), BP
|
|
VMOVDQU (BP)(R15*1), Y13
|
|
|
|
// Load and process 32 bytes from input 0 to 10 outputs
|
|
VMOVDQU (BX), Y14
|
|
ADDQ $0x20, BX
|
|
VGF2P8AFFINEQB $0x00, Y0, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VGF2P8AFFINEQB $0x00, Y1, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VGF2P8AFFINEQB $0x00, Y2, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VGF2P8AFFINEQB $0x00, Y3, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 32(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 40(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 48(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 56(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 64(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 72(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 1 to 10 outputs
|
|
VMOVDQU (SI), Y14
|
|
ADDQ $0x20, SI
|
|
VBROADCASTSD 80(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 88(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 96(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 104(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 112(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 120(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 128(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 136(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 144(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 152(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 2 to 10 outputs
|
|
VMOVDQU (DI), Y14
|
|
ADDQ $0x20, DI
|
|
VBROADCASTSD 160(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 168(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 176(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 184(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 192(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 200(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 208(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 216(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 224(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 232(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 3 to 10 outputs
|
|
VMOVDQU (R8), Y14
|
|
ADDQ $0x20, R8
|
|
VBROADCASTSD 240(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 248(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 256(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 264(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 272(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 280(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 288(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 296(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 304(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 312(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 4 to 10 outputs
|
|
VMOVDQU (R9), Y14
|
|
ADDQ $0x20, R9
|
|
VBROADCASTSD 320(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 328(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 336(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 344(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 352(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 360(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 368(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 376(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 384(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 392(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 5 to 10 outputs
|
|
VMOVDQU (R10), Y14
|
|
ADDQ $0x20, R10
|
|
VBROADCASTSD 400(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 408(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 416(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 424(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 432(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 440(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 448(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 456(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 464(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 472(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 6 to 10 outputs
|
|
VMOVDQU (R11), Y14
|
|
ADDQ $0x20, R11
|
|
VBROADCASTSD 480(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 488(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 496(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 504(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 512(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 520(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 528(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 536(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 544(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 552(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 7 to 10 outputs
|
|
VMOVDQU (R12), Y14
|
|
ADDQ $0x20, R12
|
|
VBROADCASTSD 560(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 568(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 576(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 584(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 592(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 600(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 608(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 616(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 624(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 632(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 8 to 10 outputs
|
|
VMOVDQU (R13), Y14
|
|
ADDQ $0x20, R13
|
|
VBROADCASTSD 640(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 648(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 656(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 664(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 672(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 680(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 688(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 696(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 704(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 712(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Load and process 32 bytes from input 9 to 10 outputs
|
|
VMOVDQU (DX), Y14
|
|
ADDQ $0x20, DX
|
|
VBROADCASTSD 720(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y4, Y15, Y4
|
|
VBROADCASTSD 728(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y5, Y15, Y5
|
|
VBROADCASTSD 736(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y6, Y15, Y6
|
|
VBROADCASTSD 744(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y7, Y15, Y7
|
|
VBROADCASTSD 752(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y8, Y15, Y8
|
|
VBROADCASTSD 760(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y9, Y15, Y9
|
|
VBROADCASTSD 768(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y10, Y15, Y10
|
|
VBROADCASTSD 776(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y11, Y15, Y11
|
|
VBROADCASTSD 784(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y12, Y15, Y12
|
|
VBROADCASTSD 792(CX), Y15
|
|
VGF2P8AFFINEQB $0x00, Y15, Y14, Y15
|
|
VXORPD Y13, Y15, Y13
|
|
|
|
// Store 10 outputs
|
|
MOVQ (R14), BP
|
|
VMOVDQU Y4, (BP)(R15*1)
|
|
MOVQ 24(R14), BP
|
|
VMOVDQU Y5, (BP)(R15*1)
|
|
MOVQ 48(R14), BP
|
|
VMOVDQU Y6, (BP)(R15*1)
|
|
MOVQ 72(R14), BP
|
|
VMOVDQU Y7, (BP)(R15*1)
|
|
MOVQ 96(R14), BP
|
|
VMOVDQU Y8, (BP)(R15*1)
|
|
MOVQ 120(R14), BP
|
|
VMOVDQU Y9, (BP)(R15*1)
|
|
MOVQ 144(R14), BP
|
|
VMOVDQU Y10, (BP)(R15*1)
|
|
MOVQ 168(R14), BP
|
|
VMOVDQU Y11, (BP)(R15*1)
|
|
MOVQ 192(R14), BP
|
|
VMOVDQU Y12, (BP)(R15*1)
|
|
MOVQ 216(R14), BP
|
|
VMOVDQU Y13, (BP)(R15*1)
|
|
|
|
// Prepare for next loop
|
|
ADDQ $0x20, R15
|
|
DECQ AX
|
|
JNZ mulAvxGFNI_10x10Xor_loop
|
|
VZEROUPPER
|
|
|
|
mulAvxGFNI_10x10Xor_end:
|
|
RET
|
|
|
|
// func ifftDIT48_gfni_0(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·ifftDIT48_gfni_0(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t01+32(FP), Z0
|
|
VBROADCASTF32X2 t23+40(FP), Z1
|
|
VBROADCASTF32X2 t02+48(FP), Z2
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z3
|
|
VMOVDQU64 (DI), Z4
|
|
VMOVDQU64 (R8), Z5
|
|
VMOVDQU64 (AX), Z6
|
|
VXORPD Z4, Z3, Z4
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z4, Z7
|
|
VXORPD Z3, Z7, Z3
|
|
VXORPD Z5, Z6, Z6
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z6, Z7
|
|
VPTERNLOGD $0x96, Z7, Z3, Z5
|
|
VXORPD Z4, Z6, Z6
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z2, Z5, Z7
|
|
VXORPD Z3, Z7, Z3
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
|
|
VXORPD Z4, Z7, Z4
|
|
VMOVDQU64 Z3, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z4, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z5, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z6, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func fftDIT48_gfni_0(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·fftDIT48_gfni_0(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t01+32(FP), Z0
|
|
VBROADCASTF32X2 t23+40(FP), Z1
|
|
VBROADCASTF32X2 t02+48(FP), Z2
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z3
|
|
VMOVDQU64 (DI), Z4
|
|
VMOVDQU64 (R8), Z5
|
|
VMOVDQU64 (AX), Z6
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z2, Z5, Z7
|
|
VXORPD Z3, Z7, Z3
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z2, Z6, Z7
|
|
VXORPD Z4, Z7, Z4
|
|
VXORPD Z3, Z5, Z5
|
|
VXORPD Z4, Z6, Z6
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z4, Z7
|
|
VXORPD Z3, Z7, Z3
|
|
VXORPD Z4, Z3, Z4
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z6, Z7
|
|
VXORPD Z5, Z7, Z5
|
|
VXORPD Z5, Z6, Z6
|
|
VMOVDQU64 Z3, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z4, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z5, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z6, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func ifftDIT48_gfni_1(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·ifftDIT48_gfni_1(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t23+40(FP), Z0
|
|
VBROADCASTF32X2 t02+48(FP), Z1
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z2
|
|
VMOVDQU64 (DI), Z3
|
|
VMOVDQU64 (R8), Z4
|
|
VMOVDQU64 (AX), Z5
|
|
VXORPD Z3, Z2, Z3
|
|
VXORPD Z4, Z5, Z5
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z5, Z6
|
|
VPTERNLOGD $0x96, Z6, Z2, Z4
|
|
VXORPD Z3, Z5, Z5
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z4, Z6
|
|
VXORPD Z2, Z6, Z2
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
|
|
VXORPD Z3, Z6, Z3
|
|
VMOVDQU64 Z2, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z3, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z4, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z5, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func fftDIT48_gfni_1(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·fftDIT48_gfni_1(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t01+32(FP), Z0
|
|
VBROADCASTF32X2 t23+40(FP), Z1
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z2
|
|
VMOVDQU64 (DI), Z3
|
|
VMOVDQU64 (R8), Z4
|
|
VMOVDQU64 (AX), Z5
|
|
VXORPD Z2, Z4, Z4
|
|
VXORPD Z3, Z5, Z5
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z3, Z6
|
|
VXORPD Z2, Z6, Z2
|
|
VXORPD Z3, Z2, Z3
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
|
|
VXORPD Z4, Z6, Z4
|
|
VXORPD Z4, Z5, Z5
|
|
VMOVDQU64 Z2, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z3, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z4, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z5, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func ifftDIT48_gfni_2(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·ifftDIT48_gfni_2(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t01+32(FP), Z0
|
|
VBROADCASTF32X2 t02+48(FP), Z1
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z2
|
|
VMOVDQU64 (DI), Z3
|
|
VMOVDQU64 (R8), Z4
|
|
VMOVDQU64 (AX), Z5
|
|
VXORPD Z3, Z2, Z3
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z3, Z6
|
|
VXORPD Z2, Z6, Z2
|
|
VXORPD Z4, Z5, Z5
|
|
VXORPD Z2, Z4, Z4
|
|
VXORPD Z3, Z5, Z5
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z4, Z6
|
|
VXORPD Z2, Z6, Z2
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
|
|
VXORPD Z3, Z6, Z3
|
|
VMOVDQU64 Z2, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z3, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z4, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z5, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func fftDIT48_gfni_2(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·fftDIT48_gfni_2(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t23+40(FP), Z0
|
|
VBROADCASTF32X2 t02+48(FP), Z1
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z2
|
|
VMOVDQU64 (DI), Z3
|
|
VMOVDQU64 (R8), Z4
|
|
VMOVDQU64 (AX), Z5
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z4, Z6
|
|
VXORPD Z2, Z6, Z2
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
|
|
VXORPD Z3, Z6, Z3
|
|
VXORPD Z2, Z4, Z4
|
|
VXORPD Z3, Z5, Z5
|
|
VXORPD Z3, Z2, Z3
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z5, Z6
|
|
VXORPD Z4, Z6, Z4
|
|
VXORPD Z4, Z5, Z5
|
|
VMOVDQU64 Z2, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z3, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z4, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z5, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func ifftDIT48_gfni_3(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·ifftDIT48_gfni_3(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t02+48(FP), Z0
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z1
|
|
VMOVDQU64 (DI), Z2
|
|
VMOVDQU64 (R8), Z3
|
|
VMOVDQU64 (AX), Z4
|
|
VXORPD Z2, Z1, Z2
|
|
VXORPD Z3, Z4, Z4
|
|
VXORPD Z1, Z3, Z3
|
|
VXORPD Z2, Z4, Z4
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z3, Z5
|
|
VXORPD Z1, Z5, Z1
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
|
|
VXORPD Z2, Z5, Z2
|
|
VMOVDQU64 Z1, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z2, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z3, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z4, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func fftDIT48_gfni_3(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·fftDIT48_gfni_3(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t23+40(FP), Z0
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z1
|
|
VMOVDQU64 (DI), Z2
|
|
VMOVDQU64 (R8), Z3
|
|
VMOVDQU64 (AX), Z4
|
|
VXORPD Z1, Z3, Z3
|
|
VXORPD Z2, Z4, Z4
|
|
VXORPD Z2, Z1, Z2
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
|
|
VXORPD Z3, Z5, Z3
|
|
VXORPD Z3, Z4, Z4
|
|
VMOVDQU64 Z1, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z2, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z3, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z4, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func ifftDIT48_gfni_4(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·ifftDIT48_gfni_4(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t01+32(FP), Z0
|
|
VBROADCASTF32X2 t23+40(FP), Z1
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z2
|
|
VMOVDQU64 (DI), Z3
|
|
VMOVDQU64 (R8), Z4
|
|
VMOVDQU64 (AX), Z5
|
|
VXORPD Z3, Z2, Z3
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z3, Z6
|
|
VXORPD Z2, Z6, Z2
|
|
VXORPD Z4, Z5, Z5
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
|
|
VPTERNLOGD $0x96, Z6, Z2, Z4
|
|
VXORPD Z3, Z5, Z5
|
|
VMOVDQU64 Z2, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z3, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z4, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z5, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func fftDIT48_gfni_4(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·fftDIT48_gfni_4(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t01+32(FP), Z0
|
|
VBROADCASTF32X2 t02+48(FP), Z1
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z2
|
|
VMOVDQU64 (DI), Z3
|
|
VMOVDQU64 (R8), Z4
|
|
VMOVDQU64 (AX), Z5
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z4, Z6
|
|
VXORPD Z2, Z6, Z2
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z1, Z5, Z6
|
|
VXORPD Z3, Z6, Z3
|
|
VXORPD Z2, Z4, Z4
|
|
VXORPD Z3, Z5, Z5
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z3, Z6
|
|
VXORPD Z2, Z6, Z2
|
|
VXORPD Z3, Z2, Z3
|
|
VXORPD Z4, Z5, Z5
|
|
VMOVDQU64 Z2, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z3, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z4, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z5, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func ifftDIT48_gfni_5(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·ifftDIT48_gfni_5(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t23+40(FP), Z0
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z1
|
|
VMOVDQU64 (DI), Z2
|
|
VMOVDQU64 (R8), Z3
|
|
VMOVDQU64 (AX), Z4
|
|
VXORPD Z2, Z1, Z2
|
|
VXORPD Z3, Z4, Z4
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
|
|
VPTERNLOGD $0x96, Z5, Z1, Z3
|
|
VXORPD Z2, Z4, Z4
|
|
VMOVDQU64 Z1, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z2, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z3, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z4, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func fftDIT48_gfni_5(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·fftDIT48_gfni_5(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t01+32(FP), Z0
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z1
|
|
VMOVDQU64 (DI), Z2
|
|
VMOVDQU64 (R8), Z3
|
|
VMOVDQU64 (AX), Z4
|
|
VXORPD Z1, Z3, Z3
|
|
VXORPD Z2, Z4, Z4
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z2, Z5
|
|
VXORPD Z1, Z5, Z1
|
|
VXORPD Z2, Z1, Z2
|
|
VXORPD Z3, Z4, Z4
|
|
VMOVDQU64 Z1, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z2, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z3, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z4, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func ifftDIT48_gfni_6(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·ifftDIT48_gfni_6(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t01+32(FP), Z0
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z1
|
|
VMOVDQU64 (DI), Z2
|
|
VMOVDQU64 (R8), Z3
|
|
VMOVDQU64 (AX), Z4
|
|
VXORPD Z2, Z1, Z2
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z2, Z5
|
|
VXORPD Z1, Z5, Z1
|
|
VXORPD Z3, Z4, Z4
|
|
VXORPD Z1, Z3, Z3
|
|
VXORPD Z2, Z4, Z4
|
|
VMOVDQU64 Z1, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z2, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z3, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z4, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func fftDIT48_gfni_6(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F, GFNI
|
|
TEXT ·fftDIT48_gfni_6(SB), NOSPLIT, $0-56
|
|
VBROADCASTF32X2 t02+48(FP), Z0
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z1
|
|
VMOVDQU64 (DI), Z2
|
|
VMOVDQU64 (R8), Z3
|
|
VMOVDQU64 (AX), Z4
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z3, Z5
|
|
VXORPD Z1, Z5, Z1
|
|
|
|
// LEO_MULADD_512
|
|
VGF2P8AFFINEQB $0x00, Z0, Z4, Z5
|
|
VXORPD Z2, Z5, Z2
|
|
VXORPD Z1, Z3, Z3
|
|
VXORPD Z2, Z4, Z4
|
|
VXORPD Z2, Z1, Z2
|
|
VXORPD Z3, Z4, Z4
|
|
VMOVDQU64 Z1, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z2, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z3, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z4, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func ifftDIT48_gfni_7(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F
|
|
TEXT ·ifftDIT48_gfni_7(SB), NOSPLIT, $0-56
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z0
|
|
VMOVDQU64 (DI), Z1
|
|
VMOVDQU64 (R8), Z2
|
|
VMOVDQU64 (AX), Z3
|
|
VXORPD Z1, Z0, Z1
|
|
VXORPD Z2, Z3, Z3
|
|
VXORPD Z0, Z2, Z2
|
|
VXORPD Z1, Z3, Z3
|
|
VMOVDQU64 Z0, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z1, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z2, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z3, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|
|
|
|
// func fftDIT48_gfni_7(work [][]byte, dist int, t01 uint64, t23 uint64, t02 uint64)
|
|
// Requires: AVX, AVX512DQ, AVX512F
|
|
TEXT ·fftDIT48_gfni_7(SB), NOSPLIT, $0-56
|
|
MOVQ dist+24(FP), AX
|
|
MOVQ work_base+0(FP), CX
|
|
MOVQ 8(CX), DX
|
|
XORQ BX, BX
|
|
MOVQ (CX)(BX*1), SI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), DI
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), R8
|
|
ADDQ AX, BX
|
|
MOVQ (CX)(BX*1), AX
|
|
|
|
loop:
|
|
VMOVDQU64 (SI), Z0
|
|
VMOVDQU64 (DI), Z1
|
|
VMOVDQU64 (R8), Z2
|
|
VMOVDQU64 (AX), Z3
|
|
VXORPD Z0, Z2, Z2
|
|
VXORPD Z1, Z3, Z3
|
|
VXORPD Z1, Z0, Z1
|
|
VXORPD Z2, Z3, Z3
|
|
VMOVDQU64 Z0, (SI)
|
|
ADDQ $0x40, SI
|
|
VMOVDQU64 Z1, (DI)
|
|
ADDQ $0x40, DI
|
|
VMOVDQU64 Z2, (R8)
|
|
ADDQ $0x40, R8
|
|
VMOVDQU64 Z3, (AX)
|
|
ADDQ $0x40, AX
|
|
SUBQ $0x40, DX
|
|
JA loop
|
|
VZEROUPPER
|
|
RET
|