Adding upstream version 1.16.10.upstream/1.16.10upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 230 insertions, 0 deletions

diff --git a/src/hash/crc32/crc32_s390x.s b/src/hash/crc32/crc32_s390x.s
new file mode 100644
index 0000000..a168fad
--- /dev/null
+++ b/src/hash/crc32/crc32_s390x.s
@@ -0,0 +1,230 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "textflag.h"
+
+// Vector register range containing CRC-32 constants
+
+#define CONST_PERM_LE2BE        V9
+#define CONST_R2R1              V10
+#define CONST_R4R3              V11
+#define CONST_R5                V12
+#define CONST_RU_POLY           V13
+#define CONST_CRC_POLY          V14
+
+
+// The CRC-32 constant block contains reduction constants to fold and
+// process particular chunks of the input data stream in parallel.
+//
+// Note that the constant definitions below are extended in order to compute
+// intermediate results with a single VECTOR GALOIS FIELD MULTIPLY instruction.
+// The rightmost doubleword can be 0 to prevent contribution to the result or
+// can be multiplied by 1 to perform an XOR without the need for a separate
+// VECTOR EXCLUSIVE OR instruction.
+//
+// The polynomials used are bit-reflected:
+//
+//            IEEE: P'(x) = 0x0edb88320
+//      Castagnoli: P'(x) = 0x082f63b78
+
+
+// IEEE polynomial constants
+DATA    ·crclecons+0(SB)/8,  $0x0F0E0D0C0B0A0908       // LE-to-BE mask
+DATA    ·crclecons+8(SB)/8,  $0x0706050403020100
+DATA    ·crclecons+16(SB)/8, $0x00000001c6e41596       // R2
+DATA    ·crclecons+24(SB)/8, $0x0000000154442bd4       // R1
+DATA    ·crclecons+32(SB)/8, $0x00000000ccaa009e       // R4
+DATA    ·crclecons+40(SB)/8, $0x00000001751997d0       // R3
+DATA    ·crclecons+48(SB)/8, $0x0000000000000000
+DATA    ·crclecons+56(SB)/8, $0x0000000163cd6124       // R5
+DATA    ·crclecons+64(SB)/8, $0x0000000000000000
+DATA    ·crclecons+72(SB)/8, $0x00000001F7011641       // u'
+DATA    ·crclecons+80(SB)/8, $0x0000000000000000
+DATA    ·crclecons+88(SB)/8, $0x00000001DB710641       // P'(x) << 1
+
+GLOBL    ·crclecons(SB),RODATA, $144
+
+// Castagonli Polynomial constants
+DATA    ·crcclecons+0(SB)/8,  $0x0F0E0D0C0B0A0908      // LE-to-BE mask
+DATA    ·crcclecons+8(SB)/8,  $0x0706050403020100
+DATA    ·crcclecons+16(SB)/8, $0x000000009e4addf8      // R2
+DATA    ·crcclecons+24(SB)/8, $0x00000000740eef02      // R1
+DATA    ·crcclecons+32(SB)/8, $0x000000014cd00bd6      // R4
+DATA    ·crcclecons+40(SB)/8, $0x00000000f20c0dfe      // R3
+DATA    ·crcclecons+48(SB)/8, $0x0000000000000000
+DATA    ·crcclecons+56(SB)/8, $0x00000000dd45aab8      // R5
+DATA    ·crcclecons+64(SB)/8, $0x0000000000000000
+DATA    ·crcclecons+72(SB)/8, $0x00000000dea713f1      // u'
+DATA    ·crcclecons+80(SB)/8, $0x0000000000000000
+DATA    ·crcclecons+88(SB)/8, $0x0000000105ec76f0      // P'(x) << 1
+
+GLOBL   ·crcclecons(SB),RODATA, $144
+
+// The CRC-32 function(s) use these calling conventions:
+//
+// Parameters:
+//
+//      R2:    Initial CRC value, typically ~0; and final CRC (return) value.
+//      R3:    Input buffer pointer, performance might be improved if the
+//             buffer is on a doubleword boundary.
+//      R4:    Length of the buffer, must be 64 bytes or greater.
+//
+// Register usage:
+//
+//      R5:     CRC-32 constant pool base pointer.
+//      V0:     Initial CRC value and intermediate constants and results.
+//      V1..V4: Data for CRC computation.
+//      V5..V8: Next data chunks that are fetched from the input buffer.
+//
+//      V9..V14: CRC-32 constants.
+
+// func vectorizedIEEE(crc uint32, p []byte) uint32
+TEXT ·vectorizedIEEE(SB),NOSPLIT,$0
+	MOVWZ   crc+0(FP), R2     // R2 stores the CRC value
+	MOVD    p+8(FP), R3       // data pointer
+	MOVD    p_len+16(FP), R4  // len(p)
+
+	MOVD    $·crclecons(SB), R5
+	BR      vectorizedBody<>(SB)
+
+// func vectorizedCastagnoli(crc uint32, p []byte) uint32
+TEXT ·vectorizedCastagnoli(SB),NOSPLIT,$0
+	MOVWZ   crc+0(FP), R2     // R2 stores the CRC value
+	MOVD    p+8(FP), R3       // data pointer
+	MOVD    p_len+16(FP), R4  // len(p)
+
+	// R5: crc-32 constant pool base pointer, constant is used to reduce crc
+	MOVD    $·crcclecons(SB), R5
+	BR      vectorizedBody<>(SB)
+
+TEXT vectorizedBody<>(SB),NOSPLIT,$0
+	XOR     $0xffffffff, R2 // NOTW R2
+	VLM     0(R5), CONST_PERM_LE2BE, CONST_CRC_POLY
+
+	// Load the initial CRC value into the rightmost word of V0
+	VZERO   V0
+	VLVGF   $3, R2, V0
+
+	// Crash if the input size is less than 64-bytes.
+	CMP     R4, $64
+	BLT     crash
+
+	// Load a 64-byte data chunk and XOR with CRC
+	VLM     0(R3), V1, V4    // 64-bytes into V1..V4
+
+	// Reflect the data if the CRC operation is in the bit-reflected domain
+	VPERM   V1, V1, CONST_PERM_LE2BE, V1
+	VPERM   V2, V2, CONST_PERM_LE2BE, V2
+	VPERM   V3, V3, CONST_PERM_LE2BE, V3
+	VPERM   V4, V4, CONST_PERM_LE2BE, V4
+
+	VX      V0, V1, V1     // V1 ^= CRC
+	ADD     $64, R3        // BUF = BUF + 64
+	ADD     $(-64), R4
+
+	// Check remaining buffer size and jump to proper folding method
+	CMP     R4, $64
+	BLT     less_than_64bytes
+
+fold_64bytes_loop:
+	// Load the next 64-byte data chunk into V5 to V8
+	VLM     0(R3), V5, V8
+	VPERM   V5, V5, CONST_PERM_LE2BE, V5
+	VPERM   V6, V6, CONST_PERM_LE2BE, V6
+	VPERM   V7, V7, CONST_PERM_LE2BE, V7
+	VPERM   V8, V8, CONST_PERM_LE2BE, V8
+
+
+	// Perform a GF(2) multiplication of the doublewords in V1 with
+	// the reduction constants in V0.  The intermediate result is
+	// then folded (accumulated) with the next data chunk in V5 and
+	// stored in V1.  Repeat this step for the register contents
+	// in V2, V3, and V4 respectively.
+
+	VGFMAG  CONST_R2R1, V1, V5, V1
+	VGFMAG  CONST_R2R1, V2, V6, V2
+	VGFMAG  CONST_R2R1, V3, V7, V3
+	VGFMAG  CONST_R2R1, V4, V8 ,V4
+
+	// Adjust buffer pointer and length for next loop
+	ADD     $64, R3                  // BUF = BUF + 64
+	ADD     $(-64), R4               // LEN = LEN - 64
+
+	CMP     R4, $64
+	BGE     fold_64bytes_loop
+
+less_than_64bytes:
+	// Fold V1 to V4 into a single 128-bit value in V1
+	VGFMAG  CONST_R4R3, V1, V2, V1
+	VGFMAG  CONST_R4R3, V1, V3, V1
+	VGFMAG  CONST_R4R3, V1, V4, V1
+
+	// Check whether to continue with 64-bit folding
+	CMP R4, $16
+	BLT final_fold
+
+fold_16bytes_loop:
+	VL      0(R3), V2               // Load next data chunk
+	VPERM   V2, V2, CONST_PERM_LE2BE, V2
+
+	VGFMAG  CONST_R4R3, V1, V2, V1  // Fold next data chunk
+
+	// Adjust buffer pointer and size for folding next data chunk
+	ADD     $16, R3
+	ADD     $-16, R4
+
+	// Process remaining data chunks
+	CMP     R4 ,$16
+	BGE     fold_16bytes_loop
+
+final_fold:
+	VLEIB   $7, $0x40, V9
+	VSRLB   V9, CONST_R4R3, V0
+	VLEIG   $0, $1, V0
+
+	VGFMG   V0, V1, V1
+
+	VLEIB   $7, $0x20, V9         // Shift by words
+	VSRLB   V9, V1, V2            // Store remaining bits in V2
+	VUPLLF  V1, V1                // Split rightmost doubleword
+	VGFMAG  CONST_R5, V1, V2, V1  // V1 = (V1 * R5) XOR V2
+
+
+	// The input values to the Barret reduction are the degree-63 polynomial
+	// in V1 (R(x)), degree-32 generator polynomial, and the reduction
+	// constant u.  The Barret reduction result is the CRC value of R(x) mod
+	// P(x).
+	//
+	// The Barret reduction algorithm is defined as:
+	//
+	//    1. T1(x) = floor( R(x) / x^32 ) GF2MUL u
+	//    2. T2(x) = floor( T1(x) / x^32 ) GF2MUL P(x)
+	//    3. C(x)  = R(x) XOR T2(x) mod x^32
+	//
+	// Note: To compensate the division by x^32, use the vector unpack
+	// instruction to move the leftmost word into the leftmost doubleword
+	// of the vector register.  The rightmost doubleword is multiplied
+	// with zero to not contribute to the intermediate results.
+
+
+	// T1(x) = floor( R(x) / x^32 ) GF2MUL u
+	VUPLLF  V1, V2
+	VGFMG   CONST_RU_POLY, V2, V2
+
+
+	// Compute the GF(2) product of the CRC polynomial in VO with T1(x) in
+	// V2 and XOR the intermediate result, T2(x),  with the value in V1.
+	// The final result is in the rightmost word of V2.
+
+	VUPLLF  V2 , V2
+	VGFMAG  CONST_CRC_POLY, V2, V1, V2
+
+done:
+	VLGVF   $2, V2, R2
+	XOR     $0xffffffff, R2 // NOTW R2
+	MOVWZ   R2, ret + 32(FP)
+	RET
+
+crash:
+	MOVD    $0, (R0) // input size is less than 64-bytes