From 4c12e4b5295969ee84ed0414399e79d2e7b8673b Mon Sep 17 00:00:00 2001 From: d-enk Date: Thu, 26 Mar 2026 18:31:33 +0300 Subject: [PATCH] perf(exp/simd): optimize Sum funcs with horizontal reduction - Use GetLo().Add(GetHi()) pattern for AVX2/AVX-512 reduction - Process remainder with SlicePart instead of scalar tail loops - Replace manual sum loops with sum16/sum8/sum4/sum2 helpers --- exp/simd/helpers.go | 35 ++++++ exp/simd/math_avx.go | 210 +++++++++++-------------------- exp/simd/math_avx2.go | 260 ++++++++++++++++---------------------- exp/simd/math_avx512.go | 270 ++++++++++++++++++---------------------- 4 files changed, 335 insertions(+), 440 deletions(-) create mode 100644 exp/simd/helpers.go diff --git a/exp/simd/helpers.go b/exp/simd/helpers.go new file mode 100644 index 000000000..776cee775 --- /dev/null +++ b/exp/simd/helpers.go @@ -0,0 +1,35 @@ +//go:build go1.26 && goexperiment.simd && amd64 + +package simd + +type number interface { + ~float32 | ~float64 | + ~int | ~int8 | ~int16 | ~int32 | ~int64 | + ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr +} + +// sum16 sums a vector of 16 elements. +func sum16[T number](vec *[16]T) T { + return 0 + + vec[0] + vec[1] + vec[2] + vec[3] + + vec[4] + vec[5] + vec[6] + vec[7] + + vec[8] + vec[9] + vec[10] + vec[11] + + vec[12] + vec[13] + vec[14] + vec[15] +} + +// sum8 sums a vector of 8 elements. +func sum8[T number](vec *[8]T) T { + return 0 + + vec[0] + vec[1] + vec[2] + vec[3] + + vec[4] + vec[5] + vec[6] + vec[7] +} + +// sum4 sums a vector of 4 elements. +func sum4[T number](vec *[4]T) T { + return vec[0] + vec[1] + vec[2] + vec[3] +} + +// sum2 sums a vector of 2 elements. +func sum2[T number](vec *[2]T) T { + return vec[0] + vec[1] +} diff --git a/exp/simd/math_avx.go b/exp/simd/math_avx.go index 909d6b68e..5041de94a 100644 --- a/exp/simd/math_avx.go +++ b/exp/simd/math_avx.go @@ -23,26 +23,19 @@ func SumInt8x16[T ~int8](collection []T) T { const lanes = simdLanes16 base := unsafeSliceInt8(collection, length) - var acc archsimd.Int8x16 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt8x16Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadInt8x16SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt8x16Slice(base[i : i+lanes])) } var buf [lanes]int8 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - for ; i < length; i++ { - sum += collection[i] - } - - return sum + return T(sum16(&buf)) } // SumInt16x8 sums a slice of int16 using AVX SIMD (Int16x8, 8 lanes). @@ -57,26 +50,19 @@ func SumInt16x8[T ~int16](collection []T) T { const lanes = simdLanes8 base := unsafeSliceInt16(collection, length) - var acc archsimd.Int16x8 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt16x8Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadInt16x8SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt16x8Slice(base[i : i+lanes])) } var buf [lanes]int16 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - for ; i < length; i++ { - sum += collection[i] - } - - return sum + return T(sum8(&buf)) } // SumInt32x4 sums a slice of int32 using AVX SIMD (Int32x4, 4 lanes). @@ -91,26 +77,19 @@ func SumInt32x4[T ~int32](collection []T) T { const lanes = simdLanes4 base := unsafeSliceInt32(collection, length) - var acc archsimd.Int32x4 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt32x4Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadInt32x4SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt32x4Slice(base[i : i+lanes])) } var buf [lanes]int32 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - - for ; i < length; i++ { - sum += collection[i] - } - return sum + return T(sum4(&buf)) } // SumInt64x2 sums a slice of int64 using AVX SIMD (Int64x2, 2 lanes). @@ -125,26 +104,19 @@ func SumInt64x2[T ~int64](collection []T) T { const lanes = simdLanes2 base := unsafeSliceInt64(collection, length) - var acc archsimd.Int64x2 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt64x2Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadInt64x2SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt64x2Slice(base[i : i+lanes])) } var buf [lanes]int64 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - - for ; i < length; i++ { - sum += collection[i] - } - return sum + return T(sum2(&buf)) } // SumUint8x16 sums a slice of uint8 using AVX SIMD (Uint8x16, 16 lanes). @@ -159,26 +131,19 @@ func SumUint8x16[T ~uint8](collection []T) T { const lanes = simdLanes16 base := unsafeSliceUint8(collection, length) - var acc archsimd.Uint8x16 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint8x16Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadUint8x16SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint8x16Slice(base[i : i+lanes])) } var buf [lanes]uint8 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - - for ; i < length; i++ { - sum += collection[i] - } - return sum + return T(sum16(&buf)) } // SumUint16x8 sums a slice of uint16 using AVX SIMD (Uint16x8, 8 lanes). @@ -193,26 +158,19 @@ func SumUint16x8[T ~uint16](collection []T) T { const lanes = simdLanes8 base := unsafeSliceUint16(collection, length) - var acc archsimd.Uint16x8 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint16x8Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadUint16x8SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint16x8Slice(base[i : i+lanes])) } var buf [lanes]uint16 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - for ; i < length; i++ { - sum += collection[i] - } - - return sum + return T(sum8(&buf)) } // SumUint32x4 sums a slice of uint32 using AVX SIMD (Uint32x4, 4 lanes). @@ -227,26 +185,19 @@ func SumUint32x4[T ~uint32](collection []T) T { const lanes = simdLanes4 base := unsafeSliceUint32(collection, length) - var acc archsimd.Uint32x4 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint32x4Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadUint32x4SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint32x4Slice(base[i : i+lanes])) } var buf [lanes]uint32 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - for ; i < length; i++ { - sum += collection[i] - } - - return sum + return T(sum4(&buf)) } // SumUint64x2 sums a slice of uint64 using AVX SIMD (Uint64x2, 2 lanes). @@ -261,26 +212,19 @@ func SumUint64x2[T ~uint64](collection []T) T { const lanes = simdLanes2 base := unsafeSliceUint64(collection, length) - var acc archsimd.Uint64x2 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint64x2Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadUint64x2SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint64x2Slice(base[i : i+lanes])) } var buf [lanes]uint64 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - - for ; i < length; i++ { - sum += collection[i] - } - return sum + return T(sum2(&buf)) } // SumFloat32x4 sums a slice of float32 using AVX SIMD (Float32x4, 4 lanes). @@ -294,26 +238,19 @@ func SumFloat32x4[T ~float32](collection []T) T { const lanes = simdLanes4 base := unsafeSliceFloat32(collection, length) - var acc archsimd.Float32x4 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadFloat32x4Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadFloat32x4SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadFloat32x4Slice(base[i : i+lanes])) } var buf [lanes]float32 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - - for ; i < length; i++ { - sum += collection[i] - } - return sum + return T(sum4(&buf)) } // SumFloat64x2 sums a slice of float64 using AVX SIMD (Float64x2, 2 lanes). @@ -327,26 +264,19 @@ func SumFloat64x2[T ~float64](collection []T) T { const lanes = simdLanes2 base := unsafeSliceFloat64(collection, length) - var acc archsimd.Float64x2 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadFloat64x2Slice(base[i : i+lanes]) - acc = acc.Add(v) + i := length % lanes + + acc := archsimd.LoadFloat64x2SlicePart(base[:i]) + + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadFloat64x2Slice(base[i : i+lanes])) } var buf [lanes]float64 acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } - - for ; i < length; i++ { - sum += collection[i] - } - return sum + return T(sum2(&buf)) } // MeanInt8x16 calculates the mean of a slice of int8 using AVX SIMD diff --git a/exp/simd/math_avx2.go b/exp/simd/math_avx2.go index 2f7e659c3..a4b331f2b 100644 --- a/exp/simd/math_avx2.go +++ b/exp/simd/math_avx2.go @@ -23,26 +23,22 @@ func SumInt8x32[T ~int8](collection []T) T { const lanes = simdLanes32 base := unsafeSliceInt8(collection, length) - var acc archsimd.Int8x32 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt8x32Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]int8 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadInt8x32SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt8x32Slice(base[i : i+lanes])) } - return sum + // 32 → 16 + acc16 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]int8 + acc16.Store(&buf) + + return T(sum16(&buf)) } // SumInt16x16 sums a slice of int16 using AVX2 SIMD (Int16x16, 16 lanes). @@ -57,26 +53,22 @@ func SumInt16x16[T ~int16](collection []T) T { const lanes = simdLanes16 base := unsafeSliceInt16(collection, length) - var acc archsimd.Int16x16 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt16x16Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]int16 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadInt16x16SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt16x16Slice(base[i : i+lanes])) } - return sum + // 16 → 8 + acc8 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]int16 + acc8.Store(&buf) + + return T(sum8(&buf)) } // SumInt32x8 sums a slice of int32 using AVX2 SIMD (Int32x8, 8 lanes). @@ -91,26 +83,22 @@ func SumInt32x8[T ~int32](collection []T) T { const lanes = simdLanes8 base := unsafeSliceInt32(collection, length) - var acc archsimd.Int32x8 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt32x8Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]int32 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadInt32x8SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt32x8Slice(base[i : i+lanes])) } - return sum + // 8 → 4 + acc4 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]int32 + acc4.Store(&buf) + + return T(sum4(&buf)) } // SumInt64x4 sums a slice of int64 using AVX2 SIMD (Int64x4, 4 lanes). @@ -125,26 +113,22 @@ func SumInt64x4[T ~int64](collection []T) T { const lanes = simdLanes4 base := unsafeSliceInt64(collection, length) - var acc archsimd.Int64x4 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt64x4Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]int64 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadInt64x4SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt64x4Slice(base[i : i+lanes])) } - return sum + // 4 → 2 + acc2 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]int64 + acc2.Store(&buf) + + return T(sum2(&buf)) } // SumUint8x32 sums a slice of uint8 using AVX2 SIMD (Uint8x32, 32 lanes). @@ -159,26 +143,22 @@ func SumUint8x32[T ~uint8](collection []T) T { const lanes = simdLanes32 base := unsafeSliceUint8(collection, length) - var acc archsimd.Uint8x32 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint8x32Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]uint8 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadUint8x32SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint8x32Slice(base[i : i+lanes])) } - return sum + // 32 → 16 + acc16 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]uint8 + acc16.Store(&buf) + + return T(sum16(&buf)) } // SumUint16x16 sums a slice of uint16 using AVX2 SIMD (Uint16x16, 16 lanes). @@ -193,26 +173,22 @@ func SumUint16x16[T ~uint16](collection []T) T { const lanes = simdLanes16 base := unsafeSliceUint16(collection, length) - var acc archsimd.Uint16x16 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint16x16Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]uint16 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadUint16x16SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint16x16Slice(base[i : i+lanes])) } - return sum + // 16 → 8 + acc8 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]uint16 + acc8.Store(&buf) + + return T(sum8(&buf)) } // SumUint32x8 sums a slice of uint32 using AVX2 SIMD (Uint32x8, 8 lanes). @@ -227,26 +203,22 @@ func SumUint32x8[T ~uint32](collection []T) T { const lanes = simdLanes8 base := unsafeSliceUint32(collection, length) - var acc archsimd.Uint32x8 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint32x8Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]uint32 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadUint32x8SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint32x8Slice(base[i : i+lanes])) } - return sum + // 8 → 4 + acc4 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]uint32 + acc4.Store(&buf) + + return T(sum4(&buf)) } // SumUint64x4 sums a slice of uint64 using AVX2 SIMD (Uint64x4, 4 lanes). @@ -261,26 +233,22 @@ func SumUint64x4[T ~uint64](collection []T) T { const lanes = simdLanes4 base := unsafeSliceUint64(collection, length) - var acc archsimd.Uint64x4 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint64x4Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]uint64 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadUint64x4SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint64x4Slice(base[i : i+lanes])) } - return sum + // 4 → 2 + acc2 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]uint64 + acc2.Store(&buf) + + return T(sum2(&buf)) } // SumFloat32x8 sums a slice of float32 using AVX2 SIMD (Float32x8, 8 lanes). @@ -294,26 +262,22 @@ func SumFloat32x8[T ~float32](collection []T) T { const lanes = simdLanes8 base := unsafeSliceFloat32(collection, length) - var acc archsimd.Float32x8 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadFloat32x8Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]float32 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadFloat32x8SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadFloat32x8Slice(base[i : i+lanes])) } - return sum + // 8 → 4 + acc4 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]float32 + acc4.Store(&buf) + + return T(sum4(&buf)) } // SumFloat64x4 sums a slice of float64 using AVX2 SIMD (Float64x4, 4 lanes). @@ -327,26 +291,22 @@ func SumFloat64x4[T ~float64](collection []T) T { const lanes = simdLanes4 base := unsafeSliceFloat64(collection, length) - var acc archsimd.Float64x4 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadFloat64x4Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]float64 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadFloat64x4SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadFloat64x4Slice(base[i : i+lanes])) } - return sum + // 4 → 2 + acc2 := acc.GetLo().Add(acc.GetHi()) + + var buf [lanes / 2]float64 + acc2.Store(&buf) + + return T(sum2(&buf)) } // MeanInt8x32 calculates the mean of a slice of int8 using AVX2 SIMD diff --git a/exp/simd/math_avx512.go b/exp/simd/math_avx512.go index 7714b1c9c..ccac0686c 100644 --- a/exp/simd/math_avx512.go +++ b/exp/simd/math_avx512.go @@ -23,26 +23,23 @@ func SumInt8x64[T ~int8](collection []T) T { const lanes = simdLanes64 base := unsafeSliceInt8(collection, length) - var acc archsimd.Int8x64 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt8x64Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]int8 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadInt8x64SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt8x64Slice(base[i : i+lanes])) } - return sum + // 64 → 32 → 16 + acc32 := acc.GetLo().Add(acc.GetHi()) + acc16 := acc32.GetLo().Add(acc32.GetHi()) + + var buf [lanes / 4]int8 + acc16.Store(&buf) + + return T(sum16(&buf)) } // SumInt16x32 sums a slice of int16 using AVX-512 SIMD (Int16x32, 32 lanes). @@ -57,26 +54,23 @@ func SumInt16x32[T ~int16](collection []T) T { const lanes = simdLanes32 base := unsafeSliceInt16(collection, length) - var acc archsimd.Int16x32 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt16x32Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]int16 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadInt16x32SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt16x32Slice(base[i : i+lanes])) } - return sum + // 32 → 16 → 8 + acc16 := acc.GetLo().Add(acc.GetHi()) + acc8 := acc16.GetLo().Add(acc16.GetHi()) + + var buf [lanes / 4]int16 + acc8.Store(&buf) + + return T(sum8(&buf)) } // SumInt32x16 sums a slice of int32 using AVX-512 SIMD (Int32x16, 16 lanes). @@ -91,26 +85,23 @@ func SumInt32x16[T ~int32](collection []T) T { const lanes = simdLanes16 base := unsafeSliceInt32(collection, length) - var acc archsimd.Int32x16 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt32x16Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]int32 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadInt32x16SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt32x16Slice(base[i : i+lanes])) } - return sum + // 16 → 8 → 4 + acc8 := acc.GetLo().Add(acc.GetHi()) + acc4 := acc8.GetLo().Add(acc8.GetHi()) + + var buf [lanes / 4]int32 + acc4.Store(&buf) + + return T(sum4(&buf)) } // SumInt64x8 sums a slice of int64 using AVX-512 SIMD (Int64x8, 8 lanes). @@ -125,26 +116,23 @@ func SumInt64x8[T ~int64](collection []T) T { const lanes = simdLanes8 base := unsafeSliceInt64(collection, length) - var acc archsimd.Int64x8 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadInt64x8Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]int64 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadInt64x8SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadInt64x8Slice(base[i : i+lanes])) } - return sum + // 8 → 4 → 2 + acc4 := acc.GetLo().Add(acc.GetHi()) + acc2 := acc4.GetLo().Add(acc4.GetHi()) + + var buf [lanes / 4]int64 + acc2.Store(&buf) + + return T(sum2(&buf)) } // SumUint8x64 sums a slice of uint8 using AVX-512 SIMD (Uint8x64, 64 lanes). @@ -159,26 +147,23 @@ func SumUint8x64[T ~uint8](collection []T) T { const lanes = simdLanes64 base := unsafeSliceUint8(collection, length) - var acc archsimd.Uint8x64 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint8x64Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]uint8 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadUint8x64SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint8x64Slice(base[i : i+lanes])) } - return sum + // 64 → 32 → 16 + acc32 := acc.GetLo().Add(acc.GetHi()) + acc16 := acc32.GetLo().Add(acc32.GetHi()) + + var buf [lanes / 4]uint8 + acc16.Store(&buf) + + return T(sum16(&buf)) } // SumUint16x32 sums a slice of uint16 using AVX-512 SIMD (Uint16x32, 32 lanes). @@ -193,26 +178,23 @@ func SumUint16x32[T ~uint16](collection []T) T { const lanes = simdLanes32 base := unsafeSliceUint16(collection, length) - var acc archsimd.Uint16x32 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint16x32Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]uint16 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadUint16x32SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint16x32Slice(base[i : i+lanes])) } - return sum + // 32 → 16 → 8 + acc16 := acc.GetLo().Add(acc.GetHi()) + acc8 := acc16.GetLo().Add(acc16.GetHi()) + + var buf [lanes / 4]uint16 + acc8.Store(&buf) + + return T(sum8(&buf)) } // SumUint32x16 sums a slice of uint32 using AVX-512 SIMD (Uint32x16, 16 lanes). @@ -227,26 +209,23 @@ func SumUint32x16[T ~uint32](collection []T) T { const lanes = simdLanes16 base := unsafeSliceUint32(collection, length) - var acc archsimd.Uint32x16 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint32x16Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]uint32 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadUint32x16SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint32x16Slice(base[i : i+lanes])) } - return sum + // 16 → 8 → 4 + acc8 := acc.GetLo().Add(acc.GetHi()) + acc4 := acc8.GetLo().Add(acc8.GetHi()) + + var buf [lanes / 4]uint32 + acc4.Store(&buf) + + return T(sum4(&buf)) } // SumUint64x8 sums a slice of uint64 using AVX-512 SIMD (Uint64x8, 8 lanes). @@ -261,26 +240,23 @@ func SumUint64x8[T ~uint64](collection []T) T { const lanes = simdLanes8 base := unsafeSliceUint64(collection, length) - var acc archsimd.Uint64x8 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadUint64x8Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]uint64 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadUint64x8SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadUint64x8Slice(base[i : i+lanes])) } - return sum + // 8 → 4 → 2 + acc4 := acc.GetLo().Add(acc.GetHi()) + acc2 := acc4.GetLo().Add(acc4.GetHi()) + + var buf [lanes / 4]uint64 + acc2.Store(&buf) + + return T(sum2(&buf)) } // SumFloat32x16 sums a slice of float32 using AVX-512 SIMD (Float32x16, 16 lanes). @@ -294,26 +270,23 @@ func SumFloat32x16[T ~float32](collection []T) T { const lanes = simdLanes16 base := unsafeSliceFloat32(collection, length) - var acc archsimd.Float32x16 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadFloat32x16Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]float32 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadFloat32x16SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadFloat32x16Slice(base[i : i+lanes])) } - return sum + // 16 → 8 → 4 + acc8 := acc.GetLo().Add(acc.GetHi()) + acc4 := acc8.GetLo().Add(acc8.GetHi()) + + var buf [lanes / 4]float32 + acc4.Store(&buf) + + return T(sum4(&buf)) } // SumFloat64x8 sums a slice of float64 using AVX-512 SIMD (Float64x8, 8 lanes). @@ -327,26 +300,23 @@ func SumFloat64x8[T ~float64](collection []T) T { const lanes = simdLanes8 base := unsafeSliceFloat64(collection, length) - var acc archsimd.Float64x8 - i := uint(0) - for ; i+lanes <= length; i += lanes { - v := archsimd.LoadFloat64x8Slice(base[i : i+lanes]) - acc = acc.Add(v) - } + i := length % lanes - var buf [lanes]float64 - acc.Store(&buf) - var sum T - for k := uint(0); k < lanes; k++ { - sum += T(buf[k]) - } + acc := archsimd.LoadFloat64x8SlicePart(base[:i]) - for ; i < length; i++ { - sum += collection[i] + for ; i < length; i += lanes { + acc = acc.Add(archsimd.LoadFloat64x8Slice(base[i : i+lanes])) } - return sum + // 8 → 4 → 2 + acc4 := acc.GetLo().Add(acc.GetHi()) + acc2 := acc4.GetLo().Add(acc4.GetHi()) + + var buf [lanes / 4]float64 + acc2.Store(&buf) + + return T(sum2(&buf)) } // MeanInt8x64 calculates the mean of a slice of int8 using AVX-512 SIMD