Skip to content

Remove SSE4.1 operation, reduce and reorder alignment checks #1064

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ermig1979 merged 1 commit into from
Oct 6, 2026
Merged

Remove SSE4.1 operation, reduce and reorder alignment checks #1064

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 25 additions & 47 deletions src/Simd/SimdSse41Operation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,68 +66,57 @@ namespace Simd
return _mm_adds_epu8(a, b);
}

template <bool align, SimdOperationBinary8uType type> void OperationBinary8u(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
template <SimdOperationBinary8uType type> void OperationBinary8u(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
size_t width, size_t height, size_t channelCount, uint8_t * dst, size_t dstStride)
{
assert(width*channelCount >= A);
if (align)
assert(Aligned(a) && Aligned(aStride) && Aligned(b) && Aligned(bStride) && Aligned(dst) && Aligned(dstStride));

size_t size = channelCount*width;
size_t alignedSize = Simd::AlignLo(size, A);
for (size_t row = 0; row < height; ++row)
{
for (size_t offset = 0; offset < alignedSize; offset += A)
{
const __m128i a_ = Load<align>((__m128i*)(a + offset));
const __m128i b_ = Load<align>((__m128i*)(b + offset));
Store<align>((__m128i*)(dst + offset), OperationBinary8u<type>(a_, b_));
const __m128i a_ = _mm_loadu_si128((__m128i*)(a + offset));
const __m128i b_ = _mm_loadu_si128((__m128i*)(b + offset));
_mm_storeu_si128((__m128i*)(dst + offset), OperationBinary8u<type>(a_, b_));
}
if (alignedSize != size)
{
const __m128i a_ = Load<false>((__m128i*)(a + size - A));
const __m128i b_ = Load<false>((__m128i*)(b + size - A));
Store<false>((__m128i*)(dst + size - A), OperationBinary8u<type>(a_, b_));
const __m128i a_ = _mm_loadu_si128((__m128i*)(a + size - A));
const __m128i b_ = _mm_loadu_si128((__m128i*)(b + size - A));
_mm_storeu_si128((__m128i*)(dst + size - A), OperationBinary8u<type>(a_, b_));
}
a += aStride;
b += bStride;
dst += dstStride;
}
}

template <bool align> void OperationBinary8u(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
void OperationBinary8u(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
size_t width, size_t height, size_t channelCount, uint8_t * dst, size_t dstStride, SimdOperationBinary8uType type)
{
switch (type)
{
case SimdOperationBinary8uAverage:
return OperationBinary8u<align, SimdOperationBinary8uAverage>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
return OperationBinary8u<SimdOperationBinary8uAverage>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
case SimdOperationBinary8uAnd:
return OperationBinary8u<align, SimdOperationBinary8uAnd>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
return OperationBinary8u<SimdOperationBinary8uAnd>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
case SimdOperationBinary8uOr:
return OperationBinary8u<align, SimdOperationBinary8uOr>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
return OperationBinary8u<SimdOperationBinary8uOr>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
case SimdOperationBinary8uMaximum:
return OperationBinary8u<align, SimdOperationBinary8uMaximum>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
return OperationBinary8u<SimdOperationBinary8uMaximum>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
case SimdOperationBinary8uMinimum:
return OperationBinary8u<align, SimdOperationBinary8uMinimum>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
return OperationBinary8u<SimdOperationBinary8uMinimum>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
case SimdOperationBinary8uSaturatedSubtraction:
return OperationBinary8u<align, SimdOperationBinary8uSaturatedSubtraction>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
return OperationBinary8u<SimdOperationBinary8uSaturatedSubtraction>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
case SimdOperationBinary8uSaturatedAddition:
return OperationBinary8u<align, SimdOperationBinary8uSaturatedAddition>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
return OperationBinary8u<SimdOperationBinary8uSaturatedAddition>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride);
default:
assert(0);
}
}

void OperationBinary8u(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
size_t width, size_t height, size_t channelCount, uint8_t * dst, size_t dstStride, SimdOperationBinary8uType type)
{
if (Aligned(a) && Aligned(aStride) && Aligned(b) && Aligned(bStride) && Aligned(dst) && Aligned(dstStride))
OperationBinary8u<true>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride, type);
else
OperationBinary8u<false>(a, aStride, b, bStride, width, height, channelCount, dst, dstStride, type);
}

//-----------------------------------------------------------------------------------------

template <SimdOperationBinary16iType type> SIMD_INLINE __m128i OperationBinary16i(const __m128i & a, const __m128i & b);
Expand All @@ -142,57 +131,46 @@ namespace Simd
return _mm_sub_epi16(a, b);
}

template <bool align, SimdOperationBinary16iType type> void OperationBinary16i(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
template <SimdOperationBinary16iType type> void OperationBinary16i(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
size_t width, size_t height, uint8_t * dst, size_t dstStride)
{
assert(width * sizeof(uint16_t) >= A);
if (align)
assert(Aligned(a) && Aligned(aStride) && Aligned(b) && Aligned(bStride) && Aligned(dst) && Aligned(dstStride));

size_t size = width * sizeof(int16_t);
size_t alignedSize = Simd::AlignLo(size, A);
for (size_t row = 0; row < height; ++row)
{
for (size_t offset = 0; offset < alignedSize; offset += A)
{
const __m128i a_ = Load<align>((__m128i*)(a + offset));
const __m128i b_ = Load<align>((__m128i*)(b + offset));
Store<align>((__m128i*)(dst + offset), OperationBinary16i<type>(a_, b_));
const __m128i a_ = _mm_loadu_si128((__m128i*)(a + offset));
const __m128i b_ = _mm_loadu_si128((__m128i*)(b + offset));
_mm_storeu_si128((__m128i*)(dst + offset), OperationBinary16i<type>(a_, b_));
}
if (alignedSize != size)
{
const __m128i a_ = Load<false>((__m128i*)(a + size - A));
const __m128i b_ = Load<false>((__m128i*)(b + size - A));
Store<false>((__m128i*)(dst + size - A), OperationBinary16i<type>(a_, b_));
const __m128i a_ = _mm_loadu_si128((__m128i*)(a + size - A));
const __m128i b_ = _mm_loadu_si128((__m128i*)(b + size - A));
_mm_storeu_si128((__m128i*)(dst + size - A), OperationBinary16i<type>(a_, b_));
}
a += aStride;
b += bStride;
dst += dstStride;
}
}

template <bool align> void OperationBinary16i(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
void OperationBinary16i(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
size_t width, size_t height, uint8_t * dst, size_t dstStride, SimdOperationBinary16iType type)
{
switch (type)
{
case SimdOperationBinary16iAddition:
return OperationBinary16i<align, SimdOperationBinary16iAddition>(a, aStride, b, bStride, width, height, dst, dstStride);
return OperationBinary16i<SimdOperationBinary16iAddition>(a, aStride, b, bStride, width, height, dst, dstStride);
case SimdOperationBinary16iSubtraction:
return OperationBinary16i<align, SimdOperationBinary16iSubtraction>(a, aStride, b, bStride, width, height, dst, dstStride);
return OperationBinary16i<SimdOperationBinary16iSubtraction>(a, aStride, b, bStride, width, height, dst, dstStride);
default:
assert(0);
}
}

void OperationBinary16i(const uint8_t * a, size_t aStride, const uint8_t * b, size_t bStride,
size_t width, size_t height, uint8_t * dst, size_t dstStride, SimdOperationBinary16iType type)
{
if (Aligned(a) && Aligned(aStride) && Aligned(b) && Aligned(bStride) && Aligned(dst) && Aligned(dstStride))
OperationBinary16i<true>(a, aStride, b, bStride, width, height, dst, dstStride, type);
else
OperationBinary16i<false>(a, aStride, b, bStride, width, height, dst, dstStride, type);
}
}
#endif
}
80 changes: 33 additions & 47 deletions src/Simd/SimdSse41Reduce.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,16 +60,16 @@ namespace Simd
return Average8(_mm_shuffle_epi8(s00, K8_RC4), _mm_shuffle_epi8(s01, K8_RC4), _mm_shuffle_epi8(s10, K8_RC4), _mm_shuffle_epi8(s11, K8_RC4));
}

template <size_t channelCount, bool align> SIMD_INLINE void ReduceColor2x2(const uint8_t * src0, const uint8_t * src1, uint8_t * dst)
template <size_t channelCount> SIMD_INLINE void ReduceColor2x2(const uint8_t * src0, const uint8_t * src1, uint8_t * dst)
{
__m128i s00 = Load<align>((__m128i*)src0 + 0);
__m128i s01 = Load<align>((__m128i*)src0 + 1);
__m128i s10 = Load<align>((__m128i*)src1 + 0);
__m128i s11 = Load<align>((__m128i*)src1 + 1);
Store<align>((__m128i*)dst, Average8<channelCount>(s00, s01, s10, s11));
__m128i s00 = _mm_loadu_si128((__m128i*)src0 + 0);
__m128i s01 = _mm_loadu_si128((__m128i*)src0 + 1);
__m128i s10 = _mm_loadu_si128((__m128i*)src1 + 0);
__m128i s11 = _mm_loadu_si128((__m128i*)src1 + 1);
_mm_storeu_si128((__m128i*)dst, Average8<channelCount>(s00, s01, s10, s11));
}

template <size_t channelCount, bool align> void ReduceColor2x2(const uint8_t * src, size_t srcWidth, size_t srcHeight, size_t srcStride, uint8_t * dst, size_t dstStride)
template <size_t channelCount> void ReduceColor2x2(const uint8_t * src, size_t srcWidth, size_t srcHeight, size_t srcStride, uint8_t * dst, size_t dstStride)
{
size_t evenWidth = AlignLo(srcWidth, 2);
size_t evenSize = evenWidth * channelCount;
Expand All @@ -80,12 +80,12 @@ namespace Simd
const uint8_t *src1 = (srcRow == srcHeight - 1 ? src : src + srcStride);
size_t srcOffset = 0, dstOffset = 0;
for (; srcOffset < alignedSize; srcOffset += DA, dstOffset += A)
ReduceColor2x2<channelCount, align>(src0 + srcOffset, src1 + srcOffset, dst + dstOffset);
ReduceColor2x2<channelCount>(src0 + srcOffset, src1 + srcOffset, dst + dstOffset);
if (alignedSize != evenSize)
{
srcOffset = evenSize - DA;
dstOffset = srcOffset / 2;
ReduceColor2x2<channelCount, false>(src0 + srcOffset, src1 + srcOffset, dst + dstOffset);
ReduceColor2x2<channelCount>(src0 + srcOffset, src1 + srcOffset, dst + dstOffset);
}
if (evenWidth != srcWidth)
{
Expand All @@ -105,38 +105,38 @@ namespace Simd
const __m128i K8_BGR5 = SIMD_MM_SETR_EPI8(0xF, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1);
const __m128i K8_BGR6 = SIMD_MM_SETR_EPI8(-1, 0x2, 0x0, 0x3, 0x4, 0x7, 0x5, 0x8, 0x6, 0x9, 0xA, 0xD, 0xB, 0xE, 0xC, 0xF);

template <bool align> SIMD_INLINE void ReduceBgr2x2(const uint8_t * src0, const uint8_t * src1, uint8_t * dst)
SIMD_INLINE void ReduceBgr2x2(const uint8_t * src0, const uint8_t * src1, uint8_t * dst)
{
__m128i s00 = Load<align>((__m128i*)src0 + 0);
__m128i s01 = Load<align>((__m128i*)src0 + 1);
__m128i s02 = Load<align>((__m128i*)src0 + 2);
__m128i s10 = Load<align>((__m128i*)src1 + 0);
__m128i s11 = Load<align>((__m128i*)src1 + 1);
__m128i s12 = Load<align>((__m128i*)src1 + 2);
__m128i s00 = _mm_loadu_si128((__m128i*)src0 + 0);
__m128i s01 = _mm_loadu_si128((__m128i*)src0 + 1);
__m128i s02 = _mm_loadu_si128((__m128i*)src0 + 2);
__m128i s10 = _mm_loadu_si128((__m128i*)src1 + 0);
__m128i s11 = _mm_loadu_si128((__m128i*)src1 + 1);
__m128i s12 = _mm_loadu_si128((__m128i*)src1 + 2);
__m128i m00 = _mm_or_si128(_mm_shuffle_epi8(s00, K8_BGR0), _mm_shuffle_epi8(s01, K8_BGR1));
__m128i m01 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(s00, K8_BGR2), _mm_shuffle_epi8(s01, K8_BGR3)), _mm_shuffle_epi8(s02, K8_BGR4));
__m128i m10 = _mm_or_si128(_mm_shuffle_epi8(s10, K8_BGR0), _mm_shuffle_epi8(s11, K8_BGR1));
__m128i m11 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(s10, K8_BGR2), _mm_shuffle_epi8(s11, K8_BGR3)), _mm_shuffle_epi8(s12, K8_BGR4));
Store<align>((__m128i*)dst + 0, Average8(m00, m01, m10, m11));
__m128i s03 = Load<align>((__m128i*)src0 + 3);
__m128i s04 = Load<align>((__m128i*)src0 + 4);
__m128i s13 = Load<align>((__m128i*)src1 + 3);
__m128i s14 = Load<align>((__m128i*)src1 + 4);
_mm_storeu_si128((__m128i*)dst + 0, Average8(m00, m01, m10, m11));
__m128i s03 = _mm_loadu_si128((__m128i*)src0 + 3);
__m128i s04 = _mm_loadu_si128((__m128i*)src0 + 4);
__m128i s13 = _mm_loadu_si128((__m128i*)src1 + 3);
__m128i s14 = _mm_loadu_si128((__m128i*)src1 + 4);
__m128i m02 = _mm_or_si128(_mm_shuffle_epi8(s01, K8_BGR5), _mm_shuffle_epi8(s02, K8_BGR6));
__m128i m03 = _mm_or_si128(_mm_shuffle_epi8(s03, K8_BGR0), _mm_shuffle_epi8(s04, K8_BGR1));
__m128i m12 = _mm_or_si128(_mm_shuffle_epi8(s11, K8_BGR5), _mm_shuffle_epi8(s12, K8_BGR6));
__m128i m13 = _mm_or_si128(_mm_shuffle_epi8(s13, K8_BGR0), _mm_shuffle_epi8(s14, K8_BGR1));
Store<align>((__m128i*)dst + 1, Average8(m02, m03, m12, m13));
__m128i s05 = Load<align>((__m128i*)src0 + 5);
__m128i s15 = Load<align>((__m128i*)src1 + 5);
_mm_storeu_si128((__m128i*)dst + 1, Average8(m02, m03, m12, m13));
__m128i s05 = _mm_loadu_si128((__m128i*)src0 + 5);
__m128i s15 = _mm_loadu_si128((__m128i*)src1 + 5);
__m128i m04 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(s03, K8_BGR2), _mm_shuffle_epi8(s04, K8_BGR3)), _mm_shuffle_epi8(s05, K8_BGR4));
__m128i m05 = _mm_or_si128(_mm_shuffle_epi8(s04, K8_BGR5), _mm_shuffle_epi8(s05, K8_BGR6));
__m128i m14 = _mm_or_si128(_mm_or_si128(_mm_shuffle_epi8(s13, K8_BGR2), _mm_shuffle_epi8(s14, K8_BGR3)), _mm_shuffle_epi8(s15, K8_BGR4));
__m128i m15 = _mm_or_si128(_mm_shuffle_epi8(s14, K8_BGR5), _mm_shuffle_epi8(s15, K8_BGR6));
Store<align>((__m128i*)dst + 2, Average8(m04, m05, m14, m15));
_mm_storeu_si128((__m128i*)dst + 2, Average8(m04, m05, m14, m15));
}

template <bool align> void ReduceBgr2x2(const uint8_t * src, size_t srcWidth, size_t srcHeight, size_t srcStride, uint8_t * dst, size_t dstStride)
void ReduceBgr2x2(const uint8_t * src, size_t srcWidth, size_t srcHeight, size_t srcStride, uint8_t * dst, size_t dstStride)
{
size_t evenWidth = AlignLo(srcWidth, 2);
size_t alignedWidth = AlignLo(srcWidth, DA);
Expand All @@ -149,12 +149,12 @@ namespace Simd
const uint8_t *src1 = (srcRow == srcHeight - 1 ? src : src + srcStride);
size_t srcOffset = 0, dstOffset = 0;
for (; srcOffset < alignedSize; srcOffset += srcStep, dstOffset += dstStep)
ReduceBgr2x2<align>(src0 + srcOffset, src1 + srcOffset, dst + dstOffset);
ReduceBgr2x2(src0 + srcOffset, src1 + srcOffset, dst + dstOffset);
if (alignedSize != evenSize)
{
srcOffset = evenSize - srcStep;
dstOffset = srcOffset / 2;
ReduceBgr2x2<false>(src0 + srcOffset, src1 + srcOffset, dst + dstOffset);
ReduceBgr2x2(src0 + srcOffset, src1 + srcOffset, dst + dstOffset);
}
if (evenWidth != srcWidth)
{
Expand All @@ -166,34 +166,20 @@ namespace Simd
}
}

template <bool align> void ReduceColor2x2(const uint8_t * src, size_t srcWidth, size_t srcHeight, size_t srcStride,
void ReduceColor2x2(const uint8_t * src, size_t srcWidth, size_t srcHeight, size_t srcStride,
uint8_t * dst, size_t dstWidth, size_t dstHeight, size_t dstStride, size_t channelCount)
{
assert((srcWidth + 1) / 2 == dstWidth && (srcHeight + 1) / 2 == dstHeight && srcWidth >= DA);
if (align)
{
assert(Aligned(src) && Aligned(srcStride));
assert(Aligned(dst) && Aligned(dstStride));
}

switch (channelCount)
{
case 1: ReduceColor2x2<1, align>(src, srcWidth, srcHeight, srcStride, dst, dstStride); break;
case 2: ReduceColor2x2<2, align>(src, srcWidth, srcHeight, srcStride, dst, dstStride); break;
case 3: ReduceBgr2x2<align>(src, srcWidth, srcHeight, srcStride, dst, dstStride); break;
case 4: ReduceColor2x2<4, align>(src, srcWidth, srcHeight, srcStride, dst, dstStride); break;
case 1: ReduceColor2x2<1>(src, srcWidth, srcHeight, srcStride, dst, dstStride); break;
case 2: ReduceColor2x2<2>(src, srcWidth, srcHeight, srcStride, dst, dstStride); break;
case 3: ReduceBgr2x2(src, srcWidth, srcHeight, srcStride, dst, dstStride); break;
case 4: ReduceColor2x2<4>(src, srcWidth, srcHeight, srcStride, dst, dstStride); break;
default: assert(0);
}
}

void ReduceColor2x2(const uint8_t * src, size_t srcWidth, size_t srcHeight, size_t srcStride,
uint8_t * dst, size_t dstWidth, size_t dstHeight, size_t dstStride, size_t channelCount)
{
if (Aligned(src) && Aligned(srcStride) && Aligned(dst) && Aligned(dstStride))
ReduceColor2x2<true>(src, srcWidth, srcHeight, srcStride, dst, dstWidth, dstHeight, dstStride, channelCount);
else
ReduceColor2x2<false>(src, srcWidth, srcHeight, srcStride, dst, dstWidth, dstHeight, dstStride, channelCount);
}
}
#endif
}
Loading