Skip to content

Revert "Use SIMD intrinsics for vector shifts" #2217

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adamgemmell merged 3 commits into from
Sep 9, 2026
+249 −495
Merged

Revert "Use SIMD intrinsics for vector shifts" #2217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d8e51ac
Revert "Use SIMD intrinsics for vector shifts"
RalfJung Sep 5, 2026
02f0cd7
de-constify methods that depended on the vector shift ones
RalfJung Sep 5, 2026
768db9f
fmt
RalfJung Sep 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 50 additions & 90 deletions crates/core_arch/src/x86/avx2.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2871,14 +2871,8 @@ pub const fn _mm256_bslli_epi128<const IMM8: i32>(a: __m256i) -> __m256i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsllvd))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
Comment thread
adamgemmell marked this conversation as resolved.
pub const fn _mm_sllv_epi32(a: __m128i, count: __m128i) -> __m128i {
unsafe {
let count = count.as_u32x4();
let no_overflow: u32x4 = simd_lt(count, u32x4::splat(u32::BITS));
let count = simd_select(no_overflow, count, u32x4::ZERO);
simd_select(no_overflow, simd_shl(a.as_u32x4(), count), u32x4::ZERO).as_m128i()
}
pub fn _mm_sllv_epi32(a: __m128i, count: __m128i) -> __m128i {
unsafe { transmute(psllvd(a.as_i32x4(), count.as_i32x4())) }
}

/// Shifts packed 32-bit integers in `a` left by the amount
Expand All @@ -2890,14 +2884,8 @@ pub const fn _mm_sllv_epi32(a: __m128i, count: __m128i) -> __m128i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsllvd))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_sllv_epi32(a: __m256i, count: __m256i) -> __m256i {
unsafe {
let count = count.as_u32x8();
let no_overflow: u32x8 = simd_lt(count, u32x8::splat(u32::BITS));
let count = simd_select(no_overflow, count, u32x8::ZERO);
simd_select(no_overflow, simd_shl(a.as_u32x8(), count), u32x8::ZERO).as_m256i()
}
pub fn _mm256_sllv_epi32(a: __m256i, count: __m256i) -> __m256i {
unsafe { transmute(psllvd256(a.as_i32x8(), count.as_i32x8())) }
}

/// Shifts packed 64-bit integers in `a` left by the amount
Expand All @@ -2909,14 +2897,8 @@ pub const fn _mm256_sllv_epi32(a: __m256i, count: __m256i) -> __m256i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsllvq))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm_sllv_epi64(a: __m128i, count: __m128i) -> __m128i {
unsafe {
let count = count.as_u64x2();
let no_overflow: u64x2 = simd_lt(count, u64x2::splat(u64::BITS as u64));
let count = simd_select(no_overflow, count, u64x2::ZERO);
simd_select(no_overflow, simd_shl(a.as_u64x2(), count), u64x2::ZERO).as_m128i()
}
pub fn _mm_sllv_epi64(a: __m128i, count: __m128i) -> __m128i {
unsafe { transmute(psllvq(a.as_i64x2(), count.as_i64x2())) }
}

/// Shifts packed 64-bit integers in `a` left by the amount
Expand All @@ -2928,14 +2910,8 @@ pub const fn _mm_sllv_epi64(a: __m128i, count: __m128i) -> __m128i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsllvq))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_sllv_epi64(a: __m256i, count: __m256i) -> __m256i {
unsafe {
let count = count.as_u64x4();
let no_overflow: u64x4 = simd_lt(count, u64x4::splat(u64::BITS as u64));
let count = simd_select(no_overflow, count, u64x4::ZERO);
simd_select(no_overflow, simd_shl(a.as_u64x4(), count), u64x4::ZERO).as_m256i()
}
pub fn _mm256_sllv_epi64(a: __m256i, count: __m256i) -> __m256i {
unsafe { transmute(psllvq256(a.as_i64x4(), count.as_i64x4())) }
}

/// Shifts packed 16-bit integers in `a` right by `count` while
Expand Down Expand Up @@ -3000,14 +2976,8 @@ pub const fn _mm256_srai_epi32<const IMM8: i32>(a: __m256i) -> __m256i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsravd))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm_srav_epi32(a: __m128i, count: __m128i) -> __m128i {
unsafe {
let count = count.as_u32x4();
let no_overflow: u32x4 = simd_lt(count, u32x4::splat(u32::BITS));
let count = simd_select(no_overflow, transmute(count), i32x4::splat(31));
simd_shr(a.as_i32x4(), count).as_m128i()
}
pub fn _mm_srav_epi32(a: __m128i, count: __m128i) -> __m128i {
unsafe { transmute(psravd(a.as_i32x4(), count.as_i32x4())) }
}

/// Shifts packed 32-bit integers in `a` right by the amount specified by the
Expand All @@ -3018,14 +2988,8 @@ pub const fn _mm_srav_epi32(a: __m128i, count: __m128i) -> __m128i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsravd))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_srav_epi32(a: __m256i, count: __m256i) -> __m256i {
unsafe {
let count = count.as_u32x8();
let no_overflow: u32x8 = simd_lt(count, u32x8::splat(u32::BITS));
let count = simd_select(no_overflow, transmute(count), i32x8::splat(31));
simd_shr(a.as_i32x8(), count).as_m256i()
}
pub fn _mm256_srav_epi32(a: __m256i, count: __m256i) -> __m256i {
unsafe { transmute(psravd256(a.as_i32x8(), count.as_i32x8())) }
}

/// Shifts 128-bit lanes in `a` right by `imm8` bytes while shifting in zeros.
Expand Down Expand Up @@ -3212,14 +3176,8 @@ pub const fn _mm256_srli_epi64<const IMM8: i32>(a: __m256i) -> __m256i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsrlvd))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm_srlv_epi32(a: __m128i, count: __m128i) -> __m128i {
unsafe {
let count = count.as_u32x4();
let no_overflow: u32x4 = simd_lt(count, u32x4::splat(u32::BITS));
let count = simd_select(no_overflow, count, u32x4::ZERO);
simd_select(no_overflow, simd_shr(a.as_u32x4(), count), u32x4::ZERO).as_m128i()
}
pub fn _mm_srlv_epi32(a: __m128i, count: __m128i) -> __m128i {
unsafe { transmute(psrlvd(a.as_i32x4(), count.as_i32x4())) }
}

/// Shifts packed 32-bit integers in `a` right by the amount specified by
Expand All @@ -3230,14 +3188,8 @@ pub const fn _mm_srlv_epi32(a: __m128i, count: __m128i) -> __m128i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsrlvd))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_srlv_epi32(a: __m256i, count: __m256i) -> __m256i {
unsafe {
let count = count.as_u32x8();
let no_overflow: u32x8 = simd_lt(count, u32x8::splat(u32::BITS));
let count = simd_select(no_overflow, count, u32x8::ZERO);
simd_select(no_overflow, simd_shr(a.as_u32x8(), count), u32x8::ZERO).as_m256i()
}
pub fn _mm256_srlv_epi32(a: __m256i, count: __m256i) -> __m256i {
unsafe { transmute(psrlvd256(a.as_i32x8(), count.as_i32x8())) }
}

/// Shifts packed 64-bit integers in `a` right by the amount specified by
Expand All @@ -3248,14 +3200,8 @@ pub const fn _mm256_srlv_epi32(a: __m256i, count: __m256i) -> __m256i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsrlvq))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm_srlv_epi64(a: __m128i, count: __m128i) -> __m128i {
unsafe {
let count = count.as_u64x2();
let no_overflow: u64x2 = simd_lt(count, u64x2::splat(u64::BITS as u64));
let count = simd_select(no_overflow, count, u64x2::ZERO);
simd_select(no_overflow, simd_shr(a.as_u64x2(), count), u64x2::ZERO).as_m128i()
}
pub fn _mm_srlv_epi64(a: __m128i, count: __m128i) -> __m128i {
unsafe { transmute(psrlvq(a.as_i64x2(), count.as_i64x2())) }
}

/// Shifts packed 64-bit integers in `a` right by the amount specified by
Expand All @@ -3266,14 +3212,8 @@ pub const fn _mm_srlv_epi64(a: __m128i, count: __m128i) -> __m128i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpsrlvq))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_srlv_epi64(a: __m256i, count: __m256i) -> __m256i {
unsafe {
let count = count.as_u64x4();
let no_overflow: u64x4 = simd_lt(count, u64x4::splat(u64::BITS as u64));
let count = simd_select(no_overflow, count, u64x4::ZERO);
simd_select(no_overflow, simd_shr(a.as_u64x4(), count), u64x4::ZERO).as_m256i()
}
pub fn _mm256_srlv_epi64(a: __m256i, count: __m256i) -> __m256i {
unsafe { transmute(psrlvq256(a.as_i64x4(), count.as_i64x4())) }
}

/// Load 256-bits of integer data from memory into dst using a non-temporal memory hint. mem_addr
Expand Down Expand Up @@ -3849,16 +3789,36 @@ unsafe extern "llvm-intrinsic" {
fn pslld(a: i32x8, count: i32x4) -> i32x8;
#[link_name = "llvm.x86.avx2.psll.q"]
fn psllq(a: i64x4, count: i64x2) -> i64x4;
#[link_name = "llvm.x86.avx2.psllv.d"]
fn psllvd(a: i32x4, count: i32x4) -> i32x4;
#[link_name = "llvm.x86.avx2.psllv.d.256"]
fn psllvd256(a: i32x8, count: i32x8) -> i32x8;
#[link_name = "llvm.x86.avx2.psllv.q"]
fn psllvq(a: i64x2, count: i64x2) -> i64x2;
#[link_name = "llvm.x86.avx2.psllv.q.256"]
fn psllvq256(a: i64x4, count: i64x4) -> i64x4;
#[link_name = "llvm.x86.avx2.psra.w"]
fn psraw(a: i16x16, count: i16x8) -> i16x16;
#[link_name = "llvm.x86.avx2.psra.d"]
fn psrad(a: i32x8, count: i32x4) -> i32x8;
#[link_name = "llvm.x86.avx2.psrav.d"]
fn psravd(a: i32x4, count: i32x4) -> i32x4;
#[link_name = "llvm.x86.avx2.psrav.d.256"]
fn psravd256(a: i32x8, count: i32x8) -> i32x8;
#[link_name = "llvm.x86.avx2.psrl.w"]
fn psrlw(a: i16x16, count: i16x8) -> i16x16;
#[link_name = "llvm.x86.avx2.psrl.d"]
fn psrld(a: i32x8, count: i32x4) -> i32x8;
#[link_name = "llvm.x86.avx2.psrl.q"]
fn psrlq(a: i64x4, count: i64x2) -> i64x4;
#[link_name = "llvm.x86.avx2.psrlv.d"]
fn psrlvd(a: i32x4, count: i32x4) -> i32x4;
#[link_name = "llvm.x86.avx2.psrlv.d.256"]
fn psrlvd256(a: i32x8, count: i32x8) -> i32x8;
#[link_name = "llvm.x86.avx2.psrlv.q"]
fn psrlvq(a: i64x2, count: i64x2) -> i64x2;
#[link_name = "llvm.x86.avx2.psrlv.q.256"]
fn psrlvq256(a: i64x4, count: i64x4) -> i64x4;
#[link_name = "llvm.x86.avx2.pshuf.b"]
fn pshufb(a: u8x32, b: u8x32) -> u8x32;
#[link_name = "llvm.x86.avx2.permd"]
Expand Down Expand Up @@ -5163,7 +5123,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm_sllv_epi32() {
fn test_mm_sllv_epi32() {
let a = _mm_set1_epi32(2);
let b = _mm_set1_epi32(1);
let r = _mm_sllv_epi32(a, b);
Expand All @@ -5172,7 +5132,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm256_sllv_epi32() {
fn test_mm256_sllv_epi32() {
let a = _mm256_set1_epi32(2);
let b = _mm256_set1_epi32(1);
let r = _mm256_sllv_epi32(a, b);
Expand All @@ -5181,7 +5141,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm_sllv_epi64() {
fn test_mm_sllv_epi64() {
let a = _mm_set1_epi64x(2);
let b = _mm_set1_epi64x(1);
let r = _mm_sllv_epi64(a, b);
Expand All @@ -5190,7 +5150,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm256_sllv_epi64() {
fn test_mm256_sllv_epi64() {
let a = _mm256_set1_epi64x(2);
let b = _mm256_set1_epi64x(1);
let r = _mm256_sllv_epi64(a, b);
Expand Down Expand Up @@ -5231,7 +5191,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm_srav_epi32() {
fn test_mm_srav_epi32() {
let a = _mm_set1_epi32(4);
let count = _mm_set1_epi32(1);
let r = _mm_srav_epi32(a, count);
Expand All @@ -5240,7 +5200,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm256_srav_epi32() {
fn test_mm256_srav_epi32() {
let a = _mm256_set1_epi32(4);
let count = _mm256_set1_epi32(1);
let r = _mm256_srav_epi32(a, count);
Expand Down Expand Up @@ -5317,7 +5277,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm_srlv_epi32() {
fn test_mm_srlv_epi32() {
let a = _mm_set1_epi32(2);
let count = _mm_set1_epi32(1);
let r = _mm_srlv_epi32(a, count);
Expand All @@ -5326,7 +5286,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm256_srlv_epi32() {
fn test_mm256_srlv_epi32() {
let a = _mm256_set1_epi32(2);
let count = _mm256_set1_epi32(1);
let r = _mm256_srlv_epi32(a, count);
Expand All @@ -5335,7 +5295,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm_srlv_epi64() {
fn test_mm_srlv_epi64() {
let a = _mm_set1_epi64x(2);
let count = _mm_set1_epi64x(1);
let r = _mm_srlv_epi64(a, count);
Expand All @@ -5344,7 +5304,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm256_srlv_epi64() {
fn test_mm256_srlv_epi64() {
let a = _mm256_set1_epi64x(2);
let count = _mm256_set1_epi64x(1);
let r = _mm256_srlv_epi64(a, count);
Expand Down
Loading
Loading