From ffc067b2ab8ea18d1e647d576b668462874c2a06 Mon Sep 17 00:00:00 2001 From: Julien Marrec Date: Tue, 6 Oct 2026 15:52:56 +0200 Subject: [PATCH 1/2] Initial benchmark for vector alignment, and ffast-math+march/mcpu=native ``` Products/bench_alignment_vectorization --benchmark_repetitions=5 --benchmark_report_aggregates_only=true --benchmark_out=../results/alignment_default.json --benchmark_out_format=json Products/bench_alignment_vectorization_native --benchmark_repetitions=5 --benchmark_report_aggregates_only=true --benchmark_out=../results/alignment_native.json --benchmark_out_format=json ``` Results: on macOS M1 Max Median of 5, ns. "default" = -O3 -ffp-contract=off (like EnergyPlus), "native" = -O3 -mcpu=native -ffast-math -fno-finite-math-only (like NatLabRockies/EnergyPlus#11816). Sizes 1 Ki / 32 Ki / 4 Mi doubles. | Kernel | default, 64-aligned | native, 64-aligned | native speedup | native, misaligned +8 B vs 64-aligned | |---------|-------------------------------|-----------------------------|--------------------|---------------------------------------| | Sum | 849 / 31,338 / 4,038,135 | 99 / 3,947 / 544,375 | 8.6x / 7.9x / 7.4x | +0 / +1 / +1% | | Dot | 874 / 31,398 / 4,036,528 | 138 / 6,350 / 1,072,154 | 6.3x / 4.9x / 3.8x | +2 / +3 / 0% | | Cubic | 255 / 7,850 / 1,090,068 | 182 / 5,687 / 1,086,287 | 1.4x / 1.4x / 1.0x | +3 / +7 / 0% | | Axpy | 132 / 7,529 / 1,071,804 | 131 / 7,548 / 1,074,188 | 1.0x | +32 / +14 / +2% | | Stencil | 333 / 9,142 / 2,105,869 | 325 / 9,109 / 2,056,428 | 1.0x | 0 / +1 / -24% (artifact, see below) | Findings: - Alignment: Default, Aligned64 and Aligned128 are within noise everywhere, in both builds, and std::assume_aligned<64> adds nothing. Only misalignment within the vector width costs anything, and only for streaming kernels on cache-resident data (Axpy +32% in L1, +14% in L2 in the native build, +38% / +13% in the default build). - Native + fast math: reductions vectorize thanks to reassociation (Sum 7-9x, Dot 4-6x, Dot is bandwidth bound at 4 Mi), polynomial evaluation gains 1.4x in cache from FMA contraction (disabled by -ffp-contract=off in the default build). Axpy and Stencil are unchanged: already vectorized in the default build, or memory bound. Caveats (to be addressed in a follow-up): - macOS libmalloc gave std::vector 128-byte (or better) alignment here: allocations >= 32 KiB are page-aligned (16 KiB pages), and the 8 KiB one was 128-aligned by luck (small allocations are only guaranteed 16 bytes, measured). So "Default vs Aligned64" did not test the 16-byte case that glibc/MSVC give: needs explicit Offset16 / Offset32 layouts. - Power-of-two sizes: no vector-loop tails, and cache set aliasing between in/out buffers (most likely the source of the Stencil -24% at 4 Mi, present in both builds). Real EnergyPlus array sizes are small and not powers of two (37, 251, 1003...). --- CMakeLists.txt | 15 ++ bench_alignment_vectorization.cpp | 290 ++++++++++++++++++++++++++++++ 2 files changed, 305 insertions(+) create mode 100644 bench_alignment_vectorization.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 96bbf41..4a51243 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -264,6 +264,21 @@ int main() { }" CXX_STANDARD_REQUIRED ON CXX_EXTENSIONS OFF ) + elseif (bench_name MATCHES "alignment_vectorization") + # Built twice: EnergyPlus-like default flags, and native + fast math (like NatLabRockies/EnergyPlus#11816) + add_executable( ${bench_name} ${bench_file} ) + target_link_libraries(${bench_name} PRIVATE benchmark::benchmark_main) + target_compile_options(${bench_name} PRIVATE -ffp-contract=off) + + add_executable( ${bench_name}_native ${bench_file} ) + target_link_libraries(${bench_name}_native PRIVATE benchmark::benchmark_main) + if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|ARM64") + target_compile_options(${bench_name}_native PRIVATE -mcpu=native) + else() + target_compile_options(${bench_name}_native PRIVATE -march=native) + endif() + target_compile_options(${bench_name}_native PRIVATE -ffast-math -fno-finite-math-only) + else() add_executable( ${bench_name} ${bench_file} ) target_link_libraries(${bench_name} PRIVATE diff --git a/bench_alignment_vectorization.cpp b/bench_alignment_vectorization.cpp new file mode 100644 index 0000000..00b8649 --- /dev/null +++ b/bench_alignment_vectorization.cpp @@ -0,0 +1,290 @@ +// Does array alignment matter, with and without native optimizations + fast math? +// +// Context: EnergyPlus builds ObjexxFCL arrays with OBJEXXFCL_ALIGN=64 (first element aligned to 64 bytes), a setting from +// ObjexxFCL 4.1 (NatLabRockies/EnergyPlus#5395) meant to help vectorization. Replacing ObjexxFCL arrays by std::vector / +// std::array drops that to the default allocator alignment (16 bytes on 64-bit platforms in practice). And +// NatLabRockies/EnergyPlus#11816 adds an opt-in -march/-mcpu=native + -ffast-math build, which is when vectorization (and +// so maybe alignment) can actually kick in. +// +// The same kernels run on the same data held in: +// - std::vector (default allocator) +// - std::vector> (like OBJEXXFCL_ALIGN=64, the x86 cache line) +// - std::vector> (Apple Silicon cache line) +// - a 64-byte aligned buffer viewed from its second element (data + 8 bytes): deliberately misaligned, worst case +// Kernels take a std::span, so the compiler cannot know the alignment, like EnergyPlus going through +// Array::operator(). The "AssumeAligned64" variants use std::assume_aligned<64> to show what an explicit hint gives. +// +// This file is built twice by CMakeLists.txt: +// bench_alignment_vectorization : EnergyPlus-like default flags (-O3 -ffp-contract=off) +// bench_alignment_vectorization_native : -march=native / -mcpu=native -ffast-math -fno-finite-math-only (like #11816) +// Sizes: 1 KiB-ish (L1), 256 KiB (L2), 32 MiB (memory bound). + +#include + +#include +#include +#include +#include +#include +#include +#include + +template +struct AlignedAllocator +{ + using value_type = T; + static_assert((Alignment & (Alignment - 1)) == 0, "Alignment must be a power of 2"); + + AlignedAllocator() noexcept = default; + template + explicit AlignedAllocator(const AlignedAllocator& /*other*/) noexcept {} + + template + struct rebind + { + using other = AlignedAllocator; + }; + + [[nodiscard]] T* allocate(std::size_t n) { + return static_cast(::operator new(n * sizeof(T), std::align_val_t{Alignment})); + } + + void deallocate(T* p, std::size_t /*n*/) noexcept { + ::operator delete(p, std::align_val_t{Alignment}); + } + + friend bool operator==(const AlignedAllocator& /*lhs*/, const AlignedAllocator& /*rhs*/) noexcept { + return true; + } +}; + +// Holds the buffer(s) for one benchmark run, and exposes them as spans with the requested alignment +enum class Layout +{ + Default, + Aligned64, + Aligned128, + Misaligned8, // 64-byte aligned buffer, viewed from its second element +}; + +class Buffer +{ + public: + Buffer(Layout layout, std::size_t n) { + switch (layout) { + case Layout::Default: + m_default.resize(n); + m_span = std::span(m_default); + break; + case Layout::Aligned64: + m_a64.resize(n); + m_span = std::span(m_a64); + break; + case Layout::Aligned128: + m_a128.resize(n); + m_span = std::span(m_a128); + break; + case Layout::Misaligned8: + m_a64.resize(n + 1); + m_span = std::span(m_a64.data() + 1, n); + break; + } + // Deterministic, non-trivial values in [0.5, 1.5) + for (std::size_t i = 0; i < n; ++i) { + m_span[i] = 0.5 + static_cast((i * 2654435761u) % 1000u) / 1000.0; + } + } + + [[nodiscard]] std::span span() const { + return m_span; + } + + // Offset of the first element modulo 128 bytes, reported as a counter so the output shows what each layout got + [[nodiscard]] double offsetMod128() const { + return static_cast(reinterpret_cast(m_span.data()) % 128u); + } + + private: + std::vector m_default; + std::vector> m_a64; + std::vector> m_a128; + std::span m_span; +}; + +// --------------------------------------------------------------------------------------------------------------------- +// Kernels. noinline so the compiler cannot see the allocation, and so cannot deduce the alignment. +// --------------------------------------------------------------------------------------------------------------------- + +// Reduction (like ObjexxFCL sum()): only vectorizes when the compiler may reorder additions (fast math) +[[gnu::noinline]] double kernel_sum(std::span x) { + double s = 0.0; + for (double const v : x) { + s += v; + } + return s; +} + +[[gnu::noinline]] double kernel_dot(std::span x, std::span y) { + double s = 0.0; + for (std::size_t i = 0; i < x.size(); ++i) { + s += x[i] * y[i]; + } + return s; +} + +// y += a * x: element-wise, vectorizes even without fast math (no reordering needed) +[[gnu::noinline]] void kernel_axpy(double a, std::span x, std::span y) { + for (std::size_t i = 0; i < x.size(); ++i) { + y[i] += a * x[i]; + } +} + +// Cubic curve evaluation, like a Curve:Cubic over many points +[[gnu::noinline]] void kernel_cubic(std::span x, std::span y) { + constexpr double c0 = 0.94; + constexpr double c1 = 0.012; + constexpr double c2 = -0.0003; + constexpr double c3 = 0.000002; + for (std::size_t i = 0; i < x.size(); ++i) { + double const v = x[i]; + y[i] = c0 + v * (c1 + v * (c2 + v * c3)); + } +} + +// 5-point Jacobi stencil on a row-major n x n grid (like a finite difference soil grid), interior points only +[[gnu::noinline]] void kernel_stencil(std::span in, std::span out, std::size_t n) { + for (std::size_t r = 1; r + 1 < n; ++r) { + for (std::size_t c = 1; c + 1 < n; ++c) { + std::size_t const i = r * n + c; + out[i] = 0.2 * (in[i] + in[i - 1] + in[i + 1] + in[i - n] + in[i + n]); + } + } +} + +// Same reduction, but promising the compiler a 64-byte aligned start (what ObjexxFCL's ASSUME_ALIGNED_OBJEXXFCL would do) +[[gnu::noinline]] double kernel_sum_assume_aligned64(std::span x) { + double const* p = std::assume_aligned<64>(x.data()); + double s = 0.0; + for (std::size_t i = 0; i < x.size(); ++i) { + s += p[i]; + } + return s; +} + +[[gnu::noinline]] void kernel_axpy_assume_aligned64(double a, std::span x, std::span y) { + double const* px = std::assume_aligned<64>(x.data()); + double* py = std::assume_aligned<64>(y.data()); + for (std::size_t i = 0; i < x.size(); ++i) { + py[i] += a * px[i]; + } +} + +// --------------------------------------------------------------------------------------------------------------------- +// Benchmarks. range(0) = number of doubles. +// --------------------------------------------------------------------------------------------------------------------- + +template +static void BM_Sum(benchmark::State& state) { + auto const n = static_cast(state.range(0)); + Buffer x(L, n); + for (auto _ : state) { + benchmark::DoNotOptimize(kernel_sum(x.span())); + } + state.SetBytesProcessed(static_cast(state.iterations() * n * sizeof(double))); + state.counters["offset_mod_128"] = x.offsetMod128(); +} + +template +static void BM_Dot(benchmark::State& state) { + auto const n = static_cast(state.range(0)); + Buffer x(L, n); + Buffer y(L, n); + for (auto _ : state) { + benchmark::DoNotOptimize(kernel_dot(x.span(), y.span())); + } + state.SetBytesProcessed(static_cast(state.iterations() * 2 * n * sizeof(double))); + state.counters["offset_mod_128"] = x.offsetMod128(); +} + +template +static void BM_Axpy(benchmark::State& state) { + auto const n = static_cast(state.range(0)); + Buffer x(L, n); + Buffer y(L, n); + for (auto _ : state) { + kernel_axpy(1.0e-9, x.span(), y.span()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(static_cast(state.iterations() * 3 * n * sizeof(double))); + state.counters["offset_mod_128"] = x.offsetMod128(); +} + +template +static void BM_Cubic(benchmark::State& state) { + auto const n = static_cast(state.range(0)); + Buffer x(L, n); + Buffer y(L, n); + for (auto _ : state) { + kernel_cubic(x.span(), y.span()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(static_cast(state.iterations() * 2 * n * sizeof(double))); + state.counters["offset_mod_128"] = x.offsetMod128(); +} + +template +static void BM_Stencil(benchmark::State& state) { + // range(0) is still the number of doubles: use the closest square grid + auto const side = static_cast(std::sqrt(static_cast(state.range(0)))); + Buffer in(L, side * side); + Buffer out(L, side * side); + for (auto _ : state) { + kernel_stencil(in.span(), out.span(), side); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(static_cast(state.iterations() * 2 * side * side * sizeof(double))); + state.counters["offset_mod_128"] = in.offsetMod128(); +} + +template +static void BM_SumAssumeAligned64(benchmark::State& state) { + auto const n = static_cast(state.range(0)); + Buffer x(L, n); + for (auto _ : state) { + benchmark::DoNotOptimize(kernel_sum_assume_aligned64(x.span())); + } + state.SetBytesProcessed(static_cast(state.iterations() * n * sizeof(double))); + state.counters["offset_mod_128"] = x.offsetMod128(); +} + +template +static void BM_AxpyAssumeAligned64(benchmark::State& state) { + auto const n = static_cast(state.range(0)); + Buffer x(L, n); + Buffer y(L, n); + for (auto _ : state) { + kernel_axpy_assume_aligned64(1.0e-9, x.span(), y.span()); + benchmark::ClobberMemory(); + } + state.SetBytesProcessed(static_cast(state.iterations() * 3 * n * sizeof(double))); + state.counters["offset_mod_128"] = x.offsetMod128(); +} + +// 1 Ki doubles (8 KiB, L1), 32 Ki doubles (256 KiB, L2), 4 Mi doubles (32 MiB, memory bound) +#define ALIGN_SIZES Arg(1 << 10)->Arg(1 << 15)->Arg(1 << 22) + +#define ALIGN_BENCH_ALL_LAYOUTS(BM) \ + BENCHMARK_TEMPLATE(BM, Layout::Default)->ALIGN_SIZES; \ + BENCHMARK_TEMPLATE(BM, Layout::Aligned64)->ALIGN_SIZES; \ + BENCHMARK_TEMPLATE(BM, Layout::Aligned128)->ALIGN_SIZES; \ + BENCHMARK_TEMPLATE(BM, Layout::Misaligned8)->ALIGN_SIZES + +ALIGN_BENCH_ALL_LAYOUTS(BM_Sum); +ALIGN_BENCH_ALL_LAYOUTS(BM_Dot); +ALIGN_BENCH_ALL_LAYOUTS(BM_Axpy); +ALIGN_BENCH_ALL_LAYOUTS(BM_Cubic); +ALIGN_BENCH_ALL_LAYOUTS(BM_Stencil); + +// assume_aligned is only valid on actually 64-byte aligned data +BENCHMARK_TEMPLATE(BM_SumAssumeAligned64, Layout::Aligned64)->ALIGN_SIZES; +BENCHMARK_TEMPLATE(BM_AxpyAssumeAligned64, Layout::Aligned64)->ALIGN_SIZES; From 686e76d7a5129b12160e73f9c9c6be71329e2c5a Mon Sep 17 00:00:00 2001 From: Julien Marrec Date: Tue, 6 Oct 2026 16:06:56 +0200 Subject: [PATCH 2/2] Alignment bench: OffsetN layouts, non-power-of-two sizes, per-buffer counters, and a script to compare results Offset8/16/32 view a 128-byte aligned buffer from +8/+16/+32 bytes so the alignment does not depend on the allocator (Offset16 is what std::vector gets on glibc/MSVC). Sizes are 37, 251, 1003, 30011, 4000037 doubles: powers of two hid vector-loop tails and caused cache set aliasing (the Stencil -27% at 4 Mi). compare_alignment_results.py prints Markdown tables relative to Aligned64 from one or two (default, native) result files. ``` Products/bench_alignment_vectorization --benchmark_repetitions=5 --benchmark_report_aggregates_only=true --benchmark_out=../results/alignment_default.json --benchmark_out_format=json Products/bench_alignment_vectorization_native --benchmark_repetitions=5 --benchmark_report_aggregates_only=true --benchmark_out=../results/alignment_native.json --benchmark_out_format=json python3 compare_alignment_results.py results/alignment_default.json results/alignment_native.json --offsets ``` Results on two machines: macOS M1 Max (Apple clang, NEON 16-byte vectors), and Linux Ryzen 7 1800X (Zen 1, GCC, AVX2 32-byte vectors). Median of 5, max coefficient of variation 8.7% (M1) / 5.2% (Ryzen). What the allocator actually gives std::vector (Default layout, observed): glibc (Ryzen) 16-byte aligned at all sizes (offsets 0/16/32/48/80/96/112 mod 128). macOS 16-byte guaranteed for small blocks, page-aligned for large ones (>= ~32 KiB, 16 KiB pages). __STDCPP_DEFAULT_NEW_ALIGNMENT__ is 16 on Apple clang, GCC and MSVC. Alignment, % vs Aligned64 (ie OBJEXXFCL_ALIGN=64): | Layout | M1 Max | Ryzen 1800X | |----------------------------------------|-----------------------------------------|-------------------------------------------| | Default / Aligned128 | within noise | within noise | | Offset16 (glibc/MSVC std::vector) | within noise, both builds | within noise, both builds (*) | | Offset32 (AVX2-aligned, not 64) | within noise | within noise (*) | | Offset8, Axpy, default / native build | +12 to +16% / +11 to +22% (n 251-30011) | +17 to +28% / +14 to +34% (n 37-30011) | | Offset8, Cubic | up to +8% | +6 to +10% (native) | (*) Two tiny-size anomalies on the Ryzen that are not alignment driven: Cubic native n=37 is +60/+70% for Offset16 and Default but also +58% for Aligned128 and +1% for Offset8; Stencil default n=37 Offset32 +17%. Most likely relative placement of input/output buffers (store/load address aliasing), not 16 vs 64. Native (-march/-mcpu=native -ffast-math -fno-finite-math-only) speedup vs default build (-O3 -ffp-contract=off), Aligned64: | Kernel | M1 Max | Ryzen 1800X | |-------------------|---------------------------------|----------------------------------| | Sum (n >= 251) | 7.2-8.5x (1.1x at n=37) | 2.0x (1.8x at 4M, 1.5x at n=37) | | Dot (n >= 251) | 4.5-6.3x (1.7x at n=37) | 2.0x (1.5x at 4M, 1.9x at n=37) | | Cubic (in cache) | 1.26-1.38x | 1.41-1.48x | | Axpy | 1.0-1.07x | 1.0-1.08x | | Stencil | 1.0x | 0.89-0.97x (slower at mid sizes) | std::assume_aligned<64>: no effect on the M1. On the Ryzen it helps Sum in the native build (-27% at n=37, -11% at 251, -8% at 4M: GCC drops the alignment prologue), but the hinted Axpy is 30-35% slower in the default build (n 37-1003). EnergyPlus never used such hints (ASSUME_ALIGNED_OBJEXXFCL is unused), so OBJEXXFCL_ALIGN=64 never got that benefit. Conclusions: - Going from ObjexxFCL's 64-byte alignment to std::vector's 16 bytes costs nothing measurable on either machine, in default or native builds. Alignment is not a reason to keep ObjexxFCL arrays. - Only misalignment within an element (Offset8) costs, more so on Zen 1 AVX2, but no allocator produces that for double. - Native + fast math gains depend heavily on the CPU (reductions 7-8x on M1, 2x on Zen 1 which splits 256-bit ops in two) and on size (little at tens of elements, which is typical of EnergyPlus arrays). Supports keeping NatLabRockies/EnergyPlus#11816 opt-in. Excluded outliers (single block, not reproduced in the other build): M1 default Stencil Aligned128 4M +146% (buffer placement, Aligned64/Default have the same alignment at that size), M1 default Axpy Aligned64 4M reference 15% slow (cv 5.5%; AxpyAssumeAligned64 on the same layout was normal). --- bench_alignment_vectorization.cpp | 57 ++++++++++----- compare_alignment_results.py | 113 ++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 16 deletions(-) create mode 100755 compare_alignment_results.py diff --git a/bench_alignment_vectorization.cpp b/bench_alignment_vectorization.cpp index 00b8649..3ef390b 100644 --- a/bench_alignment_vectorization.cpp +++ b/bench_alignment_vectorization.cpp @@ -10,14 +10,22 @@ // - std::vector (default allocator) // - std::vector> (like OBJEXXFCL_ALIGN=64, the x86 cache line) // - std::vector> (Apple Silicon cache line) -// - a 64-byte aligned buffer viewed from its second element (data + 8 bytes): deliberately misaligned, worst case +// - a 128-byte aligned buffer viewed from +8, +16 or +32 bytes (Offset8 / Offset16 / Offset32), whatever the allocator: +// Offset8: not even aligned to a 16-byte NEON/SSE vector (worst case) +// Offset16: what glibc and MSVC actually give std::vector (16 bytes), ie the realistic post-migration case +// Offset32: aligned for 32-byte AVX2 vectors, but not to a 64-byte cache line +// "Default" is whatever the platform allocator returns: on macOS, allocations >= ~32 KiB are page-aligned and small ones +// only 16-byte aligned (but sometimes better by luck), hence the explicit OffsetN layouts and the per-buffer +// *_offset_mod_128 counters, which report what each buffer actually got. // Kernels take a std::span, so the compiler cannot know the alignment, like EnergyPlus going through // Array::operator(). The "AssumeAligned64" variants use std::assume_aligned<64> to show what an explicit hint gives. // // This file is built twice by CMakeLists.txt: // bench_alignment_vectorization : EnergyPlus-like default flags (-O3 -ffp-contract=off) // bench_alignment_vectorization_native : -march=native / -mcpu=native -ffast-math -fno-finite-math-only (like #11816) -// Sizes: 1 KiB-ish (L1), 256 KiB (L2), 32 MiB (memory bound). +// Sizes are deliberately not powers of two, and mostly small like real EnergyPlus arrays (per zone, per surface...): +// powers of two would hide vector-loop tails and cause cache set aliasing between buffers. +// 37, 251, 1003 doubles (L1), 30011 (L2), 4000037 (~32 MB, memory bound) #include @@ -64,7 +72,9 @@ enum class Layout Default, Aligned64, Aligned128, - Misaligned8, // 64-byte aligned buffer, viewed from its second element + Offset8, // 128-byte aligned buffer, viewed from +8 bytes + Offset16, // 128-byte aligned buffer, viewed from +16 bytes + Offset32, // 128-byte aligned buffer, viewed from +32 bytes }; class Buffer @@ -84,9 +94,17 @@ class Buffer m_a128.resize(n); m_span = std::span(m_a128); break; - case Layout::Misaligned8: - m_a64.resize(n + 1); - m_span = std::span(m_a64.data() + 1, n); + case Layout::Offset8: + m_a128.resize(n + 1); + m_span = std::span(m_a128.data() + 1, n); + break; + case Layout::Offset16: + m_a128.resize(n + 2); + m_span = std::span(m_a128.data() + 2, n); + break; + case Layout::Offset32: + m_a128.resize(n + 4); + m_span = std::span(m_a128.data() + 4, n); break; } // Deterministic, non-trivial values in [0.5, 1.5) @@ -191,7 +209,7 @@ static void BM_Sum(benchmark::State& state) { benchmark::DoNotOptimize(kernel_sum(x.span())); } state.SetBytesProcessed(static_cast(state.iterations() * n * sizeof(double))); - state.counters["offset_mod_128"] = x.offsetMod128(); + state.counters["x_offset_mod_128"] = x.offsetMod128(); } template @@ -203,7 +221,8 @@ static void BM_Dot(benchmark::State& state) { benchmark::DoNotOptimize(kernel_dot(x.span(), y.span())); } state.SetBytesProcessed(static_cast(state.iterations() * 2 * n * sizeof(double))); - state.counters["offset_mod_128"] = x.offsetMod128(); + state.counters["x_offset_mod_128"] = x.offsetMod128(); + state.counters["y_offset_mod_128"] = y.offsetMod128(); } template @@ -216,7 +235,8 @@ static void BM_Axpy(benchmark::State& state) { benchmark::ClobberMemory(); } state.SetBytesProcessed(static_cast(state.iterations() * 3 * n * sizeof(double))); - state.counters["offset_mod_128"] = x.offsetMod128(); + state.counters["x_offset_mod_128"] = x.offsetMod128(); + state.counters["y_offset_mod_128"] = y.offsetMod128(); } template @@ -229,7 +249,8 @@ static void BM_Cubic(benchmark::State& state) { benchmark::ClobberMemory(); } state.SetBytesProcessed(static_cast(state.iterations() * 2 * n * sizeof(double))); - state.counters["offset_mod_128"] = x.offsetMod128(); + state.counters["x_offset_mod_128"] = x.offsetMod128(); + state.counters["y_offset_mod_128"] = y.offsetMod128(); } template @@ -243,7 +264,8 @@ static void BM_Stencil(benchmark::State& state) { benchmark::ClobberMemory(); } state.SetBytesProcessed(static_cast(state.iterations() * 2 * side * side * sizeof(double))); - state.counters["offset_mod_128"] = in.offsetMod128(); + state.counters["in_offset_mod_128"] = in.offsetMod128(); + state.counters["out_offset_mod_128"] = out.offsetMod128(); } template @@ -254,7 +276,7 @@ static void BM_SumAssumeAligned64(benchmark::State& state) { benchmark::DoNotOptimize(kernel_sum_assume_aligned64(x.span())); } state.SetBytesProcessed(static_cast(state.iterations() * n * sizeof(double))); - state.counters["offset_mod_128"] = x.offsetMod128(); + state.counters["x_offset_mod_128"] = x.offsetMod128(); } template @@ -267,17 +289,20 @@ static void BM_AxpyAssumeAligned64(benchmark::State& state) { benchmark::ClobberMemory(); } state.SetBytesProcessed(static_cast(state.iterations() * 3 * n * sizeof(double))); - state.counters["offset_mod_128"] = x.offsetMod128(); + state.counters["x_offset_mod_128"] = x.offsetMod128(); + state.counters["y_offset_mod_128"] = y.offsetMod128(); } -// 1 Ki doubles (8 KiB, L1), 32 Ki doubles (256 KiB, L2), 4 Mi doubles (32 MiB, memory bound) -#define ALIGN_SIZES Arg(1 << 10)->Arg(1 << 15)->Arg(1 << 22) +// Not powers of two (see header): 37, 251, 1003 doubles (L1), 30011 (~234 KiB, L2), 4000037 (~32 MB, memory bound) +#define ALIGN_SIZES Arg(37)->Arg(251)->Arg(1003)->Arg(30011)->Arg(4000037) #define ALIGN_BENCH_ALL_LAYOUTS(BM) \ BENCHMARK_TEMPLATE(BM, Layout::Default)->ALIGN_SIZES; \ BENCHMARK_TEMPLATE(BM, Layout::Aligned64)->ALIGN_SIZES; \ BENCHMARK_TEMPLATE(BM, Layout::Aligned128)->ALIGN_SIZES; \ - BENCHMARK_TEMPLATE(BM, Layout::Misaligned8)->ALIGN_SIZES + BENCHMARK_TEMPLATE(BM, Layout::Offset8)->ALIGN_SIZES; \ + BENCHMARK_TEMPLATE(BM, Layout::Offset16)->ALIGN_SIZES; \ + BENCHMARK_TEMPLATE(BM, Layout::Offset32)->ALIGN_SIZES ALIGN_BENCH_ALL_LAYOUTS(BM_Sum); ALIGN_BENCH_ALL_LAYOUTS(BM_Dot); diff --git a/compare_alignment_results.py b/compare_alignment_results.py new file mode 100755 index 0000000..5825dda --- /dev/null +++ b/compare_alignment_results.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Compare bench_alignment_vectorization results (Google Benchmark JSON, run with --benchmark_repetitions and +--benchmark_report_aggregates_only=true). + +Usage: + compare_alignment_results.py results/alignment_default.json [results/alignment_native.json] + +For each kernel and size, prints the median time of the reference layout (Aligned64, ie OBJEXXFCL_ALIGN=64) and the +other layouts as a % difference to it. With two files (default build, then native build), also prints the native speedup +on the reference layout. Layouts and sizes are discovered from the files, so new ones are picked up automatically. +""" + +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + +REFERENCE_LAYOUT = "Aligned64" +RUN_NAME_RE = re.compile(r"BM_(\w+)/(\d+)") + + +def load(path: Path): + """Return ({(kernel, size): {layout: median_ns}}, {(kernel, size, layout): {counter: value}}, max cv, context).""" + data = json.loads(path.read_text()) + medians = defaultdict(dict) + counters = {} + max_cv = 0.0 + for b in data["benchmarks"]: + aggregate = b.get("aggregate_name") + m = RUN_NAME_RE.match(b["run_name"]) + if not m or aggregate not in ("median", "cv"): + continue + kernel, layout, size = m.group(1), m.group(2), int(m.group(3)) + if aggregate == "median": + medians[(kernel, size)][layout] = b["real_time"] + counters[(kernel, size, layout)] = {k: v for k, v in b.items() if k.endswith("_mod_128")} + else: + max_cv = max(max_cv, b["real_time"]) + return medians, counters, max_cv, data.get("context", {}) + + +def layout_order(all_layouts): + preferred = ["Default", "Aligned64", "Aligned128", "Offset8", "Misaligned8", "Offset16", "Offset32"] + return [l for l in preferred if l in all_layouts] + sorted(set(all_layouts) - set(preferred)) + + +def print_table(title, medians, other=None): + layouts = layout_order({l for v in medians.values() for l in v}) + print(f"\n## {title}\n") + header = f"| kernel | n | {REFERENCE_LAYOUT} (ns) |" + sep = "|---|---|---|" + if other is not None: + header += " vs default build |" + sep += "---|" + for l in layouts: + if l != REFERENCE_LAYOUT: + header += f" {l} |" + sep += "---|" + print(header) + print(sep) + for kernel, size in sorted(medians): + row = medians[(kernel, size)] + ref = row.get(REFERENCE_LAYOUT) + line = f"| {kernel} | {size} | {ref:,.1f} |" if ref else f"| {kernel} | {size} | - |" + if other is not None: + other_ref = other.get((kernel, size), {}).get(REFERENCE_LAYOUT) + line += f" {other_ref / ref:.2f}x faster |" if ref and other_ref else " - |" + for l in layouts: + if l == REFERENCE_LAYOUT: + continue + line += f" {(row[l] / ref - 1) * 100:+.0f}% |" if ref and l in row else " - |" + print(line) + + +def print_alignments(counters): + """Actual first-element offset (mod 128 bytes) per layout and buffer, to catch 'lucky' allocations.""" + seen = defaultdict(set) + for (kernel, size, layout), c in counters.items(): + for name, value in c.items(): + seen[(layout, size, name)].add(int(value)) + order = {l: i for i, l in enumerate(layout_order({k[0] for k in seen}))} + print("\n## Observed offsets mod 128 bytes (per layout, size, buffer)\n") + for layout, size, name in sorted(seen, key=lambda k: (order[k[0]], k[1], k[2])): + print(f"- {layout} n={size} {name}: {sorted(seen[(layout, size, name)])}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("default_json", type=Path, help="Results of bench_alignment_vectorization") + parser.add_argument("native_json", type=Path, nargs="?", help="Results of bench_alignment_vectorization_native") + parser.add_argument("--offsets", action="store_true", help="Also list the observed buffer offsets") + args = parser.parse_args() + + default, default_counters, default_cv, ctx = load(args.default_json) + print(f"Host: {ctx.get('host_name')}, {ctx.get('num_cpus')} CPUs. Median times; % is relative to {REFERENCE_LAYOUT}.") + print(f"Max coefficient of variation: default {default_cv:.1%}", end="") + native = None + if args.native_json: + native, native_counters, native_cv, _ = load(args.native_json) + print(f", native {native_cv:.1%}") + else: + print() + + print_table(f"Default build ({args.default_json.name})", default) + if native is not None: + print_table(f"Native build ({args.native_json.name})", native, other=default) + if args.offsets: + print_alignments(default_counters) + + +if __name__ == "__main__": + main()