initial BLAS verifier harness

blas/CMakeLists.txt

@@ -0,0 +1,22 @@
+#------------------------------------------------------------------------------
+# Copyright (c) 2026 Ainekko, Co.
+# SPDX-License-Identifier: Apache-2.0
+#------------------------------------------------------------------------------
+
+function(add_etsoc_blas_kernel TARGET_NAME)
+  set(options)
+  set(oneValueArgs INSTALL_DESTINATION)
+  set(multiValueArgs)
+  cmake_parse_arguments(ARGS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+  if(NOT DEFINED ARGS_INSTALL_DESTINATION)
+    set(ARGS_INSTALL_DESTINATION kernels/blas)
+  endif()
+
+  add_etsoc_riscv_executable(${TARGET_NAME} ${ARGS_UNPARSED_ARGUMENTS})
+  target_link_libraries(${TARGET_NAME} etsoc_crt0)
+  install(TARGETS ${TARGET_NAME} ${TARGET_NAME}_dbg DESTINATION ${ARGS_INSTALL_DESTINATION})
+endfunction()
+
+add_subdirectory(reference)
+add_subdirectory(optimized)

blas/README.md

@@ -0,0 +1,34 @@
+# BLAS Kernels
+
+This directory holds custom GP-SDK device kernels for the ET-SOC1 BLAS work.
+
+It is intended to be consumed through the existing `gp-sdk/device` custom-kernel
+hook by setting:
+
+- `CUSTOM_KERNELS_SRC_DIR=<repo>/blas`
+- `CUSTOM_KERNELS_BIN_DIR=<build-dir>/blas`
+
+Layout conventions:
+
+- `blas/reference/` holds numerically conservative scalar baselines
+- `blas/optimized/` holds vector and tensor implementations
+- datatype directories sit below those roots
+- BLAS level and operation directories sit below the datatype
+
+Datatype rollout order:
+
+- `fp32` first
+- `fp16`
+- `bf16`
+- `int16`
+- `int8`
+
+Implementation policy:
+
+- every optimized kernel must have a matching reference kernel
+- reference and optimized kernels must keep the same external argument contract
+- numerical comparisons are always made against the reference implementation
+
+The intent is to always keep a trustworthy reference kernel beside each
+optimized family so we can measure error growth as operation ordering and
+reduction strategies change.

blas/optimized/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(fp32)

blas/optimized/README.md

@@ -0,0 +1,59 @@
+# Optimized Kernels
+
+Optimized kernels hold vector and tensor implementations that are validated
+against the matching reference kernels.
+
+Allowed here:
+
+- vector and tensor implementations
+- tiling, blocking, fusion, and staging for performance
+- controlled operation reordering when justified by performance
+
+Required here:
+
+- keep the same external kernel contract as the matching reference kernel
+- document any intentional numerical tradeoff in code comments or commit history
+- validate against the matching reference implementation
+
+Not allowed here:
+
+- silently changing kernel semantics
+- changing datatype interpretation relative to the reference path
+- landing an optimized kernel before a reference kernel exists
+
+Current kernels:
+
+- `fp32/level1/axpy`: scalar inner loop unrolled by 4 with the same minion
+  partitioning and external contract as the reference kernel
+- `fp32/level1/dot`: ET packed-SIMD loads plus packed multiply/add in the main
+  loop, with a masked packed-SIMD tail in the default artifact and a scalar-tail
+  comparison artifact kept alongside it
+- `fp32/level2/gemv`: ET packed-SIMD vectorization across contiguous output-row
+  blocks for the non-transpose, unit-stride path, with scalar fallback for
+  transpose and non-unit-stride cases
+- `fp32/level3/gemm`:
+  - `blas_gemm_optimized_fp32_vector`: single-minion 4x4 blocked `NN`
+    micro-kernel with K strip-mining for lower-latency execution
+  - `blas_gemm_optimized_fp32_tensor`: tensor-engine tiled `NN` path for
+    higher-throughput execution, with scalar fallback for unsupported tails and
+    non-`NN` cases
+  - current tensor bring-up keeps the tensor engine on the bulk of each tile
+    and applies a post-store scalar correction for the dropped final inner term
+    observed on the current hardware path
+
+Current validation policy:
+
+- prefer `*_dbg` kernel artifacts when validating optimized kernels
+- treat release-ELF bring-up as a separate runtime/toolchain issue
+- do not treat a release-ELF failure by itself as evidence that the optimized
+  kernel math is wrong when the matching `_dbg` artifact passes
+
+Current ET-SIMD bring-up notes:
+
+- `fp32/level1/dot` now uses ET packed-SIMD loads and packed multiply/add in
+  the main loop
+- the default optimized `dot` artifact uses a masked packed-SIMD tail to avoid
+  switching back into scalar FP arithmetic while packed state is still live in
+  the overlaid `f` register file
+- `blas_dot_optimized_fp32_scalar_tail.elf{,_dbg}` is retained as a comparison
+  artifact for validating the scalar-tail alternative

blas/optimized/fp32/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_subdirectory(level1)
+add_subdirectory(level2)
+add_subdirectory(level3)

blas/optimized/fp32/level1/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_subdirectory(axpy)
+add_subdirectory(dot)

blas/optimized/fp32/level1/axpy/CMakeLists.txt

@@ -0,0 +1,6 @@
+add_etsoc_blas_kernel(
+  blas_axpy_optimized_fp32.elf
+  axpy.cpp
+  INSTALL_DESTINATION kernels/blas/optimized/fp32/level1/axpy
+)
+target_include_directories(blas_axpy_optimized_fp32.elf PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})

blas/optimized/fp32/level1/axpy/axpy.cpp

@@ -0,0 +1,51 @@
+/*-------------------------------------------------------------------------
+ * Copyright (c) 2026 Ainekko, Co.
+ * SPDX-License-Identifier: Apache-2.0
+ *-------------------------------------------------------------------------
+ */
+
+#include <algorithm>
+#include <cstddef>
+
+#include <etsoc/common/utils.h>
+
+#include "axpy_kernel_arguments.h"
+#include "entryPoint.h"
+
+int entryPoint_0(KernelArguments* args);
+DECLARE_KERNEL_ENTRY_POINTS(entryPoint_0, nullptr);
+
+int entryPoint_0(KernelArguments* args) {
+  const auto minionId = get_relative_thread_id();
+  const size_t numWorkers = SOC_MINIONS_PER_SHIRE;
+
+  if (args->numElements == 0) {
+    return 0;
+  }
+
+  size_t elemsPerWorker = (args->numElements + numWorkers - 1) / numWorkers;
+  if (elemsPerWorker % 16) {
+    elemsPerWorker += 16 - (elemsPerWorker % 16);
+  }
+
+  const size_t begin = elemsPerWorker * minionId;
+  const size_t end = std::min(elemsPerWorker * (minionId + 1), static_cast<size_t>(args->numElements));
+  if (begin > (args->numElements - 1)) {
+    return 0;
+  }
+
+  size_t i = begin;
+  const size_t loopEnd = begin + ((end - begin) / 4) * 4;
+  for (; i < loopEnd; i += 4) {
+    args->y[i + 0] = args->alpha * args->x[i + 0] + args->y[i + 0];
+    args->y[i + 1] = args->alpha * args->x[i + 1] + args->y[i + 1];
+    args->y[i + 2] = args->alpha * args->x[i + 2] + args->y[i + 2];
+    args->y[i + 3] = args->alpha * args->x[i + 3] + args->y[i + 3];
+  }
+
+  for (; i < end; ++i) {
+    args->y[i] = args->alpha * args->x[i] + args->y[i];
+  }
+
+  return 0;
+}

blas/optimized/fp32/level1/axpy/axpy_kernel_arguments.h

@@ -0,0 +1,13 @@
+#ifndef ET_BLAS_OPTIMIZED_FP32_AXPY_KERNEL_ARGUMENTS_H
+#define ET_BLAS_OPTIMIZED_FP32_AXPY_KERNEL_ARGUMENTS_H
+
+#include <cstdint>
+
+struct KernelArguments {
+  uint64_t numElements;
+  const float* x;
+  float* y;
+  float alpha;
+} __attribute__((packed));
+
+#endif

blas/optimized/fp32/level1/dot/CMakeLists.txt

@@ -0,0 +1,12 @@
+function(add_dot_variant target_name tail_define)
+  add_etsoc_blas_kernel(
+    ${target_name}
+    dot.cpp
+    INSTALL_DESTINATION kernels/blas/optimized/fp32/level1/dot
+  )
+  target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+  target_compile_definitions(${target_name} PRIVATE ${tail_define}=1)
+endfunction()
+
+add_dot_variant(blas_dot_optimized_fp32.elf ET_BLAS_DOT_TAIL_MASKED)
+add_dot_variant(blas_dot_optimized_fp32_scalar_tail.elf ET_BLAS_DOT_TAIL_SCALAR)

blas/optimized/fp32/level1/dot/dot.cpp

@@ -0,0 +1,184 @@
+/*-------------------------------------------------------------------------
+ * Copyright (c) 2026 Ainekko, Co.
+ * SPDX-License-Identifier: Apache-2.0
+ *-------------------------------------------------------------------------
+ */
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+
+#include <etsoc/common/utils.h>
+#include <etsoc/isa/hart.h>
+#include <etsoc/isa/tensors.h>
+
+#include "CommonCode.h"
+#include "dot_kernel_arguments.h"
+#include "entryPoint.h"
+#include "sync.h"
+
+#if !defined(ET_BLAS_DOT_TAIL_SCALAR) && !defined(ET_BLAS_DOT_TAIL_MASKED)
+#define ET_BLAS_DOT_TAIL_MASKED 1
+#endif
+
+namespace {
+
+constexpr size_t kVectorLanes = 8;
+
+inline __attribute__((always_inline)) void setMaskForLaneCount(size_t activeLanes) {
+  switch (activeLanes) {
+  case 0:
+    mask_set(0, 0x00);
+    break;
+  case 1:
+    mask_set(0, 0x01);
+    break;
+  case 2:
+    mask_set(0, 0x03);
+    break;
+  case 3:
+    mask_set(0, 0x07);
+    break;
+  case 4:
+    mask_set(0, 0x0f);
+    break;
+  case 5:
+    mask_set(0, 0x1f);
+    break;
+  case 6:
+    mask_set(0, 0x3f);
+    break;
+  case 7:
+    mask_set(0, 0x7f);
+    break;
+  default:
+    mask_set(0, 0xff);
+    break;
+  }
+}
+
+inline __attribute__((always_inline)) float packedHorizontalSum(float packedValue) {
+  alignas(32) float lanes[kVectorLanes];
+  __asm__ __volatile__("fsw.ps %[packedValue], 0(%[dst])\n"
+                       :
+                       : [dst] "r"(lanes), [packedValue] "f"(packedValue)
+                       : "memory");
+
+  float result = 0.0f;
+  for (size_t lane = 0; lane < kVectorLanes; ++lane) {
+    result += lanes[lane];
+  }
+  return result;
+}
+
+inline __attribute__((always_inline)) float sdotVectorCore(
+  const float* x, const float* y, size_t begin, size_t end) {
+  float packedAcc;
+  uint32_t zeroWord = 0;
+  __asm__ __volatile__("fbcx.ps %[packedAcc], %[zeroWord]\n"
+                       : [packedAcc] "=&f"(packedAcc)
+                       : [zeroWord] "r"(zeroWord));
+
+  size_t i = begin;
+  const size_t loopEnd = begin + ((end - begin) / kVectorLanes) * kVectorLanes;
+  for (; i < loopEnd; i += kVectorLanes) {
+    float xVec;
+    float yVec;
+    float prodVec;
+    const float* xPtr = x + i;
+    const float* yPtr = y + i;
+
+    __asm__ __volatile__("flw.ps %[xVec], 0(%[xPtr])\n"
+                         "flw.ps %[yVec], 0(%[yPtr])\n"
+                         : [xVec] "=&f"(xVec), [yVec] "=&f"(yVec)
+                         : [xPtr] "r"(xPtr), [yPtr] "r"(yPtr)
+                         : "memory");
+
+    __asm__ __volatile__("fmul.ps %[prodVec], %[xVec], %[yVec]\n"
+                         "fadd.ps %[packedAcc], %[packedAcc], %[prodVec]\n"
+                         : [packedAcc] "+&f"(packedAcc), [prodVec] "=&f"(prodVec)
+                         : [xVec] "f"(xVec), [yVec] "f"(yVec));
+  }
+
+#if defined(ET_BLAS_DOT_TAIL_MASKED)
+  const size_t remaining = end - i;
+  if (remaining > 0) {
+    alignas(32) float tailX[kVectorLanes] = {};
+    alignas(32) float tailY[kVectorLanes] = {};
+    for (size_t lane = 0; lane < remaining; ++lane) {
+      tailX[lane] = x[i + lane];
+      tailY[lane] = y[i + lane];
+    }
+
+    setMaskForLaneCount(remaining);
+
+    float xVec;
+    float yVec;
+    float prodVec;
+    __asm__ __volatile__("flw.ps %[xVec], 0(%[xPtr])\n"
+                         "flw.ps %[yVec], 0(%[yPtr])\n"
+                         : [xVec] "=&f"(xVec), [yVec] "=&f"(yVec)
+                         : [xPtr] "r"(tailX), [yPtr] "r"(tailY)
+                         : "memory");
+
+    __asm__ __volatile__("fmul.ps %[prodVec], %[xVec], %[yVec]\n"
+                         "fadd.ps %[packedAcc], %[packedAcc], %[prodVec]\n"
+                         : [packedAcc] "+&f"(packedAcc), [prodVec] "=&f"(prodVec)
+                         : [xVec] "f"(xVec), [yVec] "f"(yVec));
+
+    mask_set(0, 0xff);
+  }
+#endif
+
+  float localSum = packedHorizontalSum(packedAcc);
+
+#if defined(ET_BLAS_DOT_TAIL_SCALAR)
+  for (; i < end; ++i) {
+    localSum += x[i] * y[i];
+  }
+#endif
+
+  return localSum;
+}
+
+} // namespace
+
+int entryPoint_0(KernelArguments* args);
+DECLARE_KERNEL_ENTRY_POINTS(entryPoint_0, nullptr);
+
+int entryPoint_0(KernelArguments* args) {
+  const auto minionId = get_relative_thread_id();
+  const size_t numWorkers = SOC_MINIONS_PER_SHIRE;
+
+  if (args->numElements == 0) {
+    if (minionId == 0) {
+      *(args->res) = 0.0f;
+    }
+    return 0;
+  }
+
+  size_t elemsPerWorker = (args->numElements + numWorkers - 1) / numWorkers;
+  if (elemsPerWorker % 16) {
+    elemsPerWorker += 16 - (elemsPerWorker % 16);
+  }
+
+  const size_t begin = elemsPerWorker * minionId;
+  const size_t end = std::min(elemsPerWorker * (minionId + 1), static_cast<size_t>(args->numElements));
+
+  if (begin <= (args->numElements - 1)) {
+    const float localSum = sdotVectorCore(args->x, args->y, begin, end);
+    args->partials[begin] = localSum;
+    evictCacheLine(0x1ULL, reinterpret_cast<uint8_t*>(&args->partials[begin]));
+  }
+  hart::barrier();
+
+  if (minionId == 0) {
+    float result = 0.0f;
+    for (size_t i = 0; i < args->numElements; i += elemsPerWorker) {
+      result += args->partials[i];
+    }
+    *(args->res) = result;
+  }
+
+  return 0;
+}