diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5bb9704..927d2eca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,9 +48,33 @@ jobs: if: success() || failure() run: cd rust && cargo fmt --check + # A leftover path dep or [patch.crates-io] redirect would make maturin build + # release wheels from unpublished local source instead of the published crate, + # producing a non-reproducible public wheel. Assert the POSITIVE invariant on + # the lockfile: cachekit-core must resolve to the crates.io registry. A path or + # patch dep has no `source` line, so this one check covers every Cargo.toml form + # (inline table, [dependencies.cachekit-core], [patch.crates-io]) and the + # workspace root — unlike a grep for `path =`, which only caught the inline form. + - name: Guard cachekit-core resolves to crates.io + if: success() || failure() + run: | + src=$(awk ' + /^\[\[package\]\]/ { name=""; source="" } + /^name = / { name=$3 } + /^source = / { source=$0 } + name == "\"cachekit-core\"" && source != "" { print source; exit } + ' Cargo.lock) + echo "cachekit-core resolved: ${src:-}" + if [ "$src" != 'source = "registry+https://github.com/rust-lang/crates.io-index"' ]; then + echo "::error::cachekit-core must resolve to the crates.io registry, not a local path/patch dep" + exit 1 + fi + + # --locked: fail on a stale Cargo.lock so the build provably resolves + # cachekit-core from crates.io as committed - name: Lint Rust if: success() || failure() - run: cd rust && cargo clippy -- -D warnings + run: cd rust && cargo clippy --locked -- -D warnings # PR: single Python version, critical tests only # Push: full matrix, full test suite diff --git a/Cargo.lock b/Cargo.lock index 3569ebb8..3ba54ee9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,9 +245,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cachekit-core" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01870e86fa79ca9ee521b5b9c2bcff8428eab95fb343f75d35a6fce159bb1577" +checksum = "9ee6235f73aefb0dc66b9cd0d333b9da928c8192e4234357a739fe756c2f8f23" dependencies = [ "aes", "aes-gcm", diff --git a/docs/features/rust-serialization.md b/docs/features/rust-serialization.md index d7359396..bb8d2270 100644 --- a/docs/features/rust-serialization.md +++ b/docs/features/rust-serialization.md @@ -20,7 +20,7 @@ Serializer (MessagePack / Arrow / Orjson — your choice) ↓ [Rust ByteStorage takes over here] LZ4 compression (fast, ~500MB/s) ↓ -Blake3 integrity hash (~GB/s, detects corruption) +xxHash3-64 integrity hash (~GB/s, detects corruption) ↓ [Optional] AES-256-GCM encryption (if @cache.secure) ↓ @@ -38,7 +38,7 @@ The Rust layer is transparent — you configure serializers and encryption at th | Operation | Python | Rust (ByteStorage) | |-----------|--------|---------------------| | LZ4 compression | ~50-100 MB/s | ~500 MB/s | -| Blake3 hashing | ~500 MB/s | ~15 GB/s | +| xxHash3-64 hashing | ~35 GB/s (`xxhash` C ext) | ~35 GB/s | | AES-256-GCM | ~200 MB/s | ~1-4 GB/s (AES-NI) | For most workloads the bottleneck is Redis RTT (~2-50ms), not serialization. The Rust layer matters for large payloads (DataFrames, bulk data) where serialization time approaches network time. @@ -61,16 +61,42 @@ Compression runs automatically. It can be toggled via the `CACHEKIT_ENABLE_COMPR --- -## Blake3 Integrity +## xxHash3-64 Integrity -Every value stored includes a Blake3 hash. On retrieval: +Every value stored includes an xxHash3-64 checksum (8 bytes, big-endian). On retrieval: -1. Hash of retrieved bytes is computed -2. Stored hash is compared +1. Checksum of retrieved bytes is computed +2. Stored checksum is compared 3. Mismatch → `BackendError` (corrupted data, never returned to caller) This protects against Redis memory corruption, storage bugs, and bit rot. +> **Non-cryptographic.** The checksum detects corruption, not tampering. +> Tamper-resistance comes from AES-256-GCM (`@cache.secure`), never from this checksum. + +### Standalone checksum API + +> **Available since v0.12.0.** On earlier releases, +> `from cachekit._rust_serializer import checksum` raises `ImportError`. + +The same primitive is exposed directly — decoupled from LZ4 compression — for +serializers where compression is ineffective (Arrow IPC, compact JSON): + +```python +from cachekit._rust_serializer import checksum, verify_checksum + +digest = checksum(b"payload") # 8 bytes, big-endian +assert len(digest) == 8 +assert verify_checksum(b"payload", digest) is True +assert verify_checksum(b"tampered", digest) is False +``` + +`verify_checksum` raises `ValueError` unless the expected checksum is exactly +8 bytes. Both functions accept any buffer-protocol object (`bytes`, `bytearray`, +`memoryview`), so a serializer holding its payload as a `memoryview` can hash it +without a `bytes` copy. The output is byte-identical to the checksum embedded in +every ByteStorage envelope and to `xxhash.xxh3_64_digest` from the `xxhash` package. + --- ## AES-256-GCM Encryption @@ -99,7 +125,7 @@ The Rust ByteStorage layer is orthogonal to the serializer. Mix and match: | Typed models | [Pydantic](../serializers/pydantic.md) | Optional | | Custom types | [Custom](../serializers/custom.md) | Optional | -All serializers pass through the same ByteStorage pipeline (LZ4 + Blake3 + optional AES-256-GCM). +All serializers pass through the same ByteStorage pipeline (LZ4 + xxHash3-64 + optional AES-256-GCM). --- diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dcdaf5ed..0d323bc7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -20,7 +20,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] # Compression, checksums, encryption (https://crates.io/crates/cachekit-core) -cachekit-core = { version = "0.2.0", features = ["compression", "checksum", "messagepack", "encryption"] } +cachekit-core = { version = "0.3.0", features = ["compression", "checksum", "messagepack", "encryption"] } # Python integration - optional for Rust-only builds pyo3 = { workspace = true, optional = true } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a2400ccd..feb94311 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -30,6 +30,11 @@ fn _rust_serializer(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { // Add byte storage class m.add_class::()?; + // Standalone integrity primitive — registered unconditionally (usable with + // the checksum feature alone; must not vanish when encryption is off) + m.add_function(wrap_pyfunction!(python_bindings::checksum_py, m)?)?; + m.add_function(wrap_pyfunction!(python_bindings::verify_checksum_py, m)?)?; + // Add encryption functionality if feature is enabled #[cfg(feature = "encryption")] { @@ -43,13 +48,13 @@ fn _rust_serializer(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { #[cfg(feature = "encryption")] m.add( "__description__", - "Raw byte storage with LZ4 compression, Blake3 checksums, and zero-knowledge encryption", + "Raw byte storage with LZ4 compression, xxHash3-64 checksums, and zero-knowledge encryption", )?; #[cfg(not(feature = "encryption"))] m.add( "__description__", - "Raw byte storage layer with LZ4 compression and Blake3 checksums", + "Raw byte storage layer with LZ4 compression and xxHash3-64 checksums", )?; Ok(()) diff --git a/rust/src/python_bindings.rs b/rust/src/python_bindings.rs index f94ad769..723a7458 100644 --- a/rust/src/python_bindings.rs +++ b/rust/src/python_bindings.rs @@ -4,6 +4,7 @@ //! All business logic is delegated to cachekit-core. use cachekit_core::ByteStorage; +use pyo3::buffer::PyBuffer; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyBytes; @@ -340,6 +341,47 @@ pub fn key_fingerprint_py(key: &[u8]) -> Vec { key_fingerprint(key).to_vec() } +/// Compute the standalone xxHash3-64 checksum of `data` (8 bytes, big-endian). +/// +/// Accepts any buffer-protocol object — `bytes`, `bytearray`, `memoryview`, +/// Arrow buffers — so a serializer holding its payload as a `memoryview` +/// (e.g. Arrow IPC) can hash it directly, without forcing a `bytes` copy. +/// +/// NON-cryptographic: detects corruption, not tampering. For tamper-resistance +/// use @cache.secure (AES-256-GCM), never this checksum. Produces the exact +/// bytes embedded in every StorageEnvelope, without the LZ4 compression +/// overhead — for serializers where compression is ineffective (Arrow IPC, JSON). +#[pyfunction] +#[pyo3(name = "checksum")] +pub fn checksum_py(py: Python, data: PyBuffer) -> PyResult> { + let data = data.to_vec(py)?; + Ok(PyBytes::new(py, &cachekit_core::checksum(&data)).into()) +} + +/// Verify `data` against an expected 8-byte xxHash3-64 checksum. +/// +/// Both arguments accept any buffer-protocol object (`bytes`, `bytearray`, +/// `memoryview`, …) — the Arrow verify path slices a `memoryview` (`mv[8:]`), +/// so a bytes-only signature would break the moment a serializer moves onto +/// this FFI. +/// +/// NON-cryptographic: detects corruption, not tampering (see `checksum`). +/// Raises ValueError if `expected` is not exactly 8 bytes — a truncated +/// checksum must fail loudly, never return a wrong verdict. +#[pyfunction] +#[pyo3(name = "verify_checksum")] +pub fn verify_checksum_py( + py: Python, + data: PyBuffer, + expected: PyBuffer, +) -> PyResult { + let expected: [u8; 8] = expected.to_vec(py)?.try_into().map_err(|v: Vec| { + PyValueError::new_err(format!("expected must be exactly 8 bytes, got {}", v.len())) + })?; + let data = data.to_vec(py)?; + Ok(cachekit_core::verify_checksum(&data, &expected)) +} + /// Register encryption module with Python #[cfg(feature = "encryption")] pub fn register_encryption_module(m: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/tests/benchmarks/benchmark_checksum_ffi.py b/tests/benchmarks/benchmark_checksum_ffi.py new file mode 100644 index 00000000..a177032b --- /dev/null +++ b/tests/benchmarks/benchmark_checksum_ffi.py @@ -0,0 +1,92 @@ +"""Benchmark: pure-Python xxhash package vs Rust-FFI checksum (cachekit-core#13). + +Pinned contract (spec m-cachekit-core-checksum-only-api, Task 7): fixed sizes +64 B / 1 KB / 64 KB / 1 MB, >=30 iterations per point, per-size reporting. +This is a deliverable artifact, not a pass/fail gate — the crossover size is +the go/no-go datum for migrating serializer envelopes (Arrow/orjson) from the +py-xxhash package to the shared Rust FFI. Tiny inputs favoring in-process +Python is expected information, not failure: both implementations produce +byte-identical output (see tests/unit/test_checksum_ffi.py), so the choice +is purely a per-call-overhead question. + +Run: + uv run pytest tests/benchmarks/benchmark_checksum_ffi.py \ + --benchmark-only --benchmark-group-by=group --benchmark-min-rounds=30 + +Results (2026-07-17, AMD Ryzen 9 5950X, CPython 3.13.12, median per call): + + checksum (compute) py-xxhash Rust FFI winner + 64 B 51.6 ns 39.7 ns FFI 1.30x + 1 KB 77.8 ns 130.0 ns py 1.67x + 64 KB 1.85 us 1.86 us wash (~1%) + 1 MB 28.1 us 28.7 us wash (~2%) + + verify py-xxhash Rust FFI winner + 64 B 88.1 ns 34.8 ns FFI 2.53x + 1 KB 125.0 ns 64.3 ns FFI 1.94x + 64 KB 1.87 us 1.78 us FFI ~5% + 1 MB 28.0 us 27.9 us wash + +Crossover / go-no-go datum: there is NO size where either side wins by more +than ~65 ns/call on compute; both are throughput-bound and identical from +64 KB up. py-xxhash's C library has a stronger mid-size (240 B - 8 KB) path +than xxhash-rust, hence the 1 KB compute loss; the FFI wins verify at small +sizes because it is one boundary crossing instead of hash + compare in +Python. Verdict: serializer migration py-xxhash -> FFI is performance-neutral +(worst case ~50 ns/call against envelope operations measured in us-ms); decide +it on dependency hygiene, not speed. +""" + +from __future__ import annotations + +import pytest +import xxhash + +from cachekit import _rust_serializer as rs + +SIZES = [ + (64, "64B"), + (1_024, "1KB"), + (65_536, "64KB"), + (1_048_576, "1MB"), +] + +PAYLOADS = {label: bytes(i % 251 for i in range(size)) for size, label in SIZES} + + +@pytest.mark.benchmark +@pytest.mark.parametrize("label", [label for _, label in SIZES]) +class TestChecksumComputeComparison: + """py-xxhash vs Rust FFI, grouped per size for side-by-side comparison.""" + + def test_python_xxhash(self, benchmark, label): + data = PAYLOADS[label] + benchmark.group = f"checksum-{label}" + result = benchmark(xxhash.xxh3_64_digest, data) + assert len(result) == 8 + + def test_rust_ffi(self, benchmark, label): + data = PAYLOADS[label] + benchmark.group = f"checksum-{label}" + result = benchmark(rs.checksum, data) + assert len(result) == 8 + + +@pytest.mark.benchmark +@pytest.mark.parametrize("label", [label for _, label in SIZES]) +class TestVerifyComparison: + """Verification path: python compare vs Rust FFI verify_checksum.""" + + def test_python_xxhash_verify(self, benchmark, label): + data = PAYLOADS[label] + expected = xxhash.xxh3_64_digest(data) + benchmark.group = f"verify-{label}" + result = benchmark(lambda: xxhash.xxh3_64_digest(data) == expected) + assert result is True + + def test_rust_ffi_verify(self, benchmark, label): + data = PAYLOADS[label] + expected = bytes(rs.checksum(data)) + benchmark.group = f"verify-{label}" + result = benchmark(rs.verify_checksum, data, expected) + assert result is True diff --git a/tests/unit/test_checksum_ffi.py b/tests/unit/test_checksum_ffi.py new file mode 100644 index 00000000..3ee1d636 --- /dev/null +++ b/tests/unit/test_checksum_ffi.py @@ -0,0 +1,112 @@ +"""Unit tests for the standalone checksum FFI (cachekit-core#13, Phase 2). + +The Rust core exposes checksum/verify_checksum decoupled from LZ4 compression +(cachekit-core 0.3.0). These bindings mirror them so every serializer can get +the canonical xxHash3-64 wire value without paying for compression. + +Byte-verification: the KAT constants below are pinned in cachekit-core's own +test suite (src/checksum.rs) and must match the pure-Python xxhash package — +three independent implementations agreeing on the exact wire bytes. + +Note: NON-cryptographic. Detects corruption, not tampering. Tamper-resistance +comes from AES-256-GCM (@cache.secure), never from this checksum. +""" + +from __future__ import annotations + +import pytest +import xxhash + +from cachekit import _rust_serializer as rs + +# Protocol test vectors — pinned in cachekit-core src/checksum.rs (KAT tests). +# Big-endian = xxhash canonical byte order, identical to the value embedded in +# every StorageEnvelope. +KAT_CACHEKIT = bytes([209, 35, 204, 155, 190, 157, 164, 177]) # checksum(b"cachekit-kat") +KAT_EMPTY = bytes([0x2D, 0x06, 0x80, 0x05, 0x38, 0xD3, 0x94, 0xC2]) # checksum(b"") + + +class TestChecksumFFI: + def test_checksum_is_8_bytes_and_deterministic(self): + c = rs.checksum(b"payload") + assert isinstance(c, bytes) + assert len(c) == 8 + assert rs.checksum(b"payload") == c + + def test_checksum_matches_protocol_test_vectors(self): + """Byte-verified against the KAT vectors pinned in cachekit-core.""" + assert rs.checksum(b"cachekit-kat") == KAT_CACHEKIT + assert rs.checksum(b"") == KAT_EMPTY + + def test_checksum_matches_python_xxhash_package(self): + """FFI and the pure-Python xxhash package must agree byte-for-byte. + + The Arrow/orjson serializers compute envelopes via xxhash.xxh3_64_digest; + this proves the FFI is a drop-in producer of the same wire bytes across + xxHash3's size-dependent code paths — short, the 17-240 B mid path, and + the >64 KB accumulator-merge path (all previously unchecked above ~10 KB, + where an xxhash-rust vs libxxhash divergence would ship silently). + """ + payloads = ( + b"", + b"cachekit-kat", + b"payload", + bytes(range(200)), # 200 B — xxHash3 mid-size path + bytes(range(256)) * 41, # ~10 KB + bytes(i % 251 for i in range(65_536 + 7)), # > 64 KB — accumulator merge + bytes(i % 251 for i in range(1_048_576)), # 1 MB + ) + for data in payloads: + assert rs.checksum(data) == xxhash.xxh3_64_digest(data) + + +class TestVerifyChecksumFFI: + def test_verify_round_trips(self): + assert rs.verify_checksum(b"payload", rs.checksum(b"payload")) is True + assert rs.verify_checksum(b"tampered", rs.checksum(b"payload")) is False + + def test_verify_rejects_single_bit_flip(self): + corrupted = bytearray(rs.checksum(b"payload")) + corrupted[0] ^= 0x01 + assert rs.verify_checksum(b"payload", bytes(corrupted)) is False + + @pytest.mark.parametrize("bad_len", [0, 7, 9, 32]) + def test_verify_rejects_wrong_length_expected(self, bad_len): + """expected must be exactly 8 bytes; anything else raises, never lies.""" + with pytest.raises(ValueError, match="8 bytes"): + rs.verify_checksum(b"payload", b"\x00" * bad_len) + + +class TestBufferProtocol: + """The FFI must accept any buffer-protocol object, not only `bytes`. + + The Arrow serializer hashes a `memoryview` on both write + (arrow_serializer.py:245) and verify (`body = mv[8:]`, :283). A bytes-only + signature would raise TypeError the moment a serializer migrates onto this + FFI — while a bytes-only test suite stayed green. These tests lock the + buffer-protocol contract that makes that migration safe. + """ + + PAYLOAD = b"cachekit-kat payload of some length \x00\xff\x7f" + + @pytest.mark.parametrize("wrap", [bytes, bytearray, memoryview], ids=["bytes", "bytearray", "memoryview"]) + def test_checksum_accepts_buffer_types(self, wrap): + assert rs.checksum(wrap(self.PAYLOAD)) == xxhash.xxh3_64_digest(self.PAYLOAD) + + @pytest.mark.parametrize("wrap", [bytes, bytearray, memoryview], ids=["bytes", "bytearray", "memoryview"]) + def test_verify_accepts_buffer_types(self, wrap): + digest = rs.checksum(self.PAYLOAD) + assert rs.verify_checksum(wrap(self.PAYLOAD), digest) is True + assert rs.verify_checksum(wrap(self.PAYLOAD), wrap(digest)) is True + + def test_verify_accepts_memoryview_slice_like_arrow(self): + """Mirror the Arrow verify path exactly: envelope = [8-byte checksum][body], + then verify the body (a memoryview slice) against the sliced checksum.""" + body = self.PAYLOAD + envelope = memoryview(rs.checksum(body) + body) + assert rs.verify_checksum(envelope[8:], envelope[:8]) is True + + tampered = bytearray(envelope) + tampered[-1] ^= 0x01 # corrupt the body, leave the checksum prefix intact + mv = memoryview(tampered) + assert rs.verify_checksum(mv[8:], mv[:8]) is False