From 772d2351afd190fc9671a0e38c3dc4410a645297 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Fri, 17 Jul 2026 19:20:48 +1000 Subject: [PATCH 1/5] build: bump cachekit-core to 0.3.0 for checksum API 0.3.0 ships the standalone checksum/verify_checksum primitive (cachekit-core#13) that the PyO3 bindings mirror in this PR. Verified live on crates.io via cargo add --dry-run before pinning. Co-authored-by: multica-agent --- Cargo.lock | 4 ++-- rust/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3569ebb8..3ba54ee9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -245,9 +245,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" [[package]] name = "cachekit-core" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01870e86fa79ca9ee521b5b9c2bcff8428eab95fb343f75d35a6fce159bb1577" +checksum = "9ee6235f73aefb0dc66b9cd0d333b9da928c8192e4234357a739fe756c2f8f23" dependencies = [ "aes", "aes-gcm", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dcdaf5ed..0d323bc7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -20,7 +20,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] # Compression, checksums, encryption (https://crates.io/crates/cachekit-core) -cachekit-core = { version = "0.2.0", features = ["compression", "checksum", "messagepack", "encryption"] } +cachekit-core = { version = "0.3.0", features = ["compression", "checksum", "messagepack", "encryption"] } # Python integration - optional for Rust-only builds pyo3 = { workspace = true, optional = true } From d94c59352fb8a6f4654a924c850d3081d97e0790 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Fri, 17 Jul 2026 19:29:54 +1000 Subject: [PATCH 2/5] feat: expose checksum/verify_checksum via PyO3 (cachekit-core#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Free functions registered unconditionally in the pymodule (usable with the checksum feature alone — must not vanish when encryption is off). verify_checksum rejects non-8-byte expected with ValueError instead of panicking or returning a wrong verdict. Docstrings carry the non-cryptographic warning (corruption detection, not tamper-resistance). Tests byte-verify the FFI against the protocol KAT vectors pinned in cachekit-core src/checksum.rs AND against the pure-Python xxhash package, proving three independent implementations agree on the wire bytes. Also corrects the module __description__ strings that falsely claimed Blake3 checksums (the checksum has been xxHash3-64 since 0.1.0; docs drift tracked in #168). Co-authored-by: multica-agent --- rust/src/lib.rs | 9 ++++- rust/src/python_bindings.rs | 29 +++++++++++++++ tests/unit/test_checksum_ffi.py | 66 +++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_checksum_ffi.py diff --git a/rust/src/lib.rs b/rust/src/lib.rs index a2400ccd..feb94311 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -30,6 +30,11 @@ fn _rust_serializer(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { // Add byte storage class m.add_class::()?; + // Standalone integrity primitive — registered unconditionally (usable with + // the checksum feature alone; must not vanish when encryption is off) + m.add_function(wrap_pyfunction!(python_bindings::checksum_py, m)?)?; + m.add_function(wrap_pyfunction!(python_bindings::verify_checksum_py, m)?)?; + // Add encryption functionality if feature is enabled #[cfg(feature = "encryption")] { @@ -43,13 +48,13 @@ fn _rust_serializer(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { #[cfg(feature = "encryption")] m.add( "__description__", - "Raw byte storage with LZ4 compression, Blake3 checksums, and zero-knowledge encryption", + "Raw byte storage with LZ4 compression, xxHash3-64 checksums, and zero-knowledge encryption", )?; #[cfg(not(feature = "encryption"))] m.add( "__description__", - "Raw byte storage layer with LZ4 compression and Blake3 checksums", + "Raw byte storage layer with LZ4 compression and xxHash3-64 checksums", )?; Ok(()) diff --git a/rust/src/python_bindings.rs b/rust/src/python_bindings.rs index f94ad769..edaf9bee 100644 --- a/rust/src/python_bindings.rs +++ b/rust/src/python_bindings.rs @@ -340,6 +340,35 @@ pub fn key_fingerprint_py(key: &[u8]) -> Vec { key_fingerprint(key).to_vec() } +/// Compute the standalone xxHash3-64 checksum of `data` (8 bytes, big-endian). +/// +/// NON-cryptographic: detects corruption, not tampering. For tamper-resistance +/// use @cache.secure (AES-256-GCM), never this checksum. Produces the exact +/// bytes embedded in every StorageEnvelope, without the LZ4 compression +/// overhead — for serializers where compression is ineffective (Arrow IPC, JSON). +#[pyfunction] +#[pyo3(name = "checksum")] +pub fn checksum_py(py: Python, data: &[u8]) -> Py { + PyBytes::new(py, &cachekit_core::checksum(data)).into() +} + +/// Verify `data` against an expected 8-byte xxHash3-64 checksum. +/// +/// NON-cryptographic: detects corruption, not tampering (see `checksum`). +/// Raises ValueError if `expected` is not exactly 8 bytes — a truncated +/// checksum must fail loudly, never return a wrong verdict. +#[pyfunction] +#[pyo3(name = "verify_checksum")] +pub fn verify_checksum_py(data: &[u8], expected: &[u8]) -> PyResult { + let expected: &[u8; 8] = expected.try_into().map_err(|_| { + PyValueError::new_err(format!( + "expected must be exactly 8 bytes, got {}", + expected.len() + )) + })?; + Ok(cachekit_core::verify_checksum(data, expected)) +} + /// Register encryption module with Python #[cfg(feature = "encryption")] pub fn register_encryption_module(m: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/tests/unit/test_checksum_ffi.py b/tests/unit/test_checksum_ffi.py new file mode 100644 index 00000000..8b251f64 --- /dev/null +++ b/tests/unit/test_checksum_ffi.py @@ -0,0 +1,66 @@ +"""Unit tests for the standalone checksum FFI (cachekit-core#13, Phase 2). + +The Rust core exposes checksum/verify_checksum decoupled from LZ4 compression +(cachekit-core 0.3.0). These bindings mirror them so every serializer can get +the canonical xxHash3-64 wire value without paying for compression. + +Byte-verification: the KAT constants below are pinned in cachekit-core's own +test suite (src/checksum.rs) and must match the pure-Python xxhash package — +three independent implementations agreeing on the exact wire bytes. + +Note: NON-cryptographic. Detects corruption, not tampering. Tamper-resistance +comes from AES-256-GCM (@cache.secure), never from this checksum. +""" + +from __future__ import annotations + +import pytest +import xxhash + +from cachekit import _rust_serializer as rs + +# Protocol test vectors — pinned in cachekit-core src/checksum.rs (KAT tests). +# Big-endian = xxhash canonical byte order, identical to the value embedded in +# every StorageEnvelope. +KAT_CACHEKIT = bytes([209, 35, 204, 155, 190, 157, 164, 177]) # checksum(b"cachekit-kat") +KAT_EMPTY = bytes([0x2D, 0x06, 0x80, 0x05, 0x38, 0xD3, 0x94, 0xC2]) # checksum(b"") + + +class TestChecksumFFI: + def test_checksum_is_8_bytes_and_deterministic(self): + c = rs.checksum(b"payload") + assert isinstance(c, bytes) + assert len(c) == 8 + assert rs.checksum(b"payload") == c + + def test_checksum_matches_protocol_test_vectors(self): + """Byte-verified against the KAT vectors pinned in cachekit-core.""" + assert rs.checksum(b"cachekit-kat") == KAT_CACHEKIT + assert rs.checksum(b"") == KAT_EMPTY + + def test_checksum_matches_python_xxhash_package(self): + """FFI and the pure-Python xxhash package must agree byte-for-byte. + + The Arrow/orjson serializers currently compute envelopes via + xxhash.xxh3_64_digest; this proves the FFI is a drop-in producer of + the same wire bytes. + """ + for data in (b"", b"cachekit-kat", b"payload", bytes(range(256)) * 41): + assert rs.checksum(data) == xxhash.xxh3_64_digest(data) + + +class TestVerifyChecksumFFI: + def test_verify_round_trips(self): + assert rs.verify_checksum(b"payload", rs.checksum(b"payload")) is True + assert rs.verify_checksum(b"tampered", rs.checksum(b"payload")) is False + + def test_verify_rejects_single_bit_flip(self): + corrupted = bytearray(rs.checksum(b"payload")) + corrupted[0] ^= 0x01 + assert rs.verify_checksum(b"payload", bytes(corrupted)) is False + + @pytest.mark.parametrize("bad_len", [0, 7, 9, 32]) + def test_verify_rejects_wrong_length_expected(self, bad_len): + """expected must be exactly 8 bytes; anything else raises, never lies.""" + with pytest.raises(ValueError, match="8 bytes"): + rs.verify_checksum(b"payload", b"\x00" * bad_len) From 9abac5eb4e1ddc89041290fad730df7d15a0a2a5 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Fri, 17 Jul 2026 19:32:21 +1000 Subject: [PATCH 3/5] test: benchmark py-xxhash vs Rust-FFI checksum across sizes (cachekit-core#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pinned contract: 64B/1KB/64KB/1MB, >=30 rounds, per-size groups. Deliverable artifact, not a gate — results recorded in the module docstring. Verdict: performance-neutral everywhere (worst gap ~65ns/call at 1KB compute where py-xxhash's C mid-size path beats xxhash-rust; FFI wins small-payload verify 2.5x). Serializer migration to the FFI should be decided on dependency hygiene, not speed. Co-authored-by: multica-agent --- tests/benchmarks/benchmark_checksum_ffi.py | 92 ++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 tests/benchmarks/benchmark_checksum_ffi.py diff --git a/tests/benchmarks/benchmark_checksum_ffi.py b/tests/benchmarks/benchmark_checksum_ffi.py new file mode 100644 index 00000000..a177032b --- /dev/null +++ b/tests/benchmarks/benchmark_checksum_ffi.py @@ -0,0 +1,92 @@ +"""Benchmark: pure-Python xxhash package vs Rust-FFI checksum (cachekit-core#13). + +Pinned contract (spec m-cachekit-core-checksum-only-api, Task 7): fixed sizes +64 B / 1 KB / 64 KB / 1 MB, >=30 iterations per point, per-size reporting. +This is a deliverable artifact, not a pass/fail gate — the crossover size is +the go/no-go datum for migrating serializer envelopes (Arrow/orjson) from the +py-xxhash package to the shared Rust FFI. Tiny inputs favoring in-process +Python is expected information, not failure: both implementations produce +byte-identical output (see tests/unit/test_checksum_ffi.py), so the choice +is purely a per-call-overhead question. + +Run: + uv run pytest tests/benchmarks/benchmark_checksum_ffi.py \ + --benchmark-only --benchmark-group-by=group --benchmark-min-rounds=30 + +Results (2026-07-17, AMD Ryzen 9 5950X, CPython 3.13.12, median per call): + + checksum (compute) py-xxhash Rust FFI winner + 64 B 51.6 ns 39.7 ns FFI 1.30x + 1 KB 77.8 ns 130.0 ns py 1.67x + 64 KB 1.85 us 1.86 us wash (~1%) + 1 MB 28.1 us 28.7 us wash (~2%) + + verify py-xxhash Rust FFI winner + 64 B 88.1 ns 34.8 ns FFI 2.53x + 1 KB 125.0 ns 64.3 ns FFI 1.94x + 64 KB 1.87 us 1.78 us FFI ~5% + 1 MB 28.0 us 27.9 us wash + +Crossover / go-no-go datum: there is NO size where either side wins by more +than ~65 ns/call on compute; both are throughput-bound and identical from +64 KB up. py-xxhash's C library has a stronger mid-size (240 B - 8 KB) path +than xxhash-rust, hence the 1 KB compute loss; the FFI wins verify at small +sizes because it is one boundary crossing instead of hash + compare in +Python. Verdict: serializer migration py-xxhash -> FFI is performance-neutral +(worst case ~50 ns/call against envelope operations measured in us-ms); decide +it on dependency hygiene, not speed. +""" + +from __future__ import annotations + +import pytest +import xxhash + +from cachekit import _rust_serializer as rs + +SIZES = [ + (64, "64B"), + (1_024, "1KB"), + (65_536, "64KB"), + (1_048_576, "1MB"), +] + +PAYLOADS = {label: bytes(i % 251 for i in range(size)) for size, label in SIZES} + + +@pytest.mark.benchmark +@pytest.mark.parametrize("label", [label for _, label in SIZES]) +class TestChecksumComputeComparison: + """py-xxhash vs Rust FFI, grouped per size for side-by-side comparison.""" + + def test_python_xxhash(self, benchmark, label): + data = PAYLOADS[label] + benchmark.group = f"checksum-{label}" + result = benchmark(xxhash.xxh3_64_digest, data) + assert len(result) == 8 + + def test_rust_ffi(self, benchmark, label): + data = PAYLOADS[label] + benchmark.group = f"checksum-{label}" + result = benchmark(rs.checksum, data) + assert len(result) == 8 + + +@pytest.mark.benchmark +@pytest.mark.parametrize("label", [label for _, label in SIZES]) +class TestVerifyComparison: + """Verification path: python compare vs Rust FFI verify_checksum.""" + + def test_python_xxhash_verify(self, benchmark, label): + data = PAYLOADS[label] + expected = xxhash.xxh3_64_digest(data) + benchmark.group = f"verify-{label}" + result = benchmark(lambda: xxhash.xxh3_64_digest(data) == expected) + assert result is True + + def test_rust_ffi_verify(self, benchmark, label): + data = PAYLOADS[label] + expected = bytes(rs.checksum(data)) + benchmark.group = f"verify-{label}" + result = benchmark(rs.verify_checksum, data, expected) + assert result is True From 2f575bb883d44d9e7fcf6bd24ca5af3112eaa9a1 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Fri, 17 Jul 2026 19:35:22 +1000 Subject: [PATCH 4/5] ci+docs: guard crates.io pin for cachekit-core; document standalone checksum API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI: fail if rust/Cargo.toml carries a path-dep for cachekit-core (a leftover path would make maturin build public wheels from the local workspace — release-please rewrites versions, not path->version), and run clippy --locked so a stale Cargo.lock fails instead of silently re-resolving. Docs: document checksum/verify_checksum with an executable example (markdown-docs) and the non-cryptographic warning. Corrects this page's false Blake3 claims (xxHash3-64 since core 0.1.0); the repo-wide docs sweep remains #168. Co-authored-by: multica-agent --- .github/workflows/ci.yml | 15 ++++++++++++- docs/features/rust-serialization.md | 33 +++++++++++++++++++++++------ 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5bb9704..59d0792e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,9 +48,22 @@ jobs: if: success() || failure() run: cd rust && cargo fmt --check + # A leftover path-dep would make maturin build release wheels from the local + # workspace instead of the published crate (release-please rewrites versions, + # not path -> version), producing a non-reproducible public wheel. + - name: Guard cachekit-core crates.io pin + if: success() || failure() + run: | + if grep -nE '^\s*cachekit-core\s*=.*path\s*=' rust/Cargo.toml; then + echo "::error::cachekit-core must be a crates.io version pin, not a path dep" + exit 1 + fi + + # --locked: fail on a stale Cargo.lock so the build provably resolves + # cachekit-core from crates.io as committed - name: Lint Rust if: success() || failure() - run: cd rust && cargo clippy -- -D warnings + run: cd rust && cargo clippy --locked -- -D warnings # PR: single Python version, critical tests only # Push: full matrix, full test suite diff --git a/docs/features/rust-serialization.md b/docs/features/rust-serialization.md index d7359396..08de4923 100644 --- a/docs/features/rust-serialization.md +++ b/docs/features/rust-serialization.md @@ -38,7 +38,7 @@ The Rust layer is transparent — you configure serializers and encryption at th | Operation | Python | Rust (ByteStorage) | |-----------|--------|---------------------| | LZ4 compression | ~50-100 MB/s | ~500 MB/s | -| Blake3 hashing | ~500 MB/s | ~15 GB/s | +| xxHash3-64 hashing | ~35 GB/s (`xxhash` C ext) | ~35 GB/s | | AES-256-GCM | ~200 MB/s | ~1-4 GB/s (AES-NI) | For most workloads the bottleneck is Redis RTT (~2-50ms), not serialization. The Rust layer matters for large payloads (DataFrames, bulk data) where serialization time approaches network time. @@ -61,16 +61,37 @@ Compression runs automatically. It can be toggled via the `CACHEKIT_ENABLE_COMPR --- -## Blake3 Integrity +## xxHash3-64 Integrity -Every value stored includes a Blake3 hash. On retrieval: +Every value stored includes an xxHash3-64 checksum (8 bytes, big-endian). On retrieval: -1. Hash of retrieved bytes is computed -2. Stored hash is compared +1. Checksum of retrieved bytes is computed +2. Stored checksum is compared 3. Mismatch → `BackendError` (corrupted data, never returned to caller) This protects against Redis memory corruption, storage bugs, and bit rot. +> **Non-cryptographic.** The checksum detects corruption, not tampering. +> Tamper-resistance comes from AES-256-GCM (`@cache.secure`), never from this checksum. + +### Standalone checksum API + +The same primitive is exposed directly — decoupled from LZ4 compression — for +serializers where compression is ineffective (Arrow IPC, compact JSON): + +```python +from cachekit._rust_serializer import checksum, verify_checksum + +digest = checksum(b"payload") # 8 bytes, big-endian +assert len(digest) == 8 +assert verify_checksum(b"payload", digest) is True +assert verify_checksum(b"tampered", digest) is False +``` + +`verify_checksum` raises `ValueError` unless the expected checksum is exactly +8 bytes. The output is byte-identical to the checksum embedded in every +ByteStorage envelope and to `xxhash.xxh3_64_digest` from the `xxhash` package. + --- ## AES-256-GCM Encryption @@ -99,7 +120,7 @@ The Rust ByteStorage layer is orthogonal to the serializer. Mix and match: | Typed models | [Pydantic](../serializers/pydantic.md) | Optional | | Custom types | [Custom](../serializers/custom.md) | Optional | -All serializers pass through the same ByteStorage pipeline (LZ4 + Blake3 + optional AES-256-GCM). +All serializers pass through the same ByteStorage pipeline (LZ4 + xxHash3-64 + optional AES-256-GCM). --- From d699dd861848f491b38021b03044145f91f1d3c4 Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Fri, 17 Jul 2026 23:28:54 +1000 Subject: [PATCH 5/5] fix(checksum-ffi): accept buffer protocol; harden crates.io-pin guard Apply the expert-panel review findings on the checksum FFI (LAB-132). - checksum/verify_checksum took &[u8], which in PyO3 0.29 accepts bytes only. The Arrow serializer they target hashes a memoryview (write arrow_serializer.py:245, verify body = mv[8:] :283), so the deferred serializer migration would TypeError in prod while the bytes-only test stayed green. Switch both to PyBuffer (bytes/bytearray/memoryview/Arrow buffers); wire bytes are unchanged. - CI pin guard grepped only the inline dep form. Assert the positive invariant on Cargo.lock instead: cachekit-core must resolve to the crates.io registry. A path or [patch.crates-io] redirect has no source line, so one check covers every Cargo.toml form and the workspace root. - Extend the xxhash byte-compat test to the 200 B mid-size and >64 KB accumulator-merge paths (previously unchecked above ~10 KB), and add memoryview/bytearray coverage including the Arrow mv[8:] verify shape. - docs: fix the leftover "Blake3" in the pipeline diagram; note the standalone checksum API ships in v0.12.0. Co-authored-by: multica-agent --- .github/workflows/ci.yml | 23 ++++++++---- docs/features/rust-serialization.md | 11 ++++-- rust/src/python_bindings.rs | 31 ++++++++++++----- tests/unit/test_checksum_ffi.py | 54 ++++++++++++++++++++++++++--- 4 files changed, 97 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59d0792e..927d2eca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,14 +48,25 @@ jobs: if: success() || failure() run: cd rust && cargo fmt --check - # A leftover path-dep would make maturin build release wheels from the local - # workspace instead of the published crate (release-please rewrites versions, - # not path -> version), producing a non-reproducible public wheel. - - name: Guard cachekit-core crates.io pin + # A leftover path dep or [patch.crates-io] redirect would make maturin build + # release wheels from unpublished local source instead of the published crate, + # producing a non-reproducible public wheel. Assert the POSITIVE invariant on + # the lockfile: cachekit-core must resolve to the crates.io registry. A path or + # patch dep has no `source` line, so this one check covers every Cargo.toml form + # (inline table, [dependencies.cachekit-core], [patch.crates-io]) and the + # workspace root — unlike a grep for `path =`, which only caught the inline form. + - name: Guard cachekit-core resolves to crates.io if: success() || failure() run: | - if grep -nE '^\s*cachekit-core\s*=.*path\s*=' rust/Cargo.toml; then - echo "::error::cachekit-core must be a crates.io version pin, not a path dep" + src=$(awk ' + /^\[\[package\]\]/ { name=""; source="" } + /^name = / { name=$3 } + /^source = / { source=$0 } + name == "\"cachekit-core\"" && source != "" { print source; exit } + ' Cargo.lock) + echo "cachekit-core resolved: ${src:-}" + if [ "$src" != 'source = "registry+https://github.com/rust-lang/crates.io-index"' ]; then + echo "::error::cachekit-core must resolve to the crates.io registry, not a local path/patch dep" exit 1 fi diff --git a/docs/features/rust-serialization.md b/docs/features/rust-serialization.md index 08de4923..bb8d2270 100644 --- a/docs/features/rust-serialization.md +++ b/docs/features/rust-serialization.md @@ -20,7 +20,7 @@ Serializer (MessagePack / Arrow / Orjson — your choice) ↓ [Rust ByteStorage takes over here] LZ4 compression (fast, ~500MB/s) ↓ -Blake3 integrity hash (~GB/s, detects corruption) +xxHash3-64 integrity hash (~GB/s, detects corruption) ↓ [Optional] AES-256-GCM encryption (if @cache.secure) ↓ @@ -76,6 +76,9 @@ This protects against Redis memory corruption, storage bugs, and bit rot. ### Standalone checksum API +> **Available since v0.12.0.** On earlier releases, +> `from cachekit._rust_serializer import checksum` raises `ImportError`. + The same primitive is exposed directly — decoupled from LZ4 compression — for serializers where compression is ineffective (Arrow IPC, compact JSON): @@ -89,8 +92,10 @@ assert verify_checksum(b"tampered", digest) is False ``` `verify_checksum` raises `ValueError` unless the expected checksum is exactly -8 bytes. The output is byte-identical to the checksum embedded in every -ByteStorage envelope and to `xxhash.xxh3_64_digest` from the `xxhash` package. +8 bytes. Both functions accept any buffer-protocol object (`bytes`, `bytearray`, +`memoryview`), so a serializer holding its payload as a `memoryview` can hash it +without a `bytes` copy. The output is byte-identical to the checksum embedded in +every ByteStorage envelope and to `xxhash.xxh3_64_digest` from the `xxhash` package. --- diff --git a/rust/src/python_bindings.rs b/rust/src/python_bindings.rs index edaf9bee..723a7458 100644 --- a/rust/src/python_bindings.rs +++ b/rust/src/python_bindings.rs @@ -4,6 +4,7 @@ //! All business logic is delegated to cachekit-core. use cachekit_core::ByteStorage; +use pyo3::buffer::PyBuffer; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::types::PyBytes; @@ -342,31 +343,43 @@ pub fn key_fingerprint_py(key: &[u8]) -> Vec { /// Compute the standalone xxHash3-64 checksum of `data` (8 bytes, big-endian). /// +/// Accepts any buffer-protocol object — `bytes`, `bytearray`, `memoryview`, +/// Arrow buffers — so a serializer holding its payload as a `memoryview` +/// (e.g. Arrow IPC) can hash it directly, without forcing a `bytes` copy. +/// /// NON-cryptographic: detects corruption, not tampering. For tamper-resistance /// use @cache.secure (AES-256-GCM), never this checksum. Produces the exact /// bytes embedded in every StorageEnvelope, without the LZ4 compression /// overhead — for serializers where compression is ineffective (Arrow IPC, JSON). #[pyfunction] #[pyo3(name = "checksum")] -pub fn checksum_py(py: Python, data: &[u8]) -> Py { - PyBytes::new(py, &cachekit_core::checksum(data)).into() +pub fn checksum_py(py: Python, data: PyBuffer) -> PyResult> { + let data = data.to_vec(py)?; + Ok(PyBytes::new(py, &cachekit_core::checksum(&data)).into()) } /// Verify `data` against an expected 8-byte xxHash3-64 checksum. /// +/// Both arguments accept any buffer-protocol object (`bytes`, `bytearray`, +/// `memoryview`, …) — the Arrow verify path slices a `memoryview` (`mv[8:]`), +/// so a bytes-only signature would break the moment a serializer moves onto +/// this FFI. +/// /// NON-cryptographic: detects corruption, not tampering (see `checksum`). /// Raises ValueError if `expected` is not exactly 8 bytes — a truncated /// checksum must fail loudly, never return a wrong verdict. #[pyfunction] #[pyo3(name = "verify_checksum")] -pub fn verify_checksum_py(data: &[u8], expected: &[u8]) -> PyResult { - let expected: &[u8; 8] = expected.try_into().map_err(|_| { - PyValueError::new_err(format!( - "expected must be exactly 8 bytes, got {}", - expected.len() - )) +pub fn verify_checksum_py( + py: Python, + data: PyBuffer, + expected: PyBuffer, +) -> PyResult { + let expected: [u8; 8] = expected.to_vec(py)?.try_into().map_err(|v: Vec| { + PyValueError::new_err(format!("expected must be exactly 8 bytes, got {}", v.len())) })?; - Ok(cachekit_core::verify_checksum(data, expected)) + let data = data.to_vec(py)?; + Ok(cachekit_core::verify_checksum(&data, &expected)) } /// Register encryption module with Python diff --git a/tests/unit/test_checksum_ffi.py b/tests/unit/test_checksum_ffi.py index 8b251f64..3ee1d636 100644 --- a/tests/unit/test_checksum_ffi.py +++ b/tests/unit/test_checksum_ffi.py @@ -41,11 +41,22 @@ def test_checksum_matches_protocol_test_vectors(self): def test_checksum_matches_python_xxhash_package(self): """FFI and the pure-Python xxhash package must agree byte-for-byte. - The Arrow/orjson serializers currently compute envelopes via - xxhash.xxh3_64_digest; this proves the FFI is a drop-in producer of - the same wire bytes. + The Arrow/orjson serializers compute envelopes via xxhash.xxh3_64_digest; + this proves the FFI is a drop-in producer of the same wire bytes across + xxHash3's size-dependent code paths — short, the 17-240 B mid path, and + the >64 KB accumulator-merge path (all previously unchecked above ~10 KB, + where an xxhash-rust vs libxxhash divergence would ship silently). """ - for data in (b"", b"cachekit-kat", b"payload", bytes(range(256)) * 41): + payloads = ( + b"", + b"cachekit-kat", + b"payload", + bytes(range(200)), # 200 B — xxHash3 mid-size path + bytes(range(256)) * 41, # ~10 KB + bytes(i % 251 for i in range(65_536 + 7)), # > 64 KB — accumulator merge + bytes(i % 251 for i in range(1_048_576)), # 1 MB + ) + for data in payloads: assert rs.checksum(data) == xxhash.xxh3_64_digest(data) @@ -64,3 +75,38 @@ def test_verify_rejects_wrong_length_expected(self, bad_len): """expected must be exactly 8 bytes; anything else raises, never lies.""" with pytest.raises(ValueError, match="8 bytes"): rs.verify_checksum(b"payload", b"\x00" * bad_len) + + +class TestBufferProtocol: + """The FFI must accept any buffer-protocol object, not only `bytes`. + + The Arrow serializer hashes a `memoryview` on both write + (arrow_serializer.py:245) and verify (`body = mv[8:]`, :283). A bytes-only + signature would raise TypeError the moment a serializer migrates onto this + FFI — while a bytes-only test suite stayed green. These tests lock the + buffer-protocol contract that makes that migration safe. + """ + + PAYLOAD = b"cachekit-kat payload of some length \x00\xff\x7f" + + @pytest.mark.parametrize("wrap", [bytes, bytearray, memoryview], ids=["bytes", "bytearray", "memoryview"]) + def test_checksum_accepts_buffer_types(self, wrap): + assert rs.checksum(wrap(self.PAYLOAD)) == xxhash.xxh3_64_digest(self.PAYLOAD) + + @pytest.mark.parametrize("wrap", [bytes, bytearray, memoryview], ids=["bytes", "bytearray", "memoryview"]) + def test_verify_accepts_buffer_types(self, wrap): + digest = rs.checksum(self.PAYLOAD) + assert rs.verify_checksum(wrap(self.PAYLOAD), digest) is True + assert rs.verify_checksum(wrap(self.PAYLOAD), wrap(digest)) is True + + def test_verify_accepts_memoryview_slice_like_arrow(self): + """Mirror the Arrow verify path exactly: envelope = [8-byte checksum][body], + then verify the body (a memoryview slice) against the sliced checksum.""" + body = self.PAYLOAD + envelope = memoryview(rs.checksum(body) + body) + assert rs.verify_checksum(envelope[8:], envelope[:8]) is True + + tampered = bytearray(envelope) + tampered[-1] ^= 0x01 # corrupt the body, leave the checksum prefix intact + mv = memoryview(tampered) + assert rs.verify_checksum(mv[8:], mv[:8]) is False