From 2606882f089787d85e5addc847116fa263f0deca Mon Sep 17 00:00:00 2001 From: Charlie <5764343+charlielye@users.noreply.github.com> Date: Thu, 11 Jun 2026 17:15:12 +0000 Subject: [PATCH] refactor(wsdb): migrate to generated ipc package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Generate the @aztec/wsdb TS package and the wsdb C++ client/server from wsdb_schema.jsonc via ipc-codegen instead of the hand-written bindings. Add a "bin" entry to the generated package.json (when the package wraps a native binary) so the binary lands on the user's PATH on install. The bin target is a generated JS launcher (src/bin.ts -> dest/bin.js) that resolves the native binary via the same per-arch resolution as the spawned client and execs it forwarding argv/stdio/exit code — the binary itself ships in the per-arch optional-dependency packages, and npm only links the main package's own bin. --- Makefile | 13 +- barretenberg/cpp/CMakePresets.json | 8 +- barretenberg/cpp/format.sh | 2 +- barretenberg/cpp/src/CMakeLists.txt | 12 +- .../src/barretenberg/bbapi/bbapi_chonk.cpp | 16 +- .../barretenberg/common/try_catch_shim.hpp | 11 +- .../serialize/msgpack_check_eq.hpp | 1 - .../serialize/msgpack_impl/concepts.hpp | 16 +- .../serialize/msgpack_impl/drop_keys.hpp | 5 + .../msgpack_impl/struct_map_impl.hpp | 5 + .../src/barretenberg/vm2_wsdb/CMakeLists.txt | 16 + .../vm2_wsdb/wsdb_ipc_merkle_db.cpp | 224 ++++++ .../wsdb_ipc_merkle_db.hpp | 11 +- .../cpp/src/barretenberg/wsdb/CMakeLists.txt | 118 ++- .../cpp/src/barretenberg/wsdb/cli.cpp | 27 +- .../src/barretenberg/wsdb/wsdb_commands.hpp | 536 ------------- .../src/barretenberg/wsdb/wsdb_execute.cpp | 414 ---------- .../src/barretenberg/wsdb/wsdb_execute.hpp | 115 --- .../src/barretenberg/wsdb/wsdb_handlers.cpp | 536 +++++++++++++ .../src/barretenberg/wsdb/wsdb_handlers.hpp | 80 ++ .../src/barretenberg/wsdb/wsdb_ipc_client.hpp | 3 + .../wsdb/wsdb_ipc_client_generated.cpp | 225 ------ .../wsdb/wsdb_ipc_client_generated.hpp | 65 -- .../src/barretenberg/wsdb/wsdb_ipc_server.cpp | 202 +---- .../src/barretenberg/wsdb/wsdb_request.hpp | 18 + .../src/barretenberg/wsdb/wsdb_schema.jsonc | 475 ++++++++++++ .../barretenberg/wsdb/wsdb_wire_convert.hpp | 504 ++++++++++++ .../barretenberg/wsdb_client/CMakeLists.txt | 19 - .../wsdb_client/wsdb_ipc_merkle_db.cpp | 231 ------ barretenberg/ts/.gitignore | 1 - barretenberg/ts/package.json | 7 +- barretenberg/ts/scripts/copy_native.sh | 3 +- barretenberg/ts/src/aztec-wsdb/generate.ts | 89 --- barretenberg/ts/src/aztec-wsdb/index.ts | 449 ----------- bootstrap.sh | 1 + ci3/release_prep_package_json | 2 +- .../echo_example/ts_package/.gitignore | 1 + ipc-codegen/src/generate.ts | 3 + ipc-codegen/src/typescript_package_codegen.ts | 31 + wsdb/.gitignore | 9 + wsdb/.rebuild_patterns | 6 + wsdb/.yarnrc.yml | 1 + wsdb/bootstrap.sh | 85 +++ wsdb/package.json | 16 + wsdb/yarn.lock | 396 ++++++++++ .../src/native/ipc_world_state_instance.ts | 717 ------------------ 46 files changed, 2604 insertions(+), 3121 deletions(-) create mode 100644 barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt create mode 100644 barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp rename barretenberg/cpp/src/barretenberg/{wsdb_client => vm2_wsdb}/wsdb_ipc_merkle_db.hpp (88%) delete mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_commands.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_execute.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_execute.hpp create mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp create mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp create mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.hpp create mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp create mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc create mode 100644 barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/wsdb_client/CMakeLists.txt delete mode 100644 barretenberg/cpp/src/barretenberg/wsdb_client/wsdb_ipc_merkle_db.cpp delete mode 100644 barretenberg/ts/src/aztec-wsdb/generate.ts delete mode 100644 barretenberg/ts/src/aztec-wsdb/index.ts create mode 100644 wsdb/.gitignore create mode 100644 wsdb/.rebuild_patterns create mode 100644 wsdb/.yarnrc.yml create mode 100755 wsdb/bootstrap.sh create mode 100644 wsdb/package.json create mode 100644 wsdb/yarn.lock delete mode 100644 yarn-project/world-state/src/native/ipc_world_state_instance.ts diff --git a/Makefile b/Makefile index a4cfb857e509..d40c9a8df0d1 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ endef # PHONY TARGETS - List every target that has a file/dir of the same name. #============================================================================== -.PHONY: noir barretenberg noir-projects l1-contracts release-image boxes playground docs aztec-up spartan +.PHONY: noir barretenberg noir-projects l1-contracts release-image boxes playground docs aztec-up spartan wsdb #============================================================================== # BOOTSTRAP TARGETS @@ -205,7 +205,7 @@ bb-cpp-asan: bb-cpp-smt: $(call build,$@,barretenberg/cpp,build_smt_verification) -bb-cpp-release-dir: bb-cpp-native bb-cpp-cross +bb-cpp-release-dir: bb-cpp-native bb-cpp-cross bb-cpp-wasm bb-cpp-wasm-threads $(call build,$@,barretenberg/cpp,build_release_dir) bb-cpp-full: bb-cpp bb-cpp-gcc bb-cpp-fuzzing bb-cpp-asan bb-cpp-smt bb-cpp-cross-arm64-macos bb-cpp-cross-arm64-ios bb-cpp-cross-arm64-android @@ -306,6 +306,13 @@ ipc-runtime-cross-arm64-macos: ipc-runtime-cross: ipc-runtime ipc-runtime-cross-arm64-linux ipc-runtime-cross-amd64-macos ipc-runtime-cross-arm64-macos +#============================================================================== +# WSDB +#============================================================================== + +wsdb: ipc-codegen ipc-runtime bb-cpp-native + $(call build,$@,wsdb) + #============================================================================== # .claude tooling #============================================================================== @@ -372,7 +379,7 @@ l1-contracts-tests: l1-contracts-verifier # Yarn Project - TypeScript monorepo with all TS packages #============================================================================== -yarn-project: bb-ts noir-projects l1-contracts +yarn-project: bb-ts noir-projects l1-contracts wsdb $(call build,$@,yarn-project) yarn-project-tests: yarn-project diff --git a/barretenberg/cpp/CMakePresets.json b/barretenberg/cpp/CMakePresets.json index 3d5b17a58d32..9ec185dac497 100644 --- a/barretenberg/cpp/CMakePresets.json +++ b/barretenberg/cpp/CMakePresets.json @@ -800,25 +800,25 @@ "name": "amd64-linux", "configurePreset": "amd64-linux", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external"] + "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb"] }, { "name": "arm64-linux", "configurePreset": "arm64-linux", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external"] + "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb"] }, { "name": "amd64-macos", "configurePreset": "amd64-macos", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external"] + "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb"] }, { "name": "arm64-macos", "configurePreset": "arm64-macos", "inheritConfigureEnvironment": true, - "targets": ["bb", "nodejs_module", "bb-external"] + "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb"] }, { "name": "amd64-windows", diff --git a/barretenberg/cpp/format.sh b/barretenberg/cpp/format.sh index 48d751bdcada..b64d80ba0354 100755 --- a/barretenberg/cpp/format.sh +++ b/barretenberg/cpp/format.sh @@ -23,7 +23,7 @@ elif [ "$1" == "changed" ]; then format_files "$files" fi elif [ "$1" == "check" ]; then - files=$(find ./src -iname *.hpp -o -iname *.cpp -o -iname *.tcc | grep -v bb/deps) + files=$(find ./src -iname *.hpp -o -iname *.cpp -o -iname *.tcc | grep -v bb/deps | grep -v '/generated/') echo "$files" | parallel -N10 clang-format-20 --dry-run --Werror elif [ -n "$1" ]; then files=$(git diff-index --relative --name-only $1 | grep -e '\.\(cpp\|hpp\|tcc\)$') diff --git a/barretenberg/cpp/src/CMakeLists.txt b/barretenberg/cpp/src/CMakeLists.txt index 20cfb695c920..a818ce8812d5 100644 --- a/barretenberg/cpp/src/CMakeLists.txt +++ b/barretenberg/cpp/src/CMakeLists.txt @@ -52,7 +52,7 @@ if(WASM) add_link_options(-Wl,--export-memory,--import-memory,--stack-first,-z,stack-size=1048576,--max-memory=4294967296) endif() -include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${TRACY_INCLUDE} ${LMDB_INCLUDE} ${LIBDEFLATE_INCLUDE} ${HTTPLIB_INCLUDE} ${BACKWARD_INCLUDE} ${NLOHMANN_JSON_INCLUDE}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${TRACY_INCLUDE} ${LMDB_INCLUDE} ${LIBDEFLATE_INCLUDE} ${HTTPLIB_INCLUDE} ${BACKWARD_INCLUDE} ${NLOHMANN_JSON_INCLUDE} ${CMAKE_CURRENT_SOURCE_DIR}/../../../ipc-runtime/cpp) # Add avm-transpiler include path when library is provided if(AVM_TRANSPILER_LIB) @@ -128,10 +128,18 @@ if(NOT FUZZING AND NOT WASM AND NOT BB_LITE) add_subdirectory(barretenberg/vm2) add_subdirectory(barretenberg/ipc) add_subdirectory(barretenberg/wsdb) - add_subdirectory(barretenberg/wsdb_client) + add_subdirectory(barretenberg/vm2_wsdb) add_subdirectory(barretenberg/nodejs_module) endif() +# Pull in ipc-runtime as a C++ dependency. Provides the `ipc_runtime` +# library target (static, or INTERFACE under WASM with transport sources +# stubbed) that bbapi/wsdb/etc link against for the codegen-emitted +# bb_ipc_server.hpp dispatcher. +if(NOT FUZZING AND NOT BB_LITE) + add_subdirectory(${CMAKE_SOURCE_DIR}/../../ipc-runtime/cpp ${CMAKE_BINARY_DIR}/ipc-runtime) +endif() + if(FUZZING_AVM) if(FUZZING) # Only add these if they weren't added above (when NOT FUZZING AND NOT WASM) diff --git a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_chonk.cpp b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_chonk.cpp index 60801600ff90..a17a1891c189 100644 --- a/barretenberg/cpp/src/barretenberg/bbapi/bbapi_chonk.cpp +++ b/barretenberg/cpp/src/barretenberg/bbapi/bbapi_chonk.cpp @@ -17,11 +17,13 @@ #include "barretenberg/stdlib_circuit_builders/mega_circuit_builder.hpp" #ifndef __wasm__ +#include #include #include #include #include #include +#include #include #include #include @@ -450,7 +452,9 @@ namespace { bool write_all(int fd, const uint8_t* ptr, size_t len) { while (len > 0) { - const ssize_t written = ::write(fd, ptr, len); + const auto chunk_len = + static_cast(std::min(len, std::numeric_limits::max())); + const ssize_t written = ::write(fd, ptr, chunk_len); if (written > 0) { ptr += written; len -= static_cast(written); @@ -508,7 +512,9 @@ void ChonkBatchVerifierService::start(std::vector= 0) { +#ifndef _WIN32 struct stat opened_statbuf; if (fstat(fifo_fd_, &opened_statbuf) != 0 || !S_ISFIFO(opened_statbuf.st_mode)) { info("ChonkBatchVerifierService: opened result path is not a FIFO: ", fifo_path_); @@ -590,6 +603,7 @@ bool ChonkBatchVerifierService::ensure_fifo_open() fifo_fd_ = -1; return false; } +#endif return true; } if (errno != ENXIO && errno != EINTR) { diff --git a/barretenberg/cpp/src/barretenberg/common/try_catch_shim.hpp b/barretenberg/cpp/src/barretenberg/common/try_catch_shim.hpp index 2cb08e6f2225..b1273b072ab7 100644 --- a/barretenberg/cpp/src/barretenberg/common/try_catch_shim.hpp +++ b/barretenberg/cpp/src/barretenberg/common/try_catch_shim.hpp @@ -3,7 +3,14 @@ #include // Tool to make header only libraries (i.e. CLI11 and msgpack, though it has a bundled copy) -// not use exceptions with minimally invaslive changes +// not use exceptions with minimally invaslive changes. +// +// Macros are guarded so any parent project (e.g. ipc_codegen/throw.hpp under +// codegen-emitted code) that predefines them wins. Same convention as +// ipc_codegen/throw.hpp, so the two headers can be #included in any order +// without redefinition warnings. + +#ifndef THROW #ifdef BB_NO_EXCEPTIONS struct __AbortStream { @@ -21,3 +28,5 @@ struct __AbortStream { #define THROW throw #define RETHROW THROW #endif + +#endif // THROW diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_check_eq.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_check_eq.hpp index 35c7c107cb25..18f59680f08b 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/msgpack_check_eq.hpp +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_check_eq.hpp @@ -2,7 +2,6 @@ #include "barretenberg/common/log.hpp" #include "msgpack.hpp" -#include "msgpack_impl/drop_keys.hpp" #include #include diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp index 78ca01eeaa1d..c67a98c256c4 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/concepts.hpp @@ -1,18 +1,30 @@ #pragma once +#ifndef IPC_CODEGEN_MSGPACK_CONCEPTS_DEFINED +#define IPC_CODEGEN_MSGPACK_CONCEPTS_DEFINED + struct DoNothing { void operator()(auto...) {} }; + namespace msgpack_concepts { + template concept HasMsgPack = requires(T t, DoNothing nop) { t.msgpack(nop); }; +template +concept MsgpackConstructible = requires(T object, Args... args) { T{ args... }; }; + +} // namespace msgpack_concepts + +#endif + +namespace msgpack_concepts { + template concept HasMsgPackSchema = requires(const T t, DoNothing nop) { t.msgpack_schema(nop); }; template concept HasMsgPackPack = requires(T t, DoNothing nop) { t.msgpack_pack(nop); }; -template -concept MsgpackConstructible = requires(T object, Args... args) { T{ args... }; }; } // namespace msgpack_concepts diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/drop_keys.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/drop_keys.hpp index 7f60ef7e74c4..c03b4e50bb4e 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/drop_keys.hpp +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/drop_keys.hpp @@ -1,6 +1,9 @@ #pragma once #include +#ifndef IPC_CODEGEN_MSGPACK_DROP_KEYS_DEFINED +#define IPC_CODEGEN_MSGPACK_DROP_KEYS_DEFINED + namespace msgpack { template auto drop_keys_impl(Tuple&& tuple, std::index_sequence) { @@ -20,3 +23,5 @@ template auto drop_keys(std::tuple&& tuple) return drop_keys_impl(tuple, compile_time_0_to_n_div_2); } } // namespace msgpack + +#endif diff --git a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp index a17347214cb4..78b832ca60c4 100644 --- a/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp +++ b/barretenberg/cpp/src/barretenberg/serialize/msgpack_impl/struct_map_impl.hpp @@ -10,6 +10,9 @@ #include "drop_keys.hpp" #include +#ifndef IPC_CODEGEN_MSGPACK_STRUCT_MAP_ADAPTOR_DEFINED +#define IPC_CODEGEN_MSGPACK_STRUCT_MAP_ADAPTOR_DEFINED + namespace msgpack::adaptor { // reads structs with msgpack() method from a JSON-like dictionary template struct convert { @@ -61,3 +64,5 @@ template struct pack { }; } // namespace msgpack::adaptor + +#endif diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt new file mode 100644 index 000000000000..ee67a00907e3 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/CMakeLists.txt @@ -0,0 +1,16 @@ +# VM2 adapter for native AVM simulation against aztec-wsdb. +if(TARGET vm2_sim AND TARGET wsdb_ipc_client) + add_library( + wsdb_ipc_merkle_db + STATIC + wsdb_ipc_merkle_db.cpp + ) + target_link_libraries( + wsdb_ipc_merkle_db + PUBLIC + barretenberg + vm2_sim + wsdb_ipc_client + ) + set_target_properties(wsdb_ipc_merkle_db PROPERTIES POSITION_INDEPENDENT_CODE ON) +endif() diff --git a/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp new file mode 100644 index 000000000000..912a840ca71e --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.cpp @@ -0,0 +1,224 @@ +#include "barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp" +#include "barretenberg/aztec/aztec_constants.hpp" +#include "barretenberg/common/log.hpp" +#include "barretenberg/wsdb/wsdb_wire_convert.hpp" + +#include + +namespace bb::avm2::simulation { + +// Wire <-> domain conversion helpers are shared with the server handlers +// (see wsdb_handlers.cpp) so both sides use the same encoding boundary. +using bb::wsdb::Fr; +using bb::wsdb::fr_from_wire; +using bb::wsdb::fr_to_wire; +using bb::wsdb::fr_vec_from_wire; +using bb::wsdb::indexed_nullifier_leaf_from_wire; +using bb::wsdb::indexed_public_data_leaf_from_wire; +using bb::wsdb::nullifier_leaf_to_wire; +using bb::wsdb::public_data_leaf_to_wire; +using bb::wsdb::revision_to_wire; +using bb::wsdb::sequential_nullifier_from_wire; +using bb::wsdb::sequential_public_data_from_wire; +using bb::wsdb::tree_id_to_wire; + +// --------------------------------------------------------------------------- +// Constructor +// --------------------------------------------------------------------------- + +WsdbIpcMerkleDB::WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, world_state::WorldStateRevision revision) + : client_(client) + , revision_(revision) +{} + +// --------------------------------------------------------------------------- +// Tree roots +// --------------------------------------------------------------------------- + +avm2::TreeSnapshots WsdbIpcMerkleDB::get_tree_roots() const +{ + if (cached_tree_roots_.has_value()) { + return cached_tree_roots_.value(); + } + + auto wire_rev = revision_to_wire(revision_); + + auto l1_info = client_.get_tree_info( + wsdb::WsdbGetTreeInfo{ .treeId = tree_id_to_wire(MerkleTreeId::L1_TO_L2_MESSAGE_TREE), .revision = wire_rev }); + auto nh_info = client_.get_tree_info( + wsdb::WsdbGetTreeInfo{ .treeId = tree_id_to_wire(MerkleTreeId::NOTE_HASH_TREE), .revision = wire_rev }); + auto null_info = client_.get_tree_info( + wsdb::WsdbGetTreeInfo{ .treeId = tree_id_to_wire(MerkleTreeId::NULLIFIER_TREE), .revision = wire_rev }); + auto pd_info = client_.get_tree_info( + wsdb::WsdbGetTreeInfo{ .treeId = tree_id_to_wire(MerkleTreeId::PUBLIC_DATA_TREE), .revision = wire_rev }); + + avm2::TreeSnapshots snapshots{ + .l1_to_l2_message_tree = avm2::AppendOnlyTreeSnapshot{ .root = fr_from_wire(l1_info.root), + .next_available_leaf_index = l1_info.size }, + .note_hash_tree = avm2::AppendOnlyTreeSnapshot{ .root = fr_from_wire(nh_info.root), + .next_available_leaf_index = nh_info.size }, + .nullifier_tree = avm2::AppendOnlyTreeSnapshot{ .root = fr_from_wire(null_info.root), + .next_available_leaf_index = null_info.size }, + .public_data_tree = avm2::AppendOnlyTreeSnapshot{ .root = fr_from_wire(pd_info.root), + .next_available_leaf_index = pd_info.size }, + }; + cached_tree_roots_ = snapshots; + return snapshots; +} + +void WsdbIpcMerkleDB::invalidate_tree_roots_cache() +{ + cached_tree_roots_ = std::nullopt; +} + +// --------------------------------------------------------------------------- +// Query methods +// --------------------------------------------------------------------------- + +SiblingPath WsdbIpcMerkleDB::get_sibling_path(MerkleTreeId tree_id, index_t leaf_index) const +{ + auto resp = client_.get_sibling_path(wsdb::WsdbGetSiblingPath{ + .treeId = tree_id_to_wire(tree_id), .revision = revision_to_wire(revision_), .leafIndex = leaf_index }); + return fr_vec_from_wire(resp.path); +} + +crypto::merkle_tree::GetLowIndexedLeafResponse WsdbIpcMerkleDB::get_low_indexed_leaf(MerkleTreeId tree_id, + const avm2::FF& value) const +{ + auto resp = client_.find_low_leaf(wsdb::WsdbFindLowLeaf{ + .treeId = tree_id_to_wire(tree_id), .revision = revision_to_wire(revision_), .key = fr_to_wire(value) }); + return GetLowIndexedLeafResponse(resp.alreadyPresent, resp.index); +} + +avm2::FF WsdbIpcMerkleDB::get_leaf_value(MerkleTreeId tree_id, index_t leaf_index) const +{ + auto resp = client_.get_leaf_value(wsdb::WsdbGetLeafValue{ + .treeId = tree_id_to_wire(tree_id), .revision = revision_to_wire(revision_), .leafIndex = leaf_index }); + if (!resp.value.has_value()) { + throw std::runtime_error("Invalid get_leaf_value request for tree " + + std::to_string(static_cast(tree_id)) + " index " + + std::to_string(leaf_index)); + } + return fr_from_wire(resp.value.value()); +} + +IndexedLeaf WsdbIpcMerkleDB::get_leaf_preimage_public_data_tree(index_t leaf_index) const +{ + auto resp = client_.get_public_data_leaf_preimage( + wsdb::WsdbGetPublicDataLeafPreimage{ .revision = revision_to_wire(revision_), .leafIndex = leaf_index }); + if (!resp.preimage.has_value()) { + throw std::runtime_error("Invalid get_leaf_preimage_public_data_tree request for index " + + std::to_string(leaf_index)); + } + return indexed_public_data_leaf_from_wire(resp.preimage.value()); +} + +IndexedLeaf WsdbIpcMerkleDB::get_leaf_preimage_nullifier_tree(index_t leaf_index) const +{ + auto resp = client_.get_nullifier_leaf_preimage( + wsdb::WsdbGetNullifierLeafPreimage{ .revision = revision_to_wire(revision_), .leafIndex = leaf_index }); + if (!resp.preimage.has_value()) { + throw std::runtime_error("Invalid get_leaf_preimage_nullifier_tree request for index " + + std::to_string(leaf_index)); + } + return indexed_nullifier_leaf_from_wire(resp.preimage.value()); +} + +// --------------------------------------------------------------------------- +// State modification methods +// --------------------------------------------------------------------------- + +SequentialInsertionResult WsdbIpcMerkleDB::insert_indexed_leaves_public_data_tree( + const PublicDataLeafValue& leaf_value) +{ + auto resp = client_.sequential_insert_public_data(wsdb::WsdbSequentialInsertPublicData{ + .leaves = { public_data_leaf_to_wire(leaf_value) }, .forkId = revision_.forkId }); + invalidate_tree_roots_cache(); + return sequential_public_data_from_wire(resp.result); +} + +SequentialInsertionResult WsdbIpcMerkleDB::insert_indexed_leaves_nullifier_tree( + const NullifierLeafValue& leaf_value) +{ + auto resp = client_.sequential_insert_nullifier(wsdb::WsdbSequentialInsertNullifier{ + .leaves = { nullifier_leaf_to_wire(leaf_value) }, .forkId = revision_.forkId }); + invalidate_tree_roots_cache(); + return sequential_nullifier_from_wire(resp.result); +} + +void WsdbIpcMerkleDB::append_leaves(MerkleTreeId tree_id, std::span leaves) +{ + std::vector wire_leaves; + wire_leaves.reserve(leaves.size()); + for (const auto& leaf : leaves) { + wire_leaves.push_back(fr_to_wire(leaf)); + } + client_.append_leaves(wsdb::WsdbAppendLeaves{ + .treeId = tree_id_to_wire(tree_id), .leaves = std::move(wire_leaves), .forkId = revision_.forkId }); + invalidate_tree_roots_cache(); +} + +void WsdbIpcMerkleDB::pad_tree(MerkleTreeId tree_id, size_t num_leaves) +{ + switch (tree_id) { + case MerkleTreeId::NULLIFIER_TREE: { + std::vector padding_leaves; + padding_leaves.reserve(num_leaves); + auto empty_leaf = NullifierLeafValue::empty(); + for (size_t i = 0; i < num_leaves; i++) { + padding_leaves.push_back(nullifier_leaf_to_wire(empty_leaf)); + } + client_.batch_insert_nullifier(wsdb::WsdbBatchInsertNullifier{ .leaves = std::move(padding_leaves), + .subtreeDepth = NULLIFIER_SUBTREE_HEIGHT, + .forkId = revision_.forkId }); + break; + } + case MerkleTreeId::NOTE_HASH_TREE: { + std::vector padding_leaves; + padding_leaves.reserve(num_leaves); + auto zero = avm2::FF(0); + for (size_t i = 0; i < num_leaves; i++) { + padding_leaves.push_back(fr_to_wire(zero)); + } + client_.append_leaves(wsdb::WsdbAppendLeaves{ .treeId = tree_id_to_wire(MerkleTreeId::NOTE_HASH_TREE), + .leaves = std::move(padding_leaves), + .forkId = revision_.forkId }); + break; + } + default: + throw std::runtime_error("Padding not supported for tree " + std::to_string(static_cast(tree_id))); + } + invalidate_tree_roots_cache(); +} + +// --------------------------------------------------------------------------- +// Checkpoint methods +// --------------------------------------------------------------------------- + +void WsdbIpcMerkleDB::create_checkpoint() +{ + client_.create_checkpoint(wsdb::WsdbCreateCheckpoint{ .forkId = revision_.forkId }); + uint32_t current_id = checkpoint_stack_.top(); + checkpoint_stack_.push(current_id + 1); +} + +void WsdbIpcMerkleDB::commit_checkpoint() +{ + client_.commit_checkpoint(wsdb::WsdbCommitCheckpoint{ .forkId = revision_.forkId }); + invalidate_tree_roots_cache(); + checkpoint_stack_.pop(); +} + +void WsdbIpcMerkleDB::revert_checkpoint() +{ + client_.revert_checkpoint(wsdb::WsdbRevertCheckpoint{ .forkId = revision_.forkId }); + invalidate_tree_roots_cache(); + checkpoint_stack_.pop(); +} + +uint32_t WsdbIpcMerkleDB::get_checkpoint_id() const +{ + return checkpoint_stack_.top(); +} + +} // namespace bb::avm2::simulation diff --git a/barretenberg/cpp/src/barretenberg/wsdb_client/wsdb_ipc_merkle_db.hpp b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp similarity index 88% rename from barretenberg/cpp/src/barretenberg/wsdb_client/wsdb_ipc_merkle_db.hpp rename to barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp index becfbf4d5b75..ff2056b8b6ab 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb_client/wsdb_ipc_merkle_db.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp @@ -9,14 +9,12 @@ #include "barretenberg/vm2/simulation/interfaces/db.hpp" #include "barretenberg/world_state/types.hpp" -#include "barretenberg/wsdb/wsdb_commands.hpp" -#include "barretenberg/wsdb/wsdb_execute.hpp" -#include "barretenberg/wsdb/wsdb_ipc_client_generated.hpp" +#include "barretenberg/wsdb/wsdb_ipc_client.hpp" #include #include -namespace bb::wsdb_client { +namespace bb::avm2::simulation { class WsdbIpcMerkleDB final : public avm2::simulation::LowLevelMerkleDBInterface { public: @@ -56,9 +54,6 @@ class WsdbIpcMerkleDB final : public avm2::simulation::LowLevelMerkleDBInterface uint32_t get_checkpoint_id() const override; private: - template static std::vector serialize_to_msgpack(const T& value); - template static T deserialize_from_msgpack(const std::vector& bytes); - /** Invalidate the cached tree roots (call after any write operation). */ void invalidate_tree_roots_cache(); @@ -69,4 +64,4 @@ class WsdbIpcMerkleDB final : public avm2::simulation::LowLevelMerkleDBInterface mutable std::optional cached_tree_roots_; }; -} // namespace bb::wsdb_client +} // namespace bb::avm2::simulation diff --git a/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt index f7fead2146fe..ca88f8e4b9ab 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/wsdb/CMakeLists.txt @@ -1,44 +1,88 @@ -if(NOT(FUZZING) AND NOT(WASM)) - # IPC client library (used by AVM simulator to talk to aztec-wsdb) - # Generated via: cd barretenberg/ts && npx tsx src/aztec-wsdb/generate.ts - add_library( - wsdb_ipc_client - STATIC - wsdb_ipc_client_generated.cpp - ) - target_link_libraries( - wsdb_ipc_client - PUBLIC - barretenberg - ipc - ) +# IPC client library (used by AVM simulator to talk to aztec-wsdb). +# Sources are generated by ipc-codegen from wsdb_schema.jsonc. The custom +# command below wires generation into ninja so clients and servers only +# consume the generated output directory. +set(WSDB_SCHEMA ${CMAKE_CURRENT_SOURCE_DIR}/wsdb_schema.jsonc) +set(WSDB_GEN_PARENT ${CMAKE_CURRENT_SOURCE_DIR}) +set(WSDB_GEN_DIR ${WSDB_GEN_PARENT}/generated) +set(WSDB_GEN_OUTPUTS + ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp + ${WSDB_GEN_DIR}/wsdb_ipc_client.hpp + ${WSDB_GEN_DIR}/wsdb_ipc_server.hpp + ${WSDB_GEN_DIR}/wsdb_types.hpp + ${WSDB_GEN_DIR}/ipc_codegen/msgpack_adaptor.hpp + ${WSDB_GEN_DIR}/ipc_codegen/named_union.hpp + ${WSDB_GEN_DIR}/ipc_codegen/throw.hpp +) +set(IPC_CODEGEN_DIR ${CMAKE_SOURCE_DIR}/../../ipc-codegen) +file(GLOB_RECURSE IPC_CODEGEN_SRC + ${IPC_CODEGEN_DIR}/src/*.ts + ${IPC_CODEGEN_DIR}/templates/cpp/*.hpp +) +add_custom_command( + OUTPUT ${WSDB_GEN_OUTPUTS} + COMMAND node --experimental-strip-types --experimental-transform-types --no-warnings + ${IPC_CODEGEN_DIR}/src/generate.ts + --schema ${WSDB_SCHEMA} + --lang cpp + --out ${WSDB_GEN_DIR} + --client --server + --cpp-namespace bb::wsdb + --cpp-include-dir barretenberg/wsdb/generated + DEPENDS ${WSDB_SCHEMA} ${IPC_CODEGEN_SRC} + COMMENT "Generating WSDB IPC client + server from wsdb_schema.jsonc" + VERBATIM +) +add_custom_target(wsdb_ipc_generated DEPENDS ${WSDB_GEN_OUTPUTS}) +add_library( + wsdb_ipc_client + STATIC + ${WSDB_GEN_DIR}/wsdb_ipc_client.cpp +) +add_dependencies(wsdb_ipc_client wsdb_ipc_generated) +target_include_directories( + wsdb_ipc_client + PUBLIC + ${WSDB_GEN_DIR} +) +target_link_libraries( + wsdb_ipc_client + PUBLIC + barretenberg + ipc_runtime +) - # aztec-wsdb binary (standalone world state database server) - add_executable( +# aztec-wsdb binary (standalone world state database server) +add_executable( + aztec-wsdb + main.cpp + cli.cpp + wsdb_handlers.cpp + wsdb_ipc_server.cpp +) +add_dependencies(aztec-wsdb wsdb_ipc_generated) +target_include_directories( + aztec-wsdb + PRIVATE + ${WSDB_GEN_DIR} +) +target_link_libraries( + aztec-wsdb + PRIVATE + barretenberg + world_state + ipc_runtime + env +) +if(ENABLE_STACKTRACES) + target_link_libraries( aztec-wsdb - main.cpp - cli.cpp - wsdb_execute.cpp - wsdb_ipc_server.cpp + PUBLIC + Backward::Interface ) - target_link_libraries( + target_link_options( aztec-wsdb PRIVATE - barretenberg - world_state - ipc - env + -ldw -lelf ) - if(ENABLE_STACKTRACES) - target_link_libraries( - aztec-wsdb - PUBLIC - Backward::Interface - ) - target_link_options( - aztec-wsdb - PRIVATE - -ldw -lelf - ) - endif() endif() diff --git a/barretenberg/cpp/src/barretenberg/wsdb/cli.cpp b/barretenberg/cpp/src/barretenberg/wsdb/cli.cpp index 7f9fd1899886..2cf90fb38bcb 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/cli.cpp +++ b/barretenberg/cpp/src/barretenberg/wsdb/cli.cpp @@ -2,7 +2,8 @@ #include "barretenberg/common/log.hpp" #include "barretenberg/common/throw_or_abort.hpp" #include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/wsdb/wsdb_execute.hpp" +#include "barretenberg/world_state/world_state.hpp" +#include "barretenberg/wsdb/generated/wsdb_ipc_server.hpp" #include "barretenberg/wsdb/wsdb_ipc_server.hpp" #include "barretenberg/bb/deps/cli11.hpp" @@ -17,21 +18,6 @@ namespace bb::wsdb { using namespace bb::world_state; using namespace bb::crypto::merkle_tree; -namespace { - -struct WsdbApi { - WsdbCommand commands; - WsdbCommandResponse responses; - SERIALIZATION_FIELDS(commands, responses); -}; - -std::string get_wsdb_schema_as_json() -{ - return msgpack_schema_to_string(WsdbApi{}); -} - -} // namespace - int parse_and_run_wsdb(int argc, char* argv[]) { CLI::App app{ "aztec-wsdb: Standalone world state database server" }; @@ -42,10 +28,6 @@ int parse_and_run_wsdb(int argc, char* argv[]) // ----------------------------------------------------------------------- CLI::App* msgpack_command = app.add_subcommand("msgpack", "Msgpack API interface."); - // msgpack schema - CLI::App* msgpack_schema_command = - msgpack_command->add_subcommand("schema", "Output a msgpack schema encoded as JSON to stdout."); - // msgpack run CLI::App* msgpack_run_command = msgpack_command->add_subcommand("run", "Start the world state database IPC server."); @@ -107,11 +89,6 @@ int parse_and_run_wsdb(int argc, char* argv[]) } try { - if (msgpack_schema_command->parsed()) { - std::cout << get_wsdb_schema_as_json() << std::endl; - return 0; - } - if (msgpack_run_command->parsed()) { return execute_wsdb_server(input_path, data_dir, diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_commands.hpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_commands.hpp deleted file mode 100644 index 5ccf8cc760bc..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_commands.hpp +++ /dev/null @@ -1,536 +0,0 @@ -#pragma once -/** - * @file wsdb_commands.hpp - * @brief NamedUnion command structs for the aztec-wsdb world state database API. - * - * Each command follows the bbapi pattern: - * - static constexpr MSGPACK_SCHEMA_NAME for NamedUnion dispatch - * - Nested Response struct with its own MSGPACK_SCHEMA_NAME - * - Request fields with SERIALIZATION_FIELDS - * - execute(WsdbRequest&) && method (implemented in wsdb_execute.cpp) - */ - -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/fork.hpp" -#include "barretenberg/world_state/types.hpp" -#include -#include -#include -#include - -namespace bb::wsdb { - -using namespace bb::world_state; -using namespace bb::crypto::merkle_tree; - -// Forward declaration -struct WsdbRequest; - -// --------------------------------------------------------------------------- -// Tree info / state queries -// --------------------------------------------------------------------------- - -struct WsdbGetTreeInfo { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetTreeInfo"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetTreeInfoResponse"; - MerkleTreeId treeId; - fr root; - index_t size; - uint32_t depth; - SERIALIZATION_FIELDS(treeId, root, size, depth); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - WorldStateRevision revision; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, revision); - bool operator==(const WsdbGetTreeInfo&) const = default; -}; - -struct WsdbGetStateReference { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetStateReference"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetStateReferenceResponse"; - StateReference state; - SERIALIZATION_FIELDS(state); - bool operator==(const Response&) const = default; - }; - WorldStateRevision revision; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(revision); - bool operator==(const WsdbGetStateReference&) const = default; -}; - -struct WsdbGetInitialStateReference { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetInitialStateReference"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetInitialStateReferenceResponse"; - StateReference state; - SERIALIZATION_FIELDS(state); - bool operator==(const Response&) const = default; - }; - Response execute(WsdbRequest& request) &&; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const WsdbGetInitialStateReference&) const = default; -}; - -// --------------------------------------------------------------------------- -// Leaf queries -// --------------------------------------------------------------------------- - -struct WsdbGetLeafValue { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetLeafValue"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetLeafValueResponse"; - // Polymorphic: Fr, NullifierLeafValue, or PublicDataLeafValue serialized as bytes - std::optional> value; - SERIALIZATION_FIELDS(value); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - WorldStateRevision revision; - index_t leafIndex; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, revision, leafIndex); - bool operator==(const WsdbGetLeafValue&) const = default; -}; - -struct WsdbGetLeafPreimage { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetLeafPreimage"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetLeafPreimageResponse"; - // Serialized indexed leaf (NullifierLeafValue or PublicDataLeafValue) - std::optional> preimage; - SERIALIZATION_FIELDS(preimage); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - WorldStateRevision revision; - index_t leafIndex; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, revision, leafIndex); - bool operator==(const WsdbGetLeafPreimage&) const = default; -}; - -struct WsdbGetSiblingPath { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetSiblingPath"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetSiblingPathResponse"; - fr_sibling_path path; - SERIALIZATION_FIELDS(path); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - WorldStateRevision revision; - index_t leafIndex; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, revision, leafIndex); - bool operator==(const WsdbGetSiblingPath&) const = default; -}; - -struct WsdbGetBlockNumbersForLeafIndices { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetBlockNumbersForLeafIndices"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetBlockNumbersForLeafIndicesResponse"; - std::vector> blockNumbers; - SERIALIZATION_FIELDS(blockNumbers); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - WorldStateRevision revision; - std::vector leafIndices; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, revision, leafIndices); - bool operator==(const WsdbGetBlockNumbersForLeafIndices&) const = default; -}; - -// --------------------------------------------------------------------------- -// Leaf search operations -// --------------------------------------------------------------------------- - -struct WsdbFindLeafIndices { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbFindLeafIndices"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbFindLeafIndicesResponse"; - std::vector> indices; - SERIALIZATION_FIELDS(indices); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - WorldStateRevision revision; - // Polymorphic leaves: each leaf is serialized as bytes - std::vector> leaves; - index_t startIndex; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, revision, leaves, startIndex); - bool operator==(const WsdbFindLeafIndices&) const = default; -}; - -struct WsdbFindLowLeaf { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbFindLowLeaf"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbFindLowLeafResponse"; - bool alreadyPresent; - index_t index; - SERIALIZATION_FIELDS(alreadyPresent, index); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - WorldStateRevision revision; - fr key; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, revision, key); - bool operator==(const WsdbFindLowLeaf&) const = default; -}; - -struct WsdbFindSiblingPaths { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbFindSiblingPaths"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbFindSiblingPathsResponse"; - std::vector> paths; - SERIALIZATION_FIELDS(paths); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - WorldStateRevision revision; - // Polymorphic leaves - std::vector> leaves; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, revision, leaves); - bool operator==(const WsdbFindSiblingPaths&) const = default; -}; - -// --------------------------------------------------------------------------- -// Tree mutation operations -// --------------------------------------------------------------------------- - -struct WsdbAppendLeaves { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbAppendLeaves"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbAppendLeavesResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - // Polymorphic leaves - std::vector> leaves; - Fork::Id forkId{ CANONICAL_FORK_ID }; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, leaves, forkId); - bool operator==(const WsdbAppendLeaves&) const = default; -}; - -struct WsdbBatchInsert { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbBatchInsert"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbBatchInsertResponse"; - // Serialized BatchInsertionResult - std::vector result; - SERIALIZATION_FIELDS(result); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - std::vector> leaves; - uint32_t subtreeDepth; - Fork::Id forkId{ CANONICAL_FORK_ID }; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, leaves, subtreeDepth, forkId); - bool operator==(const WsdbBatchInsert&) const = default; -}; - -struct WsdbSequentialInsert { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbSequentialInsert"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbSequentialInsertResponse"; - // Serialized SequentialInsertionResult - std::vector result; - SERIALIZATION_FIELDS(result); - bool operator==(const Response&) const = default; - }; - MerkleTreeId treeId; - std::vector> leaves; - Fork::Id forkId{ CANONICAL_FORK_ID }; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(treeId, leaves, forkId); - bool operator==(const WsdbSequentialInsert&) const = default; -}; - -struct WsdbUpdateArchive { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbUpdateArchive"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbUpdateArchiveResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - StateReference blockStateRef; - bb::fr blockHeaderHash; - Fork::Id forkId{ CANONICAL_FORK_ID }; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(blockStateRef, blockHeaderHash, forkId); - bool operator==(const WsdbUpdateArchive&) const = default; -}; - -// --------------------------------------------------------------------------- -// Transaction operations -// --------------------------------------------------------------------------- - -struct WsdbCommit { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCommit"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCommitResponse"; - WorldStateStatusFull status; - SERIALIZATION_FIELDS(status); - bool operator==(const Response&) const = default; - }; - Response execute(WsdbRequest& request) &&; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const WsdbCommit&) const = default; -}; - -struct WsdbRollback { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbRollback"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbRollbackResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - Response execute(WsdbRequest& request) &&; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const WsdbRollback&) const = default; -}; - -// --------------------------------------------------------------------------- -// Block synchronization -// --------------------------------------------------------------------------- - -struct WsdbSyncBlock { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbSyncBlock"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbSyncBlockResponse"; - WorldStateStatusFull status; - SERIALIZATION_FIELDS(status); - bool operator==(const Response&) const = default; - }; - block_number_t blockNumber; - StateReference blockStateRef; - bb::fr blockHeaderHash; - std::vector paddedNoteHashes; - std::vector paddedL1ToL2Messages; - std::vector paddedNullifiers; - std::vector publicDataWrites; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(blockNumber, - blockStateRef, - blockHeaderHash, - paddedNoteHashes, - paddedL1ToL2Messages, - paddedNullifiers, - publicDataWrites); - bool operator==(const WsdbSyncBlock&) const = default; -}; - -// --------------------------------------------------------------------------- -// Fork management -// --------------------------------------------------------------------------- - -struct WsdbCreateFork { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCreateFork"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCreateForkResponse"; - uint64_t forkId; - SERIALIZATION_FIELDS(forkId); - bool operator==(const Response&) const = default; - }; - bool latest; - block_number_t blockNumber; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(latest, blockNumber); - bool operator==(const WsdbCreateFork&) const = default; -}; - -struct WsdbDeleteFork { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbDeleteFork"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbDeleteForkResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - uint64_t forkId; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(forkId); - bool operator==(const WsdbDeleteFork&) const = default; -}; - -// --------------------------------------------------------------------------- -// Block management -// --------------------------------------------------------------------------- - -struct WsdbFinalizeBlocks { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbFinalizeBlocks"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbFinalizeBlocksResponse"; - WorldStateStatusSummary status; - SERIALIZATION_FIELDS(status); - bool operator==(const Response&) const = default; - }; - block_number_t toBlockNumber; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(toBlockNumber); - bool operator==(const WsdbFinalizeBlocks&) const = default; -}; - -struct WsdbUnwindBlocks { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbUnwindBlocks"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbUnwindBlocksResponse"; - WorldStateStatusFull status; - SERIALIZATION_FIELDS(status); - bool operator==(const Response&) const = default; - }; - block_number_t toBlockNumber; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(toBlockNumber); - bool operator==(const WsdbUnwindBlocks&) const = default; -}; - -struct WsdbRemoveHistoricalBlocks { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbRemoveHistoricalBlocks"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbRemoveHistoricalBlocksResponse"; - WorldStateStatusFull status; - SERIALIZATION_FIELDS(status); - bool operator==(const Response&) const = default; - }; - block_number_t toBlockNumber; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(toBlockNumber); - bool operator==(const WsdbRemoveHistoricalBlocks&) const = default; -}; - -// --------------------------------------------------------------------------- -// Status -// --------------------------------------------------------------------------- - -struct WsdbGetStatus { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetStatus"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbGetStatusResponse"; - WorldStateStatusSummary status; - SERIALIZATION_FIELDS(status); - bool operator==(const Response&) const = default; - }; - Response execute(WsdbRequest& request) &&; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const WsdbGetStatus&) const = default; -}; - -// --------------------------------------------------------------------------- -// Checkpoint operations -// --------------------------------------------------------------------------- - -struct WsdbCreateCheckpoint { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCreateCheckpoint"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCreateCheckpointResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - uint64_t forkId; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(forkId); - bool operator==(const WsdbCreateCheckpoint&) const = default; -}; - -struct WsdbCommitCheckpoint { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCommitCheckpoint"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCommitCheckpointResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - uint64_t forkId; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(forkId); - bool operator==(const WsdbCommitCheckpoint&) const = default; -}; - -struct WsdbRevertCheckpoint { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbRevertCheckpoint"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbRevertCheckpointResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - uint64_t forkId; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(forkId); - bool operator==(const WsdbRevertCheckpoint&) const = default; -}; - -struct WsdbCommitAllCheckpoints { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCommitAllCheckpoints"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCommitAllCheckpointsResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - uint64_t forkId; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(forkId); - bool operator==(const WsdbCommitAllCheckpoints&) const = default; -}; - -struct WsdbRevertAllCheckpoints { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbRevertAllCheckpoints"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbRevertAllCheckpointsResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - uint64_t forkId; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(forkId); - bool operator==(const WsdbRevertAllCheckpoints&) const = default; -}; - -// --------------------------------------------------------------------------- -// Database operations -// --------------------------------------------------------------------------- - -struct WsdbCopyStores { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCopyStores"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbCopyStoresResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - std::string dstPath; - std::optional compact; - Response execute(WsdbRequest& request) &&; - SERIALIZATION_FIELDS(dstPath, compact); - bool operator==(const WsdbCopyStores&) const = default; -}; - -// --------------------------------------------------------------------------- -// Lifecycle -// --------------------------------------------------------------------------- - -struct WsdbShutdown { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbShutdown"; - struct Response { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbShutdownResponse"; - void msgpack(auto&& pack_fn) { pack_fn(); } - bool operator==(const Response&) const = default; - }; - void msgpack(auto&& pack_fn) { pack_fn(); } - Response execute(WsdbRequest& request) &&; - bool operator==(const WsdbShutdown&) const = default; -}; - -} // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_execute.cpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_execute.cpp deleted file mode 100644 index 5a6282b9de8f..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_execute.cpp +++ /dev/null @@ -1,414 +0,0 @@ -#include "barretenberg/wsdb/wsdb_execute.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include -#include - -namespace bb::wsdb { - -using namespace bb::world_state; -using namespace bb::crypto::merkle_tree; - -// --------------------------------------------------------------------------- -// Helper: serialize a value to msgpack bytes -// --------------------------------------------------------------------------- - -template static std::vector serialize_to_msgpack(const T& value) -{ - msgpack::sbuffer buf; - msgpack::pack(buf, value); - return std::vector(buf.data(), buf.data() + buf.size()); -} - -// --------------------------------------------------------------------------- -// Helper: deserialize leaves from raw bytes based on tree type -// --------------------------------------------------------------------------- - -template -static std::vector deserialize_leaves(const std::vector>& raw_leaves) -{ - std::vector leaves; - leaves.reserve(raw_leaves.size()); - for (const auto& raw : raw_leaves) { - auto unpacked = msgpack::unpack(reinterpret_cast(raw.data()), raw.size()); - LeafType leaf; - unpacked.get().convert(leaf); - leaves.push_back(std::move(leaf)); - } - return leaves; -} - -// --------------------------------------------------------------------------- -// Top-level dispatch -// --------------------------------------------------------------------------- - -WsdbCommandResponse wsdb(WsdbRequest& request, WsdbCommand&& command) -{ - return execute(request, std::move(command)); -} - -// --------------------------------------------------------------------------- -// Tree info / state queries -// --------------------------------------------------------------------------- - -WsdbGetTreeInfo::Response WsdbGetTreeInfo::execute(WsdbRequest& request) && -{ - auto info = request.world_state.get_tree_info(revision, treeId); - return Response{ .treeId = treeId, .root = info.meta.root, .size = info.meta.size, .depth = info.meta.depth }; -} - -WsdbGetStateReference::Response WsdbGetStateReference::execute(WsdbRequest& request) && -{ - auto state = request.world_state.get_state_reference(revision); - return Response{ .state = state }; -} - -WsdbGetInitialStateReference::Response WsdbGetInitialStateReference::execute(WsdbRequest& request) && -{ - auto state = request.world_state.get_initial_state_reference(); - return Response{ .state = state }; -} - -// --------------------------------------------------------------------------- -// Leaf queries -// --------------------------------------------------------------------------- - -WsdbGetLeafValue::Response WsdbGetLeafValue::execute(WsdbRequest& request) && -{ - switch (treeId) { - case MerkleTreeId::NOTE_HASH_TREE: - case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: - case MerkleTreeId::ARCHIVE: { - auto leaf = request.world_state.get_leaf(revision, treeId, leafIndex); - if (!leaf.has_value()) { - return Response{ .value = std::nullopt }; - } - return Response{ .value = serialize_to_msgpack(leaf.value()) }; - } - case MerkleTreeId::PUBLIC_DATA_TREE: { - auto leaf = request.world_state.get_leaf(revision, treeId, leafIndex); - if (!leaf.has_value()) { - return Response{ .value = std::nullopt }; - } - return Response{ .value = serialize_to_msgpack(leaf.value()) }; - } - case MerkleTreeId::NULLIFIER_TREE: { - auto leaf = request.world_state.get_leaf(revision, treeId, leafIndex); - if (!leaf.has_value()) { - return Response{ .value = std::nullopt }; - } - return Response{ .value = serialize_to_msgpack(leaf.value()) }; - } - default: - throw std::runtime_error("Unsupported tree type for get_leaf_value"); - } -} - -WsdbGetLeafPreimage::Response WsdbGetLeafPreimage::execute(WsdbRequest& request) && -{ - switch (treeId) { - case MerkleTreeId::NULLIFIER_TREE: { - auto leaf = request.world_state.get_indexed_leaf(revision, treeId, leafIndex); - if (!leaf.has_value()) { - return Response{ .preimage = std::nullopt }; - } - return Response{ .preimage = serialize_to_msgpack(leaf.value()) }; - } - case MerkleTreeId::PUBLIC_DATA_TREE: { - auto leaf = request.world_state.get_indexed_leaf(revision, treeId, leafIndex); - if (!leaf.has_value()) { - return Response{ .preimage = std::nullopt }; - } - return Response{ .preimage = serialize_to_msgpack(leaf.value()) }; - } - default: - throw std::runtime_error("Unsupported tree type for get_leaf_preimage"); - } -} - -WsdbGetSiblingPath::Response WsdbGetSiblingPath::execute(WsdbRequest& request) && -{ - fr_sibling_path path = request.world_state.get_sibling_path(revision, treeId, leafIndex); - return Response{ .path = path }; -} - -WsdbGetBlockNumbersForLeafIndices::Response WsdbGetBlockNumbersForLeafIndices::execute(WsdbRequest& request) && -{ - Response response; - request.world_state.get_block_numbers_for_leaf_indices(revision, treeId, leafIndices, response.blockNumbers); - return response; -} - -// --------------------------------------------------------------------------- -// Leaf search operations -// --------------------------------------------------------------------------- - -WsdbFindLeafIndices::Response WsdbFindLeafIndices::execute(WsdbRequest& request) && -{ - Response response; - switch (treeId) { - case MerkleTreeId::NOTE_HASH_TREE: - case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: - case MerkleTreeId::ARCHIVE: { - auto typed_leaves = deserialize_leaves(leaves); - request.world_state.find_leaf_indices(revision, treeId, typed_leaves, response.indices, startIndex); - break; - } - case MerkleTreeId::PUBLIC_DATA_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - request.world_state.find_leaf_indices( - revision, treeId, typed_leaves, response.indices, startIndex); - break; - } - case MerkleTreeId::NULLIFIER_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - request.world_state.find_leaf_indices( - revision, treeId, typed_leaves, response.indices, startIndex); - break; - } - default: - throw std::runtime_error("Unsupported tree type for find_leaf_indices"); - } - return response; -} - -WsdbFindLowLeaf::Response WsdbFindLowLeaf::execute(WsdbRequest& request) && -{ - auto low_leaf_info = request.world_state.find_low_leaf_index(revision, treeId, key); - return Response{ .alreadyPresent = low_leaf_info.is_already_present, .index = low_leaf_info.index }; -} - -WsdbFindSiblingPaths::Response WsdbFindSiblingPaths::execute(WsdbRequest& request) && -{ - Response response; - switch (treeId) { - case MerkleTreeId::NOTE_HASH_TREE: - case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: - case MerkleTreeId::ARCHIVE: { - auto typed_leaves = deserialize_leaves(leaves); - request.world_state.find_sibling_paths(revision, treeId, typed_leaves, response.paths); - break; - } - case MerkleTreeId::PUBLIC_DATA_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - request.world_state.find_sibling_paths(revision, treeId, typed_leaves, response.paths); - break; - } - case MerkleTreeId::NULLIFIER_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - request.world_state.find_sibling_paths(revision, treeId, typed_leaves, response.paths); - break; - } - default: - throw std::runtime_error("Unsupported tree type for find_sibling_paths"); - } - return response; -} - -// --------------------------------------------------------------------------- -// Tree mutation operations -// --------------------------------------------------------------------------- - -WsdbAppendLeaves::Response WsdbAppendLeaves::execute(WsdbRequest& request) && -{ - switch (treeId) { - case MerkleTreeId::NOTE_HASH_TREE: - case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: - case MerkleTreeId::ARCHIVE: { - auto typed_leaves = deserialize_leaves(leaves); - request.world_state.append_leaves(treeId, typed_leaves, forkId); - break; - } - case MerkleTreeId::PUBLIC_DATA_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - request.world_state.append_leaves(treeId, typed_leaves, forkId); - break; - } - case MerkleTreeId::NULLIFIER_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - request.world_state.append_leaves(treeId, typed_leaves, forkId); - break; - } - default: - throw std::runtime_error("Unsupported tree type for append_leaves"); - } - return Response{}; -} - -WsdbBatchInsert::Response WsdbBatchInsert::execute(WsdbRequest& request) && -{ - switch (treeId) { - case MerkleTreeId::PUBLIC_DATA_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - auto result = request.world_state.batch_insert_indexed_leaves( - treeId, typed_leaves, subtreeDepth, forkId); - return Response{ .result = serialize_to_msgpack(result) }; - } - case MerkleTreeId::NULLIFIER_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - auto result = request.world_state.batch_insert_indexed_leaves( - treeId, typed_leaves, subtreeDepth, forkId); - return Response{ .result = serialize_to_msgpack(result) }; - } - default: - throw std::runtime_error("Unsupported tree type for batch_insert"); - } -} - -WsdbSequentialInsert::Response WsdbSequentialInsert::execute(WsdbRequest& request) && -{ - switch (treeId) { - case MerkleTreeId::PUBLIC_DATA_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - auto result = request.world_state.insert_indexed_leaves(treeId, typed_leaves, forkId); - return Response{ .result = serialize_to_msgpack(result) }; - } - case MerkleTreeId::NULLIFIER_TREE: { - auto typed_leaves = deserialize_leaves(leaves); - auto result = request.world_state.insert_indexed_leaves(treeId, typed_leaves, forkId); - return Response{ .result = serialize_to_msgpack(result) }; - } - default: - throw std::runtime_error("Unsupported tree type for sequential_insert"); - } -} - -WsdbUpdateArchive::Response WsdbUpdateArchive::execute(WsdbRequest& request) && -{ - request.world_state.update_archive(blockStateRef, blockHeaderHash, forkId); - return Response{}; -} - -// --------------------------------------------------------------------------- -// Transaction operations -// --------------------------------------------------------------------------- - -WsdbCommit::Response WsdbCommit::execute(WsdbRequest& request) && -{ - WorldStateStatusFull status; - request.world_state.commit(status); - return Response{ .status = status }; -} - -WsdbRollback::Response WsdbRollback::execute(WsdbRequest& request) && -{ - request.world_state.rollback(); - return Response{}; -} - -// --------------------------------------------------------------------------- -// Block synchronization -// --------------------------------------------------------------------------- - -WsdbSyncBlock::Response WsdbSyncBlock::execute(WsdbRequest& request) && -{ - WorldStateStatusFull status = request.world_state.sync_block( - blockStateRef, blockHeaderHash, paddedNoteHashes, paddedL1ToL2Messages, paddedNullifiers, publicDataWrites); - return Response{ .status = status }; -} - -// --------------------------------------------------------------------------- -// Fork management -// --------------------------------------------------------------------------- - -WsdbCreateFork::Response WsdbCreateFork::execute(WsdbRequest& request) && -{ - std::optional block = latest ? std::nullopt : std::optional(blockNumber); - uint64_t id = request.world_state.create_fork(block); - return Response{ .forkId = id }; -} - -WsdbDeleteFork::Response WsdbDeleteFork::execute(WsdbRequest& request) && -{ - request.world_state.delete_fork(forkId); - return Response{}; -} - -// --------------------------------------------------------------------------- -// Block management -// --------------------------------------------------------------------------- - -WsdbFinalizeBlocks::Response WsdbFinalizeBlocks::execute(WsdbRequest& request) && -{ - WorldStateStatusSummary status = request.world_state.set_finalized_blocks(toBlockNumber); - return Response{ .status = status }; -} - -WsdbUnwindBlocks::Response WsdbUnwindBlocks::execute(WsdbRequest& request) && -{ - WorldStateStatusFull status = request.world_state.unwind_blocks(toBlockNumber); - return Response{ .status = status }; -} - -WsdbRemoveHistoricalBlocks::Response WsdbRemoveHistoricalBlocks::execute(WsdbRequest& request) && -{ - WorldStateStatusFull status = request.world_state.remove_historical_blocks(toBlockNumber); - return Response{ .status = status }; -} - -// --------------------------------------------------------------------------- -// Status -// --------------------------------------------------------------------------- - -WsdbGetStatus::Response WsdbGetStatus::execute(WsdbRequest& request) && -{ - WorldStateStatusSummary status; - request.world_state.get_status_summary(status); - return Response{ .status = status }; -} - -// --------------------------------------------------------------------------- -// Checkpoint operations -// --------------------------------------------------------------------------- - -WsdbCreateCheckpoint::Response WsdbCreateCheckpoint::execute(WsdbRequest& request) && -{ - request.world_state.checkpoint(forkId); - return Response{}; -} - -WsdbCommitCheckpoint::Response WsdbCommitCheckpoint::execute(WsdbRequest& request) && -{ - request.world_state.commit_checkpoint(forkId); - return Response{}; -} - -WsdbRevertCheckpoint::Response WsdbRevertCheckpoint::execute(WsdbRequest& request) && -{ - request.world_state.revert_checkpoint(forkId); - return Response{}; -} - -WsdbCommitAllCheckpoints::Response WsdbCommitAllCheckpoints::execute(WsdbRequest& request) && -{ - request.world_state.commit_all_checkpoints_to(forkId, 0); - return Response{}; -} - -WsdbRevertAllCheckpoints::Response WsdbRevertAllCheckpoints::execute(WsdbRequest& request) && -{ - request.world_state.revert_all_checkpoints_to(forkId, 0); - return Response{}; -} - -// --------------------------------------------------------------------------- -// Database operations -// --------------------------------------------------------------------------- - -WsdbCopyStores::Response WsdbCopyStores::execute(WsdbRequest& request) && -{ - request.world_state.copy_stores(dstPath, compact.value_or(false)); - return Response{}; -} - -// --------------------------------------------------------------------------- -// Lifecycle -// --------------------------------------------------------------------------- - -WsdbShutdown::Response WsdbShutdown::execute(WsdbRequest& /* request */) && -{ - return Response{}; -} - -} // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_execute.hpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_execute.hpp deleted file mode 100644 index 20de7d738c4c..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_execute.hpp +++ /dev/null @@ -1,115 +0,0 @@ -#pragma once -/** - * @file wsdb_execute.hpp - * @brief WsdbCommand NamedUnion, WsdbRequest context, and dispatch function. - */ - -#include "barretenberg/common/named_union.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/wsdb_commands.hpp" - -namespace bb::wsdb { - -/** - * @brief Context passed to each command's execute() method, providing access to the WorldState. - */ -struct WsdbRequest { - world_state::WorldState& world_state; -}; - -/** - * @brief Error response returned when a command fails. - */ -struct WsdbErrorResponse { - static constexpr const char MSGPACK_SCHEMA_NAME[] = "WsdbErrorResponse"; - std::string message; - SERIALIZATION_FIELDS(message); - bool operator==(const WsdbErrorResponse&) const = default; -}; - -/** - * @brief Union of all wsdb commands (request types). - */ -using WsdbCommand = NamedUnion; - -/** - * @brief Union of all wsdb response types. - */ -using WsdbCommandResponse = NamedUnion; - -/** - * @brief Execute a wsdb command using the visitor pattern. - */ -inline WsdbCommandResponse execute(WsdbRequest& request, WsdbCommand&& command) -{ - return std::move(command).visit([&request](auto&& cmd) -> WsdbCommandResponse { - using CmdType = std::decay_t; - return std::forward(cmd).execute(request); - }); -} - -/** - * @brief Top-level wsdb API entry point. Takes a WsdbRequest and dispatches the command. - */ -WsdbCommandResponse wsdb(WsdbRequest& request, WsdbCommand&& command); - -} // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp new file mode 100644 index 000000000000..4e7ee7d47af4 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.cpp @@ -0,0 +1,536 @@ +/** + * @file wsdb_handlers.cpp + * @brief Per-command handlers consumed by the codegen-emitted server dispatch. + * + * Each handler matches the signature declared by generated/wsdb_ipc_server.hpp + * but as a non-template overload for `WsdbRequest` so the codegen's + * `make_wsdb_handler` instantiation resolves to these + * definitions via overload resolution (preferred over the unspecialized + * template). + * + * Wire <-> domain conversion happens at the entry/exit of each handler via + * the helpers in wsdb_wire_convert.hpp. + */ +#include "barretenberg/wsdb/wsdb_handlers.hpp" +#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" +#include "barretenberg/world_state/world_state.hpp" +#include "barretenberg/wsdb/generated/wsdb_ipc_server.hpp" +#include "barretenberg/wsdb/wsdb_wire_convert.hpp" + +#include +#include + +namespace bb::wsdb { + +using namespace bb::world_state; +using namespace bb::crypto::merkle_tree; + +// --------------------------------------------------------------------------- +// Tree info / state queries +// --------------------------------------------------------------------------- + +wire::WsdbGetTreeInfoResponse handle_get_tree_info(WsdbRequest& ctx, wire::WsdbGetTreeInfo&& cmd) +{ + auto info = ctx.world_state.get_tree_info(revision_from_wire(cmd.revision), tree_id_from_wire(cmd.treeId)); + return wire::WsdbGetTreeInfoResponse{ + .treeId = cmd.treeId, + .root = fr_to_wire(info.meta.root), + .size = info.meta.size, + .depth = info.meta.depth, + }; +} + +wire::WsdbGetStateReferenceResponse handle_get_state_reference(WsdbRequest& ctx, wire::WsdbGetStateReference&& cmd) +{ + auto state = ctx.world_state.get_state_reference(revision_from_wire(cmd.revision)); + return wire::WsdbGetStateReferenceResponse{ .state = state_reference_to_wire(state) }; +} + +wire::WsdbGetInitialStateReferenceResponse handle_get_initial_state_reference(WsdbRequest& ctx, + wire::WsdbGetInitialStateReference&&) +{ + auto state = ctx.world_state.get_initial_state_reference(); + return wire::WsdbGetInitialStateReferenceResponse{ .state = state_reference_to_wire(state) }; +} + +// --------------------------------------------------------------------------- +// Leaf queries +// --------------------------------------------------------------------------- + +wire::WsdbGetLeafValueResponse handle_get_leaf_value(WsdbRequest& ctx, wire::WsdbGetLeafValue&& cmd) +{ + auto revision = revision_from_wire(cmd.revision); + auto tree_id = tree_id_from_wire(cmd.treeId); + auto leaf_index = static_cast(cmd.leafIndex); + + switch (tree_id) { + case world_state::MerkleTreeId::NOTE_HASH_TREE: + case world_state::MerkleTreeId::L1_TO_L2_MESSAGE_TREE: + case world_state::MerkleTreeId::ARCHIVE: { + auto leaf = ctx.world_state.get_leaf(revision, tree_id, leaf_index); + return wire::WsdbGetLeafValueResponse{ .value = leaf.has_value() ? std::optional(fr_to_wire(*leaf)) + : std::nullopt }; + } + default: + throw std::runtime_error("Unsupported tree type for get_leaf_value"); + } +} + +wire::WsdbGetPublicDataLeafValueResponse handle_get_public_data_leaf_value(WsdbRequest& ctx, + wire::WsdbGetPublicDataLeafValue&& cmd) +{ + auto leaf = ctx.world_state.get_leaf(revision_from_wire(cmd.revision), + world_state::MerkleTreeId::PUBLIC_DATA_TREE, + static_cast(cmd.leafIndex)); + return wire::WsdbGetPublicDataLeafValueResponse{ + .value = + leaf.has_value() ? std::optional(public_data_leaf_to_wire(*leaf)) : std::nullopt + }; +} + +wire::WsdbGetNullifierLeafValueResponse handle_get_nullifier_leaf_value(WsdbRequest& ctx, + wire::WsdbGetNullifierLeafValue&& cmd) +{ + auto leaf = ctx.world_state.get_leaf(revision_from_wire(cmd.revision), + world_state::MerkleTreeId::NULLIFIER_TREE, + static_cast(cmd.leafIndex)); + return wire::WsdbGetNullifierLeafValueResponse{ .value = leaf.has_value() ? std::optional( + nullifier_leaf_to_wire(*leaf)) + : std::nullopt }; +} + +wire::WsdbGetPublicDataLeafPreimageResponse handle_get_public_data_leaf_preimage( + WsdbRequest& ctx, wire::WsdbGetPublicDataLeafPreimage&& cmd) +{ + auto leaf = ctx.world_state.get_indexed_leaf(revision_from_wire(cmd.revision), + world_state::MerkleTreeId::PUBLIC_DATA_TREE, + static_cast(cmd.leafIndex)); + return wire::WsdbGetPublicDataLeafPreimageResponse{ + .preimage = leaf.has_value() + ? std::optional(indexed_public_data_leaf_to_wire(*leaf)) + : std::nullopt + }; +} + +wire::WsdbGetNullifierLeafPreimageResponse handle_get_nullifier_leaf_preimage(WsdbRequest& ctx, + wire::WsdbGetNullifierLeafPreimage&& cmd) +{ + auto leaf = ctx.world_state.get_indexed_leaf(revision_from_wire(cmd.revision), + world_state::MerkleTreeId::NULLIFIER_TREE, + static_cast(cmd.leafIndex)); + return wire::WsdbGetNullifierLeafPreimageResponse{ .preimage = leaf.has_value() + ? std::optional( + indexed_nullifier_leaf_to_wire(*leaf)) + : std::nullopt }; +} + +wire::WsdbGetSiblingPathResponse handle_get_sibling_path(WsdbRequest& ctx, wire::WsdbGetSiblingPath&& cmd) +{ + fr_sibling_path path = ctx.world_state.get_sibling_path( + revision_from_wire(cmd.revision), tree_id_from_wire(cmd.treeId), static_cast(cmd.leafIndex)); + return wire::WsdbGetSiblingPathResponse{ .path = fr_vec_to_wire(path) }; +} + +wire::WsdbGetBlockNumbersForLeafIndicesResponse handle_get_block_numbers_for_leaf_indices( + WsdbRequest& ctx, wire::WsdbGetBlockNumbersForLeafIndices&& cmd) +{ + std::vector leaf_indices; + leaf_indices.reserve(cmd.leafIndices.size()); + for (auto i : cmd.leafIndices) { + leaf_indices.push_back(static_cast(i)); + } + std::vector> block_numbers; + ctx.world_state.get_block_numbers_for_leaf_indices( + revision_from_wire(cmd.revision), tree_id_from_wire(cmd.treeId), leaf_indices, block_numbers); + std::vector> wire_block_numbers; + wire_block_numbers.reserve(block_numbers.size()); + for (const auto& bn : block_numbers) { + wire_block_numbers.push_back(bn); + } + return wire::WsdbGetBlockNumbersForLeafIndicesResponse{ .blockNumbers = std::move(wire_block_numbers) }; +} + +// --------------------------------------------------------------------------- +// Leaf search operations +// --------------------------------------------------------------------------- + +wire::WsdbFindLeafIndicesResponse handle_find_leaf_indices(WsdbRequest& ctx, wire::WsdbFindLeafIndices&& cmd) +{ + auto revision = revision_from_wire(cmd.revision); + auto tree_id = tree_id_from_wire(cmd.treeId); + auto start_index = static_cast(cmd.startIndex); + + std::vector> indices; + switch (tree_id) { + case world_state::MerkleTreeId::NOTE_HASH_TREE: + case world_state::MerkleTreeId::L1_TO_L2_MESSAGE_TREE: + case world_state::MerkleTreeId::ARCHIVE: { + auto typed_leaves = fr_vec_from_wire(cmd.leaves); + ctx.world_state.find_leaf_indices(revision, tree_id, typed_leaves, indices, start_index); + break; + } + default: + throw std::runtime_error("Unsupported tree type for find_leaf_indices"); + } + std::vector> wire_indices; + wire_indices.reserve(indices.size()); + for (const auto& i : indices) { + wire_indices.push_back(i.has_value() ? std::optional(static_cast(*i)) : std::nullopt); + } + return wire::WsdbFindLeafIndicesResponse{ .indices = std::move(wire_indices) }; +} + +wire::WsdbFindPublicDataLeafIndicesResponse handle_find_public_data_leaf_indices( + WsdbRequest& ctx, wire::WsdbFindPublicDataLeafIndices&& cmd) +{ + std::vector> indices; + ctx.world_state.find_leaf_indices(revision_from_wire(cmd.revision), + world_state::MerkleTreeId::PUBLIC_DATA_TREE, + public_data_leaf_vec_from_wire(cmd.leaves), + indices, + static_cast(cmd.startIndex)); + std::vector> wire_indices; + wire_indices.reserve(indices.size()); + for (const auto& i : indices) { + wire_indices.push_back(i.has_value() ? std::optional(static_cast(*i)) : std::nullopt); + } + return wire::WsdbFindPublicDataLeafIndicesResponse{ .indices = std::move(wire_indices) }; +} + +wire::WsdbFindNullifierLeafIndicesResponse handle_find_nullifier_leaf_indices(WsdbRequest& ctx, + wire::WsdbFindNullifierLeafIndices&& cmd) +{ + std::vector> indices; + ctx.world_state.find_leaf_indices(revision_from_wire(cmd.revision), + world_state::MerkleTreeId::NULLIFIER_TREE, + nullifier_leaf_vec_from_wire(cmd.leaves), + indices, + static_cast(cmd.startIndex)); + std::vector> wire_indices; + wire_indices.reserve(indices.size()); + for (const auto& i : indices) { + wire_indices.push_back(i.has_value() ? std::optional(static_cast(*i)) : std::nullopt); + } + return wire::WsdbFindNullifierLeafIndicesResponse{ .indices = std::move(wire_indices) }; +} + +wire::WsdbFindLowLeafResponse handle_find_low_leaf(WsdbRequest& ctx, wire::WsdbFindLowLeaf&& cmd) +{ + auto low_leaf_info = ctx.world_state.find_low_leaf_index( + revision_from_wire(cmd.revision), tree_id_from_wire(cmd.treeId), fr_from_wire(cmd.key)); + return wire::WsdbFindLowLeafResponse{ + .alreadyPresent = low_leaf_info.is_already_present, + .index = static_cast(low_leaf_info.index), + }; +} + +wire::WsdbFindSiblingPathsResponse handle_find_sibling_paths(WsdbRequest& ctx, wire::WsdbFindSiblingPaths&& cmd) +{ + auto revision = revision_from_wire(cmd.revision); + auto tree_id = tree_id_from_wire(cmd.treeId); + std::vector> paths; + switch (tree_id) { + case world_state::MerkleTreeId::NOTE_HASH_TREE: + case world_state::MerkleTreeId::L1_TO_L2_MESSAGE_TREE: + case world_state::MerkleTreeId::ARCHIVE: { + auto typed_leaves = fr_vec_from_wire(cmd.leaves); + ctx.world_state.find_sibling_paths(revision, tree_id, typed_leaves, paths); + break; + } + default: + throw std::runtime_error("Unsupported tree type for find_sibling_paths"); + } + std::vector> wire_paths; + wire_paths.reserve(paths.size()); + for (const auto& p : paths) { + if (!p.has_value()) { + wire_paths.push_back(std::nullopt); + continue; + } + wire_paths.push_back(wire::SiblingPathAndIndex{ + .index = static_cast(p->index), + .path = fr_vec_to_wire(p->path), + }); + } + return wire::WsdbFindSiblingPathsResponse{ .paths = std::move(wire_paths) }; +} + +wire::WsdbFindPublicDataSiblingPathsResponse handle_find_public_data_sibling_paths( + WsdbRequest& ctx, wire::WsdbFindPublicDataSiblingPaths&& cmd) +{ + std::vector> paths; + ctx.world_state.find_sibling_paths(revision_from_wire(cmd.revision), + world_state::MerkleTreeId::PUBLIC_DATA_TREE, + public_data_leaf_vec_from_wire(cmd.leaves), + paths); + std::vector> wire_paths; + wire_paths.reserve(paths.size()); + for (const auto& p : paths) { + wire_paths.push_back(p.has_value() + ? std::optional(wire::SiblingPathAndIndex{ + .index = static_cast(p->index), .path = fr_vec_to_wire(p->path) }) + : std::nullopt); + } + return wire::WsdbFindPublicDataSiblingPathsResponse{ .paths = std::move(wire_paths) }; +} + +wire::WsdbFindNullifierSiblingPathsResponse handle_find_nullifier_sibling_paths( + WsdbRequest& ctx, wire::WsdbFindNullifierSiblingPaths&& cmd) +{ + std::vector> paths; + ctx.world_state.find_sibling_paths(revision_from_wire(cmd.revision), + world_state::MerkleTreeId::NULLIFIER_TREE, + nullifier_leaf_vec_from_wire(cmd.leaves), + paths); + std::vector> wire_paths; + wire_paths.reserve(paths.size()); + for (const auto& p : paths) { + wire_paths.push_back(p.has_value() + ? std::optional(wire::SiblingPathAndIndex{ + .index = static_cast(p->index), .path = fr_vec_to_wire(p->path) }) + : std::nullopt); + } + return wire::WsdbFindNullifierSiblingPathsResponse{ .paths = std::move(wire_paths) }; +} + +// --------------------------------------------------------------------------- +// Tree mutation operations +// --------------------------------------------------------------------------- + +wire::WsdbAppendLeavesResponse handle_append_leaves(WsdbRequest& ctx, wire::WsdbAppendLeaves&& cmd) +{ + auto tree_id = tree_id_from_wire(cmd.treeId); + switch (tree_id) { + case world_state::MerkleTreeId::NOTE_HASH_TREE: + case world_state::MerkleTreeId::L1_TO_L2_MESSAGE_TREE: + case world_state::MerkleTreeId::ARCHIVE: { + ctx.world_state.append_leaves(tree_id, fr_vec_from_wire(cmd.leaves), cmd.forkId); + break; + } + default: + throw std::runtime_error("Unsupported tree type for append_leaves"); + } + return wire::WsdbAppendLeavesResponse{}; +} + +wire::WsdbAppendPublicDataLeavesResponse handle_append_public_data_leaves(WsdbRequest& ctx, + wire::WsdbAppendPublicDataLeaves&& cmd) +{ + ctx.world_state.append_leaves( + world_state::MerkleTreeId::PUBLIC_DATA_TREE, public_data_leaf_vec_from_wire(cmd.leaves), cmd.forkId); + return wire::WsdbAppendPublicDataLeavesResponse{}; +} + +wire::WsdbAppendNullifierLeavesResponse handle_append_nullifier_leaves(WsdbRequest& ctx, + wire::WsdbAppendNullifierLeaves&& cmd) +{ + ctx.world_state.append_leaves( + world_state::MerkleTreeId::NULLIFIER_TREE, nullifier_leaf_vec_from_wire(cmd.leaves), cmd.forkId); + return wire::WsdbAppendNullifierLeavesResponse{}; +} + +wire::WsdbBatchInsertPublicDataResponse handle_batch_insert_public_data(WsdbRequest& ctx, + wire::WsdbBatchInsertPublicData&& cmd) +{ + auto result = + ctx.world_state.batch_insert_indexed_leaves(world_state::MerkleTreeId::PUBLIC_DATA_TREE, + public_data_leaf_vec_from_wire(cmd.leaves), + cmd.subtreeDepth, + cmd.forkId); + return wire::WsdbBatchInsertPublicDataResponse{ .result = batch_public_data_to_wire(result) }; +} + +wire::WsdbBatchInsertNullifierResponse handle_batch_insert_nullifier(WsdbRequest& ctx, + wire::WsdbBatchInsertNullifier&& cmd) +{ + auto result = + ctx.world_state.batch_insert_indexed_leaves(world_state::MerkleTreeId::NULLIFIER_TREE, + nullifier_leaf_vec_from_wire(cmd.leaves), + cmd.subtreeDepth, + cmd.forkId); + return wire::WsdbBatchInsertNullifierResponse{ .result = batch_nullifier_to_wire(result) }; +} + +wire::WsdbSequentialInsertPublicDataResponse handle_sequential_insert_public_data( + WsdbRequest& ctx, wire::WsdbSequentialInsertPublicData&& cmd) +{ + auto result = ctx.world_state.insert_indexed_leaves( + world_state::MerkleTreeId::PUBLIC_DATA_TREE, public_data_leaf_vec_from_wire(cmd.leaves), cmd.forkId); + return wire::WsdbSequentialInsertPublicDataResponse{ .result = sequential_public_data_to_wire(result) }; +} + +wire::WsdbSequentialInsertNullifierResponse handle_sequential_insert_nullifier( + WsdbRequest& ctx, wire::WsdbSequentialInsertNullifier&& cmd) +{ + auto result = ctx.world_state.insert_indexed_leaves( + world_state::MerkleTreeId::NULLIFIER_TREE, nullifier_leaf_vec_from_wire(cmd.leaves), cmd.forkId); + return wire::WsdbSequentialInsertNullifierResponse{ .result = sequential_nullifier_to_wire(result) }; +} + +wire::WsdbUpdateArchiveResponse handle_update_archive(WsdbRequest& ctx, wire::WsdbUpdateArchive&& cmd) +{ + ctx.world_state.update_archive( + state_reference_from_wire(cmd.blockStateRef), block_header_hash_from_wire(cmd.blockHeaderHash), cmd.forkId); + return wire::WsdbUpdateArchiveResponse{}; +} + +// --------------------------------------------------------------------------- +// Transaction operations +// --------------------------------------------------------------------------- + +wire::WsdbCommitResponse handle_commit(WsdbRequest& ctx, wire::WsdbCommit&&) +{ + WorldStateStatusFull status; + ctx.world_state.commit(status); + return wire::WsdbCommitResponse{ + .status = world_state_status_full_to_wire(status), + }; +} + +wire::WsdbRollbackResponse handle_rollback(WsdbRequest& ctx, wire::WsdbRollback&&) +{ + ctx.world_state.rollback(); + return wire::WsdbRollbackResponse{}; +} + +// --------------------------------------------------------------------------- +// Block synchronization +// --------------------------------------------------------------------------- + +wire::WsdbSyncBlockResponse handle_sync_block(WsdbRequest& ctx, wire::WsdbSyncBlock&& cmd) +{ + auto block_state_ref = state_reference_from_wire(cmd.blockStateRef); + auto block_header_hash = block_header_hash_from_wire(cmd.blockHeaderHash); + auto padded_note_hashes = fr_vec_from_wire(cmd.paddedNoteHashes); + auto padded_l1_to_l2_messages = fr_vec_from_wire(cmd.paddedL1ToL2Messages); + + std::vector padded_nullifiers; + padded_nullifiers.reserve(cmd.paddedNullifiers.size()); + for (const auto& w : cmd.paddedNullifiers) { + padded_nullifiers.emplace_back(nullifier_from_wire(w.nullifier)); + } + + std::vector public_data_writes; + public_data_writes.reserve(cmd.publicDataWrites.size()); + for (const auto& w : cmd.publicDataWrites) { + public_data_writes.emplace_back(public_data_slot_from_wire(w.slot), public_data_value_from_wire(w.value)); + } + + WorldStateStatusFull status = ctx.world_state.sync_block(block_state_ref, + block_header_hash, + padded_note_hashes, + padded_l1_to_l2_messages, + padded_nullifiers, + public_data_writes); + return wire::WsdbSyncBlockResponse{ + .status = world_state_status_full_to_wire(status), + }; +} + +// --------------------------------------------------------------------------- +// Fork management +// --------------------------------------------------------------------------- + +wire::WsdbCreateForkResponse handle_create_fork(WsdbRequest& ctx, wire::WsdbCreateFork&& cmd) +{ + std::optional block = cmd.latest ? std::nullopt : std::optional(cmd.blockNumber); + uint64_t id = ctx.world_state.create_fork(block); + return wire::WsdbCreateForkResponse{ .forkId = id }; +} + +wire::WsdbDeleteForkResponse handle_delete_fork(WsdbRequest& ctx, wire::WsdbDeleteFork&& cmd) +{ + ctx.world_state.delete_fork(cmd.forkId); + return wire::WsdbDeleteForkResponse{}; +} + +// --------------------------------------------------------------------------- +// Block management +// --------------------------------------------------------------------------- + +wire::WsdbFinalizeBlocksResponse handle_finalize_blocks(WsdbRequest& ctx, wire::WsdbFinalizeBlocks&& cmd) +{ + WorldStateStatusSummary status = ctx.world_state.set_finalized_blocks(cmd.toBlockNumber); + return wire::WsdbFinalizeBlocksResponse{ + .status = world_state_status_summary_to_wire(status), + }; +} + +wire::WsdbUnwindBlocksResponse handle_unwind_blocks(WsdbRequest& ctx, wire::WsdbUnwindBlocks&& cmd) +{ + WorldStateStatusFull status = ctx.world_state.unwind_blocks(cmd.toBlockNumber); + return wire::WsdbUnwindBlocksResponse{ + .status = world_state_status_full_to_wire(status), + }; +} + +wire::WsdbRemoveHistoricalBlocksResponse handle_remove_historical_blocks(WsdbRequest& ctx, + wire::WsdbRemoveHistoricalBlocks&& cmd) +{ + WorldStateStatusFull status = ctx.world_state.remove_historical_blocks(cmd.toBlockNumber); + return wire::WsdbRemoveHistoricalBlocksResponse{ + .status = world_state_status_full_to_wire(status), + }; +} + +// --------------------------------------------------------------------------- +// Status +// --------------------------------------------------------------------------- + +wire::WsdbGetStatusResponse handle_get_status(WsdbRequest& ctx, wire::WsdbGetStatus&&) +{ + WorldStateStatusSummary status; + ctx.world_state.get_status_summary(status); + return wire::WsdbGetStatusResponse{ + .status = world_state_status_summary_to_wire(status), + }; +} + +// --------------------------------------------------------------------------- +// Checkpoint operations +// --------------------------------------------------------------------------- + +wire::WsdbCreateCheckpointResponse handle_create_checkpoint(WsdbRequest& ctx, wire::WsdbCreateCheckpoint&& cmd) +{ + ctx.world_state.checkpoint(cmd.forkId); + return wire::WsdbCreateCheckpointResponse{}; +} + +wire::WsdbCommitCheckpointResponse handle_commit_checkpoint(WsdbRequest& ctx, wire::WsdbCommitCheckpoint&& cmd) +{ + ctx.world_state.commit_checkpoint(cmd.forkId); + return wire::WsdbCommitCheckpointResponse{}; +} + +wire::WsdbRevertCheckpointResponse handle_revert_checkpoint(WsdbRequest& ctx, wire::WsdbRevertCheckpoint&& cmd) +{ + ctx.world_state.revert_checkpoint(cmd.forkId); + return wire::WsdbRevertCheckpointResponse{}; +} + +wire::WsdbCommitAllCheckpointsResponse handle_commit_all_checkpoints(WsdbRequest& ctx, + wire::WsdbCommitAllCheckpoints&& cmd) +{ + ctx.world_state.commit_all_checkpoints_to(cmd.forkId, 0); + return wire::WsdbCommitAllCheckpointsResponse{}; +} + +wire::WsdbRevertAllCheckpointsResponse handle_revert_all_checkpoints(WsdbRequest& ctx, + wire::WsdbRevertAllCheckpoints&& cmd) +{ + ctx.world_state.revert_all_checkpoints_to(cmd.forkId, 0); + return wire::WsdbRevertAllCheckpointsResponse{}; +} + +// --------------------------------------------------------------------------- +// Database operations +// --------------------------------------------------------------------------- + +wire::WsdbCopyStoresResponse handle_copy_stores(WsdbRequest& ctx, wire::WsdbCopyStores&& cmd) +{ + ctx.world_state.copy_stores(cmd.dstPath, cmd.compact.value_or(false)); + return wire::WsdbCopyStoresResponse{}; +} + +} // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp new file mode 100644 index 000000000000..18c637f48896 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_handlers.hpp @@ -0,0 +1,80 @@ +#pragma once +/** + * @file wsdb_handlers.hpp + * @brief Non-template handler declarations for the wsdb service. + * + * The codegen-emitted server (generated/wsdb_ipc_server.hpp) declares + * template handle_(Ctx&, wire::Cmd&&). The free-function + * overloads below provide concrete definitions for `Ctx = WsdbRequest` + * so that overload resolution prefers them at the template's instantiation + * point inside make_wsdb_handler(...). + * + * Definitions live in wsdb_handlers.cpp. This header keeps the + * instantiation-time lookup honest. + */ +#include "barretenberg/wsdb/generated/wsdb_types.hpp" +#include "barretenberg/wsdb/wsdb_request.hpp" + +namespace bb::wsdb { + +wire::WsdbGetTreeInfoResponse handle_get_tree_info(WsdbRequest& ctx, wire::WsdbGetTreeInfo&& cmd); +wire::WsdbGetStateReferenceResponse handle_get_state_reference(WsdbRequest& ctx, wire::WsdbGetStateReference&& cmd); +wire::WsdbGetInitialStateReferenceResponse handle_get_initial_state_reference(WsdbRequest& ctx, + wire::WsdbGetInitialStateReference&& cmd); +wire::WsdbGetLeafValueResponse handle_get_leaf_value(WsdbRequest& ctx, wire::WsdbGetLeafValue&& cmd); +wire::WsdbGetPublicDataLeafValueResponse handle_get_public_data_leaf_value(WsdbRequest& ctx, + wire::WsdbGetPublicDataLeafValue&& cmd); +wire::WsdbGetNullifierLeafValueResponse handle_get_nullifier_leaf_value(WsdbRequest& ctx, + wire::WsdbGetNullifierLeafValue&& cmd); +wire::WsdbGetPublicDataLeafPreimageResponse handle_get_public_data_leaf_preimage( + WsdbRequest& ctx, wire::WsdbGetPublicDataLeafPreimage&& cmd); +wire::WsdbGetNullifierLeafPreimageResponse handle_get_nullifier_leaf_preimage(WsdbRequest& ctx, + wire::WsdbGetNullifierLeafPreimage&& cmd); +wire::WsdbGetSiblingPathResponse handle_get_sibling_path(WsdbRequest& ctx, wire::WsdbGetSiblingPath&& cmd); +wire::WsdbGetBlockNumbersForLeafIndicesResponse handle_get_block_numbers_for_leaf_indices( + WsdbRequest& ctx, wire::WsdbGetBlockNumbersForLeafIndices&& cmd); +wire::WsdbFindLeafIndicesResponse handle_find_leaf_indices(WsdbRequest& ctx, wire::WsdbFindLeafIndices&& cmd); +wire::WsdbFindPublicDataLeafIndicesResponse handle_find_public_data_leaf_indices( + WsdbRequest& ctx, wire::WsdbFindPublicDataLeafIndices&& cmd); +wire::WsdbFindNullifierLeafIndicesResponse handle_find_nullifier_leaf_indices(WsdbRequest& ctx, + wire::WsdbFindNullifierLeafIndices&& cmd); +wire::WsdbFindLowLeafResponse handle_find_low_leaf(WsdbRequest& ctx, wire::WsdbFindLowLeaf&& cmd); +wire::WsdbFindSiblingPathsResponse handle_find_sibling_paths(WsdbRequest& ctx, wire::WsdbFindSiblingPaths&& cmd); +wire::WsdbFindPublicDataSiblingPathsResponse handle_find_public_data_sibling_paths( + WsdbRequest& ctx, wire::WsdbFindPublicDataSiblingPaths&& cmd); +wire::WsdbFindNullifierSiblingPathsResponse handle_find_nullifier_sibling_paths( + WsdbRequest& ctx, wire::WsdbFindNullifierSiblingPaths&& cmd); +wire::WsdbAppendLeavesResponse handle_append_leaves(WsdbRequest& ctx, wire::WsdbAppendLeaves&& cmd); +wire::WsdbAppendPublicDataLeavesResponse handle_append_public_data_leaves(WsdbRequest& ctx, + wire::WsdbAppendPublicDataLeaves&& cmd); +wire::WsdbAppendNullifierLeavesResponse handle_append_nullifier_leaves(WsdbRequest& ctx, + wire::WsdbAppendNullifierLeaves&& cmd); +wire::WsdbBatchInsertPublicDataResponse handle_batch_insert_public_data(WsdbRequest& ctx, + wire::WsdbBatchInsertPublicData&& cmd); +wire::WsdbBatchInsertNullifierResponse handle_batch_insert_nullifier(WsdbRequest& ctx, + wire::WsdbBatchInsertNullifier&& cmd); +wire::WsdbSequentialInsertPublicDataResponse handle_sequential_insert_public_data( + WsdbRequest& ctx, wire::WsdbSequentialInsertPublicData&& cmd); +wire::WsdbSequentialInsertNullifierResponse handle_sequential_insert_nullifier( + WsdbRequest& ctx, wire::WsdbSequentialInsertNullifier&& cmd); +wire::WsdbUpdateArchiveResponse handle_update_archive(WsdbRequest& ctx, wire::WsdbUpdateArchive&& cmd); +wire::WsdbCommitResponse handle_commit(WsdbRequest& ctx, wire::WsdbCommit&& cmd); +wire::WsdbRollbackResponse handle_rollback(WsdbRequest& ctx, wire::WsdbRollback&& cmd); +wire::WsdbSyncBlockResponse handle_sync_block(WsdbRequest& ctx, wire::WsdbSyncBlock&& cmd); +wire::WsdbCreateForkResponse handle_create_fork(WsdbRequest& ctx, wire::WsdbCreateFork&& cmd); +wire::WsdbDeleteForkResponse handle_delete_fork(WsdbRequest& ctx, wire::WsdbDeleteFork&& cmd); +wire::WsdbFinalizeBlocksResponse handle_finalize_blocks(WsdbRequest& ctx, wire::WsdbFinalizeBlocks&& cmd); +wire::WsdbUnwindBlocksResponse handle_unwind_blocks(WsdbRequest& ctx, wire::WsdbUnwindBlocks&& cmd); +wire::WsdbRemoveHistoricalBlocksResponse handle_remove_historical_blocks(WsdbRequest& ctx, + wire::WsdbRemoveHistoricalBlocks&& cmd); +wire::WsdbGetStatusResponse handle_get_status(WsdbRequest& ctx, wire::WsdbGetStatus&& cmd); +wire::WsdbCreateCheckpointResponse handle_create_checkpoint(WsdbRequest& ctx, wire::WsdbCreateCheckpoint&& cmd); +wire::WsdbCommitCheckpointResponse handle_commit_checkpoint(WsdbRequest& ctx, wire::WsdbCommitCheckpoint&& cmd); +wire::WsdbRevertCheckpointResponse handle_revert_checkpoint(WsdbRequest& ctx, wire::WsdbRevertCheckpoint&& cmd); +wire::WsdbCommitAllCheckpointsResponse handle_commit_all_checkpoints(WsdbRequest& ctx, + wire::WsdbCommitAllCheckpoints&& cmd); +wire::WsdbRevertAllCheckpointsResponse handle_revert_all_checkpoints(WsdbRequest& ctx, + wire::WsdbRevertAllCheckpoints&& cmd); +wire::WsdbCopyStoresResponse handle_copy_stores(WsdbRequest& ctx, wire::WsdbCopyStores&& cmd); + +} // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp new file mode 100644 index 000000000000..ed738a4c419b --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client.hpp @@ -0,0 +1,3 @@ +#pragma once + +#include "barretenberg/wsdb/generated/wsdb_ipc_client.hpp" diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.cpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.cpp deleted file mode 100644 index 08da0e939635..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.cpp +++ /dev/null @@ -1,225 +0,0 @@ -// AUTOGENERATED FILE - DO NOT EDIT - -#include "barretenberg/wsdb/wsdb_ipc_client_generated.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/serialize/msgpack_impl.hpp" -#include "barretenberg/wsdb/wsdb_execute.hpp" - -#include -#include - -namespace bb::wsdb { - -WsdbIpcClient::WsdbIpcClient(const std::string& socket_path) - : client_(ipc::IpcClient::create_socket(socket_path)) -{ - if (!client_->connect()) { - throw std::runtime_error("Failed to connect to server at " + socket_path); - } -} - -WsdbIpcClient::~WsdbIpcClient() -{ - if (client_) { - client_->close(); - } -} - -template typename Cmd::Response WsdbIpcClient::send(Cmd&& cmd) const -{ - // Wrap command in WsdbCommand NamedUnion, then in a 1-element tuple (matches server expectations) - WsdbCommand command = std::forward(cmd); - auto wrapped = std::make_tuple(std::move(command)); - - // Serialize to msgpack - msgpack::sbuffer send_buffer; - msgpack::pack(send_buffer, wrapped); - - // Send to server - constexpr uint64_t timeout_ns = 30'000'000'000ULL; // 30 seconds - if (!client_->send(send_buffer.data(), send_buffer.size(), timeout_ns)) { - throw std::runtime_error("Failed to send command to server"); - } - - // Receive response - auto response_span = client_->receive(timeout_ns); - if (response_span.empty()) { - throw std::runtime_error("Empty response from server"); - } - - // Deserialize response - auto unpacked = msgpack::unpack(reinterpret_cast(response_span.data()), response_span.size()); - auto response_obj = unpacked.get(); - - WsdbCommandResponse response; - response_obj.convert(response); - - // Release the receive buffer - client_->release(response_span.size()); - - // Check for error response - return std::move(response).visit([](auto&& resp) -> typename Cmd::Response { - using RespType = std::decay_t; - - if constexpr (std::is_same_v) { - throw std::runtime_error("Server error: " + resp.message); - } else if constexpr (std::is_same_v) { - return std::forward(resp); - } else { - throw std::runtime_error("Unexpected response type from server"); - } - }); -} - -WsdbGetTreeInfo::Response WsdbIpcClient::get_tree_info(WsdbGetTreeInfo cmd) const -{ - return send(std::move(cmd)); -} - -WsdbGetStateReference::Response WsdbIpcClient::get_state_reference(WsdbGetStateReference cmd) const -{ - return send(std::move(cmd)); -} - -WsdbGetInitialStateReference::Response WsdbIpcClient::get_initial_state_reference() const -{ - return send(WsdbGetInitialStateReference{}); -} - -WsdbGetLeafValue::Response WsdbIpcClient::get_leaf_value(WsdbGetLeafValue cmd) const -{ - return send(std::move(cmd)); -} - -WsdbGetLeafPreimage::Response WsdbIpcClient::get_leaf_preimage(WsdbGetLeafPreimage cmd) const -{ - return send(std::move(cmd)); -} - -WsdbGetSiblingPath::Response WsdbIpcClient::get_sibling_path(WsdbGetSiblingPath cmd) const -{ - return send(std::move(cmd)); -} - -WsdbGetBlockNumbersForLeafIndices::Response WsdbIpcClient::get_block_numbers_for_leaf_indices( - WsdbGetBlockNumbersForLeafIndices cmd) const -{ - return send(std::move(cmd)); -} - -WsdbFindLeafIndices::Response WsdbIpcClient::find_leaf_indices(WsdbFindLeafIndices cmd) const -{ - return send(std::move(cmd)); -} - -WsdbFindLowLeaf::Response WsdbIpcClient::find_low_leaf(WsdbFindLowLeaf cmd) const -{ - return send(std::move(cmd)); -} - -WsdbFindSiblingPaths::Response WsdbIpcClient::find_sibling_paths(WsdbFindSiblingPaths cmd) const -{ - return send(std::move(cmd)); -} - -void WsdbIpcClient::append_leaves(WsdbAppendLeaves cmd) const -{ - send(std::move(cmd)); -} - -WsdbBatchInsert::Response WsdbIpcClient::batch_insert(WsdbBatchInsert cmd) const -{ - return send(std::move(cmd)); -} - -WsdbSequentialInsert::Response WsdbIpcClient::sequential_insert(WsdbSequentialInsert cmd) const -{ - return send(std::move(cmd)); -} - -void WsdbIpcClient::update_archive(WsdbUpdateArchive cmd) const -{ - send(std::move(cmd)); -} - -WsdbCommit::Response WsdbIpcClient::commit() -{ - return send(WsdbCommit{}); -} - -void WsdbIpcClient::rollback() -{ - send(WsdbRollback{}); -} - -WsdbSyncBlock::Response WsdbIpcClient::sync_block(WsdbSyncBlock cmd) -{ - return send(std::move(cmd)); -} - -WsdbCreateFork::Response WsdbIpcClient::create_fork(WsdbCreateFork cmd) -{ - return send(std::move(cmd)); -} - -void WsdbIpcClient::delete_fork(WsdbDeleteFork cmd) -{ - send(std::move(cmd)); -} - -WsdbFinalizeBlocks::Response WsdbIpcClient::finalize_blocks(WsdbFinalizeBlocks cmd) const -{ - return send(std::move(cmd)); -} - -WsdbUnwindBlocks::Response WsdbIpcClient::unwind_blocks(WsdbUnwindBlocks cmd) -{ - return send(std::move(cmd)); -} - -WsdbRemoveHistoricalBlocks::Response WsdbIpcClient::remove_historical_blocks(WsdbRemoveHistoricalBlocks cmd) const -{ - return send(std::move(cmd)); -} - -WsdbGetStatus::Response WsdbIpcClient::get_status() const -{ - return send(WsdbGetStatus{}); -} - -void WsdbIpcClient::create_checkpoint(WsdbCreateCheckpoint cmd) -{ - send(std::move(cmd)); -} - -void WsdbIpcClient::commit_checkpoint(WsdbCommitCheckpoint cmd) -{ - send(std::move(cmd)); -} - -void WsdbIpcClient::revert_checkpoint(WsdbRevertCheckpoint cmd) -{ - send(std::move(cmd)); -} - -void WsdbIpcClient::commit_all_checkpoints(WsdbCommitAllCheckpoints cmd) -{ - send(std::move(cmd)); -} - -void WsdbIpcClient::revert_all_checkpoints(WsdbRevertAllCheckpoints cmd) -{ - send(std::move(cmd)); -} - -void WsdbIpcClient::copy_stores(WsdbCopyStores cmd) const -{ - send(std::move(cmd)); -} - -void WsdbIpcClient::shutdown() -{ - send(WsdbShutdown{}); -} - -} // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.hpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.hpp deleted file mode 100644 index cc5fede28845..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.hpp +++ /dev/null @@ -1,65 +0,0 @@ -// AUTOGENERATED FILE - DO NOT EDIT -#pragma once - -#include "barretenberg/common/try_catch_shim.hpp" -#include "barretenberg/ipc/ipc_client.hpp" -#include "barretenberg/wsdb/wsdb_execute.hpp" - -#include -#include - -namespace bb::wsdb { - -/** - * @brief Auto-generated IPC client. - * - * Each method sends a msgpack-serialized command to the server over UDS - * and returns the typed response. All methods block until the response arrives. - */ -class WsdbIpcClient { - public: - explicit WsdbIpcClient(const std::string& socket_path); - ~WsdbIpcClient(); - - WsdbIpcClient(const WsdbIpcClient&) = delete; - WsdbIpcClient& operator=(const WsdbIpcClient&) = delete; - - WsdbGetTreeInfo::Response get_tree_info(WsdbGetTreeInfo cmd) const; - WsdbGetStateReference::Response get_state_reference(WsdbGetStateReference cmd) const; - WsdbGetInitialStateReference::Response get_initial_state_reference() const; - WsdbGetLeafValue::Response get_leaf_value(WsdbGetLeafValue cmd) const; - WsdbGetLeafPreimage::Response get_leaf_preimage(WsdbGetLeafPreimage cmd) const; - WsdbGetSiblingPath::Response get_sibling_path(WsdbGetSiblingPath cmd) const; - WsdbGetBlockNumbersForLeafIndices::Response get_block_numbers_for_leaf_indices( - WsdbGetBlockNumbersForLeafIndices cmd) const; - WsdbFindLeafIndices::Response find_leaf_indices(WsdbFindLeafIndices cmd) const; - WsdbFindLowLeaf::Response find_low_leaf(WsdbFindLowLeaf cmd) const; - WsdbFindSiblingPaths::Response find_sibling_paths(WsdbFindSiblingPaths cmd) const; - void append_leaves(WsdbAppendLeaves cmd) const; - WsdbBatchInsert::Response batch_insert(WsdbBatchInsert cmd) const; - WsdbSequentialInsert::Response sequential_insert(WsdbSequentialInsert cmd) const; - void update_archive(WsdbUpdateArchive cmd) const; - WsdbCommit::Response commit(); - void rollback(); - WsdbSyncBlock::Response sync_block(WsdbSyncBlock cmd); - WsdbCreateFork::Response create_fork(WsdbCreateFork cmd); - void delete_fork(WsdbDeleteFork cmd); - WsdbFinalizeBlocks::Response finalize_blocks(WsdbFinalizeBlocks cmd) const; - WsdbUnwindBlocks::Response unwind_blocks(WsdbUnwindBlocks cmd); - WsdbRemoveHistoricalBlocks::Response remove_historical_blocks(WsdbRemoveHistoricalBlocks cmd) const; - WsdbGetStatus::Response get_status() const; - void create_checkpoint(WsdbCreateCheckpoint cmd); - void commit_checkpoint(WsdbCommitCheckpoint cmd); - void revert_checkpoint(WsdbRevertCheckpoint cmd); - void commit_all_checkpoints(WsdbCommitAllCheckpoints cmd); - void revert_all_checkpoints(WsdbRevertAllCheckpoints cmd); - void copy_stores(WsdbCopyStores cmd) const; - void shutdown(); - - private: - template typename Cmd::Response send(Cmd&& cmd) const; - - mutable std::unique_ptr client_; -}; - -} // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp index b124326e8347..4221a2abfad0 100644 --- a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp +++ b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_ipc_server.cpp @@ -1,78 +1,36 @@ #include "barretenberg/wsdb/wsdb_ipc_server.hpp" #include "barretenberg/common/log.hpp" #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/ipc/ipc_server.hpp" #include "barretenberg/serialize/msgpack.hpp" #include "barretenberg/world_state/world_state.hpp" -#include "barretenberg/wsdb/wsdb_execute.hpp" +#include "barretenberg/wsdb/generated/wsdb_ipc_server.hpp" +#include "barretenberg/wsdb/wsdb_handlers.hpp" +#include "barretenberg/wsdb/wsdb_request.hpp" +#include "ipc_runtime/ipc_server.hpp" +#include "ipc_runtime/serve_helper.hpp" +#include "ipc_runtime/signal_handlers.hpp" -#include #include #include #include +#include #include -#include -#include #include #include -#ifdef __linux__ -#include -#elif defined(__APPLE__) -#include -#endif - -// Use nlohmann/json if available, otherwise minimal parsing -#include - namespace bb::wsdb { using namespace bb::world_state; using namespace bb::crypto::merkle_tree; -// --------------------------------------------------------------------------- -// Platform-specific parent death monitoring -// (Same pattern as api_msgpack.cpp) -// --------------------------------------------------------------------------- - -static void setup_parent_death_monitoring() -{ -#ifdef __linux__ - if (prctl(PR_SET_PDEATHSIG, SIGTERM) == -1) { - std::cerr << "Warning: Could not set parent death signal" << '\n'; - } -#elif defined(__APPLE__) - pid_t parent_pid = getppid(); - std::thread([parent_pid]() { - int kq = kqueue(); - if (kq == -1) { - std::cerr << "Warning: Could not create kqueue for parent monitoring" << '\n'; - return; - } - struct kevent change; - EV_SET(&change, parent_pid, EVFILT_PROC, EV_ADD | EV_ENABLE, NOTE_EXIT, 0, nullptr); - if (kevent(kq, &change, 1, nullptr, 0, nullptr) == -1) { - std::cerr << "Warning: Could not monitor parent process" << '\n'; - close(kq); - return; - } - struct kevent event; - kevent(kq, nullptr, 0, &event, 1, nullptr); - std::cerr << "Parent process exited, shutting down..." << '\n'; - close(kq); - std::exit(0); - }).detach(); -#endif -} - // --------------------------------------------------------------------------- // Simple JSON-like parsing for config maps // Parses "{0:1024,1:2048,...}" into unordered_map // --------------------------------------------------------------------------- -static std::unordered_map parse_tree_uint64_map(const std::string& json) +static std::unordered_map parse_tree_uint64_map(const std::string& json) { - std::unordered_map result; + std::unordered_map result; if (json.empty()) { return result; } @@ -87,7 +45,7 @@ static std::unordered_map parse_tree_uint64_map(const st while (std::getline(ss, pair, ',')) { auto colon_pos = pair.find(':'); if (colon_pos != std::string::npos) { - auto key = static_cast(std::stoi(pair.substr(0, colon_pos))); + auto key = static_cast(std::stoi(pair.substr(0, colon_pos))); auto value = static_cast(std::stoull(pair.substr(colon_pos + 1))); result[key] = value; } @@ -95,9 +53,9 @@ static std::unordered_map parse_tree_uint64_map(const st return result; } -static std::unordered_map parse_tree_uint32_map(const std::string& json) +static std::unordered_map parse_tree_uint32_map(const std::string& json) { - std::unordered_map result; + std::unordered_map result; if (json.empty()) { return result; } @@ -108,9 +66,9 @@ static std::unordered_map parse_tree_uint32_map(const st return result; } -static std::unordered_map parse_tree_index_map(const std::string& json) +static std::unordered_map parse_tree_index_map(const std::string& json) { - std::unordered_map result; + std::unordered_map result; if (json.empty()) { return result; } @@ -190,12 +148,12 @@ int execute_wsdb_server(const std::string& input_path, auto tree_height = parse_tree_uint32_map(tree_heights_json); auto tree_prefill = parse_tree_index_map(tree_prefill_json); - std::unordered_map map_size{ - { MerkleTreeId::ARCHIVE, DEFAULT_MAP_SIZE }, - { MerkleTreeId::NULLIFIER_TREE, DEFAULT_MAP_SIZE }, - { MerkleTreeId::NOTE_HASH_TREE, DEFAULT_MAP_SIZE }, - { MerkleTreeId::PUBLIC_DATA_TREE, DEFAULT_MAP_SIZE }, - { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, DEFAULT_MAP_SIZE }, + std::unordered_map map_size{ + { world_state::MerkleTreeId::ARCHIVE, DEFAULT_MAP_SIZE }, + { world_state::MerkleTreeId::NULLIFIER_TREE, DEFAULT_MAP_SIZE }, + { world_state::MerkleTreeId::NOTE_HASH_TREE, DEFAULT_MAP_SIZE }, + { world_state::MerkleTreeId::PUBLIC_DATA_TREE, DEFAULT_MAP_SIZE }, + { world_state::MerkleTreeId::L1_TO_L2_MESSAGE_TREE, DEFAULT_MAP_SIZE }, }; if (!map_sizes_json.empty()) { auto parsed = parse_tree_uint64_map(map_sizes_json); @@ -224,47 +182,20 @@ int execute_wsdb_server(const std::string& input_path, WsdbRequest request{ .world_state = *ws }; - // Create IPC server based on path suffix - std::unique_ptr server; - - if (input_path.size() >= 4 && input_path.substr(input_path.size() - 4) == ".shm") { - std::string base_name = input_path.substr(0, input_path.size() - 4); - constexpr size_t MAX_SHM_CLIENTS = 2; // TS backend (client 0) + AVM binary (client 1) - server = ipc::IpcServer::create_mpsc_shm(base_name, MAX_SHM_CLIENTS, request_ring_size, response_ring_size); - std::cerr << "MPSC shared memory server at " << base_name << " (max " << MAX_SHM_CLIENTS << " clients)\n"; - } else if (input_path.size() >= 5 && input_path.substr(input_path.size() - 5) == ".sock") { - server = ipc::IpcServer::create_socket(input_path, 1); - std::cerr << "Socket server at " << input_path << '\n'; - } else { - std::cerr << "Error: --input path must end with .sock or .shm" << '\n'; + // Pick UDS vs MPSC-SHM by path suffix; install the runtime's default + // lifecycle signal handlers (SIGTERM/SIGINT → request_shutdown, SIGBUS/SIGSEGV + // → close+exit, plus parent-death monitoring via prctl/kqueue). + ipc::ServerOptions opts; + opts.max_shm_clients = 2; // TS backend (client 0) + AVM binary (client 1) + opts.shm_request_ring_size = request_ring_size; + opts.shm_response_ring_size = response_ring_size; + auto server = ipc::make_server(input_path, opts); + if (!server) { + std::cerr << "Error: --input path must end with .sock or .shm: " << input_path << '\n'; return 1; } - - // Set up signal handlers - static ipc::IpcServer* global_server = server.get(); - - auto graceful_shutdown_handler = [](int signal) { - std::cerr << "\nReceived signal " << signal << ", shutting down gracefully..." << '\n'; - if (global_server) { - global_server->request_shutdown(); - } - }; - - auto fatal_error_handler = [](int signal) { - const char* signal_name = (signal == SIGBUS) ? "SIGBUS" : (signal == SIGSEGV) ? "SIGSEGV" : "UNKNOWN"; - std::cerr << "\nFatal error: received " << signal_name << '\n'; - if (global_server) { - global_server->close(); - } - std::exit(1); - }; - - (void)std::signal(SIGTERM, graceful_shutdown_handler); - (void)std::signal(SIGINT, graceful_shutdown_handler); - (void)std::signal(SIGBUS, fatal_error_handler); - (void)std::signal(SIGSEGV, fatal_error_handler); - - setup_parent_death_monitoring(); + std::cerr << "aztec-wsdb listening on " << input_path << '\n'; + ipc::install_default_signal_handlers(*server); if (!server->listen()) { std::cerr << "Error: Could not start IPC server" << '\n'; @@ -273,72 +204,9 @@ int execute_wsdb_server(const std::string& input_path, std::cerr << "aztec-wsdb IPC server ready" << '\n'; - // Run server with wsdb command handler - server->run([&request](int client_id, std::span raw_request) -> std::vector { - try { - // Deserialize msgpack command - // Format: [["CommandName", {payload}]] - a 1-element tuple containing the NamedUnion - auto unpacked = msgpack::unpack(reinterpret_cast(raw_request.data()), raw_request.size()); - auto obj = unpacked.get(); - - // Expect array of size 1 (tuple wrapping) - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) - if (obj.type != msgpack::type::ARRAY || obj.via.array.size != 1) { - std::cerr << "Error: Expected array of size 1 from client " << client_id << '\n'; - return {}; - } - - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) - auto& command_obj = obj.via.array.ptr[0]; - - // Check for shutdown before converting - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) - if (command_obj.type == msgpack::type::ARRAY && command_obj.via.array.size == 2 && - command_obj.via.array.ptr[0].type == msgpack::type::STR) { - // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) - std::string_view command_name(command_obj.via.array.ptr[0].via.str.ptr, - command_obj.via.array.ptr[0].via.str.size); - bool is_shutdown = (command_name == "WsdbShutdown"); - - // Convert and execute - WsdbCommand command; - command_obj.convert(command); - auto response = wsdb(request, std::move(command)); - - // Serialize response - msgpack::sbuffer response_buffer; - msgpack::pack(response_buffer, response); - std::vector result(response_buffer.data(), response_buffer.data() + response_buffer.size()); - - if (is_shutdown) { - throw ipc::ShutdownRequested(std::move(result)); - } - - return result; - } - - // Fallback: try converting directly - WsdbCommand command; - command_obj.convert(command); - auto response = wsdb(request, std::move(command)); - - msgpack::sbuffer response_buffer; - msgpack::pack(response_buffer, response); - return std::vector(response_buffer.data(), response_buffer.data() + response_buffer.size()); - - } catch (const ipc::ShutdownRequested&) { - throw; - } catch (const std::exception& e) { - std::cerr << "Error processing request from client " << client_id << ": " << e.what() << '\n'; - std::cerr.flush(); - - WsdbErrorResponse error_response{ .message = std::string(e.what()) }; - WsdbCommandResponse response = error_response; - - msgpack::sbuffer response_buffer; - msgpack::pack(response_buffer, response); - return std::vector(response_buffer.data(), response_buffer.data() + response_buffer.size()); - } + auto handler = make_wsdb_handler(request); + server->run([&handler](int /*client_id*/, std::span raw) { + return handler(std::vector(raw.begin(), raw.end())); }); server->close(); diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp new file mode 100644 index 000000000000..a0a4bb847e93 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_request.hpp @@ -0,0 +1,18 @@ +#pragma once +/** + * @file wsdb_request.hpp + * @brief Service-level context passed to every wsdb handler. + * + * Each codegen-emitted handler in wsdb_handlers.hpp takes a WsdbRequest& + * as its `Ctx`. The struct owns no state of its own — it just bundles the + * WorldState reference handlers need to do their work. + */ +#include "barretenberg/world_state/world_state.hpp" + +namespace bb::wsdb { + +struct WsdbRequest { + world_state::WorldState& world_state; +}; + +} // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc new file mode 100644 index 000000000000..4eaa8d4c86f9 --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc @@ -0,0 +1,475 @@ +{ + "service": "Wsdb", + "aliases": { + "MerkleTreeId": "u32", + "ForkId": "u64", + "BlockNumber": "u32", + "Fr": "bin32", + "LeafIndex": "u64", + "PublicDataSlot": "bin32", + "PublicDataValue": "bin32", + "Nullifier": "bin32", + "BlockHeaderHash": "bin32" + }, + "types": { + "WorldStateRevision": { + "forkId": "ForkId", + "blockNumber": "BlockNumber", + "includeUncommitted": "bool" + }, + "TreeStateReference": { + "treeId": "MerkleTreeId", + "root": "Fr", + "size": "LeafIndex" + }, + "PublicDataLeafValue": { + "slot": "PublicDataSlot", + "value": "PublicDataValue" + }, + "NullifierLeafValue": { + "nullifier": "Nullifier" + }, + "IndexedPublicDataLeafValue": { + "leaf": "PublicDataLeafValue", + "nextIndex": "LeafIndex", + "nextKey": "Fr" + }, + "IndexedNullifierLeafValue": { + "leaf": "NullifierLeafValue", + "nextIndex": "LeafIndex", + "nextKey": "Fr" + }, + "SiblingPathAndIndex": { + "index": "LeafIndex", + "path": "Fr[]" + }, + "BatchInsertionResultPublicData": { + "lowLeafWitnessData": "PublicDataLeafUpdateWitnessData[]", + "sortedLeaves": "SortedPublicDataLeaf[]", + "subtreePath": "Fr[]" + }, + "PublicDataLeafUpdateWitnessData": { + "leaf": "IndexedPublicDataLeafValue", + "index": "LeafIndex", + "path": "Fr[]" + }, + "SortedPublicDataLeaf": { + "leaf": "PublicDataLeafValue", + "index": "LeafIndex" + }, + "BatchInsertionResultNullifier": { + "lowLeafWitnessData": "NullifierLeafUpdateWitnessData[]", + "sortedLeaves": "SortedNullifierLeaf[]", + "subtreePath": "Fr[]" + }, + "NullifierLeafUpdateWitnessData": { + "leaf": "IndexedNullifierLeafValue", + "index": "LeafIndex", + "path": "Fr[]" + }, + "SortedNullifierLeaf": { + "leaf": "NullifierLeafValue", + "index": "LeafIndex" + }, + "SequentialInsertionResultPublicData": { + "lowLeafWitnessData": "PublicDataLeafUpdateWitnessData[]", + "insertionWitnessData": "PublicDataLeafUpdateWitnessData[]" + }, + "SequentialInsertionResultNullifier": { + "lowLeafWitnessData": "NullifierLeafUpdateWitnessData[]", + "insertionWitnessData": "NullifierLeafUpdateWitnessData[]" + }, + "WorldStateStatusFull": { + "summary": "WorldStateStatusSummary", + "dbStats": "WorldStateDBStats", + "meta": "WorldStateMeta" + }, + "WorldStateStatusSummary": { + "unfinalizedBlockNumber": "u64", + "finalizedBlockNumber": "u64", + "oldestHistoricalBlock": "u64", + "treesAreSynched": "bool" + }, + "WorldStateDBStats": { + "noteHashTreeStats": "TreeDBStats", + "messageTreeStats": "TreeDBStats", + "archiveTreeStats": "TreeDBStats", + "publicDataTreeStats": "TreeDBStats", + "nullifierTreeStats": "TreeDBStats" + }, + "TreeDBStats": { + "mapSize": "u64", + "physicalFileSize": "u64", + "blocksDBStats": "DBStats", + "nodesDBStats": "DBStats", + "leafPreimagesDBStats": "DBStats", + "leafIndicesDBStats": "DBStats", + "blockIndicesDBStats": "DBStats" + }, + "DBStats": { + "name": "string", + "numDataItems": "u64", + "totalUsedSize": "u64" + }, + "WorldStateMeta": { + "noteHashTreeMeta": "TreeMeta", + "messageTreeMeta": "TreeMeta", + "archiveTreeMeta": "TreeMeta", + "publicDataTreeMeta": "TreeMeta", + "nullifierTreeMeta": "TreeMeta" + }, + "TreeMeta": { + "name": "string", + "depth": "u32", + "size": "u64", + "committedSize": "u64", + "root": "Fr", + "initialSize": "u64", + "initialRoot": "Fr", + "oldestHistoricBlock": "u32", + "unfinalizedBlockHeight": "u32", + "finalizedBlockHeight": "u32" + } + }, + "error": { + "message": "string" + }, + "commands": { + "GetTreeInfo": { + "request": { + "treeId": "MerkleTreeId", + "revision": "WorldStateRevision" + }, + "response": { + "treeId": "MerkleTreeId", + "root": "Fr", + "size": "LeafIndex", + "depth": "u32" + } + }, + "GetStateReference": { + "request": { + "revision": "WorldStateRevision" + }, + "response": { + "state": "TreeStateReference[]" + } + }, + "GetInitialStateReference": { + "request": {}, + "response": { + "state": "TreeStateReference[]" + } + }, + "GetLeafValue": { + "request": { + "treeId": "MerkleTreeId", + "revision": "WorldStateRevision", + "leafIndex": "LeafIndex" + }, + "response": { + "value": "Fr?" + } + }, + "GetPublicDataLeafValue": { + "request": { + "revision": "WorldStateRevision", + "leafIndex": "LeafIndex" + }, + "response": { + "value": "PublicDataLeafValue?" + } + }, + "GetNullifierLeafValue": { + "request": { + "revision": "WorldStateRevision", + "leafIndex": "LeafIndex" + }, + "response": { + "value": "NullifierLeafValue?" + } + }, + "GetPublicDataLeafPreimage": { + "request": { + "revision": "WorldStateRevision", + "leafIndex": "LeafIndex" + }, + "response": { + "preimage": "IndexedPublicDataLeafValue?" + } + }, + "GetNullifierLeafPreimage": { + "request": { + "revision": "WorldStateRevision", + "leafIndex": "LeafIndex" + }, + "response": { + "preimage": "IndexedNullifierLeafValue?" + } + }, + "GetSiblingPath": { + "request": { + "treeId": "MerkleTreeId", + "revision": "WorldStateRevision", + "leafIndex": "LeafIndex" + }, + "response": { + "path": "Fr[]" + } + }, + "GetBlockNumbersForLeafIndices": { + "request": { + "treeId": "MerkleTreeId", + "revision": "WorldStateRevision", + "leafIndices": "LeafIndex[]" + }, + "response": { + "blockNumbers": "BlockNumber?[]" + } + }, + "FindLeafIndices": { + "request": { + "treeId": "MerkleTreeId", + "revision": "WorldStateRevision", + "leaves": "Fr[]", + "startIndex": "LeafIndex" + }, + "response": { + "indices": "LeafIndex?[]" + } + }, + "FindPublicDataLeafIndices": { + "request": { + "revision": "WorldStateRevision", + "leaves": "PublicDataLeafValue[]", + "startIndex": "LeafIndex" + }, + "response": { + "indices": "LeafIndex?[]" + } + }, + "FindNullifierLeafIndices": { + "request": { + "revision": "WorldStateRevision", + "leaves": "NullifierLeafValue[]", + "startIndex": "LeafIndex" + }, + "response": { + "indices": "LeafIndex?[]" + } + }, + "FindLowLeaf": { + "request": { + "treeId": "MerkleTreeId", + "revision": "WorldStateRevision", + "key": "Fr" + }, + "response": { + "alreadyPresent": "bool", + "index": "LeafIndex" + } + }, + "FindSiblingPaths": { + "request": { + "treeId": "MerkleTreeId", + "revision": "WorldStateRevision", + "leaves": "Fr[]" + }, + "response": { + "paths": "SiblingPathAndIndex?[]" + } + }, + "FindPublicDataSiblingPaths": { + "request": { + "revision": "WorldStateRevision", + "leaves": "PublicDataLeafValue[]" + }, + "response": { + "paths": "SiblingPathAndIndex?[]" + } + }, + "FindNullifierSiblingPaths": { + "request": { + "revision": "WorldStateRevision", + "leaves": "NullifierLeafValue[]" + }, + "response": { + "paths": "SiblingPathAndIndex?[]" + } + }, + "AppendLeaves": { + "request": { + "treeId": "MerkleTreeId", + "leaves": "Fr[]", + "forkId": "ForkId" + }, + "response": {} + }, + "AppendPublicDataLeaves": { + "request": { + "leaves": "PublicDataLeafValue[]", + "forkId": "ForkId" + }, + "response": {} + }, + "AppendNullifierLeaves": { + "request": { + "leaves": "NullifierLeafValue[]", + "forkId": "ForkId" + }, + "response": {} + }, + "BatchInsertPublicData": { + "request": { + "leaves": "PublicDataLeafValue[]", + "subtreeDepth": "u32", + "forkId": "ForkId" + }, + "response": { + "result": "BatchInsertionResultPublicData" + } + }, + "BatchInsertNullifier": { + "request": { + "leaves": "NullifierLeafValue[]", + "subtreeDepth": "u32", + "forkId": "ForkId" + }, + "response": { + "result": "BatchInsertionResultNullifier" + } + }, + "SequentialInsertPublicData": { + "request": { + "leaves": "PublicDataLeafValue[]", + "forkId": "ForkId" + }, + "response": { + "result": "SequentialInsertionResultPublicData" + } + }, + "SequentialInsertNullifier": { + "request": { + "leaves": "NullifierLeafValue[]", + "forkId": "ForkId" + }, + "response": { + "result": "SequentialInsertionResultNullifier" + } + }, + "UpdateArchive": { + "request": { + "blockStateRef": "TreeStateReference[]", + "blockHeaderHash": "BlockHeaderHash", + "forkId": "ForkId" + }, + "response": {} + }, + "Commit": { + "request": {}, + "response": { + "status": "WorldStateStatusFull" + } + }, + "Rollback": { + "request": {}, + "response": {} + }, + "SyncBlock": { + "request": { + "blockNumber": "BlockNumber", + "blockStateRef": "TreeStateReference[]", + "blockHeaderHash": "BlockHeaderHash", + "paddedNoteHashes": "Fr[]", + "paddedL1ToL2Messages": "Fr[]", + "paddedNullifiers": "NullifierLeafValue[]", + "publicDataWrites": "PublicDataLeafValue[]" + }, + "response": { + "status": "WorldStateStatusFull" + } + }, + "CreateFork": { + "request": { + "latest": "bool", + "blockNumber": "BlockNumber" + }, + "response": { + "forkId": "ForkId" + } + }, + "DeleteFork": { + "request": { + "forkId": "ForkId" + }, + "response": {} + }, + "FinalizeBlocks": { + "request": { + "toBlockNumber": "BlockNumber" + }, + "response": { + "status": "WorldStateStatusSummary" + } + }, + "UnwindBlocks": { + "request": { + "toBlockNumber": "BlockNumber" + }, + "response": { + "status": "WorldStateStatusFull" + } + }, + "RemoveHistoricalBlocks": { + "request": { + "toBlockNumber": "BlockNumber" + }, + "response": { + "status": "WorldStateStatusFull" + } + }, + "GetStatus": { + "request": {}, + "response": { + "status": "WorldStateStatusSummary" + } + }, + "CreateCheckpoint": { + "request": { + "forkId": "ForkId" + }, + "response": {} + }, + "CommitCheckpoint": { + "request": { + "forkId": "ForkId" + }, + "response": {} + }, + "RevertCheckpoint": { + "request": { + "forkId": "ForkId" + }, + "response": {} + }, + "CommitAllCheckpoints": { + "request": { + "forkId": "ForkId" + }, + "response": {} + }, + "RevertAllCheckpoints": { + "request": { + "forkId": "ForkId" + }, + "response": {} + }, + "CopyStores": { + "request": { + "dstPath": "string", + "compact": "bool?" + }, + "response": {} + } + } +} diff --git a/barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp new file mode 100644 index 000000000000..6de14909e38e --- /dev/null +++ b/barretenberg/cpp/src/barretenberg/wsdb/wsdb_wire_convert.hpp @@ -0,0 +1,504 @@ +#pragma once +/** + * @file wsdb_wire_convert.hpp + * @brief Wire <-> domain conversion helpers for the aztec-wsdb service. + */ +#include "barretenberg/crypto/merkle_tree/node_store/tree_meta.hpp" +#include "barretenberg/crypto/merkle_tree/response.hpp" +#include "barretenberg/crypto/merkle_tree/types.hpp" +#include "barretenberg/ecc/curves/bn254/fr.hpp" +#include "barretenberg/lmdblib/types.hpp" +#include "barretenberg/world_state/types.hpp" +#include "barretenberg/world_state/world_state.hpp" +#include "barretenberg/wsdb/generated/wsdb_types.hpp" + +namespace bb::wsdb { + +inline Fr fr_to_wire(const bb::fr& d) +{ + Fr r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr fr_from_wire(const Fr& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline BlockHeaderHash block_header_hash_to_wire(const bb::fr& d) +{ + BlockHeaderHash r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr block_header_hash_from_wire(const BlockHeaderHash& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline PublicDataSlot public_data_slot_to_wire(const bb::fr& d) +{ + PublicDataSlot r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr public_data_slot_from_wire(const PublicDataSlot& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline PublicDataValue public_data_value_to_wire(const bb::fr& d) +{ + PublicDataValue r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr public_data_value_from_wire(const PublicDataValue& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline Nullifier nullifier_to_wire(const bb::fr& d) +{ + Nullifier r{}; + bb::fr::serialize_to_buffer(d, r.data()); + return r; +} + +inline bb::fr nullifier_from_wire(const Nullifier& w) +{ + return bb::fr::serialize_from_buffer(w.data()); +} + +inline std::vector fr_vec_to_wire(const std::vector& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(fr_to_wire(x)); + } + return r; +} + +inline std::vector fr_vec_from_wire(const std::vector& w) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(fr_from_wire(x)); + } + return r; +} + +inline wire::WorldStateRevision revision_to_wire(const world_state::WorldStateRevision& d) +{ + return wire::WorldStateRevision{ + .forkId = d.forkId, + .blockNumber = d.blockNumber, + .includeUncommitted = d.includeUncommitted, + }; +} + +inline world_state::WorldStateRevision revision_from_wire(const wire::WorldStateRevision& w) +{ + return world_state::WorldStateRevision{ + .forkId = w.forkId, + .blockNumber = w.blockNumber, + .includeUncommitted = w.includeUncommitted, + }; +} + +inline MerkleTreeId tree_id_to_wire(world_state::MerkleTreeId d) +{ + return static_cast(d); +} + +inline world_state::MerkleTreeId tree_id_from_wire(MerkleTreeId w) +{ + return static_cast(w); +} + +inline wire::PublicDataLeafValue public_data_leaf_to_wire(const crypto::merkle_tree::PublicDataLeafValue& d) +{ + return { .slot = public_data_slot_to_wire(d.slot), .value = public_data_value_to_wire(d.value) }; +} + +inline crypto::merkle_tree::PublicDataLeafValue public_data_leaf_from_wire(const wire::PublicDataLeafValue& w) +{ + return { public_data_slot_from_wire(w.slot), public_data_value_from_wire(w.value) }; +} + +inline std::vector public_data_leaf_vec_to_wire( + const std::vector& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(public_data_leaf_to_wire(x)); + } + return r; +} + +inline std::vector public_data_leaf_vec_from_wire( + const std::vector& w) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(public_data_leaf_from_wire(x)); + } + return r; +} + +inline wire::NullifierLeafValue nullifier_leaf_to_wire(const crypto::merkle_tree::NullifierLeafValue& d) +{ + return { .nullifier = nullifier_to_wire(d.nullifier) }; +} + +inline crypto::merkle_tree::NullifierLeafValue nullifier_leaf_from_wire(const wire::NullifierLeafValue& w) +{ + return { nullifier_from_wire(w.nullifier) }; +} + +inline std::vector nullifier_leaf_vec_to_wire( + const std::vector& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(nullifier_leaf_to_wire(x)); + } + return r; +} + +inline std::vector nullifier_leaf_vec_from_wire( + const std::vector& w) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(nullifier_leaf_from_wire(x)); + } + return r; +} + +inline wire::IndexedPublicDataLeafValue indexed_public_data_leaf_to_wire( + const crypto::merkle_tree::IndexedLeaf& d) +{ + return { .leaf = public_data_leaf_to_wire(d.leaf), .nextIndex = d.nextIndex, .nextKey = fr_to_wire(d.nextKey) }; +} + +inline crypto::merkle_tree::IndexedLeaf indexed_public_data_leaf_from_wire( + const wire::IndexedPublicDataLeafValue& w) +{ + return { public_data_leaf_from_wire(w.leaf), w.nextIndex, fr_from_wire(w.nextKey) }; +} + +inline wire::IndexedNullifierLeafValue indexed_nullifier_leaf_to_wire( + const crypto::merkle_tree::IndexedLeaf& d) +{ + return { .leaf = nullifier_leaf_to_wire(d.leaf), .nextIndex = d.nextIndex, .nextKey = fr_to_wire(d.nextKey) }; +} + +inline crypto::merkle_tree::IndexedLeaf indexed_nullifier_leaf_from_wire( + const wire::IndexedNullifierLeafValue& w) +{ + return { nullifier_leaf_from_wire(w.leaf), w.nextIndex, fr_from_wire(w.nextKey) }; +} + +inline wire::PublicDataLeafUpdateWitnessData public_data_witness_to_wire( + const crypto::merkle_tree::LeafUpdateWitnessData& d) +{ + return { .leaf = indexed_public_data_leaf_to_wire(d.leaf), .index = d.index, .path = fr_vec_to_wire(d.path) }; +} + +inline crypto::merkle_tree::LeafUpdateWitnessData +public_data_witness_from_wire(const wire::PublicDataLeafUpdateWitnessData& w) +{ + return { indexed_public_data_leaf_from_wire(w.leaf), w.index, fr_vec_from_wire(w.path) }; +} + +inline wire::NullifierLeafUpdateWitnessData nullifier_witness_to_wire( + const crypto::merkle_tree::LeafUpdateWitnessData& d) +{ + return { .leaf = indexed_nullifier_leaf_to_wire(d.leaf), .index = d.index, .path = fr_vec_to_wire(d.path) }; +} + +inline crypto::merkle_tree::LeafUpdateWitnessData nullifier_witness_from_wire( + const wire::NullifierLeafUpdateWitnessData& w) +{ + return { indexed_nullifier_leaf_from_wire(w.leaf), w.index, fr_vec_from_wire(w.path) }; +} + +template +inline std::vector vec_to_wire(const std::vector& d, Fn fn) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& x : d) { + r.push_back(fn(x)); + } + return r; +} + +template +inline std::vector vec_from_wire(const std::vector& w, Fn fn) +{ + std::vector r; + r.reserve(w.size()); + for (const auto& x : w) { + r.push_back(fn(x)); + } + return r; +} + +inline wire::BatchInsertionResultPublicData batch_public_data_to_wire( + const world_state::BatchInsertionResult& d) +{ + std::vector sorted; + sorted.reserve(d.sorted_leaves.size()); + for (const auto& [leaf, index] : d.sorted_leaves) { + sorted.push_back({ .leaf = public_data_leaf_to_wire(leaf), .index = index }); + } + return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, + public_data_witness_to_wire), + .sortedLeaves = std::move(sorted), + .subtreePath = fr_vec_to_wire(d.subtree_path) }; +} + +inline world_state::BatchInsertionResult batch_public_data_from_wire( + const wire::BatchInsertionResultPublicData& w) +{ + world_state::BatchInsertionResult r; + r.low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, public_data_witness_from_wire); + r.sorted_leaves.reserve(w.sortedLeaves.size()); + for (const auto& x : w.sortedLeaves) { + r.sorted_leaves.emplace_back(public_data_leaf_from_wire(x.leaf), x.index); + } + r.subtree_path = fr_vec_from_wire(w.subtreePath); + return r; +} + +inline wire::BatchInsertionResultNullifier batch_nullifier_to_wire( + const world_state::BatchInsertionResult& d) +{ + std::vector sorted; + sorted.reserve(d.sorted_leaves.size()); + for (const auto& [leaf, index] : d.sorted_leaves) { + sorted.push_back({ .leaf = nullifier_leaf_to_wire(leaf), .index = index }); + } + return { .lowLeafWitnessData = + vec_to_wire(d.low_leaf_witness_data, nullifier_witness_to_wire), + .sortedLeaves = std::move(sorted), + .subtreePath = fr_vec_to_wire(d.subtree_path) }; +} + +inline world_state::BatchInsertionResult batch_nullifier_from_wire( + const wire::BatchInsertionResultNullifier& w) +{ + world_state::BatchInsertionResult r; + r.low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, nullifier_witness_from_wire); + r.sorted_leaves.reserve(w.sortedLeaves.size()); + for (const auto& x : w.sortedLeaves) { + r.sorted_leaves.emplace_back(nullifier_leaf_from_wire(x.leaf), x.index); + } + r.subtree_path = fr_vec_from_wire(w.subtreePath); + return r; +} + +inline wire::SequentialInsertionResultPublicData sequential_public_data_to_wire( + const world_state::SequentialInsertionResult& d) +{ + return { .lowLeafWitnessData = vec_to_wire(d.low_leaf_witness_data, + public_data_witness_to_wire), + .insertionWitnessData = vec_to_wire(d.insertion_witness_data, + public_data_witness_to_wire) }; +} + +inline world_state::SequentialInsertionResult +sequential_public_data_from_wire(const wire::SequentialInsertionResultPublicData& w) +{ + return { .low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, public_data_witness_from_wire), + .insertion_witness_data = + vec_from_wire>( + w.insertionWitnessData, public_data_witness_from_wire) }; +} + +inline wire::SequentialInsertionResultNullifier sequential_nullifier_to_wire( + const world_state::SequentialInsertionResult& d) +{ + return { .lowLeafWitnessData = + vec_to_wire(d.low_leaf_witness_data, nullifier_witness_to_wire), + .insertionWitnessData = vec_to_wire(d.insertion_witness_data, + nullifier_witness_to_wire) }; +} + +inline world_state::SequentialInsertionResult sequential_nullifier_from_wire( + const wire::SequentialInsertionResultNullifier& w) +{ + return { .low_leaf_witness_data = + vec_from_wire>( + w.lowLeafWitnessData, nullifier_witness_from_wire), + .insertion_witness_data = + vec_from_wire>( + w.insertionWitnessData, nullifier_witness_from_wire) }; +} + +inline std::vector state_reference_to_wire(const world_state::StateReference& d) +{ + std::vector r; + r.reserve(d.size()); + for (const auto& [tree_id, tree_ref] : d) { + r.push_back( + { .treeId = tree_id_to_wire(tree_id), .root = fr_to_wire(tree_ref.first), .size = tree_ref.second }); + } + return r; +} + +inline world_state::StateReference state_reference_from_wire(const std::vector& w) +{ + world_state::StateReference r; + r.reserve(w.size()); + for (const auto& entry : w) { + r.emplace(tree_id_from_wire(entry.treeId), + world_state::TreeStateReference{ fr_from_wire(entry.root), entry.size }); + } + return r; +} + +inline wire::DBStats db_stats_to_wire(const bb::lmdblib::DBStats& d) +{ + return { .name = d.name, .numDataItems = d.numDataItems, .totalUsedSize = d.totalUsedSize }; +} + +inline bb::lmdblib::DBStats db_stats_from_wire(const wire::DBStats& w) +{ + return bb::lmdblib::DBStats(w.name, w.numDataItems, w.totalUsedSize); +} + +inline wire::TreeDBStats tree_db_stats_to_wire(const bb::crypto::merkle_tree::TreeDBStats& d) +{ + return { .mapSize = d.mapSize, + .physicalFileSize = d.physicalFileSize, + .blocksDBStats = db_stats_to_wire(d.blocksDBStats), + .nodesDBStats = db_stats_to_wire(d.nodesDBStats), + .leafPreimagesDBStats = db_stats_to_wire(d.leafPreimagesDBStats), + .leafIndicesDBStats = db_stats_to_wire(d.leafIndicesDBStats), + .blockIndicesDBStats = db_stats_to_wire(d.blockIndicesDBStats) }; +} + +inline bb::crypto::merkle_tree::TreeDBStats tree_db_stats_from_wire(const wire::TreeDBStats& w) +{ + return { w.mapSize, + w.physicalFileSize, + db_stats_from_wire(w.blocksDBStats), + db_stats_from_wire(w.nodesDBStats), + db_stats_from_wire(w.leafPreimagesDBStats), + db_stats_from_wire(w.leafIndicesDBStats), + db_stats_from_wire(w.blockIndicesDBStats) }; +} + +inline wire::TreeMeta tree_meta_to_wire(const bb::crypto::merkle_tree::TreeMeta& d) +{ + return { .name = d.name, + .depth = d.depth, + .size = d.size, + .committedSize = d.committedSize, + .root = fr_to_wire(d.root), + .initialSize = d.initialSize, + .initialRoot = fr_to_wire(d.initialRoot), + .oldestHistoricBlock = d.oldestHistoricBlock, + .unfinalizedBlockHeight = d.unfinalizedBlockHeight, + .finalizedBlockHeight = d.finalizedBlockHeight }; +} + +inline bb::crypto::merkle_tree::TreeMeta tree_meta_from_wire(const wire::TreeMeta& w) +{ + return { w.name, + w.depth, + w.size, + w.committedSize, + fr_from_wire(w.root), + w.initialSize, + fr_from_wire(w.initialRoot), + w.oldestHistoricBlock, + w.unfinalizedBlockHeight, + w.finalizedBlockHeight }; +} + +inline wire::WorldStateDBStats world_state_db_stats_to_wire(const bb::world_state::WorldStateDBStats& d) +{ + return { .noteHashTreeStats = tree_db_stats_to_wire(d.noteHashTreeStats), + .messageTreeStats = tree_db_stats_to_wire(d.messageTreeStats), + .archiveTreeStats = tree_db_stats_to_wire(d.archiveTreeStats), + .publicDataTreeStats = tree_db_stats_to_wire(d.publicDataTreeStats), + .nullifierTreeStats = tree_db_stats_to_wire(d.nullifierTreeStats) }; +} + +inline bb::world_state::WorldStateDBStats world_state_db_stats_from_wire(const wire::WorldStateDBStats& w) +{ + return { tree_db_stats_from_wire(w.noteHashTreeStats), + tree_db_stats_from_wire(w.messageTreeStats), + tree_db_stats_from_wire(w.archiveTreeStats), + tree_db_stats_from_wire(w.publicDataTreeStats), + tree_db_stats_from_wire(w.nullifierTreeStats) }; +} + +inline wire::WorldStateMeta world_state_meta_to_wire(const bb::world_state::WorldStateMeta& d) +{ + return { .noteHashTreeMeta = tree_meta_to_wire(d.noteHashTreeMeta), + .messageTreeMeta = tree_meta_to_wire(d.messageTreeMeta), + .archiveTreeMeta = tree_meta_to_wire(d.archiveTreeMeta), + .publicDataTreeMeta = tree_meta_to_wire(d.publicDataTreeMeta), + .nullifierTreeMeta = tree_meta_to_wire(d.nullifierTreeMeta) }; +} + +inline bb::world_state::WorldStateMeta world_state_meta_from_wire(const wire::WorldStateMeta& w) +{ + return { tree_meta_from_wire(w.noteHashTreeMeta), + tree_meta_from_wire(w.messageTreeMeta), + tree_meta_from_wire(w.archiveTreeMeta), + tree_meta_from_wire(w.publicDataTreeMeta), + tree_meta_from_wire(w.nullifierTreeMeta) }; +} + +inline wire::WorldStateStatusSummary world_state_status_summary_to_wire( + const bb::world_state::WorldStateStatusSummary& d) +{ + return { .unfinalizedBlockNumber = d.unfinalizedBlockNumber, + .finalizedBlockNumber = d.finalizedBlockNumber, + .oldestHistoricalBlock = d.oldestHistoricalBlock, + .treesAreSynched = d.treesAreSynched }; +} + +inline bb::world_state::WorldStateStatusSummary world_state_status_summary_from_wire( + const wire::WorldStateStatusSummary& w) +{ + return { w.unfinalizedBlockNumber, w.finalizedBlockNumber, w.oldestHistoricalBlock, w.treesAreSynched }; +} + +inline wire::WorldStateStatusFull world_state_status_full_to_wire(const bb::world_state::WorldStateStatusFull& d) +{ + return { .summary = world_state_status_summary_to_wire(d.summary), + .dbStats = world_state_db_stats_to_wire(d.dbStats), + .meta = world_state_meta_to_wire(d.meta) }; +} + +inline bb::world_state::WorldStateStatusFull world_state_status_full_from_wire(const wire::WorldStateStatusFull& w) +{ + return { world_state_status_summary_from_wire(w.summary), + world_state_db_stats_from_wire(w.dbStats), + world_state_meta_from_wire(w.meta) }; +} + +} // namespace bb::wsdb diff --git a/barretenberg/cpp/src/barretenberg/wsdb_client/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/wsdb_client/CMakeLists.txt deleted file mode 100644 index f7647019cce0..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb_client/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -if(NOT(FUZZING) AND NOT(WASM)) - # WSDB IPC client library - bridges callers to aztec-wsdb over IPC. - # Implements LowLevelMerkleDBInterface so the AVM simulator can talk to a - # standalone aztec-wsdb process instead of an in-process WorldState. - add_library( - wsdb_client - STATIC - wsdb_ipc_merkle_db.cpp - ) - target_link_libraries( - wsdb_client - PUBLIC - barretenberg - wsdb_ipc_client - ipc - vm2_sim - ) - set_target_properties(wsdb_client PROPERTIES POSITION_INDEPENDENT_CODE ON) -endif() diff --git a/barretenberg/cpp/src/barretenberg/wsdb_client/wsdb_ipc_merkle_db.cpp b/barretenberg/cpp/src/barretenberg/wsdb_client/wsdb_ipc_merkle_db.cpp deleted file mode 100644 index a6c5c7df4fd5..000000000000 --- a/barretenberg/cpp/src/barretenberg/wsdb_client/wsdb_ipc_merkle_db.cpp +++ /dev/null @@ -1,231 +0,0 @@ -#include "barretenberg/wsdb_client/wsdb_ipc_merkle_db.hpp" -#include "barretenberg/aztec/aztec_constants.hpp" -#include "barretenberg/common/log.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/serialize/msgpack_impl.hpp" -#include "barretenberg/wsdb/wsdb_commands.hpp" - -namespace bb::wsdb_client { - -// Use avm2::simulation for interface types, but NOT world_state (it transitively -// imports crypto::merkle_tree which conflicts with avm2::simulation aliases). -using namespace avm2::simulation; - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -template std::vector WsdbIpcMerkleDB::serialize_to_msgpack(const T& value) -{ - msgpack::sbuffer buf; - msgpack::pack(buf, value); - return std::vector(buf.data(), buf.data() + buf.size()); -} - -template T WsdbIpcMerkleDB::deserialize_from_msgpack(const std::vector& bytes) -{ - auto unpacked = msgpack::unpack(reinterpret_cast(bytes.data()), bytes.size()); - T value; - unpacked.get().convert(value); - return value; -} - -// --------------------------------------------------------------------------- -// Constructor -// --------------------------------------------------------------------------- - -WsdbIpcMerkleDB::WsdbIpcMerkleDB(wsdb::WsdbIpcClient& client, world_state::WorldStateRevision revision) - : client_(client) - , revision_(revision) -{} - -// --------------------------------------------------------------------------- -// Tree roots -// --------------------------------------------------------------------------- - -avm2::TreeSnapshots WsdbIpcMerkleDB::get_tree_roots() const -{ - if (cached_tree_roots_.has_value()) { - return cached_tree_roots_.value(); - } - - auto l1_info = client_.get_tree_info( - wsdb::WsdbGetTreeInfo{ .treeId = MerkleTreeId::L1_TO_L2_MESSAGE_TREE, .revision = revision_ }); - auto nh_info = - client_.get_tree_info(wsdb::WsdbGetTreeInfo{ .treeId = MerkleTreeId::NOTE_HASH_TREE, .revision = revision_ }); - auto null_info = - client_.get_tree_info(wsdb::WsdbGetTreeInfo{ .treeId = MerkleTreeId::NULLIFIER_TREE, .revision = revision_ }); - auto pd_info = - client_.get_tree_info(wsdb::WsdbGetTreeInfo{ .treeId = MerkleTreeId::PUBLIC_DATA_TREE, .revision = revision_ }); - - avm2::TreeSnapshots snapshots{ - .l1_to_l2_message_tree = - avm2::AppendOnlyTreeSnapshot{ .root = l1_info.root, .next_available_leaf_index = l1_info.size }, - .note_hash_tree = - avm2::AppendOnlyTreeSnapshot{ .root = nh_info.root, .next_available_leaf_index = nh_info.size }, - .nullifier_tree = - avm2::AppendOnlyTreeSnapshot{ .root = null_info.root, .next_available_leaf_index = null_info.size }, - .public_data_tree = - avm2::AppendOnlyTreeSnapshot{ .root = pd_info.root, .next_available_leaf_index = pd_info.size }, - }; - cached_tree_roots_ = snapshots; - return snapshots; -} - -void WsdbIpcMerkleDB::invalidate_tree_roots_cache() -{ - cached_tree_roots_ = std::nullopt; -} - -// --------------------------------------------------------------------------- -// Query methods -// --------------------------------------------------------------------------- - -SiblingPath WsdbIpcMerkleDB::get_sibling_path(MerkleTreeId tree_id, index_t leaf_index) const -{ - auto resp = client_.get_sibling_path( - wsdb::WsdbGetSiblingPath{ .treeId = tree_id, .revision = revision_, .leafIndex = leaf_index }); - return resp.path; -} - -crypto::merkle_tree::GetLowIndexedLeafResponse WsdbIpcMerkleDB::get_low_indexed_leaf(MerkleTreeId tree_id, - const avm2::FF& value) const -{ - auto resp = client_.find_low_leaf(wsdb::WsdbFindLowLeaf{ .treeId = tree_id, .revision = revision_, .key = value }); - return GetLowIndexedLeafResponse(resp.alreadyPresent, resp.index); -} - -avm2::FF WsdbIpcMerkleDB::get_leaf_value(MerkleTreeId tree_id, index_t leaf_index) const -{ - auto resp = client_.get_leaf_value( - wsdb::WsdbGetLeafValue{ .treeId = tree_id, .revision = revision_, .leafIndex = leaf_index }); - if (!resp.value.has_value()) { - throw std::runtime_error("Invalid get_leaf_value request for tree " + - std::to_string(static_cast(tree_id)) + " index " + - std::to_string(leaf_index)); - } - return deserialize_from_msgpack(resp.value.value()); -} - -IndexedLeaf WsdbIpcMerkleDB::get_leaf_preimage_public_data_tree(index_t leaf_index) const -{ - auto resp = client_.get_leaf_preimage(wsdb::WsdbGetLeafPreimage{ - .treeId = MerkleTreeId::PUBLIC_DATA_TREE, .revision = revision_, .leafIndex = leaf_index }); - if (!resp.preimage.has_value()) { - throw std::runtime_error("Invalid get_leaf_preimage_public_data_tree request for index " + - std::to_string(leaf_index)); - } - return deserialize_from_msgpack>(resp.preimage.value()); -} - -IndexedLeaf WsdbIpcMerkleDB::get_leaf_preimage_nullifier_tree(index_t leaf_index) const -{ - auto resp = client_.get_leaf_preimage(wsdb::WsdbGetLeafPreimage{ - .treeId = MerkleTreeId::NULLIFIER_TREE, .revision = revision_, .leafIndex = leaf_index }); - if (!resp.preimage.has_value()) { - throw std::runtime_error("Invalid get_leaf_preimage_nullifier_tree request for index " + - std::to_string(leaf_index)); - } - return deserialize_from_msgpack>(resp.preimage.value()); -} - -// --------------------------------------------------------------------------- -// State modification methods -// --------------------------------------------------------------------------- - -SequentialInsertionResult WsdbIpcMerkleDB::insert_indexed_leaves_public_data_tree( - const PublicDataLeafValue& leaf_value) -{ - std::vector> serialized_leaves = { serialize_to_msgpack(leaf_value) }; - auto resp = client_.sequential_insert(wsdb::WsdbSequentialInsert{ - .treeId = MerkleTreeId::PUBLIC_DATA_TREE, .leaves = std::move(serialized_leaves), .forkId = revision_.forkId }); - invalidate_tree_roots_cache(); - return deserialize_from_msgpack>(resp.result); -} - -SequentialInsertionResult WsdbIpcMerkleDB::insert_indexed_leaves_nullifier_tree( - const NullifierLeafValue& leaf_value) -{ - std::vector> serialized_leaves = { serialize_to_msgpack(leaf_value) }; - auto resp = client_.sequential_insert(wsdb::WsdbSequentialInsert{ - .treeId = MerkleTreeId::NULLIFIER_TREE, .leaves = std::move(serialized_leaves), .forkId = revision_.forkId }); - invalidate_tree_roots_cache(); - return deserialize_from_msgpack>(resp.result); -} - -void WsdbIpcMerkleDB::append_leaves(MerkleTreeId tree_id, std::span leaves) -{ - std::vector> serialized_leaves; - serialized_leaves.reserve(leaves.size()); - for (const auto& leaf : leaves) { - serialized_leaves.push_back(serialize_to_msgpack(leaf)); - } - client_.append_leaves(wsdb::WsdbAppendLeaves{ - .treeId = tree_id, .leaves = std::move(serialized_leaves), .forkId = revision_.forkId }); - invalidate_tree_roots_cache(); -} - -void WsdbIpcMerkleDB::pad_tree(MerkleTreeId tree_id, size_t num_leaves) -{ - switch (tree_id) { - case MerkleTreeId::NULLIFIER_TREE: { - std::vector> padding_leaves; - padding_leaves.reserve(num_leaves); - auto empty_leaf = NullifierLeafValue::empty(); - for (size_t i = 0; i < num_leaves; i++) { - padding_leaves.push_back(serialize_to_msgpack(empty_leaf)); - } - client_.batch_insert(wsdb::WsdbBatchInsert{ .treeId = MerkleTreeId::NULLIFIER_TREE, - .leaves = std::move(padding_leaves), - .subtreeDepth = NULLIFIER_SUBTREE_HEIGHT, - .forkId = revision_.forkId }); - break; - } - case MerkleTreeId::NOTE_HASH_TREE: { - std::vector> padding_leaves; - padding_leaves.reserve(num_leaves); - auto zero = avm2::FF(0); - for (size_t i = 0; i < num_leaves; i++) { - padding_leaves.push_back(serialize_to_msgpack(zero)); - } - client_.append_leaves(wsdb::WsdbAppendLeaves{ - .treeId = MerkleTreeId::NOTE_HASH_TREE, .leaves = std::move(padding_leaves), .forkId = revision_.forkId }); - break; - } - default: - throw std::runtime_error("Padding not supported for tree " + std::to_string(static_cast(tree_id))); - } - invalidate_tree_roots_cache(); -} - -// --------------------------------------------------------------------------- -// Checkpoint methods -// --------------------------------------------------------------------------- - -void WsdbIpcMerkleDB::create_checkpoint() -{ - client_.create_checkpoint(wsdb::WsdbCreateCheckpoint{ .forkId = revision_.forkId }); - uint32_t current_id = checkpoint_stack_.top(); - checkpoint_stack_.push(current_id + 1); -} - -void WsdbIpcMerkleDB::commit_checkpoint() -{ - client_.commit_checkpoint(wsdb::WsdbCommitCheckpoint{ .forkId = revision_.forkId }); - invalidate_tree_roots_cache(); - checkpoint_stack_.pop(); -} - -void WsdbIpcMerkleDB::revert_checkpoint() -{ - client_.revert_checkpoint(wsdb::WsdbRevertCheckpoint{ .forkId = revision_.forkId }); - invalidate_tree_roots_cache(); - checkpoint_stack_.pop(); -} - -uint32_t WsdbIpcMerkleDB::get_checkpoint_id() const -{ - return checkpoint_stack_.top(); -} - -} // namespace bb::wsdb_client diff --git a/barretenberg/ts/.gitignore b/barretenberg/ts/.gitignore index c4cb49ce017e..cc254d3c8714 100644 --- a/barretenberg/ts/.gitignore +++ b/barretenberg/ts/.gitignore @@ -12,4 +12,3 @@ package # Generated files src/cbind/generated/ -src/aztec-wsdb/generated/ diff --git a/barretenberg/ts/package.json b/barretenberg/ts/package.json index 8f537d1f3750..671d34b7efc3 100644 --- a/barretenberg/ts/package.json +++ b/barretenberg/ts/package.json @@ -12,9 +12,6 @@ "browser": "./dest/browser/index.js", "default": "./dest/node/index.js" }, - "./aztec-wsdb": { - "default": "./dest/node/aztec-wsdb/index.js" - }, "./platform": { "default": "./dest/node/bb_backends/node/platform.js" } @@ -29,14 +26,14 @@ "README.md" ], "scripts": { - "clean": "rm -rf ./dest .tsbuildinfo .tsbuildinfo.cjs ./src/cbind/generated ./src/aztec-wsdb/generated", + "clean": "rm -rf ./dest .tsbuildinfo .tsbuildinfo.cjs ./src/cbind/generated", "build": "yarn clean && yarn generate && yarn build:wasm && yarn build:native && yarn build:esm && yarn build:cjs && yarn build:browser", "build:wasm": "./scripts/copy_wasm.sh", "build:native": "./scripts/copy_native.sh", "build:esm": "tsgo -b tsconfig.esm.json && chmod +x ./dest/node/bin/index.js", "build:cjs": "tsgo -b tsconfig.cjs.json && ./scripts/cjs_postprocess.sh", "build:browser": "tsgo -b tsconfig.browser.json && ./scripts/browser_postprocess.sh", - "generate": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 ts-node src/cbind/generate.ts && npx tsx src/aztec-wsdb/generate.ts", + "generate": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 ts-node src/cbind/generate.ts", "formatting": "prettier --check ./src && eslint --max-warnings 0 ./src", "formatting:fix": "prettier -w ./src", "test": "NODE_OPTIONS='--loader ts-node/esm' NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests", diff --git a/barretenberg/ts/scripts/copy_native.sh b/barretenberg/ts/scripts/copy_native.sh index 07ed0065f37f..3fdd7aa2f10e 100755 --- a/barretenberg/ts/scripts/copy_native.sh +++ b/barretenberg/ts/scripts/copy_native.sh @@ -9,11 +9,10 @@ cd $(dirname $0)/.. target="$(arch)-$(os)" if [ "${BUILD_CPP:-0}" -eq 1 ]; then - ../cpp/bootstrap.sh build_preset clang20 --target bb --target nodejs_module --target aztec-wsdb + ../cpp/bootstrap.sh build_preset clang20 --target bb --target nodejs_module fi mkdir -p ./build/$target cp ../cpp/build/bin/bb ./build/$target -cp ../cpp/build/bin/aztec-wsdb ./build/$target cp ../cpp/build/lib/nodejs_module.node ./build/$target diff --git a/barretenberg/ts/src/aztec-wsdb/generate.ts b/barretenberg/ts/src/aztec-wsdb/generate.ts deleted file mode 100644 index 510a0179eb54..000000000000 --- a/barretenberg/ts/src/aztec-wsdb/generate.ts +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Code generation for aztec-wsdb TypeScript bindings. - * - * Uses the same codegen pipeline as bb.js but targets the aztec-wsdb binary schema. - * Run: npx tsx src/aztec-wsdb/generate.ts - */ - -import { writeFileSync, mkdirSync } from 'fs'; -import { dirname, join } from 'path'; -import { exec } from 'child_process'; -import { promisify } from 'util'; -import { fileURLToPath } from 'url'; -import { SchemaVisitor } from '../cbind/schema_visitor.js'; -import { TypeScriptCodegen } from '../cbind/typescript_codegen.js'; -import { CppCodegen } from '../cbind/cpp_codegen.js'; - -const execAsync = promisify(exec); - -// @ts-ignore -const __dirname = dirname(fileURLToPath(import.meta.url)); - -async function generate() { - const wsdbBuildPath = process.env.WSDB_BINARY_PATH || join(__dirname, '../../../cpp/build/bin/aztec-wsdb'); - - // Get schema from aztec-wsdb - console.log('Fetching msgpack schema from aztec-wsdb...'); - const { stdout } = await execAsync(`${wsdbBuildPath} msgpack schema`); - const schema = JSON.parse(stdout.trim()); - - if (!schema.commands || !schema.responses) { - throw new Error('Invalid schema: missing commands or responses'); - } - - // Compile schema using the shared visitor - console.log('Compiling schema...'); - const visitor = new SchemaVisitor(); - const compiled = visitor.visit(schema.commands, schema.responses); - - console.log(`Found ${compiled.commands.length} commands, ${compiled.structs.size} structs\n`); - - // Generate TypeScript bindings - const tsGen = new TypeScriptCodegen(); - - // Generate C++ IPC client - const cppGen = new CppCodegen({ - namespace: 'bb::wsdb', - prefix: 'Wsdb', - executeHeader: 'barretenberg/wsdb/wsdb_execute.hpp', - commandsHeader: 'barretenberg/wsdb/wsdb_commands.hpp', - }); - - const files = [ - { path: 'generated/api_types.ts', content: tsGen.generateTypes(compiled) }, - { path: 'generated/async.ts', content: tsGen.generateAsyncApi(compiled) }, - { path: '../../../cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.hpp', content: cppGen.generateHeader(compiled) }, - { path: '../../../cpp/src/barretenberg/wsdb/wsdb_ipc_client_generated.cpp', content: cppGen.generateImpl(compiled) }, - ]; - - // Ensure output directory exists - const outputDir = join(__dirname, 'generated'); - mkdirSync(outputDir, { recursive: true }); - - const cppFiles: string[] = []; - for (const file of files) { - const outputPath = join(__dirname, file.path); - mkdirSync(dirname(outputPath), { recursive: true }); - writeFileSync(outputPath, file.content); - console.log(` ${outputPath}`); - if (file.path.endsWith('.hpp') || file.path.endsWith('.cpp')) { - cppFiles.push(outputPath); - } - } - - // Run clang-format on generated C++ files - if (cppFiles.length > 0) { - try { - await execAsync(`clang-format-20 -i ${cppFiles.join(' ')}`); - } catch { - // clang-format-20 may not be available in all environments - } - } - - console.log('\nWsdb codegen complete.'); -} - -generate().catch(error => { - console.error('Generation failed:', error); - process.exit(1); -}); diff --git a/barretenberg/ts/src/aztec-wsdb/index.ts b/barretenberg/ts/src/aztec-wsdb/index.ts deleted file mode 100644 index 6260156cdeb8..000000000000 --- a/barretenberg/ts/src/aztec-wsdb/index.ts +++ /dev/null @@ -1,449 +0,0 @@ -/** - * aztec-wsdb TypeScript client. - * - * Spawns the aztec-wsdb binary and communicates via Unix Domain Socket or - * shared memory IPC. Implements IMsgpackBackendAsync so it can be used with - * the generated WsdbAsyncApi. - */ - -import { spawn, ChildProcess } from 'child_process'; -import { createRequire } from 'module'; -import * as net from 'net'; -import * as fs from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { IMsgpackBackendAsync } from '../bb_backends/interface.js'; -import { findNapiBinary, findPackageRoot } from '../bb_backends/node/platform.js'; -import { threadId } from 'worker_threads'; - -let instanceCounter = 0; - -export interface WsdbOptions { - /** Path to the aztec-wsdb binary */ - binaryPath: string; - /** Data directory for LMDB stores */ - dataDir: string; - /** Tree heights map: { treeId: height } */ - treeHeights?: Record; - /** Tree prefill sizes: { treeId: size } */ - treePrefill?: Record; - /** LMDB map sizes in KB: { treeId: sizeKb } */ - mapSizes?: Record; - /** Thread pool size */ - threads?: number; - /** Initial header generator point */ - initialHeaderGeneratorPoint?: number; - /** Prefilled public data as array of [slotBuffer, valueBuffer] pairs */ - prefilledPublicData?: Array<[Buffer, Buffer]>; - /** Genesis block timestamp (must match TS-side buildInitialHeader) */ - genesisTimestamp?: number; - /** Optional logger function */ - logger?: (msg: string) => void; - /** Use shared memory instead of UDS for IPC (lower latency). */ - useShm?: boolean; - /** Path to NAPI binary (required when useShm=true, auto-detected if omitted). */ - napiPath?: string; -} - -/** - * Formats a Record as a CLI-friendly JSON string: {0:1024,1:2048,...} - */ -function formatMap(map: Record | undefined): string | undefined { - if (!map || Object.keys(map).length === 0) { - return undefined; - } - const entries = Object.entries(map).map(([k, v]) => `${k}:${v}`); - return `{${entries.join(',')}}`; -} - -/** Build CLI args common to both UDS and SHM modes. */ -function buildWsdbArgs(inputPath: string, options: WsdbOptions, threads: number): string[] { - const args = [ - 'msgpack', - 'run', - '--input', - inputPath, - '--data-dir', - options.dataDir, - '--threads', - threads.toString(), - ]; - - if (options.initialHeaderGeneratorPoint !== undefined) { - args.push('--initial-header-generator-point', options.initialHeaderGeneratorPoint.toString()); - } - - const treeHeightsStr = formatMap(options.treeHeights); - if (treeHeightsStr) { - args.push('--tree-heights', treeHeightsStr); - } - - const treePrefillStr = formatMap(options.treePrefill); - if (treePrefillStr) { - args.push('--tree-prefill', treePrefillStr); - } - - const mapSizesStr = formatMap(options.mapSizes); - if (mapSizesStr) { - args.push('--map-sizes', mapSizesStr); - } - - if (options.prefilledPublicData && options.prefilledPublicData.length > 0) { - const pairs = options.prefilledPublicData.map(([slot, value]) => [slot.toString('hex'), value.toString('hex')]); - args.push('--prefilled-public-data', JSON.stringify(pairs)); - } - - if (options.genesisTimestamp !== undefined && options.genesisTimestamp !== 0) { - args.push('--genesis-timestamp', options.genesisTimestamp.toString()); - } - - return args; -} - -export { AsyncApi } from './generated/async.js'; -export * from './generated/api_types.js'; - -/** - * IPC backend that communicates with the aztec-wsdb binary. - * Supports both Unix Domain Socket and shared memory transports. - */ -export class WsdbBackend implements IMsgpackBackendAsync { - private process: ChildProcess; - /** For UDS mode */ - private socket: net.Socket | null = null; - /** For SHM mode */ - private shmClient: any = null; - private inputPath: string; - private useShm: boolean; - private connectionPromise: Promise; - private connectionTimeout: NodeJS.Timeout | null = null; - /** Resolves when the child process exits (for clean destroy). */ - private processExitPromise: Promise; - - private pendingCallbacks: Array<{ - resolve: (data: Uint8Array) => void; - reject: (error: Error) => void; - }> = []; - - // State machine for reading UDS responses - private readingLength: boolean = true; - private lengthBuffer: Buffer = Buffer.alloc(4); - private lengthBytesRead: number = 0; - private responseLength: number = 0; - private responseBuffer: Buffer | null = null; - private responseBytesRead: number = 0; - - constructor(options: WsdbOptions) { - this.useShm = options.useShm ?? false; - const instanceId = `wsdb-${process.pid}-${threadId}-${instanceCounter++}`; - - if (this.useShm) { - // SHM mode: use shared memory name (no path, just a name for /dev/shm/) - this.inputPath = `${instanceId}.shm`; - } else { - // UDS mode: use socket file in tmpdir - this.inputPath = path.join(os.tmpdir(), `${instanceId}.sock`); - if (fs.existsSync(this.inputPath)) { - fs.unlinkSync(this.inputPath); - } - } - - let connectionResolve: (() => void) | null = null; - let connectionReject: ((error: Error) => void) | null = null; - - this.connectionPromise = new Promise((resolve, reject) => { - connectionResolve = resolve; - connectionReject = reject; - }); - - const threads = options.threads ?? Math.min(16, os.cpus().length); - const env = { ...process.env, HARDWARE_CONCURRENCY: threads.toString() }; - - const args = buildWsdbArgs(this.inputPath, options, threads); - - // SHM mode needs larger ring buffers for pipelining - if (this.useShm) { - args.push('--request-ring-size', `${1024 * 1024 * 4}`); - args.push('--response-ring-size', `${1024 * 1024 * 4}`); - } - - this.process = spawn(options.binaryPath, args, { - stdio: ['ignore', options.logger ? 'pipe' : 'ignore', options.logger ? 'pipe' : 'ignore'], - env, - }); - - if (options.logger) { - const logger = options.logger; - if (this.process.stdout) { - this.process.stdout.on('data', (data: Buffer) => logger(`[wsdb stdout] ${data.toString().trimEnd()}`)); - } - if (this.process.stderr) { - this.process.stderr.on('data', (data: Buffer) => logger(`[wsdb stderr] ${data.toString().trimEnd()}`)); - } - } - - this.process.on('error', (err: Error) => { - for (const cb of this.pendingCallbacks) { - cb.reject(new Error(`aztec-wsdb process error: ${err.message}`)); - } - this.pendingCallbacks = []; - connectionReject?.(err); - }); - - this.processExitPromise = new Promise(resolve => { - this.process.on('exit', (code: number | null) => { - const error = new Error(`aztec-wsdb process exited with code ${code}`); - for (const cb of this.pendingCallbacks) { - cb.reject(error); - } - this.pendingCallbacks = []; - resolve(); - }); - }); - - if (this.useShm) { - this.connectShm(connectionResolve!, connectionReject!, options.napiPath); - } else { - this.connectUdsPoll(connectionResolve!, connectionReject!); - } - } - - /** Returns the IPC path for the running wsdb server (for other IPC clients to connect). */ - getSocketPath(): string { - return this.inputPath; - } - - /** Wait until the backend is connected and ready to accept commands. */ - waitUntilReady(): Promise { - return this.connectionPromise; - } - - // ——— SHM connection ——— - - private connectShm( - resolve: () => void, - reject: (error: Error) => void, - napiPath?: string, - ) { - const shmName = this.inputPath.replace(/\.shm$/, ''); - const addonPath = findNapiBinary(napiPath); - if (!addonPath) { - reject(new Error('NAPI binary not found — required for shared memory mode')); - return; - } - - let addon: any; - try { - const require = createRequire(findPackageRoot()!); - addon = require(addonPath); - } catch (err: any) { - reject(new Error(`Failed to load NAPI module for SHM: ${err.message}`)); - return; - } - - // Retry connecting until wsdb creates the shared memory region - const retryInterval = 100; - const maxAttempts = 100; // 10s total - let attempt = 0; - - const tryConnect = () => { - attempt++; - try { - // TS backend is client 0 in the MPSC SHM system (AVM is client 1) - this.shmClient = new addon.MsgpackClientAsync(shmName, 0); - // Register response callback - this.shmClient.setResponseCallback((responseBuffer: Buffer) => { - const callback = this.pendingCallbacks.shift(); - if (callback) { - callback.resolve(new Uint8Array(responseBuffer)); - } - if (this.pendingCallbacks.length === 0) { - this.shmClient.release(); - } - }); - resolve(); - } catch (e: any) { - if (attempt >= maxAttempts) { - reject(new Error(`Timeout connecting to wsdb shared memory after ${maxAttempts * retryInterval}ms: ${e?.message ?? e}`)); - } else { - this.connectionTimeout = setTimeout(tryConnect, retryInterval); - } - } - }; - - this.connectionTimeout = setTimeout(tryConnect, retryInterval); - } - - // ——— UDS connection ——— - - private connectUdsPoll(resolve: () => void, reject: (error: Error) => void) { - const pollInterval = 50; - const maxWait = 10000; - let waited = 0; - - const poll = () => { - if (fs.existsSync(this.inputPath)) { - this.connectUds(resolve, reject); - } else if (waited >= maxWait) { - reject(new Error(`Timeout waiting for aztec-wsdb socket at ${this.inputPath}`)); - } else { - waited += pollInterval; - this.connectionTimeout = setTimeout(poll, pollInterval); - } - }; - - this.connectionTimeout = setTimeout(poll, pollInterval); - } - - private connectUds(resolve: () => void, reject: (error: Error) => void) { - this.socket = net.createConnection(this.inputPath); - - this.socket.on('connect', () => { - resolve(); - }); - - this.socket.on('error', (err: Error) => { - reject(err); - for (const cb of this.pendingCallbacks) { - cb.reject(err); - } - this.pendingCallbacks = []; - }); - - this.socket.on('data', (chunk: Buffer) => { - this.handleData(chunk); - }); - - this.socket.on('close', () => { - const error = new Error('aztec-wsdb socket closed'); - for (const cb of this.pendingCallbacks) { - cb.reject(error); - } - this.pendingCallbacks = []; - }); - } - - private handleData(chunk: Buffer) { - let offset = 0; - - while (offset < chunk.length) { - if (this.readingLength) { - const bytesNeeded = 4 - this.lengthBytesRead; - const bytesAvailable = chunk.length - offset; - const bytesToCopy = Math.min(bytesNeeded, bytesAvailable); - - chunk.copy(this.lengthBuffer, this.lengthBytesRead, offset, offset + bytesToCopy); - this.lengthBytesRead += bytesToCopy; - offset += bytesToCopy; - - if (this.lengthBytesRead === 4) { - this.responseLength = this.lengthBuffer.readUInt32LE(0); - this.responseBuffer = Buffer.alloc(this.responseLength); - this.responseBytesRead = 0; - this.readingLength = false; - } - } else { - const bytesNeeded = this.responseLength - this.responseBytesRead; - const bytesAvailable = chunk.length - offset; - const bytesToCopy = Math.min(bytesNeeded, bytesAvailable); - - chunk.copy(this.responseBuffer!, this.responseBytesRead, offset, offset + bytesToCopy); - this.responseBytesRead += bytesToCopy; - offset += bytesToCopy; - - if (this.responseBytesRead === this.responseLength) { - const callback = this.pendingCallbacks.shift(); - if (callback) { - callback.resolve(new Uint8Array(this.responseBuffer!)); - } - - // Reset state for next message - this.readingLength = true; - this.lengthBytesRead = 0; - this.responseBuffer = null; - } - } - } - } - - // ——— Unified call/destroy ——— - - async call(inputBuffer: Uint8Array): Promise { - await this.connectionPromise; - - if (this.useShm) { - return new Promise((resolve, reject) => { - if (this.pendingCallbacks.length === 0) { - this.shmClient.acquire(); - } - this.pendingCallbacks.push({ resolve, reject }); - try { - this.shmClient.call(Buffer.from(inputBuffer)); - } catch (err: any) { - this.pendingCallbacks.pop(); - if (this.pendingCallbacks.length === 0) { - this.shmClient.release(); - } - reject(new Error(`SHM call failed: ${err.message}`)); - } - }); - } - - // UDS mode - return new Promise((resolve, reject) => { - this.pendingCallbacks.push({ resolve, reject }); - - const lengthBuf = Buffer.alloc(4); - lengthBuf.writeUInt32LE(inputBuffer.length, 0); - - this.socket!.write(lengthBuf); - this.socket!.write(Buffer.from(inputBuffer)); - }); - } - - async destroy(): Promise { - // Suppress any pending connection promise rejection to avoid unhandled rejections - // when destroying before the IPC connection is established. - this.connectionPromise?.catch(() => {}); - - if (this.connectionTimeout) { - clearTimeout(this.connectionTimeout); - this.connectionTimeout = null; - } - - if (this.socket) { - this.socket.destroy(); - this.socket = null; - } - - if (this.process && this.process.exitCode === null) { - this.process.kill('SIGTERM'); - } - await this.processExitPromise; - - // Clean up stdio streams and remove all listeners to allow the event loop to exit. - if (this.process) { - this.process.stdout?.destroy(); - this.process.stderr?.destroy(); - this.process.removeAllListeners(); - } - - // Clean up socket/shm files - try { - if (!this.useShm && fs.existsSync(this.inputPath)) { - fs.unlinkSync(this.inputPath); - } - if (this.useShm) { - const shmName = this.inputPath.replace(/\.shm$/, ''); - for (const suffix of ['_request', '_response']) { - const shmPath = `/dev/shm/${shmName}${suffix}`; - if (fs.existsSync(shmPath)) { - fs.unlinkSync(shmPath); - } - } - } - } catch { - // Ignore cleanup errors - } - } -} diff --git a/bootstrap.sh b/bootstrap.sh index f2639d4985db..81e3909d28af 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -537,6 +537,7 @@ function release { projects=( barretenberg/cpp ipc-runtime + wsdb barretenberg/ts barretenberg/rust noir diff --git a/ci3/release_prep_package_json b/ci3/release_prep_package_json index 70a727502687..53e88d7a8b38 100755 --- a/ci3/release_prep_package_json +++ b/ci3/release_prep_package_json @@ -9,7 +9,7 @@ jq --arg v $version '.version = $v' package.json >$tmp && mv $tmp package.json # We update every category of dependency. # While we don't strictly need to update devDependencies, 'workspace:^' is not a valid URL in npm. -for deps in dependencies devDependencies peerDependencies; do +for deps in dependencies devDependencies peerDependencies optionalDependencies; do # Update each dependency @aztec package version in package.json. for pkg in $(jq --raw-output "(.$deps // {}) | keys[] | select(contains(\"@aztec/\"))" package.json); do jq --arg v $version ".$deps[\"$pkg\"] = \$v" package.json >$tmp diff --git a/ipc-codegen/echo_example/ts_package/.gitignore b/ipc-codegen/echo_example/ts_package/.gitignore index 2f10a5bc825a..79f99d29e23c 100644 --- a/ipc-codegen/echo_example/ts_package/.gitignore +++ b/ipc-codegen/echo_example/ts_package/.gitignore @@ -8,4 +8,5 @@ tsconfig.json src/generated/ src/index.ts src/platform.ts +src/bin.ts scripts/prepare_arch_packages.sh diff --git a/ipc-codegen/src/generate.ts b/ipc-codegen/src/generate.ts index 79ef1a15e1b6..e6d146522013 100644 --- a/ipc-codegen/src/generate.ts +++ b/ipc-codegen/src/generate.ts @@ -418,6 +418,9 @@ function generate(args: Args) { writePackage("README.md", packageGen.generateReadme()); writePackage("src/index.ts", packageGen.generateIndex()); writePackage("src/platform.ts", packageGen.generatePlatform()); + if (binaryName) { + writePackage("src/bin.ts", packageGen.generateBin()); + } for (const manifest of packageGen.generateArchPackageManifests()) { writePackage(manifest.path, manifest.content); } diff --git a/ipc-codegen/src/typescript_package_codegen.ts b/ipc-codegen/src/typescript_package_codegen.ts index 18fa48b1c1f1..971d87cfaa1a 100644 --- a/ipc-codegen/src/typescript_package_codegen.ts +++ b/ipc-codegen/src/typescript_package_codegen.ts @@ -71,6 +71,13 @@ export class TypeScriptPackageCodegen { name: this.opts.packageName, version: "0.1.0", type: "module", + // Put the native binary on the consumer's PATH. The bin target is a JS + // launcher (dest/bin.js), not the binary itself: the binary ships in the + // per-arch optional-dependency packages and is resolved at runtime, and + // npm only links the main package's own bin onto PATH. + ...(this.opts.binaryName + ? { bin: { [this.opts.binaryName]: "./dest/bin.js" } } + : {}), exports: { ".": { types: "./dest/index.d.ts", @@ -95,6 +102,30 @@ export class TypeScriptPackageCodegen { return JSON.stringify(pkg, null, 2) + "\n"; } + generateBin(): string { + const findBinary = binaryFinderName(this.opts.prefix); + return `#!/usr/bin/env node +import { spawnSync } from 'node:child_process'; +import { ${findBinary} } from './platform.js'; + +const binaryPath = ${findBinary}(); +if (!binaryPath) { + console.error( + "${this.opts.binaryName}: native binary not found. Install the matching " + + "'${this.opts.packageName}-' package, set ${this.opts.binaryEnvVar}, or pass its path.", + ); + process.exit(1); +} + +const result = spawnSync(binaryPath, process.argv.slice(2), { stdio: 'inherit' }); +if (result.error) { + console.error(result.error.message); + process.exit(1); +} +process.exit(result.status ?? 1); +`; + } + generateArchPackageJson(suffix: string, os: string, cpu: string): string { const pkg = { name: `${this.opts.packageName}-${suffix}`, diff --git a/wsdb/.gitignore b/wsdb/.gitignore new file mode 100644 index 000000000000..5e538a0ae7de --- /dev/null +++ b/wsdb/.gitignore @@ -0,0 +1,9 @@ +ts/ +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/sdks +!.yarn/versions +node_modules +*.log diff --git a/wsdb/.rebuild_patterns b/wsdb/.rebuild_patterns new file mode 100644 index 000000000000..1652d3c01a63 --- /dev/null +++ b/wsdb/.rebuild_patterns @@ -0,0 +1,6 @@ +^wsdb/bootstrap\.sh$ +^wsdb/package\.json$ +^wsdb/\.rebuild_patterns$ +^ipc-codegen/src/.*\.ts$ +^ipc-codegen/templates/ +^barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema\.json$ diff --git a/wsdb/.yarnrc.yml b/wsdb/.yarnrc.yml new file mode 100644 index 000000000000..3186f3f0795a --- /dev/null +++ b/wsdb/.yarnrc.yml @@ -0,0 +1 @@ +nodeLinker: node-modules diff --git a/wsdb/bootstrap.sh b/wsdb/bootstrap.sh new file mode 100755 index 000000000000..577cc1e5779e --- /dev/null +++ b/wsdb/bootstrap.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +source $(git rev-parse --show-toplevel)/ci3/source_bootstrap + +ROOT=$(git rev-parse --show-toplevel) +WSDB_BINARY=aztec-wsdb + +hash=$(hash_str \ + $(../barretenberg/cpp/bootstrap.sh hash) \ + $(../ipc-runtime/bootstrap.sh hash) \ + $(cache_content_hash .rebuild_patterns)) + +function generate_ts_package { + node --experimental-strip-types --experimental-transform-types --no-warnings \ + "$ROOT/ipc-codegen/src/generate.ts" \ + --schema "$ROOT/barretenberg/cpp/src/barretenberg/wsdb/wsdb_schema.jsonc" \ + --lang ts \ + --client \ + --out "$ROOT/wsdb/ts/src/generated" \ + --package "$ROOT/wsdb/ts" \ + --package-name @aztec/wsdb \ + --binary-name "$WSDB_BINARY" \ + --package-transports uds,shm \ + --package-ipc-path-args 'msgpack,run,--input,{path}' +} + +function copy_native { + local target_dir="ts/build/$(arch)-$(os)" + mkdir -p "$target_dir" + cp "$ROOT/barretenberg/cpp/build/bin/$WSDB_BINARY" "$target_dir/$WSDB_BINARY" +} + +function copy_cross { + if [ -n "${1:-}" ]; then + local cross_arch="$1" + mkdir -p "ts/build/$cross_arch" + cp "$ROOT/barretenberg/cpp/build-$cross_arch/bin/$WSDB_BINARY" "ts/build/$cross_arch/$WSDB_BINARY" + elif semver check "${REF_NAME:-}" && [ "$(arch)" == "amd64" ]; then + for cross_arch in arm64-linux amd64-macos arm64-macos; do + mkdir -p "ts/build/$cross_arch" + cp "$ROOT/barretenberg/cpp/build-$cross_arch/bin/$WSDB_BINARY" "ts/build/$cross_arch/$WSDB_BINARY" + done + else + echo "This task is expected to be run with an explicit arch or in an x86 release context." + fi +} + +function build { + echo_header "wsdb build" + generate_ts_package + copy_native + npm_install_deps + yarn build + (cd ts && ./scripts/prepare_arch_packages.sh "$(arch)-$(os)=build/$(arch)-$(os)/$WSDB_BINARY") +} + +function clean { + rm -rf ts node_modules +} + +function release { + generate_ts_package + copy_native + copy_cross + npm_install_deps + yarn build + (cd ts && ./scripts/prepare_arch_packages.sh) + for package_dir in ts/packages/*; do + (cd "$package_dir" && retry "deploy_npm ${REF_NAME#v}") + done + (cd ts && retry "deploy_npm ${REF_NAME#v}") +} + +export -f generate_ts_package copy_native copy_cross build clean release + +case "$cmd" in + "") + build + ;; + "hash") + echo "$hash" + ;; + *) + default_cmd_handler "$@" + ;; +esac diff --git a/wsdb/package.json b/wsdb/package.json new file mode 100644 index 000000000000..d2a88a1534f9 --- /dev/null +++ b/wsdb/package.json @@ -0,0 +1,16 @@ +{ + "name": "@aztec/wsdb-packages", + "packageManager": "yarn@4.13.0", + "private": true, + "scripts": { + "build": "yarn workspace @aztec/wsdb build", + "clean": "yarn workspaces foreach -A -p -v run clean" + }, + "workspaces": [ + "ts", + "ts/packages/*" + ], + "resolutions": { + "@aztec/ipc-runtime": "portal:../ipc-runtime/ts" + } +} diff --git a/wsdb/yarn.lock b/wsdb/yarn.lock new file mode 100644 index 000000000000..d6301b5a25d7 --- /dev/null +++ b/wsdb/yarn.lock @@ -0,0 +1,396 @@ +# This file is generated by running "yarn install" inside your project. +# Manual changes might be lost - proceed with caution! + +__metadata: + version: 8 + cacheKey: 10c0 + +"@aztec/ipc-runtime@portal:../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A.": + version: 0.0.0-use.local + resolution: "@aztec/ipc-runtime@portal:../ipc-runtime/ts::locator=%40aztec%2Fwsdb-packages%40workspace%3A." + languageName: node + linkType: soft + +"@aztec/wsdb-darwin-arm64@npm:0.1.0, @aztec/wsdb-darwin-arm64@workspace:ts/packages/wsdb-darwin-arm64": + version: 0.0.0-use.local + resolution: "@aztec/wsdb-darwin-arm64@workspace:ts/packages/wsdb-darwin-arm64" + languageName: unknown + linkType: soft + +"@aztec/wsdb-darwin-x64@npm:0.1.0, @aztec/wsdb-darwin-x64@workspace:ts/packages/wsdb-darwin-x64": + version: 0.0.0-use.local + resolution: "@aztec/wsdb-darwin-x64@workspace:ts/packages/wsdb-darwin-x64" + languageName: unknown + linkType: soft + +"@aztec/wsdb-linux-arm64@npm:0.1.0, @aztec/wsdb-linux-arm64@workspace:ts/packages/wsdb-linux-arm64": + version: 0.0.0-use.local + resolution: "@aztec/wsdb-linux-arm64@workspace:ts/packages/wsdb-linux-arm64" + languageName: unknown + linkType: soft + +"@aztec/wsdb-linux-x64@npm:0.1.0, @aztec/wsdb-linux-x64@workspace:ts/packages/wsdb-linux-x64": + version: 0.0.0-use.local + resolution: "@aztec/wsdb-linux-x64@workspace:ts/packages/wsdb-linux-x64" + languageName: unknown + linkType: soft + +"@aztec/wsdb-packages@workspace:.": + version: 0.0.0-use.local + resolution: "@aztec/wsdb-packages@workspace:." + languageName: unknown + linkType: soft + +"@aztec/wsdb@workspace:ts": + version: 0.0.0-use.local + resolution: "@aztec/wsdb@workspace:ts" + dependencies: + "@aztec/ipc-runtime": "@aztec/ipc-runtime" + "@aztec/wsdb-darwin-arm64": "npm:0.1.0" + "@aztec/wsdb-darwin-x64": "npm:0.1.0" + "@aztec/wsdb-linux-arm64": "npm:0.1.0" + "@aztec/wsdb-linux-x64": "npm:0.1.0" + "@types/node": "npm:^22.15.17" + msgpackr: "npm:^1.11.2" + tslib: "npm:^2.4.0" + typescript: "npm:^5.3.3" + dependenciesMeta: + "@aztec/wsdb-darwin-arm64": + optional: true + "@aztec/wsdb-darwin-x64": + optional: true + "@aztec/wsdb-linux-arm64": + optional: true + "@aztec/wsdb-linux-x64": + optional: true + languageName: unknown + linkType: soft + +"@isaacs/fs-minipass@npm:^4.0.0": + version: 4.0.1 + resolution: "@isaacs/fs-minipass@npm:4.0.1" + dependencies: + minipass: "npm:^7.0.4" + checksum: 10c0/c25b6dc1598790d5b55c0947a9b7d111cfa92594db5296c3b907e2f533c033666f692a3939eadac17b1c7c40d362d0b0635dc874cbfe3e70db7c2b07cc97a5d2 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-arm64@npm:3.0.4" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-darwin-x64@npm:3.0.4" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm64@npm:3.0.4" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-arm@npm:3.0.4" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-linux-x64@npm:3.0.4" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4": + version: 3.0.4 + resolution: "@msgpackr-extract/msgpackr-extract-win32-x64@npm:3.0.4" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@types/node@npm:^22.15.17": + version: 22.19.20 + resolution: "@types/node@npm:22.19.20" + dependencies: + undici-types: "npm:~6.21.0" + checksum: 10c0/933d4466f1a498dd7c8e173af7265a53e9d410cab4a827ccc348414d5065a9a40ba7a7c994a71b3ee651188111db3b43573b830dc30a61a7489f3e6efc537bf7 + languageName: node + linkType: hard + +"abbrev@npm:^4.0.0": + version: 4.0.0 + resolution: "abbrev@npm:4.0.0" + checksum: 10c0/b4cc16935235e80702fc90192e349e32f8ef0ed151ef506aa78c81a7c455ec18375c4125414b99f84b2e055199d66383e787675f0bcd87da7a4dbd59f9eac1d5 + languageName: node + linkType: hard + +"chownr@npm:^3.0.0": + version: 3.0.0 + resolution: "chownr@npm:3.0.0" + checksum: 10c0/43925b87700f7e3893296c8e9c56cc58f926411cce3a6e5898136daaf08f08b9a8eb76d37d3267e707d0dcc17aed2e2ebdf5848c0c3ce95cf910a919935c1b10 + languageName: node + linkType: hard + +"detect-libc@npm:^2.0.1": + version: 2.1.2 + resolution: "detect-libc@npm:2.1.2" + checksum: 10c0/acc675c29a5649fa1fb6e255f993b8ee829e510b6b56b0910666949c80c364738833417d0edb5f90e4e46be17228b0f2b66a010513984e18b15deeeac49369c4 + languageName: node + linkType: hard + +"env-paths@npm:^2.2.0": + version: 2.2.1 + resolution: "env-paths@npm:2.2.1" + checksum: 10c0/285325677bf00e30845e330eec32894f5105529db97496ee3f598478e50f008c5352a41a30e5e72ec9de8a542b5a570b85699cd63bd2bc646dbcb9f311d83bc4 + languageName: node + linkType: hard + +"exponential-backoff@npm:^3.1.1": + version: 3.1.3 + resolution: "exponential-backoff@npm:3.1.3" + checksum: 10c0/77e3ae682b7b1f4972f563c6dbcd2b0d54ac679e62d5d32f3e5085feba20483cf28bd505543f520e287a56d4d55a28d7874299941faf637e779a1aa5994d1267 + languageName: node + linkType: hard + +"fdir@npm:^6.5.0": + version: 6.5.0 + resolution: "fdir@npm:6.5.0" + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + checksum: 10c0/e345083c4306b3aed6cb8ec551e26c36bab5c511e99ea4576a16750ddc8d3240e63826cc624f5ae17ad4dc82e68a253213b60d556c11bfad064b7607847ed07f + languageName: node + linkType: hard + +"graceful-fs@npm:^4.2.6": + version: 4.2.11 + resolution: "graceful-fs@npm:4.2.11" + checksum: 10c0/386d011a553e02bc594ac2ca0bd6d9e4c22d7fa8cfbfc448a6d148c59ea881b092db9dbe3547ae4b88e55f1b01f7c4a2ecc53b310c042793e63aa44cf6c257f2 + languageName: node + linkType: hard + +"isexe@npm:^4.0.0": + version: 4.0.0 + resolution: "isexe@npm:4.0.0" + checksum: 10c0/5884815115bceac452877659a9c7726382531592f43dc29e5d48b7c4100661aed54018cb90bd36cb2eaeba521092570769167acbb95c18d39afdccbcca06c5ce + languageName: node + linkType: hard + +"minipass@npm:^7.0.4, minipass@npm:^7.1.2": + version: 7.1.3 + resolution: "minipass@npm:7.1.3" + checksum: 10c0/539da88daca16533211ea5a9ee98dc62ff5742f531f54640dd34429e621955e91cc280a91a776026264b7f9f6735947629f920944e9c1558369e8bf22eb33fbb + languageName: node + linkType: hard + +"minizlib@npm:^3.1.0": + version: 3.1.0 + resolution: "minizlib@npm:3.1.0" + dependencies: + minipass: "npm:^7.1.2" + checksum: 10c0/5aad75ab0090b8266069c9aabe582c021ae53eb33c6c691054a13a45db3b4f91a7fb1bd79151e6b4e9e9a86727b522527c0a06ec7d45206b745d54cd3097bcec + languageName: node + linkType: hard + +"msgpackr-extract@npm:^3.0.2": + version: 3.0.4 + resolution: "msgpackr-extract@npm:3.0.4" + dependencies: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-darwin-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-arm64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-linux-x64": "npm:3.0.4" + "@msgpackr-extract/msgpackr-extract-win32-x64": "npm:3.0.4" + node-gyp: "npm:latest" + node-gyp-build-optional-packages: "npm:5.2.2" + dependenciesMeta: + "@msgpackr-extract/msgpackr-extract-darwin-arm64": + optional: true + "@msgpackr-extract/msgpackr-extract-darwin-x64": + optional: true + "@msgpackr-extract/msgpackr-extract-linux-arm": + optional: true + "@msgpackr-extract/msgpackr-extract-linux-arm64": + optional: true + "@msgpackr-extract/msgpackr-extract-linux-x64": + optional: true + "@msgpackr-extract/msgpackr-extract-win32-x64": + optional: true + bin: + download-msgpackr-prebuilds: bin/download-prebuilds.js + checksum: 10c0/582a9d17abbf3019e600e948736695056280ce401fd0235ee2474e95f9952208b9f6cce4d0e355b03b7d3c5630e6c3d11fe5fc27fdedb2311cce48de464338d8 + languageName: node + linkType: hard + +"msgpackr@npm:^1.11.2": + version: 1.11.13 + resolution: "msgpackr@npm:1.11.13" + dependencies: + msgpackr-extract: "npm:^3.0.2" + dependenciesMeta: + msgpackr-extract: + optional: true + checksum: 10c0/0efcf35235494763882e44579a5e6bb94fae964d9fdc226934a6259a52cbb6136137f32290415885eb7728ffb4a70841e0fde36592d5117fb6cff9d010d49bf1 + languageName: node + linkType: hard + +"node-gyp-build-optional-packages@npm:5.2.2": + version: 5.2.2 + resolution: "node-gyp-build-optional-packages@npm:5.2.2" + dependencies: + detect-libc: "npm:^2.0.1" + bin: + node-gyp-build-optional-packages: bin.js + node-gyp-build-optional-packages-optional: optional.js + node-gyp-build-optional-packages-test: build-test.js + checksum: 10c0/c81128c6f91873381be178c5eddcbdf66a148a6a89a427ce2bcd457593ce69baf2a8662b6d22cac092d24aa9c43c230dec4e69b3a0da604503f4777cd77e282b + languageName: node + linkType: hard + +"node-gyp@npm:latest": + version: 12.4.0 + resolution: "node-gyp@npm:12.4.0" + dependencies: + env-paths: "npm:^2.2.0" + exponential-backoff: "npm:^3.1.1" + graceful-fs: "npm:^4.2.6" + nopt: "npm:^9.0.0" + proc-log: "npm:^6.0.0" + semver: "npm:^7.3.5" + tar: "npm:^7.5.4" + tinyglobby: "npm:^0.2.12" + undici: "npm:^6.25.0" + which: "npm:^6.0.0" + bin: + node-gyp: bin/node-gyp.js + checksum: 10c0/9acb7c798e124275a6f9c1f7eb64b5abd6196bb885a3945fb44ee0dccf435514e88cdfb0f228ee7ff76ef25107c1f39ff37a067bf92fd00b9aff9234db29ff9e + languageName: node + linkType: hard + +"nopt@npm:^9.0.0": + version: 9.0.0 + resolution: "nopt@npm:9.0.0" + dependencies: + abbrev: "npm:^4.0.0" + bin: + nopt: bin/nopt.js + checksum: 10c0/1822eb6f9b020ef6f7a7516d7b64a8036e09666ea55ac40416c36e4b2b343122c3cff0e2f085675f53de1d2db99a2a89a60ccea1d120bcd6a5347bf6ceb4a7fd + languageName: node + linkType: hard + +"picomatch@npm:^4.0.4": + version: 4.0.4 + resolution: "picomatch@npm:4.0.4" + checksum: 10c0/e2c6023372cc7b5764719a5ffb9da0f8e781212fa7ca4bd0562db929df8e117460f00dff3cb7509dacfc06b86de924b247f504d0ce1806a37fac4633081466b0 + languageName: node + linkType: hard + +"proc-log@npm:^6.0.0": + version: 6.1.0 + resolution: "proc-log@npm:6.1.0" + checksum: 10c0/4f178d4062733ead9d71a9b1ab24ebcecdfe2250916a5b1555f04fe2eda972a0ec76fbaa8df1ad9c02707add6749219d118a4fc46dc56bdfe4dde4b47d80bb82 + languageName: node + linkType: hard + +"semver@npm:^7.3.5": + version: 7.8.2 + resolution: "semver@npm:7.8.2" + bin: + semver: bin/semver.js + checksum: 10c0/8e8c193fa75b938e5b3ccf6707c6447e4b34f73e493e72b03f3185393489f45e049144052f624217c346d6c6e0a301dda8eeab2f14413e337218ecb1cbd2de16 + languageName: node + linkType: hard + +"tar@npm:^7.5.4": + version: 7.5.16 + resolution: "tar@npm:7.5.16" + dependencies: + "@isaacs/fs-minipass": "npm:^4.0.0" + chownr: "npm:^3.0.0" + minipass: "npm:^7.1.2" + minizlib: "npm:^3.1.0" + yallist: "npm:^5.0.0" + checksum: 10c0/4f37f3c4bd2ca2755fd736a5df1d573c1a868ec1b1e893346aeafa95ac510f9e2fd1469420bd866cc7904799e5bd4ac62b5d4f03fe27747d6e1e373b44505c5c + languageName: node + linkType: hard + +"tinyglobby@npm:^0.2.12": + version: 0.2.17 + resolution: "tinyglobby@npm:0.2.17" + dependencies: + fdir: "npm:^6.5.0" + picomatch: "npm:^4.0.4" + checksum: 10c0/7f7bb0f197c88bc4b20c231e0deca4240ca3bf313a88f5a7fee93a872b84966a4d50220947c0455ad07a60b3b360961c5b7fd979222aeb716a9f99b412002e4c + languageName: node + linkType: hard + +"tslib@npm:^2.4.0": + version: 2.8.1 + resolution: "tslib@npm:2.8.1" + checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62 + languageName: node + linkType: hard + +"typescript@npm:^5.3.3": + version: 5.9.3 + resolution: "typescript@npm:5.9.3" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: 10c0/6bd7552ce39f97e711db5aa048f6f9995b53f1c52f7d8667c1abdc1700c68a76a308f579cd309ce6b53646deb4e9a1be7c813a93baaf0a28ccd536a30270e1c5 + languageName: node + linkType: hard + +"typescript@patch:typescript@npm%3A^5.3.3#optional!builtin": + version: 5.9.3 + resolution: "typescript@patch:typescript@npm%3A5.9.3#optional!builtin::version=5.9.3&hash=5786d5" + bin: + tsc: bin/tsc + tsserver: bin/tsserver + checksum: 10c0/ad09fdf7a756814dce65bc60c1657b40d44451346858eea230e10f2e95a289d9183b6e32e5c11e95acc0ccc214b4f36289dcad4bf1886b0adb84d711d336a430 + languageName: node + linkType: hard + +"undici-types@npm:~6.21.0": + version: 6.21.0 + resolution: "undici-types@npm:6.21.0" + checksum: 10c0/c01ed51829b10aa72fc3ce64b747f8e74ae9b60eafa19a7b46ef624403508a54c526ffab06a14a26b3120d055e1104d7abe7c9017e83ced038ea5cf52f8d5e04 + languageName: node + linkType: hard + +"undici@npm:^6.25.0": + version: 6.26.0 + resolution: "undici@npm:6.26.0" + checksum: 10c0/cf2b4caf58c33d6582970991290cc7a6486d6e738845f25dcdd16952d708ec844815c6d30362919764fcaf30f719891289341f1ada496f003ce2700310453a47 + languageName: node + linkType: hard + +"which@npm:^6.0.0": + version: 6.0.1 + resolution: "which@npm:6.0.1" + dependencies: + isexe: "npm:^4.0.0" + bin: + node-which: bin/which.js + checksum: 10c0/7e710e54ea36d2d6183bee2f9caa27a3b47b9baf8dee55a199b736fcf85eab3b9df7556fca3d02b50af7f3dfba5ea3a45644189836df06267df457e354da66d5 + languageName: node + linkType: hard + +"yallist@npm:^5.0.0": + version: 5.0.0 + resolution: "yallist@npm:5.0.0" + checksum: 10c0/a499c81ce6d4a1d260d4ea0f6d49ab4da09681e32c3f0472dee16667ed69d01dae63a3b81745a24bd78476ec4fcf856114cb4896ace738e01da34b2c42235416 + languageName: node + linkType: hard diff --git a/yarn-project/world-state/src/native/ipc_world_state_instance.ts b/yarn-project/world-state/src/native/ipc_world_state_instance.ts deleted file mode 100644 index cd4d20f1a003..000000000000 --- a/yarn-project/world-state/src/native/ipc_world_state_instance.ts +++ /dev/null @@ -1,717 +0,0 @@ -import { AsyncApi } from '@aztec/bb.js/aztec-wsdb'; -import type { - WorldStateDBStats as WsdbDBStats, - DBStats as WsdbDBStatsInner, - WorldStateMeta as WsdbMeta, - SiblingPathAndIndex as WsdbSiblingPathAndIndex, - WorldStateStatusFull as WsdbStatusFull, - WorldStateStatusSummary as WsdbStatusSummary, - TreeDBStats as WsdbTreeDBStats, - TreeMeta as WsdbTreeMeta, -} from '@aztec/bb.js/aztec-wsdb'; -import { - ARCHIVE_HEIGHT, - DomainSeparator, - L1_TO_L2_MSG_TREE_HEIGHT, - MAX_NULLIFIERS_PER_TX, - MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - NOTE_HASH_TREE_HEIGHT, - NULLIFIER_TREE_HEIGHT, - PUBLIC_DATA_TREE_HEIGHT, -} from '@aztec/constants'; -import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log'; -import { MerkleTreeId } from '@aztec/stdlib/trees'; -import type { WorldStateRevision } from '@aztec/stdlib/world-state'; - -import assert from 'assert'; -import { Decoder, Encoder } from 'msgpackr'; - -import type { WorldStateInstrumentation } from '../instrumentation/instrumentation.js'; -import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js'; -import { - type DBStats, - type SerializedIndexedLeaf, - type SerializedLeafValue, - type TreeDBStats, - type TreeMeta, - type WorldStateDBStats, - WorldStateMessageType, - type WorldStateMeta, - type WorldStateRequest, - type WorldStateRequestCategories, - type WorldStateResponse, - type WorldStateStatusFull, - type WorldStateStatusSummary, - isWithCanonical, - isWithForkId, - isWithRevision, -} from './message.js'; -import type { NativeWorldStateInstance } from './native_world_state_instance.js'; -import { WorldStateOpsQueue } from './world_state_ops_queue.js'; - -// ————— Msgpack helpers ————— - -const msgpackEncoder = new Encoder({ useRecords: false }); -const msgpackDecoder = new Decoder({ useRecords: false }); - -/** Msgpack-encode a SerializedLeafValue into bytes for IPC transport. */ -function serializeLeafToBytes(leaf: SerializedLeafValue): Uint8Array { - return Buffer.from(msgpackEncoder.pack(leaf)); -} - -// ————— Request conversion helpers ————— - -function toWsdbRevision(rev: WorldStateRevision): { forkid: number; blocknumber: number; includeuncommitted: boolean } { - return { - forkid: rev.forkId, - blocknumber: Number(rev.blockNumber), - includeuncommitted: rev.includeUncommitted, - }; -} - -function blockStateRefToMap(ref: Map): Map { - const result = new Map(); - for (const [treeId, [root, size]] of ref.entries()) { - result.set(treeId, [new Uint8Array(root), Number(size)]); - } - return result; -} - -// ————— Response conversion helpers ————— - -/** Convert Uint8Array fields to Buffer recursively (for opaque blob responses). */ -function convertUint8ArraysToBuffers(obj: unknown): unknown { - if (obj instanceof Uint8Array) { - return Buffer.from(obj); - } - if (Array.isArray(obj)) { - return obj.map(convertUint8ArraysToBuffers); - } - if (obj !== null && typeof obj === 'object') { - const result: Record = {}; - for (const [key, value] of Object.entries(obj)) { - result[key] = convertUint8ArraysToBuffers(value); - } - return result; - } - return obj; -} - -/** Decode a msgpack-encoded leaf value blob and convert Uint8Arrays to Buffers. */ -function decodeLeafValue(encoded: Uint8Array): SerializedLeafValue { - const decoded = msgpackDecoder.unpack(Buffer.from(encoded)); - return convertUint8ArraysToBuffers(decoded) as SerializedLeafValue; -} - -/** Decode a msgpack-encoded indexed leaf preimage blob. */ -function decodeLeafPreimage(encoded: Uint8Array): SerializedIndexedLeaf { - const decoded = msgpackDecoder.unpack(Buffer.from(encoded)); - return convertUint8ArraysToBuffers(decoded) as SerializedIndexedLeaf; -} - -/** Convert Wsdb state reference (Record) to NAPI format. */ -function convertStateRef( - state: Record, -): Record { - const result: Record = {}; - for (const [key, [root, size]] of Object.entries(state)) { - result[Number(key)] = [Buffer.from(root), BigInt(size)] as const; - } - return result; -} - -/** Convert Wsdb WorldStateStatusSummary (lowercase) to NAPI format (camelCase). */ -function convertStatusSummary(s: WsdbStatusSummary): WorldStateStatusSummary { - return { - unfinalizedBlockNumber: s.unfinalizedblocknumber, - finalizedBlockNumber: s.finalizedblocknumber, - oldestHistoricalBlock: s.oldesthistoricalblock, - treesAreSynched: s.treesaresynched, - } as unknown as WorldStateStatusSummary; -} - -function convertDBStats(s: WsdbDBStatsInner): DBStats { - return { - name: s.name, - numDataItems: s.numdataitems, - totalUsedSize: s.totalusedsize, - } as unknown as DBStats; -} - -function convertTreeDBStats(s: WsdbTreeDBStats): TreeDBStats { - return { - mapSize: s.mapsize, - physicalFileSize: s.physicalfilesize, - blocksDBStats: convertDBStats(s.blocksdbstats), - nodesDBStats: convertDBStats(s.nodesdbstats), - leafPreimagesDBStats: convertDBStats(s.leafpreimagesdbstats), - leafIndicesDBStats: convertDBStats(s.leafindicesdbstats), - blockIndicesDBStats: convertDBStats(s.blockindicesdbstats), - } as unknown as TreeDBStats; -} - -function convertWorldStateDBStats(s: WsdbDBStats): WorldStateDBStats { - return { - noteHashTreeStats: convertTreeDBStats(s.notehashtreestats), - messageTreeStats: convertTreeDBStats(s.messagetreestats), - archiveTreeStats: convertTreeDBStats(s.archivetreestats), - publicDataTreeStats: convertTreeDBStats(s.publicdatatreestats), - nullifierTreeStats: convertTreeDBStats(s.nullifiertreestats), - } as unknown as WorldStateDBStats; -} - -function convertTreeMeta(m: WsdbTreeMeta): TreeMeta { - return { - name: m.name, - depth: m.depth, - size: m.size, - committedSize: m.committedsize, - root: m.root, - initialSize: m.initialsize, - initialRoot: m.initialroot, - oldestHistoricBlock: m.oldesthistoricblock, - unfinalizedBlockHeight: m.unfinalizedblockheight, - finalizedBlockHeight: m.finalizedblockheight, - } as unknown as TreeMeta; -} - -function convertWorldStateMeta(m: WsdbMeta): WorldStateMeta { - return { - noteHashTreeMeta: convertTreeMeta(m.notehashtreemeta), - messageTreeMeta: convertTreeMeta(m.messagetreemeta), - archiveTreeMeta: convertTreeMeta(m.archivetreemeta), - publicDataTreeMeta: convertTreeMeta(m.publicdatatreemeta), - nullifierTreeMeta: convertTreeMeta(m.nullifiertreemeta), - } as unknown as WorldStateMeta; -} - -function convertStatusFull(s: WsdbStatusFull): WorldStateStatusFull { - return { - summary: convertStatusSummary(s.summary), - dbStats: convertWorldStateDBStats(s.dbstats), - meta: convertWorldStateMeta(s.meta), - } as unknown as WorldStateStatusFull; -} - -/** Convert Wsdb SiblingPathAndIndex to NAPI format. */ -function convertSiblingPathAndIndex( - s: WsdbSiblingPathAndIndex | undefined, -): { index: bigint; path: Buffer[] } | undefined { - if (!s) { - return undefined; - } - return { - index: BigInt(s.index), - path: s.path.map(p => Buffer.from(p)), - }; -} - -// ————— Public API ————— - -/** Backend interface matching WsdbBackend from bb.js. */ -export interface WsdbIpcBackend { - call(inputBuffer: Uint8Array): Promise; - getSocketPath(): string; - destroy?(): Promise; -} - -/** - * IPC-backed world state instance. - * Uses WsdbBackend (spawns aztec-wsdb binary) and the generated AsyncApi - * to communicate via the NamedUnion IPC protocol. - */ -export class IpcWorldState implements NativeWorldStateInstance { - private open = true; - private queues = new Map(); - private api: AsyncApi; - /** Tracks checkpoint depth per fork (WSDB IPC doesn't return depth in response). */ - private checkpointDepths = new Map(); - - constructor( - private readonly wsdbBackend: WsdbIpcBackend, - private readonly instrumentation: WorldStateInstrumentation, - bindings?: LoggerBindings, - private readonly log: Logger = createLogger('world-state:ipc-database', bindings), - ) { - this.api = new AsyncApi(wsdbBackend as any); - this.queues.set(0, new WorldStateOpsQueue()); - this.log.info('Created IPC-backed world state instance'); - } - - /** Returns the socket path of the underlying wsdb server. */ - getSocketPath(): string { - return this.wsdbBackend.getSocketPath(); - } - - /** - * Required by `NativeWorldStateInstance` for compatibility with the in-process - * NAPI path. The IPC backend does not expose an in-process pointer; callers that - * need to reach the WSDB process must use {@link getSocketPath} instead. - */ - getHandle(): any { - throw new Error('IpcWorldState has no in-process handle; use getSocketPath() instead'); - } - - async call( - messageType: T, - body: WorldStateRequest[T] & WorldStateRequestCategories, - responseHandler = (response: WorldStateResponse[T]): WorldStateResponse[T] => response, - errorHandler = (_: string) => {}, - ): Promise { - let forkId = -1; - let committedOnly = false; - - if (isWithCanonical(body)) { - forkId = 0; - } else if (isWithForkId(body)) { - forkId = body.forkId; - } else if (isWithRevision(body)) { - forkId = body.revision.forkId; - committedOnly = body.revision.includeUncommitted === false; - } else { - const _: never = body; - throw new Error(`Unable to determine forkId for message=${WorldStateMessageType[messageType]}`); - } - - let requestQueue = this.queues.get(forkId); - if (requestQueue === undefined) { - requestQueue = new WorldStateOpsQueue(); - this.queues.set(forkId, requestQueue); - } - - // The per-fork queue is cleaned up in `finally` even on error, so the JS-side queues map cannot outlive - // the native fork (e.g. when the native fork was already destroyed by an unwind/historical-prune and - // DELETE_FORK rejects with "Fork not found"). - try { - const response = await requestQueue.execute( - async () => { - assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the IPC instance'); - assert.equal(this.open, true, 'IPC instance is closed'); - let response: WorldStateResponse[T]; - try { - response = await this._sendMessage(messageType, body); - } catch (error: any) { - errorHandler(error.message); - throw error; - } - return responseHandler(response); - }, - messageType, - committedOnly, - ); - return response; - } finally { - if (messageType === WorldStateMessageType.DELETE_FORK) { - await requestQueue.stop(); - this.queues.delete(forkId); - } - } - } - - async close(): Promise { - if (!this.open) { - return; - } - this.open = false; - const queue = this.queues.get(0)!; - - // Send shutdown command. Under normal operation, the WSDB process sends its - // response before exiting (via ShutdownRequested in ipc_server.hpp). The - // try/catch is defensive: if the process is killed externally (SIGKILL, OOM) - // before responding, the pending IPC callback would be rejected by the socket - // close handler. We proceed to destroy the backend regardless. - try { - await queue.execute( - async () => { - await this.api.wsdbShutdown({}); - }, - WorldStateMessageType.CLOSE, - false, - ); - } catch (err: any) { - this.log.debug(`wsdbShutdown completed with error: ${err.message}`); - } - await queue.stop(); - - if (this.wsdbBackend.destroy) { - await this.wsdbBackend.destroy(); - } - } - - private async _sendMessage( - messageType: T, - body: WorldStateRequest[T] & WorldStateRequestCategories, - ): Promise { - const start = performance.now(); - try { - const response = await this.dispatch(messageType, body); - const durationMs = performance.now() - start; - this.log.trace(`Call ${WorldStateMessageType[messageType]} took (ms)`, { duration: durationMs }); - this.instrumentation.recordRoundTrip(durationMs * 1000, messageType); - return response; - } catch (error) { - this.log.error(`Call ${WorldStateMessageType[messageType]} failed: ${error}`, error); - throw error; - } - } - - private async dispatch( - messageType: T, - body: WorldStateRequest[T] & WorldStateRequestCategories, - ): Promise { - switch (messageType) { - // ——— Tree info & state reference ——— - - case WorldStateMessageType.GET_TREE_INFO: { - const b = body as WorldStateRequest[WorldStateMessageType.GET_TREE_INFO]; - const resp = await this.api.wsdbGetTreeInfo({ - treeid: b.treeId, - revision: toWsdbRevision(b.revision), - }); - return { - treeId: resp.treeid, - root: Buffer.from(resp.root), - size: resp.size, - depth: resp.depth, - } as WorldStateResponse[T]; - } - - case WorldStateMessageType.GET_STATE_REFERENCE: { - const b = body as WorldStateRequest[WorldStateMessageType.GET_STATE_REFERENCE]; - const resp = await this.api.wsdbGetStateReference({ - revision: toWsdbRevision(b.revision), - }); - return { state: convertStateRef(resp.state) } as WorldStateResponse[T]; - } - - case WorldStateMessageType.GET_INITIAL_STATE_REFERENCE: { - const resp = await this.api.wsdbGetInitialStateReference({}); - return { state: convertStateRef(resp.state) } as WorldStateResponse[T]; - } - - // ——— Leaf queries ——— - - case WorldStateMessageType.GET_LEAF_VALUE: { - const b = body as WorldStateRequest[WorldStateMessageType.GET_LEAF_VALUE]; - const resp = await this.api.wsdbGetLeafValue({ - treeid: b.treeId, - revision: toWsdbRevision(b.revision), - leafindex: Number(b.leafIndex), - }); - if (!resp.value) { - return undefined as WorldStateResponse[T]; - } - return decodeLeafValue(resp.value) as WorldStateResponse[T]; - } - - case WorldStateMessageType.GET_LEAF_PREIMAGE: { - const b = body as WorldStateRequest[WorldStateMessageType.GET_LEAF_PREIMAGE]; - const resp = await this.api.wsdbGetLeafPreimage({ - treeid: b.treeId, - revision: toWsdbRevision(b.revision), - leafindex: Number(b.leafIndex), - }); - if (!resp.preimage) { - return undefined as WorldStateResponse[T]; - } - return decodeLeafPreimage(resp.preimage) as WorldStateResponse[T]; - } - - case WorldStateMessageType.GET_SIBLING_PATH: { - const b = body as WorldStateRequest[WorldStateMessageType.GET_SIBLING_PATH]; - const resp = await this.api.wsdbGetSiblingPath({ - treeid: b.treeId, - revision: toWsdbRevision(b.revision), - leafindex: Number(b.leafIndex), - }); - return resp.path.map(p => Buffer.from(p)) as WorldStateResponse[T]; - } - - case WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES: { - const b = body as WorldStateRequest[WorldStateMessageType.GET_BLOCK_NUMBERS_FOR_LEAF_INDICES]; - const resp = await this.api.wsdbGetBlockNumbersForLeafIndices({ - treeid: b.treeId, - revision: toWsdbRevision(b.revision), - leafindices: b.leafIndices.map(Number), - }); - return { - blockNumbers: resp.blocknumbers.map(n => (n != null ? BigInt(n) : undefined)), - } as WorldStateResponse[T]; - } - - // ——— Find operations ——— - - case WorldStateMessageType.FIND_LEAF_INDICES: { - const b = body as WorldStateRequest[WorldStateMessageType.FIND_LEAF_INDICES]; - const resp = await this.api.wsdbFindLeafIndices({ - treeid: b.treeId, - revision: toWsdbRevision(b.revision), - leaves: b.leaves.map(serializeLeafToBytes), - startindex: Number(b.startIndex), - }); - return { - indices: resp.indices.map(n => (n != null ? BigInt(n) : undefined)), - } as WorldStateResponse[T]; - } - - case WorldStateMessageType.FIND_LOW_LEAF: { - const b = body as WorldStateRequest[WorldStateMessageType.FIND_LOW_LEAF]; - const resp = await this.api.wsdbFindLowLeaf({ - treeid: b.treeId, - revision: toWsdbRevision(b.revision), - key: new Uint8Array(b.key.toBuffer()), - }); - return { - alreadyPresent: resp.alreadypresent, - index: BigInt(resp.index), - } as WorldStateResponse[T]; - } - - case WorldStateMessageType.FIND_SIBLING_PATHS: { - const b = body as WorldStateRequest[WorldStateMessageType.FIND_SIBLING_PATHS]; - const resp = await this.api.wsdbFindSiblingPaths({ - treeid: b.treeId, - revision: toWsdbRevision(b.revision), - leaves: b.leaves.map(serializeLeafToBytes), - }); - return { - paths: resp.paths.map(convertSiblingPathAndIndex), - } as WorldStateResponse[T]; - } - - // ——— Mutations ——— - - case WorldStateMessageType.APPEND_LEAVES: { - const b = body as WorldStateRequest[WorldStateMessageType.APPEND_LEAVES]; - await this.api.wsdbAppendLeaves({ - treeid: b.treeId, - leaves: b.leaves.map(serializeLeafToBytes), - forkid: b.forkId, - }); - return undefined as WorldStateResponse[T]; - } - - case WorldStateMessageType.BATCH_INSERT: { - const b = body as WorldStateRequest[WorldStateMessageType.BATCH_INSERT]; - const resp = await this.api.wsdbBatchInsert({ - treeid: b.treeId, - leaves: b.leaves.map(serializeLeafToBytes), - subtreedepth: b.subtreeDepth, - forkid: b.forkId, - }); - const decoded = msgpackDecoder.unpack(Buffer.from(resp.result)); - return convertUint8ArraysToBuffers(decoded) as WorldStateResponse[T]; - } - - case WorldStateMessageType.SEQUENTIAL_INSERT: { - const b = body as WorldStateRequest[WorldStateMessageType.SEQUENTIAL_INSERT]; - const resp = await this.api.wsdbSequentialInsert({ - treeid: b.treeId, - leaves: b.leaves.map(serializeLeafToBytes), - forkid: b.forkId, - }); - const decoded = msgpackDecoder.unpack(Buffer.from(resp.result)); - return convertUint8ArraysToBuffers(decoded) as WorldStateResponse[T]; - } - - case WorldStateMessageType.UPDATE_ARCHIVE: { - const b = body as WorldStateRequest[WorldStateMessageType.UPDATE_ARCHIVE]; - await this.api.wsdbUpdateArchive({ - blockstateref: blockStateRefToMap(b.blockStateRef as Map) as any, - blockheaderhash: new Uint8Array(b.blockHeaderHash), - forkid: b.forkId, - }); - return undefined as WorldStateResponse[T]; - } - - // ——— Commit / Rollback ——— - - case WorldStateMessageType.COMMIT: { - await this.api.wsdbCommit({}); - return undefined as WorldStateResponse[T]; - } - - case WorldStateMessageType.ROLLBACK: { - await this.api.wsdbRollback({}); - return undefined as WorldStateResponse[T]; - } - - // ——— Block sync ——— - - case WorldStateMessageType.SYNC_BLOCK: { - const b = body as WorldStateRequest[WorldStateMessageType.SYNC_BLOCK]; - const resp = await this.api.wsdbSyncBlock({ - blocknumber: Number(b.blockNumber), - blockstateref: blockStateRefToMap(b.blockStateRef as Map) as any, - blockheaderhash: new Uint8Array(b.blockHeaderHash), - paddednotehashes: b.paddedNoteHashes.map(l => new Uint8Array(l as Buffer)), - paddedl1tol2messages: b.paddedL1ToL2Messages.map(l => new Uint8Array(l as Buffer)), - paddednullifiers: b.paddedNullifiers.map(l => ({ - nullifier: new Uint8Array((l as { nullifier: Buffer }).nullifier), - })), - publicdatawrites: b.publicDataWrites.map(l => ({ - slot: new Uint8Array((l as { slot: Buffer; value: Buffer }).slot), - value: new Uint8Array((l as { slot: Buffer; value: Buffer }).value), - })), - }); - return convertStatusFull(resp.status) as WorldStateResponse[T]; - } - - // ——— Fork management ——— - - case WorldStateMessageType.CREATE_FORK: { - const b = body as WorldStateRequest[WorldStateMessageType.CREATE_FORK]; - const resp = await this.api.wsdbCreateFork({ - latest: b.latest, - blocknumber: Number(b.blockNumber), - }); - return { forkId: resp.forkid } as WorldStateResponse[T]; - } - - case WorldStateMessageType.DELETE_FORK: { - const b = body as WorldStateRequest[WorldStateMessageType.DELETE_FORK]; - await this.api.wsdbDeleteFork({ forkid: b.forkId }); - return undefined as WorldStateResponse[T]; - } - - // ——— Block finalization ——— - - case WorldStateMessageType.FINALIZE_BLOCKS: { - const b = body as WorldStateRequest[WorldStateMessageType.FINALIZE_BLOCKS]; - const resp = await this.api.wsdbFinalizeBlocks({ toblocknumber: Number(b.toBlockNumber) }); - return convertStatusSummary(resp.status) as WorldStateResponse[T]; - } - - case WorldStateMessageType.UNWIND_BLOCKS: { - const b = body as WorldStateRequest[WorldStateMessageType.UNWIND_BLOCKS]; - const resp = await this.api.wsdbUnwindBlocks({ toblocknumber: Number(b.toBlockNumber) }); - return convertStatusFull(resp.status) as WorldStateResponse[T]; - } - - case WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS: { - const b = body as WorldStateRequest[WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS]; - const resp = await this.api.wsdbRemoveHistoricalBlocks({ toblocknumber: Number(b.toBlockNumber) }); - return convertStatusFull(resp.status) as WorldStateResponse[T]; - } - - // ——— Status ——— - - case WorldStateMessageType.GET_STATUS: { - const resp = await this.api.wsdbGetStatus({}); - return convertStatusSummary(resp.status) as WorldStateResponse[T]; - } - - // ——— Checkpoints ——— - - case WorldStateMessageType.CREATE_CHECKPOINT: { - const b = body as WorldStateRequest[WorldStateMessageType.CREATE_CHECKPOINT]; - await this.api.wsdbCreateCheckpoint({ forkid: b.forkId }); - const depth = (this.checkpointDepths.get(b.forkId) ?? 0) + 1; - this.checkpointDepths.set(b.forkId, depth); - return { depth } as WorldStateResponse[T]; - } - - case WorldStateMessageType.COMMIT_CHECKPOINT: { - const b = body as WorldStateRequest[WorldStateMessageType.COMMIT_CHECKPOINT]; - await this.api.wsdbCommitCheckpoint({ forkid: b.forkId }); - const depth = Math.max(0, (this.checkpointDepths.get(b.forkId) ?? 0) - 1); - this.checkpointDepths.set(b.forkId, depth); - return undefined as WorldStateResponse[T]; - } - - case WorldStateMessageType.REVERT_CHECKPOINT: { - const b = body as WorldStateRequest[WorldStateMessageType.REVERT_CHECKPOINT]; - await this.api.wsdbRevertCheckpoint({ forkid: b.forkId }); - const depth = Math.max(0, (this.checkpointDepths.get(b.forkId) ?? 0) - 1); - this.checkpointDepths.set(b.forkId, depth); - return undefined as WorldStateResponse[T]; - } - - case WorldStateMessageType.COMMIT_ALL_CHECKPOINTS: { - const b = body as WorldStateRequest[WorldStateMessageType.COMMIT_ALL_CHECKPOINTS]; - const targetDepth = b.depth ?? 0; - const currentDepth = this.checkpointDepths.get(b.forkId) ?? 0; - if (targetDepth === 0) { - // Commit everything — use the bulk operation - await this.api.wsdbCommitAllCheckpoints({ forkid: b.forkId }); - } else { - // Commit one level at a time down to target depth - for (let d = currentDepth; d > targetDepth; d--) { - await this.api.wsdbCommitCheckpoint({ forkid: b.forkId }); - } - } - this.checkpointDepths.set(b.forkId, targetDepth); - return undefined as WorldStateResponse[T]; - } - - case WorldStateMessageType.REVERT_ALL_CHECKPOINTS: { - const b = body as WorldStateRequest[WorldStateMessageType.REVERT_ALL_CHECKPOINTS]; - const targetDepth = b.depth ?? 0; - const currentDepth = this.checkpointDepths.get(b.forkId) ?? 0; - if (targetDepth === 0) { - // Revert everything — use the bulk operation - await this.api.wsdbRevertAllCheckpoints({ forkid: b.forkId }); - } else { - // Revert one level at a time down to target depth - for (let d = currentDepth; d > targetDepth; d--) { - await this.api.wsdbRevertCheckpoint({ forkid: b.forkId }); - } - } - this.checkpointDepths.set(b.forkId, targetDepth); - return undefined as WorldStateResponse[T]; - } - - // ——— Misc ——— - - case WorldStateMessageType.COPY_STORES: { - const b = body as WorldStateRequest[WorldStateMessageType.COPY_STORES]; - await this.api.wsdbCopyStores({ dstpath: b.dstPath, compact: b.compact }); - return undefined as WorldStateResponse[T]; - } - - case WorldStateMessageType.CLOSE: { - await this.api.wsdbShutdown({}); - return undefined as WorldStateResponse[T]; - } - - default: - throw new Error(`Unknown message type: ${messageType}`); - } - } -} - -/** - * Helper to create WsdbOptions from standard world state config. - * Returns the options needed to construct a WsdbBackend. - */ -export function getWsdbOptions( - dataDir: string, - wsTreeMapSizes: WorldStateTreeMapSizes, -): { - treeHeights: Record; - treePrefill: Record; - mapSizes: Record; - initialHeaderGeneratorPoint: number; -} { - return { - treeHeights: { - [MerkleTreeId.NULLIFIER_TREE]: NULLIFIER_TREE_HEIGHT, - [MerkleTreeId.NOTE_HASH_TREE]: NOTE_HASH_TREE_HEIGHT, - [MerkleTreeId.PUBLIC_DATA_TREE]: PUBLIC_DATA_TREE_HEIGHT, - [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: L1_TO_L2_MSG_TREE_HEIGHT, - [MerkleTreeId.ARCHIVE]: ARCHIVE_HEIGHT, - }, - treePrefill: { - [MerkleTreeId.NULLIFIER_TREE]: 2 * MAX_NULLIFIERS_PER_TX, - [MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, - }, - mapSizes: { - [MerkleTreeId.NULLIFIER_TREE]: wsTreeMapSizes.nullifierTreeMapSizeKb, - [MerkleTreeId.NOTE_HASH_TREE]: wsTreeMapSizes.noteHashTreeMapSizeKb, - [MerkleTreeId.PUBLIC_DATA_TREE]: wsTreeMapSizes.publicDataTreeMapSizeKb, - [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: wsTreeMapSizes.messageTreeMapSizeKb, - [MerkleTreeId.ARCHIVE]: wsTreeMapSizes.archiveTreeMapSizeKb, - }, - initialHeaderGeneratorPoint: DomainSeparator.BLOCK_HEADER_HASH, - }; -}