refactor(barretenberg-rs): migrate to ipc-codegen + ipc-runtime; drop PipeBackend

Makefile

@@ -47,21 +47,21 @@ endef
 # PHONY TARGETS - List every target that has a file/dir of the same name.
 #==============================================================================
 
-.PHONY: noir barretenberg noir-projects l1-contracts release-image boxes playground docs aztec-up spartan
+.PHONY: noir barretenberg noir-projects l1-contracts release-image boxes playground docs aztec-up spartan wsdb
 
 #==============================================================================
 # BOOTSTRAP TARGETS
 #==============================================================================
 
 # Fast bootstrap.
 fast: release-image barretenberg boxes playground docs aztec-up \
-		  bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests release-image-tests spartan claude-tests
+		  bb-tests l1-contracts-tests yarn-project-tests boxes-tests playground-tests aztec-up-tests docs-tests noir-protocol-circuits-tests release-image-tests spartan claude-tests ipc-codegen-tests
 
 # Full bootstrap.
 full: fast bb-full-tests bb-cpp-full yarn-project-benches
 
 # Release. Everything plus copy bb cross compiles to ts projects.
-release: fast bb-cpp-release-dir bb-ts-cross-copy
+release: fast bb-cpp-release-dir bb-ts-cross-copy ipc-runtime-cross
 
 #==============================================================================
 # Noir
@@ -205,13 +205,13 @@ bb-cpp-asan:
 bb-cpp-smt:
 	$(call build,$@,barretenberg/cpp,build_smt_verification)
 
-bb-cpp-release-dir: bb-cpp-native bb-cpp-cross
+bb-cpp-release-dir: bb-cpp-native bb-cpp-cross bb-cpp-wasm bb-cpp-wasm-threads
 	$(call build,$@,barretenberg/cpp,build_release_dir)
 
 bb-cpp-full: bb-cpp bb-cpp-gcc bb-cpp-fuzzing bb-cpp-asan bb-cpp-smt bb-cpp-cross-arm64-macos bb-cpp-cross-arm64-ios bb-cpp-cross-arm64-android
 
 # BB TypeScript - TypeScript bindings
-bb-ts: bb-cpp-wasm bb-cpp-wasm-threads bb-cpp-native
+bb-ts: bb-cpp-wasm bb-cpp-wasm-threads bb-cpp-native ipc-runtime
 	$(call build,$@,barretenberg/ts)
 
 # Copies the cross-compiles into bb.js.
@@ -275,6 +275,44 @@ bb-tests: bb-cpp-native-tests bb-acir-tests bb-ts-tests bb-sol-tests bb-bbup-tes
 
 bb-full-tests: bb-cpp-wasm-threads-tests bb-cpp-asan-tests bb-cpp-smt-tests
 
+#==============================================================================
+# IPC Codegen
+#==============================================================================
+
+.PHONY: ipc-codegen ipc-codegen-tests
+ipc-codegen:
+	$(call build,$@,ipc-codegen)
+
+ipc-codegen-tests: ipc-codegen
+	$(call test,$@,ipc-codegen)
+
+.PHONY: ipc-runtime ipc-runtime-tests ipc-runtime-cross
+ipc-runtime:
+	$(call build,$@,ipc-runtime)
+
+ipc-runtime-tests: ipc-runtime
+	$(call test,$@,ipc-runtime)
+
+# Cross-compile the NAPI addon for the 3 non-host release targets.
+# Host (amd64-linux) addon is produced by the standalone `ipc-runtime` target.
+ipc-runtime-cross-arm64-linux:
+	$(call build,$@,ipc-runtime,build_cross arm64-linux)
+
+ipc-runtime-cross-amd64-macos:
+	$(call build,$@,ipc-runtime,build_cross amd64-macos)
+
+ipc-runtime-cross-arm64-macos:
+	$(call build,$@,ipc-runtime,build_cross arm64-macos)
+
+ipc-runtime-cross: ipc-runtime ipc-runtime-cross-arm64-linux ipc-runtime-cross-amd64-macos ipc-runtime-cross-arm64-macos
+
+#==============================================================================
+# WSDB
+#==============================================================================
+
+wsdb: ipc-codegen ipc-runtime bb-cpp-native
+	$(call build,$@,wsdb)
+
 #==============================================================================
 # .claude tooling
 #==============================================================================
@@ -341,7 +379,7 @@ l1-contracts-tests: l1-contracts-verifier
 # Yarn Project - TypeScript monorepo with all TS packages
 #==============================================================================
 
-yarn-project: bb-ts noir-projects l1-contracts
+yarn-project: bb-ts noir-projects l1-contracts wsdb
 	$(call build,$@,yarn-project)
 
 yarn-project-tests: yarn-project

aztec-up/bootstrap.sh

@@ -103,7 +103,7 @@ EOF
     # TODO(AD): we have kludged a retry here. a local NPM install ought to be robust enough not to.
     echo "Deploying packages to local npm registry (version: $version)..."
     {
-      echo $root/barretenberg/ts
+      (cd $root/barretenberg/ts && ./bootstrap.sh get_projects)
       $root/noir/bootstrap.sh get_projects
       $root/yarn-project/bootstrap.sh get_projects
     } | DRY_RUN= parallel --tag --line-buffer --halt now,fail=1 "retry 'cd {} && dump_fail \"deploy_npm $version\" >/dev/null'"

barretenberg/.gitignore

@@ -13,3 +13,6 @@ bench-out
 rust/barretenberg-rs/src/generated_types.rs
 rust/barretenberg-rs/src/api.rs
 ts/src/cbind/generated/
+
+# Codegen output dirs (ipc-codegen emits into a `generated/` subdir under each consumer)
+**/generated/

barretenberg/acir_tests/yarn.lock

@@ -24,6 +24,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@aztec/bb.js@portal:../../ts::locator=bbjs-test%40workspace%3Abbjs-test"
   dependencies:
+    "@aztec/ipc-runtime": "portal:../../ipc-runtime/ts"
     comlink: "npm:^4.4.1"
     commander: "npm:^12.1.0"
     idb-keyval: "npm:^6.2.1"
@@ -39,6 +40,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@aztec/bb.js@portal:../../ts::locator=browser-test-app%40workspace%3Abrowser-test-app"
   dependencies:
+    "@aztec/ipc-runtime": "portal:../../ipc-runtime/ts"
     comlink: "npm:^4.4.1"
     commander: "npm:^12.1.0"
     idb-keyval: "npm:^6.2.1"
@@ -50,6 +52,18 @@ __metadata:
   languageName: node
   linkType: soft
 
+"@aztec/ipc-runtime@portal:../../ipc-runtime/ts::locator=%40aztec%2Fbb.js%40portal%3A..%2F..%2Fts%3A%3Alocator%3Dbbjs-test%2540workspace%253Abbjs-test":
+  version: 0.0.0-use.local
+  resolution: "@aztec/ipc-runtime@portal:../../ipc-runtime/ts::locator=%40aztec%2Fbb.js%40portal%3A..%2F..%2Fts%3A%3Alocator%3Dbbjs-test%2540workspace%253Abbjs-test"
+  languageName: node
+  linkType: soft
+
+"@aztec/ipc-runtime@portal:../../ipc-runtime/ts::locator=%40aztec%2Fbb.js%40portal%3A..%2F..%2Fts%3A%3Alocator%3Dbrowser-test-app%2540workspace%253Abrowser-test-app":
+  version: 0.0.0-use.local
+  resolution: "@aztec/ipc-runtime@portal:../../ipc-runtime/ts::locator=%40aztec%2Fbb.js%40portal%3A..%2F..%2Fts%3A%3Alocator%3Dbrowser-test-app%2540workspace%253Abrowser-test-app"
+  languageName: node
+  linkType: soft
+
 "@babel/code-frame@npm:^7.0.0":
   version: 7.26.2
   resolution: "@babel/code-frame@npm:7.26.2"

barretenberg/cpp/.rebuild_patterns

@@ -4,3 +4,9 @@
 ^barretenberg/cpp/scripts/
 ^barretenberg/cpp/bootstrap.sh
 ^barretenberg/cpp/CMakePresets.json
+# bbapi and ipc_runtime generate C++ headers via ipc-codegen at CMake-time.
+# Treat the codegen sources + templates as part of bb-cpp's input so the CI
+# cache key invalidates when the codegen changes.
+^ipc-codegen/src/.*\.ts$
+^ipc-codegen/templates/cpp/.*$
+^ipc-runtime/cpp/.*$

barretenberg/cpp/CMakePresets.json

@@ -409,6 +409,7 @@
         "CC": "$env{WASI_SDK_PREFIX}/bin/clang",
         "CXX": "$env{WASI_SDK_PREFIX}/bin/clang++",
         "CXXFLAGS": "-DBB_VERBOSE -fvisibility=hidden",
+        "LDFLAGS": "--no-wasm-opt",
         "AR": "$env{WASI_SDK_PREFIX}/bin/llvm-ar",
         "RANLIB": "$env{WASI_SDK_PREFIX}/bin/llvm-ranlib"
       },
@@ -800,25 +801,25 @@
       "name": "amd64-linux",
       "configurePreset": "amd64-linux",
       "inheritConfigureEnvironment": true,
-      "targets": ["bb", "nodejs_module", "bb-external"]
+      "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb"]
     },
     {
       "name": "arm64-linux",
       "configurePreset": "arm64-linux",
       "inheritConfigureEnvironment": true,
-      "targets": ["bb", "nodejs_module", "bb-external"]
+      "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb"]
     },
     {
       "name": "amd64-macos",
       "configurePreset": "amd64-macos",
       "inheritConfigureEnvironment": true,
-      "targets": ["bb", "nodejs_module", "bb-external"]
+      "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb"]
     },
     {
       "name": "arm64-macos",
       "configurePreset": "arm64-macos",
       "inheritConfigureEnvironment": true,
-      "targets": ["bb", "nodejs_module", "bb-external"]
+      "targets": ["bb", "nodejs_module", "bb-external", "aztec-wsdb"]
     },
     {
       "name": "amd64-windows",

barretenberg/cpp/cmake/module.cmake

@@ -274,6 +274,7 @@ function(barretenberg_module_with_sources MODULE_NAME)
             target_link_libraries(
                 ${BENCHMARK_NAME}_bench_objects
                 PRIVATE
+                ${MODULE_DEPENDENCIES}
                 benchmark::benchmark
                 ${TRACY_LIBS}
                 ${TBB_IMPORTED_TARGETS}

barretenberg/cpp/format.sh

@@ -23,7 +23,7 @@ elif [ "$1" == "changed" ]; then
     format_files "$files"
   fi
 elif [ "$1" == "check" ]; then
-  files=$(find ./src -iname *.hpp -o -iname *.cpp -o -iname *.tcc | grep -v bb/deps)
+  files=$(find ./src -iname *.hpp -o -iname *.cpp -o -iname *.tcc | grep -v bb/deps | grep -v '/generated/')
   echo "$files" | parallel -N10 clang-format-20 --dry-run --Werror
 elif [ -n "$1" ]; then
   files=$(git diff-index --relative --name-only $1 | grep -e '\.\(cpp\|hpp\|tcc\)$')

barretenberg/cpp/src/CMakeLists.txt

@@ -52,7 +52,7 @@ if(WASM)
     add_link_options(-Wl,--export-memory,--import-memory,--stack-first,-z,stack-size=1048576,--max-memory=4294967296)
 endif()
 
-include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${TRACY_INCLUDE} ${LMDB_INCLUDE} ${LIBDEFLATE_INCLUDE} ${HTTPLIB_INCLUDE} ${BACKWARD_INCLUDE} ${NLOHMANN_JSON_INCLUDE})
+include_directories(${CMAKE_CURRENT_SOURCE_DIR} ${MSGPACK_INCLUDE} ${TRACY_INCLUDE} ${LMDB_INCLUDE} ${LIBDEFLATE_INCLUDE} ${HTTPLIB_INCLUDE} ${BACKWARD_INCLUDE} ${NLOHMANN_JSON_INCLUDE} ${CMAKE_CURRENT_SOURCE_DIR}/../../../ipc-runtime/cpp)
 
 # Add avm-transpiler include path when library is provided
 if(AVM_TRANSPILER_LIB)
@@ -128,10 +128,15 @@ if(NOT FUZZING AND NOT WASM AND NOT BB_LITE)
     add_subdirectory(barretenberg/vm2)
     add_subdirectory(barretenberg/ipc)
     add_subdirectory(barretenberg/wsdb)
-    add_subdirectory(barretenberg/wsdb_client)
+    add_subdirectory(barretenberg/vm2_wsdb)
     add_subdirectory(barretenberg/nodejs_module)
 endif()
 
+# Pull in ipc-runtime for native IPC servers and clients.
+if(NOT FUZZING AND NOT WASM AND NOT BB_LITE)
+    add_subdirectory(${CMAKE_SOURCE_DIR}/../../ipc-runtime/cpp ${CMAKE_BINARY_DIR}/ipc-runtime)
+endif()
+
 if(FUZZING_AVM)
     if(FUZZING)
         # Only add these if they weren't added above (when NOT FUZZING AND NOT WASM)

barretenberg/cpp/src/barretenberg/api/CMakeLists.txt

@@ -6,5 +6,5 @@ if(AVM_TRANSPILER_LIB)
 endif()
 
 if(NOT WASM AND NOT BB_LITE)
-    target_link_libraries(api_objects PRIVATE ipc)
+    target_link_libraries(api_objects PRIVATE ipc_runtime)
 endif()

barretenberg/cpp/src/barretenberg/api/api_chonk.cpp

@@ -2,7 +2,10 @@
 #include "barretenberg/api/file_io.hpp"
 #include "barretenberg/api/json_output.hpp"
 #include "barretenberg/api/log.hpp"
-#include "barretenberg/bbapi/bbapi.hpp"
+#include "barretenberg/bbapi/bbapi_handlers.hpp"
+#include "barretenberg/bbapi/bbapi_shared.hpp"
+#include "barretenberg/bbapi/bbapi_wire_convert.hpp"
+#include "barretenberg/bbapi/generated/bb_types.hpp"
 #include "barretenberg/chonk/chonk.hpp"
 #include "barretenberg/chonk/chonk_verifier.hpp"
 #include "barretenberg/chonk/mock_circuit_producer.hpp"
@@ -33,16 +36,21 @@ namespace { // anonymous namespace
  */
 void write_chonk_vk(std::vector<uint8_t> bytecode, const std::filesystem::path& output_path, const API::Flags& flags)
 {
+    bbapi::BBApiRequest request;
     auto response =
-        bbapi::ChonkComputeVk{ .circuit = { .bytecode = std::move(bytecode) }, .use_zk_flavor = flags.use_zk_flavor }
-            .execute();
+        bbapi::handle_chonk_compute_vk(request,
+                                       bbapi::wire::ChonkComputeVk{
+                                           .circuit = bbapi::wire::CircuitInputNoVK{ .bytecode = std::move(bytecode) },
+                                           .use_zk_flavor = flags.use_zk_flavor,
+                                       });
 
     const bool is_stdout = output_path == "-";
     if (is_stdout) {
         write_bytes_to_stdout(response.bytes);
     } else if (flags.output_format == "json") {
-        // Note: Chonk VK doesn't have a hash, so we pass an empty string
-        std::string json_content = VkJson::build(response.fields, "", flags.scheme);
+        // Note: Chonk VK doesn't have a hash, so we pass an empty string.
+        auto fields = bbapi::fr_vec_from_wire(response.fields);
+        std::string json_content = VkJson::build(fields, "", flags.scheme);
         write_file(output_path / "vk.json", std::vector<uint8_t>(json_content.begin(), json_content.end()));
         info("VK (JSON) saved to ", output_path / "vk.json");
     } else {
@@ -60,21 +68,25 @@ void ChonkAPI::prove(const Flags& flags,
     request.vk_policy = bbapi::parse_vk_policy(flags.vk_policy);
     std::vector<PrivateExecutionStepRaw> raw_steps = PrivateExecutionStepRaw::load_and_decompress(input_path);
 
-    bbapi::ChonkStart{ .num_circuits = static_cast<uint32_t>(raw_steps.size()) }.execute(request);
+    bbapi::handle_chonk_start(request,
+                              bbapi::wire::ChonkStart{ .num_circuits = static_cast<uint32_t>(raw_steps.size()) });
     info("Chonk: starting with ", raw_steps.size(), " circuits");
     for (size_t i = 0; i < raw_steps.size(); ++i) {
         const auto& step = raw_steps[i];
-        bbapi::ChonkLoad{
-            .circuit = { .name = step.function_name, .bytecode = step.bytecode, .verification_key = step.vk },
-        }
-            .execute(request);
+        bbapi::handle_chonk_load(request,
+                                 bbapi::wire::ChonkLoad{ .circuit = bbapi::wire::CircuitInput{
+                                                             .name = step.function_name,
+                                                             .bytecode = step.bytecode,
+                                                             .verification_key = step.vk,
+                                                         } });
 
         // NOLINTNEXTLINE(bugprone-unchecked-optional-access): we know the optional has been set here.
         info("Chonk: accumulating " + step.function_name);
-        bbapi::ChonkAccumulate{ .witness = step.witness }.execute(request);
+        bbapi::handle_chonk_accumulate(request, bbapi::wire::ChonkAccumulate{ .witness = step.witness });
     }
 
-    auto proof = bbapi::ChonkProve{}.execute(request).proof;
+    auto wire_proof = bbapi::handle_chonk_prove(request, bbapi::wire::ChonkProve{}).proof;
+    auto proof = bbapi::chonk_proof_from_wire(std::move(wire_proof));
 
     const bool output_to_stdout = output_dir == "-";
 
@@ -117,7 +129,9 @@ bool ChonkAPI::verify([[maybe_unused]] const Flags& flags,
 
     auto vk_buffer = read_vk_file(vk_path);
 
-    auto response = bbapi::ChonkVerify{ .proof = std::move(proof), .vk = std::move(vk_buffer) }.execute();
+    bbapi::BBApiRequest request;
+    auto response = bbapi::handle_chonk_verify(
+        request, bbapi::wire::ChonkVerify{ .proof = bbapi::chonk_proof_to_wire(proof), .vk = std::move(vk_buffer) });
     return response.valid;
 }
 
@@ -147,7 +161,14 @@ bool ChonkAPI::batch_verify([[maybe_unused]] const Flags& flags, const std::file
 
     info("ChonkAPI::batch_verify - found ", proofs.size(), " proof/vk pairs in ", proofs_dir.string());
 
-    auto response = bbapi::ChonkBatchVerify{ .proofs = std::move(proofs), .vks = std::move(vks) }.execute();
+    std::vector<bbapi::wire::ChonkProof> wire_proofs;
+    wire_proofs.reserve(proofs.size());
+    for (const auto& p : proofs) {
+        wire_proofs.push_back(bbapi::chonk_proof_to_wire(p));
+    }
+    bbapi::BBApiRequest request;
+    auto response = bbapi::handle_chonk_batch_verify(
+        request, bbapi::wire::ChonkBatchVerify{ .proofs = std::move(wire_proofs), .vks = std::move(vks) });
     return response.valid;
 }
 
@@ -221,12 +242,14 @@ bool ChonkAPI::check_precomputed_vks(const Flags& flags, const std::filesystem::
             return false;
         }
         const bool use_zk_flavor = (i == raw_steps.size() - 1);
-        auto response =
-            bbapi::ChonkCheckPrecomputedVk{
-                .circuit = { .name = step.function_name, .bytecode = step.bytecode, .verification_key = step.vk },
+        auto response = bbapi::handle_chonk_check_precomputed_vk(
+            request,
+            bbapi::wire::ChonkCheckPrecomputedVk{
+                .circuit = bbapi::wire::CircuitInput{ .name = step.function_name,
+                                                      .bytecode = step.bytecode,
+                                                      .verification_key = step.vk },
                 .use_zk_flavor = use_zk_flavor,
-            }
-                .execute();
+            });
 
         if (!response.valid) {
             info("VK mismatch detected for function ", step.function_name);
@@ -271,9 +294,12 @@ void chonk_gate_count(const std::string& bytecode_path, bool include_gates_per_o
     bbapi::BBApiRequest request;
 
     auto bytecode = get_bytecode(bytecode_path);
-    auto response = bbapi::ChonkStats{ .circuit = { .name = "ivc_circuit", .bytecode = std::move(bytecode) },
-                                       .include_gates_per_opcode = include_gates_per_opcode }
-                        .execute(request);
+    auto response = bbapi::handle_chonk_stats(
+        request,
+        bbapi::wire::ChonkStats{
+            .circuit = bbapi::wire::CircuitInputNoVK{ .name = "ivc_circuit", .bytecode = std::move(bytecode) },
+            .include_gates_per_opcode = include_gates_per_opcode,
+        });
 
     // Build the circuit report. It always has one function, corresponding to the ACIR constraint systems.
     // NOTE: can be reconsidered