Make bitcoind optional: add transaction-sync as an alternative chain backend

.github/workflows/build.yaml

@@ -41,9 +41,3 @@ jobs:
           rustflags: ""
       - name: Build
         run: cargo build --release
-      - name: Build (electrum only)
-        if: matrix.name == 'linux' && matrix.toolchain == 'stable'
-        run: cargo build --release --no-default-features --features electrum
-      - name: Build (esplora only)
-        if: matrix.name == 'linux' && matrix.toolchain == 'stable'
-        run: cargo build --release --no-default-features --features esplora

.github/workflows/lint.yaml

@@ -12,7 +12,20 @@ env:
 jobs:
   lint:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        features:
+          - block-sync,electrum
+          - block-sync,esplora
+          - block-sync,electrum,esplora
+          - transaction-sync,electrum
+          - transaction-sync,esplora
+          - transaction-sync,electrum,esplora
+          - block-sync,transaction-sync,electrum
+          - block-sync,transaction-sync,esplora
+          - block-sync,transaction-sync,electrum,esplora
     steps:
       - uses: actions/checkout@v6
         with:
@@ -21,9 +34,5 @@ jobs:
         with:
           components: clippy
           rustflags: ""
-      - name: Lint
-        run: cargo clippy --all-targets -- -D warnings
-      - name: Lint (electrum only)
-        run: cargo clippy --all-targets --no-default-features --features electrum -- -D warnings
-      - name: Lint (esplora only)
-        run: cargo clippy --all-targets --no-default-features --features esplora -- -D warnings
+      - name: Lint (${{ matrix.features }})
+        run: cargo clippy --all-targets --no-default-features --features ${{ matrix.features }} -- -D warnings

.gitignore

@@ -1,5 +1,6 @@
 # Test files
 /datacore
+/dataesplora
 /dataindex
 /dataldk0
 /dataldk1

Cargo.toml

@@ -8,11 +8,34 @@ rust-version = "1.88.0"
 name = "rgb-lightning-node"
 
 [features]
-default = ["electrum", "esplora"]
+default = [
+    "block-sync",
+    "transaction-sync",
+    "electrum",
+    "esplora",
+]
+# sync LDK from a bitcoind instance, consuming full blocks over JSON-RPC
+block-sync = [
+    "dep:lightning-block-sync",
+]
+# sync LDK from the indexer, without requiring a bitcoind instance
+transaction-sync = [
+    "dep:lightning-transaction-sync",
+]
 # support indexers implementing the electrum protocol
-electrum = ["rgb-lib/electrum", "lightning/electrum"]
+electrum = [
+    "rgb-lib/electrum",
+    "lightning/electrum",
+    "lightning-transaction-sync?/electrum",
+    "dep:electrum-client",
+]
 # support indexers implementing the esplora protocol
-esplora = ["rgb-lib/esplora", "lightning/esplora"]
+esplora = [
+    "rgb-lib/esplora",
+    "lightning/esplora",
+    "lightning-transaction-sync?/esplora-blocking",
+    "dep:esplora-client",
+]
 
 [dependencies]
 amplify = { version = "=4.8.1", default-features = false }
@@ -29,17 +52,20 @@ chacha20poly1305 = { version = "0.10.1", features = ["stream"] }
 chrono = { version = "0.4", default-features = false, features = ["clock"] }
 clap = "4.5.20"
 dirs = "5.0.1"
+electrum-client = { version = "0.24.0", default-features = false, features = ["use-rustls"], optional = true }
+esplora-client = { version = "0.12", default-features = false, features = ["blocking-https-rustls"], optional = true }
 futures = "0.3"
 hex = { package = "hex-conservative", version = "0.3.0", default-features = false }
 lightning = { version = "0.2.0", path = "./rust-lightning/lightning", features = ["dnssec"] }
 lightning-background-processor = { version = "0.2.0", path = "./rust-lightning/lightning-background-processor" }
-lightning-block-sync = { version = "0.2.0", features = ["rpc-client", "tokio"] }
+lightning-block-sync = { version = "0.2.0", features = ["rpc-client", "tokio"], optional = true }
 lightning-dns-resolver = { version = "0.3.0", path = "./rust-lightning/lightning-dns-resolver" }
 lightning-invoice = { version = "0.34.0", features = ["std"], path = "./rust-lightning/lightning-invoice" }
 lightning-macros = { version = "0.2.0" }
 lightning-net-tokio = { version = "0.2.0" }
 lightning-persister = { version = "0.2.0", path = "./rust-lightning/lightning-persister", features = ["tokio"] }
 lightning-rapid-gossip-sync = { version = "0.2.0", path = "./rust-lightning/lightning-rapid-gossip-sync" }
+lightning-transaction-sync = { version = "0.2.0", path = "./rust-lightning/lightning-transaction-sync", optional = true }
 rand = "0.8.5"
 regex = { version = "1.11", default-features = false }
 rgb-lib = { version = "=0.3.0-beta.7", default-features = false }
@@ -61,7 +87,7 @@ zip = { version = "2.2.0", default-features = false, features = ["time", "zstd"]
 
 [dev-dependencies]
 dircmp = "0.2.0"
-electrum-client = "0.20.0"
+electrum-client = "0.24.0"
 http = "1.4.0"
 lazy_static = { version = "1.5.0", default-features = false }
 lightning = { version = "0.2.0", path = "./rust-lightning/lightning", features = ["_rln_test_hooks"] }

README.md

@@ -50,18 +50,34 @@ Support for the indexer protocols is behind cargo features, `electrum` and
 
 To support electrum indexers only:
 ```sh
-cargo install --locked --path . --no-default-features --features electrum
+cargo install --locked --path . --no-default-features --features electrum,block-sync,transaction-sync
 ```
 
 To support esplora indexers only:
 ```sh
-cargo install --locked --path . --no-default-features --features esplora
+cargo install --locked --path . --no-default-features --features esplora,block-sync,transaction-sync
+```
+
+### Chain sync support
+
+Support for the chain sync backends is behind cargo features, `block-sync` and
+`transaction-sync` (both enabled by default). At least one of them needs to be
+enabled. See [Sync modes](#sync-modes) for what each backend does.
+
+To support the block-sync backend only:
+```sh
+cargo install --locked --path . --no-default-features --features block-sync,electrum,esplora
+```
+
+To support the transaction-sync backend only:
+```sh
+cargo install --locked --path . --no-default-features --features transaction-sync,electrum,esplora
 ```
 
 ## Run
 
 In order to operate, the node will need:
-- a bitcoind node
+- a bitcoind node (only for the `BlockSync` [sync mode](#sync-modes))
 - an indexer instance (electrum or esplora)
 
 Once services are running, daemons can be started.
@@ -401,6 +417,23 @@ Example proxy URLs (only when using proxy transport):
 | Local | `rpc://127.0.0.1:3000/json-rpc` |
 | Public | `rpcs://proxy.iriswallet.com/0.2/json-rpc` |
 
+## Sync modes
+
+The node keeps LDK in sync with the chain in one of two ways, selected at unlock
+time via the `ldk_chain_sync` field of the `/unlock` payload (see the
+`UnlockRequest` schema in `openapi.yaml` for the exact shape):
+
+- `BlockSync`: consume full blocks from a trusted/local `bitcoind` over JSON-RPC.
+  The `bitcoind_rpc_*` parameters are provided under this mode's `config`. This
+  is the more trust-minimized option, since the node does not rely on an indexer
+  to tell it which transactions are relevant.
+- `TransactionSync`: sync through an electrum/esplora indexer, so no `bitcoind`
+  is needed. The indexer LDK syncs against is given under this mode's `config`
+  via `indexer_url` and can differ from the one the RGB wallet uses.
+
+Both modes are available in a stock build. See
+[Chain sync support](#chain-sync-support) to build with only one of them.
+
 ## Test
 
 Tests for a few scenarios using the regtest network are included. The same

compose.yaml

@@ -22,6 +22,27 @@ services:
       - bitcoind
     ports:
       - 50001:50001
+  # its electrum RPC is only bound inside the container, so it doesn't clash with electrs
+  esplora:
+    image: blockstream/esplora:latest
+    entrypoint: []
+    command: >
+      /srv/explorer/electrs_bitcoin/bin/electrs
+      --timestamp
+      --network regtest
+      --jsonrpc-import
+      --cookie user:password
+      --daemon-rpc-addr bitcoind:18443
+      --http-addr 0.0.0.0:3002
+      --electrum-rpc-addr 0.0.0.0:60401
+      --db-dir /data/db
+      --cors '*'
+    volumes:
+      - ./dataesplora:/data
+    depends_on:
+      - bitcoind
+    ports:
+      - 3002:3002
   proxy:
     image: ghcr.io/rgb-tools/rgb-proxy-server:0.3.0
     ports:

openapi.yaml

@@ -2228,6 +2228,61 @@ components:
           example: 89d28bd306aa9bb906fd0ac31092d04c37c919a171b343083167e2a3cdc60578
         status:
           $ref: '#/components/schemas/HTLCStatus'
+    LdkChainSync:
+      oneOf:
+        - $ref: '#/components/schemas/LdkChainSyncBlockSync'
+        - $ref: '#/components/schemas/LdkChainSyncTransactionSync'
+      discriminator:
+        propertyName: mode
+        mapping:
+          BlockSync: '#/components/schemas/LdkChainSyncBlockSync'
+          TransactionSync: '#/components/schemas/LdkChainSyncTransactionSync'
+    LdkChainSyncBlockSync:
+      type: object
+      required:
+        - mode
+        - config
+      properties:
+        mode:
+          type: string
+          enum: [BlockSync]
+        config:
+          type: object
+          required:
+            - bitcoind_rpc_username
+            - bitcoind_rpc_password
+            - bitcoind_rpc_host
+            - bitcoind_rpc_port
+          properties:
+            bitcoind_rpc_username:
+              type: string
+              example: user
+            bitcoind_rpc_password:
+              type: string
+              example: password
+            bitcoind_rpc_host:
+              type: string
+              example: localhost
+            bitcoind_rpc_port:
+              type: integer
+              example: 18443
+    LdkChainSyncTransactionSync:
+      type: object
+      required:
+        - mode
+        - config
+      properties:
+        mode:
+          type: string
+          enum: [TransactionSync]
+        config:
+          type: object
+          required:
+            - indexer_url
+          properties:
+            indexer_url:
+              type: string
+              example: 127.0.0.1:50001
     ListAssetsRequest:
       type: object
       required:
@@ -3436,28 +3491,15 @@ components:
       type: object
       required:
         - password
-        - bitcoind_rpc_username
-        - bitcoind_rpc_password
-        - bitcoind_rpc_host
-        - bitcoind_rpc_port
+        - ldk_chain_sync
         - indexer_url
         - announce_addresses
       properties:
         password:
           type: string
           example: nodepassword
-        bitcoind_rpc_username:
-          type: string
-          example: user
-        bitcoind_rpc_password:
-          type: string
-          example: password
-        bitcoind_rpc_host:
-          type: string
-          example: localhost
-        bitcoind_rpc_port:
-          type: integer
-          example: 18443
+        ldk_chain_sync:
+          $ref: '#/components/schemas/LdkChainSync'
         indexer_url:
           type: string
           example: 127.0.0.1:50001

regtest.sh

@@ -65,12 +65,26 @@ _wait_for_electrs() {
     done
 }
 
+_wait_for_esplora() {
+    # wait for the esplora REST API to become responsive
+    start_time=$(date +%s)
+    until curl -sf http://localhost:3002/blocks/tip/height >/dev/null 2>&1; do
+        current_time=$(date +%s)
+        if [ $((current_time - start_time)) -gt $TIMEOUT ]; then
+            echo "Timeout waiting for esplora to start"
+            $COMPOSE logs esplora
+            exit 1
+        fi
+        sleep 1
+    done
+}
+
 _start_services() {
     _stop_services
 
-    mkdir -p data{core,index,ldk0,ldk1,ldk2}
+    mkdir -p data{core,index,esplora,ldk0,ldk1,ldk2}
     # see compose.yaml for the exposed ports
-    EXPOSED_PORTS=(3000 50001)
+    EXPOSED_PORTS=(3000 3002 50001)
     for port in "${EXPOSED_PORTS[@]}"; do
         if _is_port_bound "$port"; then
             _die "port $port is already bound, services can't be started"
@@ -85,11 +99,13 @@ _start_services() {
     $COMPOSE up -d
     echo "waiting for electrs to have completed startup"
     _wait_for_electrs
+    echo "waiting for esplora to have completed startup"
+    _wait_for_esplora
 }
 
 _stop_services() {
     $COMPOSE down -v --remove-orphans
-    rm -rf data{core,index,ldk0,ldk1,ldk2}
+    rm -rf data{core,index,esplora,ldk0,ldk1,ldk2}
 }
 
 _mine() {