DuckSync v0.3.0 — Quack server, two-stage invalidation, transparent cached-table routing

.env.example

@@ -9,3 +9,4 @@ SNOWFLAKE_WAREHOUSE=COMPUTE_WH
 SNOWFLAKE_DATABASE=DUCKSYNC_TEST
 SNOWFLAKE_SCHEMA=TEST_DATA
 SNOWFLAKE_ROLE=your_existing_role
+SNOWFLAKE_SCHEMA_ROLE=your_no_compute_role_password_or_token

.github/workflows/MainDistributionPipeline.yml

@@ -16,8 +16,8 @@ jobs:
     name: Build extension binaries
     uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.5.4
     with:
-      duckdb_version: v1.5.0
-      ci_tools_version: v1.5.0
+      duckdb_version: v1.5.4
+      ci_tools_version: v1.5.4
       extension_name: ducksync
 
   duckdb-v15-build:
@@ -40,7 +40,7 @@ jobs:
     name: Code Quality Check
     uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@v1.5.4
     with:
-      duckdb_version: v1.5.0
-      ci_tools_version: v1.5.0
+      duckdb_version: v1.5.4
+      ci_tools_version: v1.5.4
       extension_name: ducksync
       format_checks: 'format;tidy'

.gitignore

@@ -4,6 +4,7 @@ cmake-build-*/
 
 # IDE
 .idea/
+.cache/
 .vscode/
 *.swp
 *.swo
@@ -28,3 +29,5 @@ venv/
 # Docker volumes
 postgres_data/
 minio_data/
+
+plans/

Makefile

@@ -7,7 +7,7 @@ EXT_CONFIG=${PROJ_DIR}extension_config.cmake
 # Include the Makefile from extension-ci-tools
 include extension-ci-tools/makefiles/duckdb_extension.Makefile
 
-.PHONY: test integration-test test-docker-up test-docker-down clean-test-data clean-all update-version test-compat
+.PHONY: test integration-test test-docker-up test-docker-down clean-test-data clean-all update-version test-compat install-adbc-driver
 
 test: release test-docker-up
 	@chmod +x $(PROJ_DIR)test/run_tests.sh
@@ -89,3 +89,22 @@ endif
 	@echo ""
 	@echo "NOTE: Restore submodule to stable version when done:"
 	@echo "  cd duckdb && git checkout v1.4.4"
+
+install-adbc-driver:
+	@src=$$(find "$$HOME/.duckdb/extensions" -name libadbc_driver_snowflake.so -print -quit 2>/dev/null); \
+	if [ -z "$$src" ]; then \
+		echo "ERROR: Could not find libadbc_driver_snowflake.so under $$HOME/.duckdb/extensions/"; \
+		echo "Run 'INSTALL snowflake FROM community' in DuckDB first."; \
+		exit 1; \
+	fi; \
+	if cp "$$src" /usr/local/lib/libadbc_driver_snowflake.so 2>/dev/null; then \
+		echo "Installed (copy)"; \
+		exit 0; \
+	fi; \
+	if sudo ln -sf "$$src" /usr/local/lib/libadbc_driver_snowflake.so; then \
+		echo "Installed (symlink via sudo)"; \
+		exit 0; \
+	fi; \
+	echo "ERROR: Failed to install libadbc_driver_snowflake.so from $$src"; \
+	echo "Copy or symlink it manually to /usr/local/lib/libadbc_driver_snowflake.so"; \
+	exit 1

README.md

@@ -3,19 +3,22 @@
 ### 📉 [How to Reduce Snowflake Compute Costs with Smart DuckDB Caching](https://danjsiegel.substack.com/p/how-to-reduce-snowflake-compute-costs)
 *A technical breakdown of intelligent query result caching and architectural cost optimization.*
 
-**DuckSync** is a DuckDB extension that provides intelligent query result caching between DuckDB and Snowflake. It uses **DuckLake** for storage (PostgreSQL catalog + Parquet files) and features transparent query routing, TTL-based expiration, and smart refresh based on source table metadata.
+**DuckSync** is a DuckDB extension that provides intelligent query result caching between DuckDB and Snowflake. It uses **DuckLake** for storage (PostgreSQL catalog + Parquet files) and supports explicit Quack serving, two-stage freshness checks, and transparent cached-table reads for DuckDB-native clients.
 
 ## Features
 
-- **Transparent Query Routing**: Queries automatically route to cached data when available
-- **Smart Refresh**: Only refreshes when source tables have changed (checks `last_altered`)
+- **Quack Server Lifecycle**: Start and stop a DuckDB Quack endpoint with `ducksync_serve(...)` and `ducksync_stop(...)`
+- **Transparent Query Routing**: `ducksync_query(...)` routes fully cached queries to local DuckLake data
+- **Transparent Cached-Table Reads**: After a cache is refreshed, `SELECT * FROM orders` can resolve directly to `orders_cache`
+- **Two-Stage Invalidation**: Use `invalidation_mode = 'two_stage'` with `metadata_secret` to skip warehouse-backed checks when `SHOW TABLES` rows/bytes are unchanged
+- **Smart Refresh**: Refreshes only when source tables have changed
 - **TTL Support**: Configurable cache expiration with time-to-live
 - **DuckLake Storage**: Uses DuckLake for efficient Parquet-based storage
 - **PostgreSQL Catalog**: Metadata stored in DuckLake's PostgreSQL catalog
 
 ## Prerequisites
 
-DuckSync automatically installs the required extensions (DuckLake, Snowflake) on first use.
+DuckSync automatically installs the required extensions it needs on demand: DuckLake and Snowflake for storage/querying, and Quack when you start a server with `ducksync_serve(...)`.
 
 You'll need:
 1. **PostgreSQL database** - for DuckLake catalog storage
@@ -66,6 +69,9 @@ SELECT * FROM ducksync_refresh('sales_summary');
 
 -- Query via ducksync_query (smart routing)
 SELECT * FROM ducksync_query('SELECT * FROM PROD.PUBLIC.SALES WHERE region = ''US''', 'prod');
+
+-- Or read the cached table transparently once refreshed
+SELECT * FROM SALES;
 ```
 
 ### Option B: Full Setup (New Users)
@@ -104,6 +110,27 @@ Full setup - attaches DuckLake and initializes DuckSync. Use if you don't have D
 - `pg_connection_string`: PostgreSQL connection string (libpq format)
 - `data_path`: Local path or S3 path for Parquet data files
 
+### `ducksync_serve(listen_uri [, token := ..., allow_other_hostname := ..., disable_ssl := ...])`
+
+Start a Quack listener explicitly for DuckDB-native clients.
+
+**Example:**
+```sql
+SELECT * FROM ducksync_serve('quack:localhost');
+SELECT * FROM ducksync_serve('quack:localhost', token := 'mytoken');
+```
+
+Then connect from another DuckDB client with Quack using the same listen URI.
+
+### `ducksync_stop(listen_uri)`
+
+Stop a previously started Quack listener.
+
+**Example:**
+```sql
+SELECT * FROM ducksync_stop('quack:localhost');
+```
+
 ### `ducksync_add_source(source_name, driver_type, secret_name, [passthrough_enabled])`
 
 Register a Snowflake data source.
@@ -124,6 +151,20 @@ Define a cached query result.
 - `source_query`: SQL query to cache results from
 - `monitor_tables`: List of tables to monitor for changes (e.g., `['DB.SCHEMA.TABLE']`)
 - `ttl_seconds` (optional): Cache TTL in seconds (NULL = no expiration)
+- `invalidation_mode` (named, optional): `last_altered`, `two_stage`, `ttl_only`, or `manual`
+- `metadata_secret` (named, required for `two_stage`): no-warehouse Snowflake secret for Stage 1 `SHOW TABLES`
+
+**Two-stage invalidation example:**
+```sql
+SELECT * FROM ducksync_create_cache(
+    'orders_cache',
+    'prod',
+    'SELECT * FROM ORDERS',
+    ['DUCKSYNC_TEST.TEST_DATA.ORDERS'],
+    invalidation_mode := 'two_stage',
+    metadata_secret := 'sf_meta'
+);
+```
 
 ### `ducksync_refresh(cache_name, [force])`
 
@@ -191,6 +232,29 @@ SELECT * FROM ducksync_query(
 
 **Best Practice:** Create caches with `monitor_tables` matching your Snowflake table names for automatic routing.
 
+### Transparent cached-table reads
+
+Once a cache has been refreshed, DuckSync also installs a `ReplacementScan` hook for DuckDB-native reads:
+
+```sql
+SELECT * FROM ducksync_create_cache(
+    'orders_cache',
+    'prod',
+    'SELECT * FROM ORDERS',
+    ['DUCKSYNC_TEST.TEST_DATA.ORDERS']
+);
+SELECT * FROM ducksync_refresh('orders_cache');
+
+-- Transparent cache hit
+SELECT * FROM ORDERS;
+SELECT * FROM main.orders;
+SELECT * FROM TEST_DATA.ORDERS;
+```
+
+**Naming convention:** keep the source table name and cache table name distinct, e.g. `orders` routes to `orders_cache`.
+
+**Miss behavior:** if the monitored table has not been refreshed yet, DuckSync raises an explicit error telling you to run `ducksync_refresh(...)` or use `ducksync_query(...)`. Unknown tables still return the normal DuckDB catalog error.
+
 ### Direct DuckLake Access
 
 Cached data is stored in standard DuckLake tables. Query them directly with normal DuckDB SQL:
@@ -224,14 +288,15 @@ JOIN local_segments l ON c.id = l.customer_id;
 DuckSync uses a "smart check" approach to minimize unnecessary data transfers:
 
 1. **TTL Check**: If `expires_at < now()`, trigger refresh
-2. **Metadata Check**: Query `information_schema.tables` for `last_altered` timestamps
-3. **Hash Comparison**: Compare hash of current metadata vs stored `source_state_hash`
-4. **Skip if Match**: If hashes match and TTL not expired, skip refresh
-5. **Refresh if Changed**: Execute query, write to DuckLake, update state
+2. **Stage 1 (optional)**: With `invalidation_mode = 'two_stage'`, run `SHOW TABLES` using `metadata_secret` and compare stored `rows`/`bytes`
+3. **Stage 2**: If needed, query `information_schema.tables.last_altered` with the warehouse-backed data secret
+4. **Hash Comparison**: Compare hash of current metadata vs stored `source_state_hash`
+5. **Skip if Match**: If hashes match and TTL not expired, skip refresh
+6. **Refresh if Changed**: Execute query, write to DuckLake, update state
 
 This approach means:
-- Zero Snowflake compute cost when data hasn't changed
-- Sub-second metadata checks vs full query execution
+- Zero warehouse wake-up when Stage 1 rows/bytes are unchanged
+- Lower-cost false-positive filtering with Stage 2 `last_altered`
 - Automatic refresh when source tables are modified
 
 ## Architecture
@@ -273,7 +338,7 @@ This approach means:
 ### Prerequisites
 
 - Docker (for PostgreSQL)
-- DuckDB v1.5.0+
+- DuckDB v1.5.4+
 - ADBC Snowflake driver (for Snowflake integration tests)
 
 ### Run Tests
@@ -313,7 +378,7 @@ make release
 
 ## Dependencies
 
-- **DuckDB v1.5.0+**
+- **DuckDB v1.5.4+**
 - **DuckLake extension** - auto-installed ([docs](https://ducklake.select/docs/))
 - **Snowflake extension** - auto-installed ([docs](https://duckdb.org/community_extensions/extensions/snowflake))
 

docs/COMPATIBILITY.md

@@ -71,14 +71,14 @@ DuckLake does not support `DEFAULT` expressions in `CREATE TABLE`. DuckSync pass
 
 ## DuckDB API Surface
 
-These are the DuckDB C++ API surfaces DuckSync uses. Check these when upgrading DuckDB (e.g., v1.4.4 → v1.5.0).
+These are the DuckDB C++ API surfaces DuckSync uses. Check these when upgrading DuckDB (e.g., v1.4.4 → v1.5.4).
 
 ### Extension loading
 
 | API | Location | Notes |
 |-----|----------|-------|
-| `ExtensionLoader::RegisterFunction(TableFunction)` | [`src/ducksync_extension.cpp:764`](../src/ducksync_extension.cpp) | Used to register all 6 table functions |
-| `ExtensionLoader::GetDatabaseInstance()` | [`src/ducksync_extension.cpp:800`](../src/ducksync_extension.cpp) | Used to register replacement scan hook |
+| `ExtensionLoader::RegisterFunction(TableFunction)` | [`src/ducksync_extension.cpp`](../src/ducksync_extension.cpp) | Used to register DuckSync table functions, including `ducksync_serve` and `ducksync_stop` |
+| `ExtensionLoader::GetDatabaseInstance()` | [`src/ducksync_extension.cpp`](../src/ducksync_extension.cpp) | Used to register the ReplacementScan hook |
 | `DucksyncExtension::Load(ExtensionLoader &)` | [`src/ducksync_extension.cpp:804`](../src/ducksync_extension.cpp) | Extension entry point |
 
 ### Connection and query execution
@@ -105,20 +105,40 @@ These are the DuckDB C++ API surfaces DuckSync uses. Check these when upgrading
 
 | API | Location | Notes |
 |-----|----------|-------|
-| `ClientContextState` (base class) | [`src/include/query_router.hpp:11`](../src/include/query_router.hpp) | `DuckSyncState` inherits from this |
-| `context.registered_state->GetOrCreate<T>(key)` | [`src/query_router.cpp:8`](../src/query_router.cpp) | Per-connection state management |
+| `ClientContextState` (base class) | [`src/include/query_router.hpp`](../src/include/query_router.hpp) | `DuckSyncState` inherits from this |
+| `context.registered_state->GetOrCreate<T>(key)` | [`src/query_router.cpp`](../src/query_router.cpp) | Per-connection state management |
 
 ### Table function registration
 
 | API | Location | Notes |
 |-----|----------|-------|
-| `TableFunction(name, arg_types, func, bind)` | [`src/ducksync_extension.cpp:766`](../src/ducksync_extension.cpp) | 6 functions registered (2 overloads each for init/setup_storage) |
-| `TableFunction::named_parameters` | [`src/ducksync_extension.cpp:778`](../src/ducksync_extension.cpp) | Used for `passthrough_enabled` and `force` named params |
-| `TableFunctionInitGlobal` | [`src/ducksync_extension.cpp:795`](../src/ducksync_extension.cpp) | Used for `ducksync_query` global state |
+| `TableFunction(name, arg_types, func, bind)` | [`src/ducksync_extension.cpp`](../src/ducksync_extension.cpp) | Used to register the DuckSync table-function surface, including init/setup overloads |
+| `TableFunction::named_parameters` | [`src/ducksync_extension.cpp`](../src/ducksync_extension.cpp) | Used for `passthrough_enabled`, `force`, `metadata_secret`, and `ducksync_serve` options |
+| `TableFunctionInitGlobal` | [`src/ducksync_extension.cpp`](../src/ducksync_extension.cpp) | Used for `ducksync_query` and `ducksync_serve` global state |
+
+### ReplacementScan
+
+| API | Location | Notes |
+|-----|----------|-------|
+| `DBConfig::GetConfig(db).replacement_scans.emplace_back(...)` | [`src/query_router.cpp`](../src/query_router.cpp) | Registers DuckSync's cached-table routing hook |
+| `ReplacementScanInput` | [`src/query_router.cpp`](../src/query_router.cpp) | Preserves naked, two-part, and three-part table-name forms |
+| `BaseTableRef` | [`src/query_router.cpp`](../src/query_router.cpp) | Rewrites cache hits to `{ducklake_catalog}.{source_name}.{cache_name}` |
 
 ---
 
-## Upgrading to DuckDB v1.5
+## Upgrading to DuckDB v1.5.4
+
+DuckSync's v1.5.4 upgrade adds support for the Quack remote-protocol API surface without changing DuckSync's own extension API:
+
+- `quack_serve`, `quack_stop`, and `quack_query` are now available through DuckDB's Quack extension
+- DuckSync exposes these via `ducksync_serve(...)` and `ducksync_stop(...)`
+- Transparent cached-table routing now relies on DuckDB `ReplacementScan`
+- There are no breaking changes to existing DuckSync SQL entry points
+
+### Arrow scanner constraint for `SHOW TABLES`
+
+`SHOW TABLES` results from Snowflake must still be consumed with `SELECT *` through `snowflake_query(...)`.
+Projecting individual columns from that Arrow result shape can crash the scanner. DuckSync's two-stage invalidation path is written to avoid this by selecting the full result set first and only then reading the returned columns in DuckDB.
 
 ### Before the release (proactive)
 
@@ -129,7 +149,7 @@ These are the DuckDB C++ API surfaces DuckSync uses. Check these when upgrading
 ### On release day
 
 ```bash
-make update-version VERSION=v1.5.0
+make update-version VERSION=v1.5.4
 make clean-all && make release
 make test
 bash docs/manual_test.sh
@@ -177,6 +197,11 @@ SELECT * FROM ducksync_setup_storage('host=...', 's3://...', 'my_ducksync_schema
 
 The metadata schema parameter only affects where DuckSync stores its own `sources`, `caches`, and `state` tables. It does **not** affect where cached data tables are stored (those always live in `catalog.source_name.cache_name`).
 
-### Future: Replacement scan
+### Transparent cached-table routing
+
+DuckSync now uses [`QueryRouter::Register`](../src/query_router.cpp) to install a `ReplacementScan` hook for cached source tables. The current contract is intentionally narrow:
 
-A true "drop-in" experience where `SELECT * FROM snowflake_table` is intercepted without `ducksync_query()` would require implementing [`QueryRouter::Register`](../src/query_router.cpp) (currently a no-op). This is tracked as future work.
+- cache hits route reads like `orders`, `main.orders`, and fully qualified monitored names to the local DuckLake cache table
+- misses still surface the normal DuckDB catalog error
+- monitored-but-unrefreshed tables raise a DuckSync-specific error that instructs the user to run `ducksync_refresh(...)` or use `ducksync_query(...)`
+- the callback must remain planner-only: no live network I/O, refresh work, or background activity

docs/manual_test.sh

@@ -86,7 +86,10 @@ CREATE SECRET sf_test (
 
 -- Test Snowflake connection
 SELECT '=== Testing Snowflake Connection ===' as section;
-SELECT * FROM snowflake_query('SELECT CURRENT_USER() as user, CURRENT_ROLE() as role', 'sf_test');
+-- Use single-column snowflake_query() calls here to avoid the known Arrow scanner crash
+-- for bare multi-column results from the Snowflake community extension.
+SELECT * FROM snowflake_query('SELECT CURRENT_USER() as current_user', 'sf_test');
+SELECT * FROM snowflake_query('SELECT CURRENT_ROLE() as current_role', 'sf_test');
 
 -- Initialize DuckSync
 SELECT '=== Initializing DuckSync ===' as section;

extension_config.cmake

@@ -6,6 +6,12 @@ duckdb_extension_load(ducksync
     LOAD_TESTS
 )
 
+if(MSVC)
+    set(CMAKE_CXX_STANDARD 17 CACHE STRING "C++ standard to enforce" FORCE)
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
+    add_compile_options($<$<COMPILE_LANGUAGE:CXX>:/std:c++17>)
+endif()
+
 # Any extra extensions that should be built
 # These are dependencies that DuckSync needs at runtime
 # duckdb_extension_load(postgres)