fix(vector): stream large local vector files through DuckDB

apps/geolibre-desktop/src/components/layout/DesktopShell.tsx

@@ -1372,20 +1372,20 @@
         // pyramid, which a path-less browser File cannot support.
         const layers =
           paths.length === imports.length
-            ? await loadDroppedVectorPaths(paths, {
-                onLargeDataset: confirmLargeVectorDataset,
-              })
+            ? await loadDroppedVectorPaths(paths, { onLargeDataset: confirmLargeVectorDataset })
             : await loadDroppedVectorFiles(
                 imports.map(({ file }) => file),
-                { onLargeDataset: confirmLargeVectorDataset },
+                {
+                  onLargeDataset: confirmLargeVectorDataset,
+                },
               );
         addImportedVectorLayers(layers);
       } catch (error) {
         setDropError(error instanceof Error ? error.message : t("kml.importFailed"));
       }
     });
     return () => setKmlFileImportHandler(null);
-  }, [addImportedVectorLayers, confirmLargeVectorDataset, t]);
+  }, [addImportedVectorLayers, t]);
 
   const addDroppedPhotos = useCallback(
     (result: GeotaggedPhotoResult | null): number => {
@@ -1661,7 +1661,7 @@
       disposed = true;
       unlisten?.();
     };
   }, [
     clearDropMessageLater,
     finishDrop,
     addDroppedRasters,
@@ -1805,7 +1805,7 @@
         clearDropMessageLater();
       }
     },
     [
       clearDropMessageLater,
       finishDrop,
       addDroppedRasters,

apps/geolibre-desktop/src/lib/duckdb-vector-guard.ts

@@ -16,7 +16,41 @@
  * (`osm-pbf-loader.ts`); unlike a raw byte size it is accurate for compressed
  * formats like GeoParquet, where a small file can hold millions of rows.
  */
-export const DUCKDB_VECTOR_FEATURE_WARN_COUNT = 500_000;
+export const DUCKDB_VECTOR_FEATURE_WARN_COUNT = 100_000;
+
+/**
+ * Local vector files at or above this size skip the in-memory JavaScript
+ * readers and stream through DuckDB instead.
+ *
+ * The JS readers all materialize the whole file on the main thread before
+ * yielding anything: `JSON.parse` over one giant string for GeoJSON, and shpjs's
+ * `parseShp`, which is fully synchronous and applies the `.prj` proj4 transform
+ * **per coordinate**. Past this size that is a visible freeze with no progress
+ * and no way to cancel. DuckDB reads off the main thread (native on desktop, the
+ * DuckDB-WASM worker in the browser) and reports a feature count first, so
+ * {@link DUCKDB_VECTOR_FEATURE_WARN_COUNT} gets a chance to fire.
+ *
+ * One threshold covers every format. For a zipped shapefile it is measured on
+ * the **uncompressed** `.shp`, which is what governs the parse cost — shapefiles
+ * compress heavily, so the archive's own size says little about it.
+ *
+ * Routing trades total time for responsiveness rather than being a pure win.
+ * Measured on a 197 MB / 170k-polygon `.shp` in the browser: with a projected
+ * `.prj` the worst main-thread stall drops from 8.9s to 1.6s while the whole
+ * load goes from 10.3s to 13.3s; with an already-WGS84 `.prj` (where proj4 is
+ * nearly free) it is 4.7s → 1.9s of stall for 6.1s → 10.3s overall. A UI that
+ * keeps responding is worth the extra seconds; a nine-second freeze reads as a
+ * crash. Below the threshold the JS readers stay the default: they are faster
+ * end to end and preserve field-name fidelity without a DuckDB round-trip.
+ *
+ * This also subsumes a hard engine limit. V8 caps a single string at
+ * `2**29 - 24` bytes (~537 MB), so a text file at or above *that* size could
+ * never be read by the text path at all — `readTextFile` / `File.text()` throw
+ * `RangeError: Invalid string length` before `JSON.parse` runs. Since 100 MB is
+ * far below the cap, such files are already routed away and the RangeError is
+ * now unreachable.
+ */
+export const DUCKDB_VECTOR_ROUTE_BYTES = 100 * 1024 * 1024; // 100 MB
 
 /** Details passed to {@link DuckDbVectorLoadOptions.onLargeDataset}. */
 export interface LargeVectorDataset {
@@ -96,3 +130,15 @@ export async function confirmLargeDataset(
   const proceed = await onLargeDataset(dataset);
   if (!proceed) throw new VectorLoadCancelledError();
 }
+
+/**
+ * Whether a file of this size should skip the in-memory JavaScript readers and
+ * stream through DuckDB instead. An unknown size (undefined) reads as "small",
+ * so a failed `stat` leaves the existing behaviour untouched rather than
+ * diverting every file to DuckDB on a metadata hiccup.
+ *
+ * @see DUCKDB_VECTOR_ROUTE_BYTES
+ */
+export function shouldRouteToDuckDb(sizeBytes: number | undefined): boolean {
+  return sizeBytes !== undefined && sizeBytes >= DUCKDB_VECTOR_ROUTE_BYTES;
+}

apps/geolibre-desktop/src/lib/tauri-io.ts

@@ -11,6 +11,7 @@ import {
   readFile,
   readTextFile,
   readTextFileLines,
+  stat,
   writeFile,
   writeTextFile,
 } from "@tauri-apps/plugin-fs";
@@ -30,6 +31,7 @@ import { IS_MAS_BUILD } from "./build-flags";
 import type { DuckDbVectorFile } from "./duckdb-vector-loader";
 import {
   confirmLargeDataset,
+  shouldRouteToDuckDb,
   type DuckDbVectorLoadOptions,
   type LargeVectorDataset,
 } from "./duckdb-vector-guard";
@@ -545,6 +547,23 @@ async function readLocalFileText(path: string): Promise<string> {
   }
 }
 
+/**
+ * A local file's size in bytes, read from filesystem metadata so the size is
+ * known *before* the file is read into memory. Returns undefined outside Tauri
+ * (the browser has no path-based `stat`; those callers use `File.size`) or when
+ * the `stat` fails — an unreadable path surfaces its own error at read time, so
+ * a metadata failure must not block the load.
+ */
+async function localFileSizeBytes(path: string): Promise<number | undefined> {
+  if (!isTauri()) return undefined;
+  try {
+    return (await stat(path)).size;
+  } catch (error) {
+    console.debug(`[GeoLibre] Could not stat "${path}" for the large-file guard.`, error);
+    return undefined;
+  }
+}
+
 function parseGpxText(text: string): FeatureCollection {
   const result = parseGpxLayer(text);
   return mergeFeatureCollections([result.waypoints, result.tracks, result.routes]);
@@ -718,6 +737,13 @@ function parseShapefileComponents({ file, sidecar }: UnzippedShapefile): Feature
  * already-extracted buffers, retrying through DuckDB if shpjs cannot read it. A
  * corrupt archive or one without a `.shp` throws, since GeoLibre reads only
  * shapefile `.zip`s.
+ *
+ * A `.shp` at or above {@link DUCKDB_VECTOR_ROUTE_BYTES} skips shpjs and streams
+ * through DuckDB: shpjs would otherwise freeze the main thread reprojecting
+ * every coordinate synchronously, with no progress, no cancel, and no
+ * feature-count guard. The threshold is measured on the *uncompressed* `.shp`,
+ * which is the number that governs the parse cost — shapefiles compress heavily,
+ * so the zip's own size says little about it.
  */
 async function loadShapefileZip(
   data: ArrayBuffer | Uint8Array,
@@ -730,6 +756,12 @@ async function loadShapefileZip(
   if (unzipped.isMultiPatch) {
     return loadDuckDbVector(unzipped.file, options);
   }
+  if (shouldRouteToDuckDb(unzipped.file.data.byteLength)) {
+    console.info(
+      `[GeoLibre] "${unzipped.file.name}" is ${Math.round(unzipped.file.data.byteLength / (1024 * 1024))} MB uncompressed; reading it with DuckDB instead of shpjs to keep the parse off the main thread.`,
+    );
+    return loadDuckDbVector(unzipped.file, options);
+  }
   try {
     return parseShapefileComponents(unzipped);
   } catch {
@@ -1772,7 +1804,16 @@ async function loadBrowserVectorFile(
   options?: DuckDbVectorLoadOptions,
 ): Promise<LoadedVectorLayer> {
   const extension = fileExtension(file.name);
-  if (extension === "geojson" || extension === "json") {
+  // Browser counterpart to the metadata preflight in `loadTauriVectorFile`;
+  // `File.size` is known without reading the blob, so the same rule applies.
+  const streamViaDuckDb = shouldRouteToDuckDb(file.size);
+  if (streamViaDuckDb) {
+    console.info(
+      `[GeoLibre] "${file.name}" is ${Math.round(file.size / (1024 * 1024))} MB; streaming it through DuckDB instead of the in-memory reader.`,
+    );
+  }
+
+  if (!streamViaDuckDb && (extension === "geojson" || extension === "json")) {
     try {
       return {
         data: await parseGeoJsonText(await file.text()),
@@ -1798,7 +1839,7 @@ async function loadBrowserVectorFile(
     };
   }
 
-  if (extension === "kml") {
+  if (!streamViaDuckDb && extension === "kml") {
     try {
       return {
         data: parseKmlText(await file.text()),
@@ -1809,14 +1850,14 @@ async function loadBrowserVectorFile(
     }
   }
 
-  if (extension === "gpx") {
+  if (!streamViaDuckDb && extension === "gpx") {
     return {
       data: parseGpxText(await file.text()),
       path: file.name,
     };
   }
 
-  if (isDelimitedTextFileName(file.name)) {
+  if (!streamViaDuckDb && isDelimitedTextFileName(file.name)) {
     const points = parseDelimitedTextFile(await file.text(), file.name);
     // No lon/lat columns: fall through to DuckDB so spatial CSV variants
     // (e.g. a WKT geometry column) still load.
@@ -2031,7 +2072,17 @@ async function loadTauriVectorFile(
   path: string;
 }> {
   const extension = fileExtension(path);
-  if (extension === "geojson" || extension === "json") {
+  // Decided from filesystem metadata, before the first byte is read, so an
+  // oversized file never starts a text parse that would freeze the UI.
+  const sizeBytes = await localFileSizeBytes(path);
+  const streamViaDuckDb = shouldRouteToDuckDb(sizeBytes);
+  if (streamViaDuckDb) {
+    console.info(
+      `[GeoLibre] "${browserSafeFileName(path)}" is ${Math.round((sizeBytes ?? 0) / (1024 * 1024))} MB; streaming it through DuckDB instead of the in-memory reader.`,
+    );
+  }
+
+  if (!streamViaDuckDb && (extension === "geojson" || extension === "json")) {
     try {
       return {
         data: await parseGeoJsonText(await readLocalFileText(path)),
@@ -2063,7 +2114,7 @@ async function loadTauriVectorFile(
     }
   }
 
-  if (extension === "kml") {
+  if (!streamViaDuckDb && extension === "kml") {
     try {
       return {
         data: parseKmlText(await readLocalFileText(path)),
@@ -2074,7 +2125,7 @@ async function loadTauriVectorFile(
     }
   }
 
-  if (extension === "gpx") {
+  if (!streamViaDuckDb && extension === "gpx") {
     try {
       return {
         data: parseGpxText(await readLocalFileText(path)),
@@ -2086,7 +2137,7 @@ async function loadTauriVectorFile(
     }
   }
 
-  if (isDelimitedTextFileName(path)) {
+  if (!streamViaDuckDb && isDelimitedTextFileName(path)) {
     const points = parseDelimitedTextFile(await readLocalFileText(path), path);
     // No lon/lat columns: fall through to DuckDB so spatial CSV variants
     // (e.g. a WKT geometry column) still load.

docs/user-guide/adding-data.md

@@ -16,6 +16,26 @@ The **Add Data** menu is the main way to bring layers into GeoLibre. It groups s
 
 Vector files are reprojected to EPSG:4326 on load. In the browser, vector import relies on DuckDB-WASM Spatial, with direct handling for GeoJSON, zipped Shapefiles, and KMZ archives.
 
+!!! warning "Large vector files"
+    There is no fixed size limit. Files **under 100 MB** are read by the
+    in-memory JavaScript readers, which are fastest for everyday data. At
+    **100 MB or larger**, GeoLibre streams the file through DuckDB instead —
+    off the main thread, so the interface keeps responding. For a zipped
+    Shapefile the threshold applies to the *uncompressed* `.shp`, since
+    shapefiles compress heavily and the archive's size says little about the
+    parse cost. This happens automatically; nothing is asked of you.
+
+    A separate check counts features once the source is open. Past
+    **100,000 features**, GeoLibre asks before converting every one to GeoJSON
+    in memory, because that is where memory rather than file size becomes the
+    limit — a small GeoParquet can hold millions of rows.
+
+    For very large data, converting first still pays: **Processing → Conversion
+    → Vector to PMTiles** (or GeoParquet) writes a format the map streams a tile
+    at a time instead of loading whole. GeoJSON is the most expensive option at
+    any size — it expands several-fold in memory — so prefer a Shapefile,
+    GeoParquet, or FlatGeobuf source when you have the choice.
+
 !!! tip "KML and KMZ"
     KML is read by an in-house parser that keeps the file's own symbology, so styled KML renders the way it does in Google Earth. A file that parser cannot handle falls back to the DuckDB Spatial reader, which loads the geometry without the styling.
 

tests/duckdb-vector-guard.test.ts

@@ -3,6 +3,8 @@ import { describe, it } from "node:test";
 import {
   confirmLargeDataset,
   DUCKDB_VECTOR_FEATURE_WARN_COUNT,
+  DUCKDB_VECTOR_ROUTE_BYTES,
+  shouldRouteToDuckDb,
   VectorLoadCancelledError,
 } from "../apps/geolibre-desktop/src/lib/duckdb-vector-guard";
 
@@ -59,3 +61,33 @@ describe("confirmLargeDataset", () => {
     );
   });
 });
+
+describe("shouldRouteToDuckDb", () => {
+  it("treats an unknown size as small", () => {
+    // A failed `stat` must not divert every file to DuckDB.
+    assert.equal(shouldRouteToDuckDb(undefined), false);
+  });
+
+  it("routes at the threshold and keeps one byte under it in-memory", () => {
+    assert.equal(shouldRouteToDuckDb(DUCKDB_VECTOR_ROUTE_BYTES), true);
+    assert.equal(shouldRouteToDuckDb(DUCKDB_VECTOR_ROUTE_BYTES - 1), false);
+  });
+
+  it("routes the reported 148 MB shapefile and 539 MB GeoJSON", () => {
+    assert.equal(shouldRouteToDuckDb(148 * 1024 * 1024), true);
+    assert.equal(shouldRouteToDuckDb(539 * 1000 * 1000), true);
+  });
+
+  it("stays below V8's maximum string length", () => {
+    // Text files at or above 2**29 - 24 bytes cannot be read into a string at
+    // all; routing far below that is what makes the RangeError unreachable.
+    assert.ok(DUCKDB_VECTOR_ROUTE_BYTES < 2 ** 29 - 24);
+  });
+});
+
+describe("configured defaults", () => {
+  it("routes at 100 MB and warns at 100k features", () => {
+    assert.equal(DUCKDB_VECTOR_ROUTE_BYTES, 100 * 1024 * 1024);
+    assert.equal(DUCKDB_VECTOR_FEATURE_WARN_COUNT, 100_000);
+  });
+});