feat: tpcgen-cli early access

.github/copilot-instructions.md

@@ -25,7 +25,7 @@ src/lakebench/
 │
 ├── datagen/
 │   ├── tpch.py                  # TPCHDataGenerator (uses tpchgen-rs, ~10x faster than alternatives)
-│   ├── tpcds.py                 # TPCDSDataGenerator (wraps DuckDB TPC-DS extension)
+│   ├── tpcds.py                 # TPCDSDataGenerator (bundled tpcgen-cli; DuckDB fallback)
 │   └── clickbench.py            # Downloads dataset from ClickHouse host
 │
 ├── engines/
@@ -135,7 +135,8 @@ Install only what you need:
 | `duckdb` | `duckdb`, `deltalake`, `pyarrow` |
 | `polars` | `polars`, `deltalake`, `pyarrow` |
 | `daft` | `daft`, `deltalake`, `pyarrow` |
-| `tpcds_datagen` | `duckdb`, `pyarrow` |
+| `tpcds_datagen` | Vendored `tpcgen-cli` binary selected for the host platform |
+| `tpcds_duckdb_datagen` | Legacy DuckDB TPC-DS generator |
 | `tpch_datagen` | `tpchgen-cli` |
 | `sparkmeasure` | `sparkmeasure` |
 | `sail` | `pysail`, `pyspark[connect]`, `deltalake`, `pyarrow` |

.github/workflows/native-wheels.yml

@@ -0,0 +1,103 @@
+name: Native LakeBench wheels
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  TPCGEN_COMMIT: a0e30d55358bd5a21f2981e92b5978cdcb669d95
+  TPCGEN_LOCK_COMMIT: 1ec501fc4f23ee527b5da5dce646fb3c5dba2385
+
+jobs:
+  build:
+    name: ${{ matrix.platform }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux
+            runner: ubuntu-22.04
+            target: x86_64
+            wheel_tag: py3-none-manylinux_2_17_x86_64
+          - platform: windows
+            runner: windows-latest
+            target: x64
+            wheel_tag: py3-none-win_amd64
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Check out pinned tpcgen-rs
+        uses: actions/checkout@v4
+        with:
+          repository: datafusion-contrib/tpcgen-rs
+          ref: ${{ env.TPCGEN_COMMIT }}
+          path: .build/tpcgen-rs
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          python-version: "3.11"
+          enable-cache: true
+
+      - name: Add pinned upstream lockfile
+        shell: pwsh
+        run: |
+          Invoke-WebRequest `
+            "https://raw.githubusercontent.com/datafusion-contrib/tpcgen-rs/${{ env.TPCGEN_LOCK_COMMIT }}/Cargo.lock" `
+            -OutFile .build/tpcgen-rs/Cargo.lock
+
+      - name: Build pinned tpcgen-cli wheel
+        uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b
+        with:
+          target: ${{ matrix.target }}
+          args: --locked --release --out dist
+          working-directory: .build/tpcgen-rs/tpcgen-cli
+          manylinux: ${{ matrix.platform == 'linux' && '2_17' || '' }}
+
+      - name: Stage and verify tpcgen-cli
+        id: stage
+        shell: pwsh
+        run: |
+          $lines = python scripts/stage_tpcgen_binary.py `
+            --wheel-directory .build/tpcgen-rs/tpcgen-cli/dist `
+            --upstream-root .build/tpcgen-rs `
+            --output-directory .build/tpcgen-bundle `
+            --platform ${{ matrix.platform }}
+          $lines | ForEach-Object {
+            Write-Host $_
+            Add-Content -Path $env:GITHUB_OUTPUT -Value $_
+          }
+
+      - name: Build self-contained LakeBench wheel
+        shell: pwsh
+        env:
+          LAKEBENCH_TPCGEN_BINARY: ${{ steps.stage.outputs.binary }}
+          LAKEBENCH_TPCGEN_PROVENANCE: ${{ steps.stage.outputs.provenance }}
+          LAKEBENCH_TPCGEN_LICENSE: ${{ steps.stage.outputs.license }}
+          LAKEBENCH_WHEEL_TAG: ${{ matrix.wheel_tag }}
+        run: uv build --wheel
+
+      - name: Install and smoke test LakeBench wheel
+        shell: pwsh
+        run: |
+          $wheel = (Get-ChildItem dist\lakebench-*.whl | Select-Object -First 1).FullName
+          uv venv .build/wheel-test
+          if ("${{ matrix.platform }}" -eq "windows") {
+            $python = ".build\wheel-test\Scripts\python.exe"
+          } else {
+            $python = ".build/wheel-test/bin/python"
+          }
+          uv pip install --python $python $wheel
+          & $python -c "from lakebench.datagen._tpcgen_cli import TpcgenCli; print(TpcgenCli().run(['tpcds', '--help']).returncode)"
+          & $python scripts/smoke_test_tpcgen_bundle.py
+
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: lakebench-${{ matrix.platform }}-x86_64
+          path: dist/lakebench-*.whl
+          if-no-files-found: error

.github/workflows/tests.yml

@@ -58,20 +58,20 @@ jobs:
       matrix:
         include:
           - engine: duckdb
-            extras_flags: "--extra duckdb --extra tpcds_datagen --extra tpch_datagen"
+            extras_flags: "--extra duckdb --extra tpcds_duckdb_datagen --extra tpch_datagen"
             test_file: "tests/integration/test_duckdb.py"
           - engine: daft
-            extras_flags: "--extra daft --extra tpcds_datagen --extra tpch_datagen"
+            extras_flags: "--extra daft --extra tpcds_duckdb_datagen --extra tpch_datagen"
             test_file: "tests/integration/test_daft.py"
           - engine: polars
-            extras_flags: "--extra polars --extra tpcds_datagen --extra tpch_datagen"
+            extras_flags: "--extra polars --extra tpcds_duckdb_datagen --extra tpch_datagen"
             test_file: "tests/integration/test_polars.py"
           - engine: spark
-            extras_flags: "--extra spark --extra tpcds_datagen --extra tpch_datagen"
+            extras_flags: "--extra spark --extra tpcds_duckdb_datagen --extra tpch_datagen"
             test_file: "tests/integration/test_spark.py"
             java: "17"
           - engine: sail
-            extras_flags: "--extra sail --extra tpcds_datagen --extra tpch_datagen"
+            extras_flags: "--extra sail --extra tpcds_duckdb_datagen --extra tpch_datagen"
             test_file: "tests/integration/test_sail.py"
 
     steps:

.gitignore

@@ -86,3 +86,4 @@ scratch/
 # uv lockfile - not committed; resolution is intentionally left floating so
 # each environment picks the newest packages available in its configured index.
 uv.lock
+.cibuildwheel-build-requirements/

README.md

@@ -167,12 +167,16 @@ pip install lakebench[duckdb,polars,tpcds_datagen,tpch_datagen,sparkmeasure]
 ```
 
 > _Note: the `daft` extra pins `deltalake` to 1.5.x (Daft cannot read the Arrow `Utf8View` parquet that `deltalake` 1.6.x emits from `MERGE`), so it must be installed in its own environment rather than alongside `duckdb`, `polars`, or `sail`._
+>
+> `tpcds_datagen` uses the self-contained Rust `tpcgen-cli` binary bundled in
+> the Windows x86_64 and Linux x86_64 LakeBench wheels. The legacy DuckDB
+> generator is available separately through `tpcds_duckdb_datagen`.
 
 ## Example Usage
 To run any LakeBench benchmark, first do a one time generation of the data required for the benchmark and scale of interest. LakeBench provides datagen classes to quickly generate parquet datasets required by the benchmarks.
 
 ### Data Generation
-- **TPC-H** data generation is provided via the (tpchgen-rs)[https://github.com/clflushopt/tpchgen-rs] project. The project is currently about 10x+ faster than the next closest method of generating TPC-H datasets. _The TPC-DS version of project is currently under development._
+- **TPC-H** data generation is provided via the [tpcgen-rs](https://github.com/datafusion-contrib/tpcgen-rs) project's legacy `tpchgen-cli` package. The project is currently about 10x+ faster than the next closest method of generating TPC-H datasets. LakeBench will migrate this path to the unified `tpcgen-cli` package after its official release.
 
     _The below are generation runtimes on a 64 v-core VM writing to OneLake. Scale factors below 1000 can easily be generated on a 2 v-core machine._
     | Scale Factor | Duration (hh:mm:ss)|
@@ -182,7 +186,7 @@ To run any LakeBench benchmark, first do a one time generation of the data requi
     | 100          | 00:01:09           |
     | 1000         | 00:10:15           |
 
-- **TPC-DS** data generation is provided via the DuckDB [TPC-DS](https://duckdb.org/docs/stable/core_extensions/tpcds) extension. The LakeBench wrapper around DuckDB adds support for writing out parquet files with a provided row-group target file size as normally the files generated by DuckDB are atypically small (i.e. 10MB) and are most suitable for ultra-small scale scenarios. LakeBench defaults to target 128MB row groups but can be configured via the `target_row_group_size_mb` parameter of both TPC-H and TPC-DS DataGenerator classes.
+- **TPC-DS** data generation defaults to a pinned build of the unified Rust `tpcgen-cli` from [datafusion-contrib/tpcgen-rs#406](https://github.com/datafusion-contrib/tpcgen-rs/pull/406). The temporary Windows x86_64 and manylinux 2.17 x86_64 executables are committed under `native/tpcgen`. LakeBench will migrate to taking a dependency straight from `tpcgen-cli` once it's released in PyPi.
 - **ClickBench** data is downloaded directly from the Clickhouse host site.
 
 #### TPC-H Data Generation
@@ -208,6 +212,26 @@ datagen.run()
 ```
 
 _Notes:_
+- By default, each table is split automatically using its estimated total
+  compressed size: 128 MiB files below 10 GiB, 256 MiB below 1 TiB, 512 MiB
+  below 5 TiB, and 1 GiB for larger tables. Estimated physical size is
+  calculated directly from each table's SF1000 baseline for any supported
+  scale factor; part counts are always selected automatically.
+- `target_row_group_size_mb` is an on-disk compressed-size target. LakeBench
+  converts it to the uncompressed-byte value expected by `tpcgen-cli` using
+  per-table ZSTD(1) or Snappy compression ratios measured from SF10
+  C-compatible output on September 3, 2026. Automatic part counts are adjusted
+  for the selected codec. The row-group conversion includes a 5% planning
+  margin for upstream's estimated bytes-per-source-row model. Other compressed
+  codecs require an explicit `compression_factor`, which is used for both row
+  groups and file estimates.
+- Output remains organized as `<root>/<table>/*.parquet`.
+- To use the legacy implementation, install
+  `lakebench[tpcds_duckdb_datagen]` on Python 3.10+ and pass
+  `backend="duckdb"`.
+- Editable/source installations use the matching executable from
+  `native/tpcgen`. An explicit `executable=...` override and `PATH` fallback
+  remain available for generator development.
 - TPC-DS data up to SF1000 can be generated on a 32-vCore machine. 
 - TPC-H datasets are generated extremely fast (i.e. SF1000 in 10 minutes on an 64-vCore machine).
 - The ClickBench dataset (only 1 size) should download with partitioned files in ~ 1 minute and ~ 6 minutes as a single file. 

hatch_build.py

@@ -0,0 +1,57 @@
+import os
+import platform
+import stat
+from pathlib import Path
+
+from hatchling.builders.hooks.plugin.interface import BuildHookInterface
+
+
+class CustomBuildHook(BuildHookInterface):
+    VENDORED_BINARIES = {
+        ("windows", "amd64"): ("windows-x86_64/tpcgen-cli.exe", "py3-none-win_amd64"),
+        ("windows", "x86_64"): ("windows-x86_64/tpcgen-cli.exe", "py3-none-win_amd64"),
+        ("linux", "amd64"): ("linux-x86_64/tpcgen-cli", "py3-none-manylinux_2_17_x86_64"),
+        ("linux", "x86_64"): ("linux-x86_64/tpcgen-cli", "py3-none-manylinux_2_17_x86_64"),
+    }
+
+    def initialize(self, version, build_data):
+        if self.target_name != "wheel":
+            return
+
+        binary = os.environ.get("LAKEBENCH_TPCGEN_BINARY")
+        if not binary:
+            platform_key = (platform.system().lower(), platform.machine().lower())
+            vendored = self.VENDORED_BINARIES.get(platform_key)
+            if vendored is None:
+                return
+
+            relative_binary, default_wheel_tag = vendored
+            bundle_root = Path(self.root) / "native" / "tpcgen"
+            binary = bundle_root / relative_binary
+            provenance = binary.parent / "provenance.json"
+            license_file = bundle_root / "LICENSE.tpcgen-rs"
+            wheel_tag = default_wheel_tag
+        else:
+            wheel_tag = os.environ.get("LAKEBENCH_WHEEL_TAG")
+            provenance = os.environ.get("LAKEBENCH_TPCGEN_PROVENANCE")
+            license_file = os.environ.get("LAKEBENCH_TPCGEN_LICENSE")
+            if not wheel_tag or not provenance or not license_file:
+                raise RuntimeError(
+                    "Native wheel builds require LAKEBENCH_WHEEL_TAG, "
+                    "LAKEBENCH_TPCGEN_PROVENANCE, and LAKEBENCH_TPCGEN_LICENSE."
+                )
+
+        binary_path = Path(binary).resolve()
+        provenance_path = Path(provenance).resolve()
+        license_path = Path(license_file).resolve()
+        for required_path in (binary_path, provenance_path, license_path):
+            if not required_path.is_file():
+                raise RuntimeError(f"Native wheel input is missing: {required_path}")
+
+        binary_path.chmod(binary_path.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+
+        build_data["pure_python"] = False
+        build_data["tag"] = wheel_tag
+        build_data["force_include"][str(binary_path)] = f"lakebench/datagen/_bin/{binary_path.name}"
+        build_data["force_include"][str(provenance_path)] = "lakebench/datagen/_bin/provenance.json"
+        build_data["force_include"][str(license_path)] = "lakebench/datagen/_bin/LICENSE.tpcgen-rs"