diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8c359859987..7a3385dc5a8 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -46,12 +46,12 @@ jobs: name: cargo-clippy (${{ strategy.job-index == 0 && 'Linux x86-64' || (strategy.job-index == 1 && 'Windows x86-64' || 'macOS arm64') }}) strategy: matrix: - os: ["ubuntu-22.04", "windows-2022", "macos-14"] + os: ["ubuntu-22.04", "windows-2022", "macos-26"] runs-on: ${{ matrix.os }} steps: - # Enable the sccache runs-on S3 backend for Linux runners, to cache C/C++/CUDA and Rust. + # Enable the sccache runs-on S3 backend for Linux runners, to cache C/C++ and Rust. # On GitHub-hosted or fork runners, the runs-on action is ignored. - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60 # v2.1.2 with: @@ -101,44 +101,6 @@ jobs: run: brew install automake if: runner.os == 'macOS' - # TODO: cache CUDA from Program Files on Windows, because it takes 6+ minutes to install. - - name: CUDA toolchain - uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 - with: - cuda: '12.4.1' - method: network - sub-packages: '["nvcc", "cudart"]' - if: runner.os == 'Linux'|| runner.os == 'Windows' - - - name: Configure ROCm cache (Windows) - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - id: rocm-cache - with: - path: C:\Program Files\AMD\ROCm - key: ${{ runner.os }}-rocm - if: runner.os == 'Windows' && false - - - name: ROCm toolchain - run: | - ROCM_VERSION=6.2.2 - sudo mkdir -p --mode=0755 /etc/apt/keyrings - curl -L https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION jammy main" | sudo tee /etc/apt/sources.list.d/rocm.list > /dev/null - echo -e "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" | sudo tee /etc/apt/preferences.d/rocm-pin-600 > /dev/null - sudo apt-get update - DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends rocm-hip-runtime-dev - echo "/opt/rocm/lib" | sudo tee /etc/ld.so.conf.d/rocm.conf > /dev/null - sudo ldconfig - if: runner.os == 'Linux' - - - name: ROCm toolchain - run: | - $ErrorActionPreference = "Stop" - Invoke-WebRequest -Uri https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe -OutFile "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" - Start-Process "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" -ArgumentList '-install' -NoNewWindow -Wait - Remove-Item "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" - if: runner.os == 'Windows' && steps.rocm-cache.outputs.cache-hit != 'true' - - name: Configure source deps cache uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -169,26 +131,6 @@ jobs: - name: cargo clippy run: | cargo -Zgitoxide -Zgit clippy --locked --all-targets --features runtime-benchmarks -- -D warnings - if: runner.os == 'macOS' - - - name: cargo clippy - run: | - cargo -Zgitoxide -Zgit clippy --locked --all-targets --features runtime-benchmarks,cuda -- -D warnings - if: runner.os == 'Linux' || runner.os == 'Windows' - - - name: cargo clippy (ROCm) - run: | - cargo -Zgitoxide -Zgit clippy --locked --all-targets --features rocm -- -D warnings - if: runner.os == 'Linux' - - - name: cargo clippy (ROCm) - env: - # Why `PROGRA~1` instead of `Program Files`? Because Windows! - HIPCC: C:\PROGRA~1\AMD\ROCm\6.1\bin\hipcc.bin.exe - HIP_PLATFORM: amd - run: | - cargo -Zgitoxide -Zgit clippy --locked --all-targets --features rocm -- -D warnings - if: runner.os == 'Windows' cargo-runtime-build: name: cargo-runtime-build (Linux x86-64) @@ -242,7 +184,7 @@ jobs: name: cargo-test (${{ strategy.job-index == 0 && 'Linux x86-64' || (strategy.job-index == 1 && 'Windows x86-64' || 'macOS arm64') }}) strategy: matrix: - os: ["ubuntu-22.04", "windows-2022", "macos-14"] + os: ["ubuntu-22.04", "windows-2022", "macos-26"] runs-on: ${{ matrix.os }} # Don't use the full 6 hours if a test hangs timeout-minutes: 120 @@ -314,6 +256,77 @@ jobs: run: echo "RUSTFLAGS=$env:RUSTFLAGS -C debuginfo=0 -C link-arg=/DEBUG:NONE" >> $env:GITHUB_ENV if: runner.os == 'Windows' + # Software Vulkan (lavapipe) so the wgpu byte-identity test runs on the GPU-less runners; + # macOS uses the runner's native Metal adapter and needs nothing here. + - name: Install Vulkan runtime (Linux) + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends --yes mesa-vulkan-drivers + if: runner.os == 'Linux' + + - name: Install Vulkan runtime (Windows) + run: | + & { + # See https://vulkan.lunarg.com/sdk/home for updates + $sdkVersion = "1.4.341.0" + if ("${{ runner.arch }}" -eq "ARM64") { + $arch = "ARM64" + $urlPath = "warm" + $expectedHash = "4a350fbeec743683494f746e9cf7e2052494ca9ae618931afc2abb8985e072b0" + } else { + $arch = "X64" + $urlPath = "windows" + $expectedHash = "20571dcc8292b80a47d68bbbfa4094d87044041f50f4e2e1f326a01e3732b226" + } + + $url = "https://sdk.lunarg.com/sdk/download/$sdkVersion/$urlPath/VulkanRT-$arch-$sdkVersion-Components.zip" + Invoke-WebRequest -Uri $url -OutFile VulkanRT.zip + + $actualHash = (Get-FileHash VulkanRT.zip -Algorithm SHA256).Hash.ToLower() + if ($actualHash -ne $expectedHash.ToLower()) { + throw "SHA256 mismatch: expected $expectedHash, got $actualHash" + } + + 7z e VulkanRT.zip -o"${{ runner.temp }}\vulkan" -r "vulkan-1.*" + Remove-Item VulkanRT.zip + + $vulkanPath = "${{ runner.temp }}\vulkan" + "$vulkanPath" >> $env:GITHUB_PATH + } + + & { + # See https://github.com/mmozeiko/build-mesa/releases for updates + $version = "26.0.5" + if ("${{ runner.arch }}" -eq "ARM64") { + $arch = "arm64" + $archJson = "aarch64" + $expectedHash = "6980c51ce19cff6318d001d05a5fbd9c53cff64d29ea04b5e0285a09b6fe7ae4" + } else { + $arch = "x64" + $archJson = "x86_64" + $expectedHash = "289b283fd13a0029d0226d9695807e62343bb25da2499545bb5f973c97146262" + } + + $url = "https://github.com/mmozeiko/build-mesa/releases/download/$version/mesa-lavapipe-$arch-$version.7z" + Invoke-WebRequest -Uri $url -OutFile mesa-lavapipe.7z + + $actualHash = (Get-FileHash mesa-lavapipe.7z -Algorithm SHA256).Hash.ToLower() + if ($actualHash -ne $expectedHash.ToLower()) { + throw "SHA256 mismatch: expected $expectedHash, got $actualHash" + } + + 7z x mesa-lavapipe.7z -o"${{ runner.temp }}\mesa" + Remove-Item mesa-lavapipe.7z + + $mesaPath = "${{ runner.temp }}\mesa" + "$mesaPath" >> $env:GITHUB_PATH + + $icdPath = "$mesaPath\lvp_icd.$archJson.json" + New-Item -Path "HKLM:\SOFTWARE\Khronos\Vulkan\Drivers" -Force| Out-Null + Set-ItemProperty -Path "HKLM:\SOFTWARE\Khronos\Vulkan\Drivers" -Name $icdPath -Value 0 -Type DWord + } + if: runner.os == 'Windows' + - name: cargo nextest run --locked run: | cargo -Zgitoxide -Zgit nextest run --locked @@ -461,25 +474,6 @@ jobs: with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: CUDA toolchain - uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 - with: - cuda: '12.4.1' - method: network - sub-packages: '["nvcc", "cudart"]' - - - name: ROCm toolchain - run: | - ROCM_VERSION=6.2.2 - sudo mkdir -p --mode=0755 /etc/apt/keyrings - curl -L https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION jammy main" | sudo tee /etc/apt/sources.list.d/rocm.list > /dev/null - echo -e "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" | sudo tee /etc/apt/preferences.d/rocm-pin-600 > /dev/null - sudo apt-get update - DEBIAN_FRONTEND=noninteractive sudo apt-get install -y --no-install-recommends rocm-hip-runtime-dev - echo "/opt/rocm/lib" | sudo tee /etc/ld.so.conf.d/rocm.conf > /dev/null - sudo ldconfig - - name: Install cargo-udeps uses: taiki-e/install-action@7a79fe8c3a13344501c80d99cae481c1c9085912 # 2.81.10 with: diff --git a/.github/workflows/rustdoc.yml b/.github/workflows/rustdoc.yml index 3d9b07a4962..9f83b3992a6 100644 --- a/.github/workflows/rustdoc.yml +++ b/.github/workflows/rustdoc.yml @@ -42,16 +42,9 @@ jobs: restore-keys: | ${{ runner.os }}-cargo- - - name: CUDA toolchain - uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 - with: - cuda: '12.4.1' - method: network - sub-packages: '["nvcc", "cudart"]' - - # Build the rust crate docs + # Build the rust crate docs (wgpu GPU plotting is a default feature) - name: Build Documentation - run: cargo -Zgitoxide -Zgit doc --locked --all --no-deps --lib --features cuda + run: cargo -Zgitoxide -Zgit doc --locked --all --no-deps --lib env: RUSTDOCFLAGS: "-D rustdoc::broken-intra-doc-links -D rustdoc::private_intra_doc_links -Z unstable-options --enable-index-page" diff --git a/.github/workflows/snapshot-build.yml b/.github/workflows/snapshot-build.yml index e8ef8aac9a4..1cbc46fce9a 100644 --- a/.github/workflows/snapshot-build.yml +++ b/.github/workflows/snapshot-build.yml @@ -133,12 +133,6 @@ jobs: # TODO: Pull is a workaround for https://github.com/moby/moby/issues/48197#issuecomment-2472265028 docker pull --platform linux/amd64/v2 $IMAGE docker run --rm --platform linux/amd64/v2 --entrypoint /bin/cat $IMAGE /$ARTIFACT > $ARTIFACT-ubuntu-x86_64-v2-$SAFE_REF_NAME - if [ "${{ matrix.build.image }}" == "farmer" ]; then - docker run --rm --platform linux/amd64 --entrypoint /bin/cat $IMAGE /$ARTIFACT-rocm > $ARTIFACT-rocm-ubuntu-x86_64-skylake-$SAFE_REF_NAME - # TODO: Pull is a workaround for https://github.com/moby/moby/issues/48197#issuecomment-2472265028 - docker pull --platform linux/amd64/v2 $IMAGE - docker run --rm --platform linux/amd64/v2 --entrypoint /bin/cat $IMAGE /$ARTIFACT-rocm > $ARTIFACT-rocm-ubuntu-x86_64-v2-$SAFE_REF_NAME - fi docker run --rm --platform linux/arm64 --entrypoint /bin/cat $IMAGE /$ARTIFACT > $ARTIFACT-ubuntu-aarch64-$SAFE_REF_NAME if: matrix.build.upload-executables @@ -163,7 +157,7 @@ jobs: strategy: matrix: build: - - os: macos-14 + - os: macos-26 target: aarch64-apple-darwin suffix: macos-aarch64-${{ github.ref_name }} - os: windows-2022 @@ -223,30 +217,6 @@ jobs: run: brew install automake if: runner.os == 'macOS' - - name: CUDA toolchain - uses: Jimver/cuda-toolkit@3d45d157f327c09c04b50ee6ccdea2d9d017ec76 # v0.2.35 - with: - cuda: '12.4.1' - method: network - sub-packages: '["nvcc", "cudart"]' - if: runner.os == 'Windows' - - - name: Configure ROCm cache (Windows) - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - id: rocm-cache - with: - path: C:\Program Files\AMD\ROCm - key: ${{ runner.os }}-rocm - if: runner.os == 'Windows' - - - name: ROCm toolchain - run: | - $ErrorActionPreference = "Stop" - Invoke-WebRequest -Uri https://download.amd.com/developer/eula/rocm-hub/AMD-Software-PRO-Edition-24.Q3-WinSvr2022-For-HIP.exe -OutFile "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" - Start-Process "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" -ArgumentList '-install' -NoNewWindow -Wait - Remove-Item "${env:RUNNER_TEMP}\HIP-SDK-Installer.exe" - if: runner.os == 'Windows' && steps.rocm-cache.outputs.cache-hit != 'true' - - name: Configure cache uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -260,24 +230,6 @@ jobs: - name: Build farmer run: | cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer - if: runner.os == 'macOS' || !startsWith(matrix.build.target, 'x86_64') - - # ROCm can't be enabled together with CUDA for now - - name: Build farmer (ROCm, Windows) - env: - # Why `PROGRA~1` instead of `Program Files`? Because Windows! - HIPCC: C:\PROGRA~1\AMD\ROCm\6.1\bin\hipcc.bin.exe - HIP_PLATFORM: amd - run: | - cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features rocm - move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe - if: runner.os == 'Windows' && startsWith(matrix.build.target, 'x86_64') - - - name: Build farmer - run: | - cargo -Zgitoxide -Zgit build --locked -Z build-std --target ${{ matrix.build.target }} --profile production --bin subspace-farmer --features cuda - # TODO: We don't configure CUDA for cross-compilation purposes, hence only x86-64 for now - if: runner.os == 'Windows' && startsWith(matrix.build.target, 'x86_64') - name: Build node run: | @@ -319,7 +271,6 @@ jobs: - name: Sign Application (Windows) run: | AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe" - AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe" AzureSignTool sign --azure-key-vault-url "${{ secrets.AZURE_KEY_VAULT_URI }}" --azure-key-vault-client-id "${{ secrets.AZURE_CLIENT_ID }}" --azure-key-vault-client-secret "${{ secrets.AZURE_CLIENT_SECRET }}" --azure-key-vault-tenant-id "${{ secrets.AZURE_TENANT_ID }}" --azure-key-vault-certificate "${{ secrets.AZURE_CERT_NAME }}" --file-digest sha512 --timestamp-rfc3161 http://timestamp.digicert.com -v "${{ env.PRODUCTION_TARGET }}/subspace-node.exe" # Allow code signing to fail on non-release builds and in non-subspace repos (forks) continue-on-error: ${{ github.repository_owner != 'autonomys' || github.event_name != 'push' || github.ref_type != 'tag' }} @@ -342,7 +293,6 @@ jobs: run: | mkdir executables move ${{ env.PRODUCTION_TARGET }}/subspace-farmer.exe executables/subspace-farmer-${{ env.SAFE_SUFFIX }}.exe - move ${{ env.PRODUCTION_TARGET }}/subspace-farmer-rocm.exe executables/subspace-farmer-rocm-${{ env.SAFE_SUFFIX }}.exe move ${{ env.PRODUCTION_TARGET }}/subspace-node.exe executables/subspace-node-${{ env.SAFE_SUFFIX }}.exe if: runner.os == 'Windows' diff --git a/Cargo.lock b/Cargo.lock index 1d4d002808e..cf96197ba0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,12 +12,54 @@ dependencies = [ "regex", ] +[[package]] +name = "ab-blake3" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b86172b9a93eed540aa8bcabf8e4f3f6fcfdbfcd3573dcc284fed94722c5da" +dependencies = [ + "blake3", +] + [[package]] name = "ab-chacha8" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78928d0e8969c1b539a98de5cf3a678d136f160415c765f3f4c464b5eee39976" +[[package]] +name = "ab-proof-of-space" +version = "0.1.0" +dependencies = [ + "ab-blake3", + "ab-chacha8", + "chacha20 0.10.1", + "derive_more 2.1.1", + "rayon", + "rclite", + "seq-macro", + "sha2 0.11.0", + "subspace-core-primitives", +] + +[[package]] +name = "ab-proof-of-space-gpu" +version = "0.1.0" +dependencies = [ + "ab-blake3", + "ab-chacha8", + "cargo-gpu-install", + "derive_more 2.1.1", + "futures", + "parking_lot 0.12.5", + "rclite", + "spirv-std", + "subspace-core-primitives", + "thiserror 2.0.18", + "tracing", + "wgpu", +] + [[package]] name = "actix-codec" version = "0.5.2" @@ -73,7 +115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -187,7 +229,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -372,9 +414,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "approx" @@ -396,7 +438,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -547,7 +589,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -573,7 +615,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -648,7 +690,7 @@ checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -731,6 +773,15 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "ash" +version = "0.38.0+1.3.281" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bb44936d800fea8f016d7f2311c6a4f97aebd5dc86f09906139ec848cf3a46f" +dependencies = [ + "libloading", +] + [[package]] name = "asn1-rs" version = "0.7.1" @@ -755,7 +806,7 @@ checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure 0.13.2", ] @@ -767,7 +818,7 @@ checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -953,7 +1004,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1114,7 +1165,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -1267,7 +1318,16 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" dependencies = [ - "bit-vec", + "bit-vec 0.8.0", +] + +[[package]] +name = "bit-set" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ddef2995421ab6a5c779542c81ee77c115206f4ad9d5a8e05f4ff49716a3dd" +dependencies = [ + "bit-vec 0.9.1", ] [[package]] @@ -1276,6 +1336,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" + [[package]] name = "bitcoin-io" version = "0.1.4" @@ -1303,6 +1369,9 @@ name = "bitflags" version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +dependencies = [ + "serde_core", +] [[package]] name = "bitvec" @@ -1401,6 +1470,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + [[package]] name = "blocking" version = "1.6.2" @@ -1448,6 +1526,15 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "branches" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e426eb5cc1900033930ec955317b302e68f19f326cc7bb0c8a86865a826cdf0c" +dependencies = [ + "rustc_version", +] + [[package]] name = "bs58" version = "0.5.1" @@ -1493,6 +1580,20 @@ name = "bytemuck" version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] [[package]] name = "byteorder" @@ -1543,6 +1644,19 @@ dependencies = [ "serde_core", ] +[[package]] +name = "cargo-gpu-install" +version = "0.10.0-alpha.1" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=bd49568baf08301107c9f698494735724980dee0#bd49568baf08301107c9f698494735724980dee0" +dependencies = [ + "anyhow", + "cargo_metadata 0.21.0", + "directories 6.0.0", + "log", + "serde", + "spirv-builder", +] + [[package]] name = "cargo-platform" version = "0.1.9" @@ -1552,6 +1666,31 @@ dependencies = [ "serde", ] +[[package]] +name = "cargo-platform" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84982c6c0ae343635a3a4ee6dedef965513735c8b183caa7289fa6e27399ebd4" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-util-schemas" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dc1a6f7b5651af85774ae5a34b4e8be397d9cf4bc063b7e6dbd99a841837830" +dependencies = [ + "semver 1.0.27", + "serde", + "serde-untagged", + "serde-value", + "thiserror 2.0.18", + "toml 0.8.23", + "unicode-xid", + "url", +] + [[package]] name = "cargo_metadata" version = "0.15.4" @@ -1559,13 +1698,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", - "cargo-platform", + "cargo-platform 0.1.9", "semver 1.0.27", "serde", "serde_json", "thiserror 1.0.69", ] +[[package]] +name = "cargo_metadata" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cfca2aaa699835ba88faf58a06342a314a950d2b9686165e038286c30316868" +dependencies = [ + "camino", + "cargo-platform 0.2.0", + "cargo-util-schemas", + "semver 1.0.27", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "case" version = "1.0.0" @@ -1646,9 +1800,9 @@ dependencies = [ [[package]] name = "chacha20" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", "cipher 0.5.1", @@ -1785,7 +1939,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2468,7 +2622,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2588,7 +2742,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2618,7 +2772,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2632,7 +2786,7 @@ dependencies = [ "indexmap", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2650,7 +2804,7 @@ dependencies = [ "indexmap", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2684,7 +2838,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2697,7 +2851,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2708,7 +2862,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2719,7 +2873,7 @@ checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core 0.23.0", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2758,7 +2912,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2824,7 +2978,7 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2835,7 +2989,7 @@ checksum = "d08b3a0bcc0d079199cd476b2cae8435016ec11d1c0986c6901c5ac223041534" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2848,7 +3002,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2877,7 +3031,7 @@ checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -2890,7 +3044,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn 2.0.117", + "syn 2.0.118", "unicode-xid", ] @@ -2941,7 +3095,16 @@ version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" dependencies = [ - "dirs-sys", + "dirs-sys 0.4.1", +] + +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys 0.5.0", ] [[package]] @@ -2960,7 +3123,7 @@ version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys", + "dirs-sys 0.4.1", ] [[package]] @@ -2971,10 +3134,22 @@ checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", "option-ext", - "redox_users", + "redox_users 0.4.6", "windows-sys 0.48.0", ] +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.5.2", + "windows-sys 0.61.2", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -2982,10 +3157,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" dependencies = [ "libc", - "redox_users", + "redox_users 0.4.6", "winapi", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.11.0", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -2994,7 +3179,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3018,12 +3203,21 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.117", + "syn 2.0.118", "termcolor", "toml 0.8.23", "walkdir", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "domain-block-builder" version = "0.1.0" @@ -3447,7 +3641,7 @@ checksum = "7e8671d54058979a37a26f3511fbf8d198ba1aa35ffb202c42587d918d77213a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3522,7 +3716,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3587,7 +3781,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3627,7 +3821,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -3675,6 +3869,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + [[package]] name = "errno" version = "0.3.14" @@ -3968,7 +4173,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4004,7 +4209,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4517,7 +4722,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4639,7 +4844,7 @@ dependencies = [ "proc-macro2", "quote", "sp-crypto-hashing 0.1.0 (git+https://github.com/autonomys/polkadot-sdk?rev=962a09e060d0d07dacadca5cd3a2864fb456d610)", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4651,7 +4856,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4661,7 +4866,7 @@ source = "git+https://github.com/autonomys/polkadot-sdk?rev=962a09e060d0d07dacad dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -4841,7 +5046,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -5037,6 +5242,15 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "glam" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f22fb22f065b308be0d8724e3706c7fa3fc2a6c7d6899df4cad7860e7a75436" +dependencies = [ + "libm", +] + [[package]] name = "glob" version = "0.3.3" @@ -5075,6 +5289,39 @@ dependencies = [ "spinning_top", ] +[[package]] +name = "gpu-allocator" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51255ea7cfaadb6c5f1528d43e92a82acb2b96c43365989a28b2d44ee38f8795" +dependencies = [ + "ash", + "hashbrown 0.16.1", + "log", + "presser", + "thiserror 2.0.18", +] + +[[package]] +name = "gpu-descriptor" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b89c83349105e3732062a895becfc71a8f921bb71ecbbdd8ff99263e3b53a0ca" +dependencies = [ + "bitflags 2.11.0", + "gpu-descriptor-types", + "hashbrown 0.15.5", +] + +[[package]] +name = "gpu-descriptor-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdf242682df893b86f33a73828fb09ca4b2d3bb6cc95249707fc684d27484b91" +dependencies = [ + "bitflags 2.11.0", +] + [[package]] name = "group" version = "0.13.0" @@ -5113,6 +5360,7 @@ checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", + "num-traits", "zerocopy", ] @@ -5268,6 +5516,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" +[[package]] +name = "hexf-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" + [[package]] name = "hickory-net" version = "0.26.1" @@ -5423,15 +5677,6 @@ dependencies = [ "hmac 0.8.1", ] -[[package]] -name = "home" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "http" version = "0.2.12" @@ -5839,7 +6084,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6066,7 +6311,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6090,7 +6335,7 @@ checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6136,7 +6381,7 @@ dependencies = [ "quote", "rustc_version", "simd_cesu8", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6164,7 +6409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6264,7 +6509,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -6424,6 +6669,16 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -6807,7 +7062,7 @@ source = "git+https://github.com/autonomys/rust-libp2p?rev=5be7c854f715da73189a7 dependencies = [ "heck 0.5.0", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7081,6 +7336,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "local-waker" version = "0.1.4" @@ -7163,7 +7424,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7177,7 +7438,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7188,7 +7449,7 @@ checksum = "b02abfe41815b5bd98dbd4260173db2c116dda171dc0fe7838cb206333b83308" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7199,7 +7460,7 @@ checksum = "73ea28ee64b88876bf45277ed9a5817c1817df061a74f2b988971a12570e5869" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7210,7 +7471,7 @@ checksum = "757aee279b8bdbb9f9e676796fd459e4207a1f986e87886700abf589f5abf771" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7413,7 +7674,7 @@ dependencies = [ "cfg-if", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -7548,6 +7809,33 @@ dependencies = [ "unsigned-varint 0.8.0", ] +[[package]] +name = "naga" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2bf919621e7975acb27d881bae2fb993e0d45c8e0446e85e6272971e00dc8df" +dependencies = [ + "arrayvec 0.7.6", + "bit-set 0.9.1", + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases 0.2.1", + "codespan-reporting", + "half", + "hashbrown 0.16.1", + "hexf-parse", + "indexmap", + "libm", + "log", + "num-traits", + "once_cell", + "petgraph 0.8.3", + "rustc-hash 1.1.0", + "spirv", + "thiserror 2.0.18", + "unicode-ident", +] + [[package]] name = "nalgebra" version = "0.33.2" @@ -7866,7 +8154,69 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", +] + +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.11.0", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.11.0", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-metal" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0125f776a10d00af4152d74616409f0d4a2053a6f57fa5b7d6aa2854ac04794" +dependencies = [ + "bitflags 2.11.0", + "block2", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c1358452b371bf9f104e21ec536d37a650eb10f7ee379fff67d2e08d537f1f" +dependencies = [ + "bitflags 2.11.0", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-metal", ] [[package]] @@ -7978,6 +8328,24 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7d950ca161dc355eaf28f82b11345ed76c6e1f6eb1f4f4479e0323b9e2fbd0e" +dependencies = [ + "num-traits", +] + [[package]] name = "ouroboros" version = "0.18.5" @@ -7999,7 +8367,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8793,7 +9161,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -8945,7 +9313,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9012,7 +9380,7 @@ checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9398,7 +9766,7 @@ dependencies = [ "polkavm-common", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9408,7 +9776,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a4f5352e13c1ca5f0e4d7b4a804fbb85b0e02c45cae435d101fe71081bc8ed8" dependencies = [ "polkavm-derive-impl", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9556,7 +9924,7 @@ dependencies = [ "proc-macro2", "quote", "sp-crypto-hashing 0.1.0 (git+https://github.com/autonomys/polkadot-sdk?rev=962a09e060d0d07dacadca5cd3a2864fb456d610)", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9596,6 +9964,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "presser" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf8e6a8aa66ce33f63993ffc4ea4271eb5b0530a9002db8455ea6050c77bfa" + [[package]] name = "prettyplease" version = "0.2.37" @@ -9603,7 +9977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9699,7 +10073,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9710,7 +10084,7 @@ checksum = "75eea531cfcd120e0851a3f8aed42c4841f78c889eefafd96339c72677ae42c3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9730,11 +10104,17 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "version_check", "yansi", ] +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" + [[package]] name = "prometheus" version = "0.13.4" @@ -9769,7 +10149,7 @@ checksum = "9adf1691c04c0a5ff46ff8f262b58beb07b0dbb61f96f9f54f6cbd82106ed87f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9787,8 +10167,8 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" dependencies = [ - "bit-set", - "bit-vec", + "bit-set 0.8.0", + "bit-vec 0.8.0", "bitflags 2.11.0", "num-traits", "rand 0.9.4", @@ -9846,7 +10226,7 @@ dependencies = [ "prost 0.13.5", "prost-types 0.13.5", "regex", - "syn 2.0.117", + "syn 2.0.118", "tempfile", ] @@ -9865,7 +10245,7 @@ dependencies = [ "prost 0.14.3", "prost-types 0.14.3", "regex", - "syn 2.0.117", + "syn 2.0.118", "tempfile", ] @@ -9891,7 +10271,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9904,7 +10284,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9917,7 +10297,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -9958,7 +10338,7 @@ checksum = "8bccae89ed67a40989e780105fab43e6c71a077b9fc8ae4c805ff5f73d2a79c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10040,9 +10420,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.45" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" dependencies = [ "proc-macro2", ] @@ -10092,7 +10472,7 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ - "chacha20 0.10.0", + "chacha20 0.10.1", "getrandom 0.4.2", "rand_core 0.10.1", ] @@ -10178,6 +10558,30 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "raw-string" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0501e134c6905fee1f10fed25b0a7e1261bf676cffac9543a7d0730dec01af2" + +[[package]] +name = "raw-window-handle" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" + +[[package]] +name = "raw-window-metal" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40d213455a5f1dc59214213c7330e074ddf8114c9a42411eb890c767357ce135" +dependencies = [ + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-quartz-core", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -10217,6 +10621,15 @@ dependencies = [ "yasna 0.5.2", ] +[[package]] +name = "rclite" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6716a637a8a17bce72da6b6ddab8548619cc748658c41ab6ee5c4a3c1001385" +dependencies = [ + "branches", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -10255,6 +10668,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + [[package]] name = "ref-cast" version = "1.0.25" @@ -10272,7 +10696,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10337,6 +10761,12 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "renderdoc-sys" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b30a45b0cd0bcca8037f3d0dc3421eaf95327a17cad11964fb8179b4fc4832" + [[package]] name = "reqwest" version = "0.12.28" @@ -10433,7 +10863,7 @@ checksum = "652db34deaaa57929e10ca18e5454a32cb0efc351ae80d320334bbf907b908b3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -10462,6 +10892,16 @@ dependencies = [ "sha2 0.10.9", ] +[[package]] +name = "rspirv" +version = "0.13.0+sdk-1.4.341.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091dca2e1d6fd3098417b5ec88e77e80d1ba5945750943419dc976858082c296" +dependencies = [ + "rustc-hash 1.1.0", + "spirv", +] + [[package]] name = "rtnetlink" version = "0.20.0" @@ -10529,6 +10969,19 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustc_codegen_spirv-types" +version = "0.10.0-alpha.1" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=bd49568baf08301107c9f698494735724980dee0#bd49568baf08301107c9f698494735724980dee0" +dependencies = [ + "rspirv", + "semver 1.0.27", + "serde", + "serde_json", + "spirv", + "thiserror 2.0.18", +] + [[package]] name = "rustc_version" version = "0.4.1" @@ -10829,7 +11282,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11627,7 +12080,7 @@ version = "0.56.0" source = "git+https://github.com/autonomys/polkadot-sdk?rev=962a09e060d0d07dacadca5cd3a2864fb456d610#962a09e060d0d07dacadca5cd3a2864fb456d610" dependencies = [ "async-trait", - "directories", + "directories 5.0.1", "exit-future", "futures", "futures-timer", @@ -11829,7 +12282,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11932,7 +12385,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11960,7 +12413,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -11986,7 +12439,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12008,7 +12461,7 @@ dependencies = [ "proc-macro2", "quote", "scale-info", - "syn 2.0.117", + "syn 2.0.118", "thiserror 2.0.18", ] @@ -12262,6 +12715,28 @@ dependencies = [ "serde", ] +[[package]] +name = "serde-untagged" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9faf48a4a2d2693be24c6289dbe26552776eb7737074e6722891fadbe6c5058" +dependencies = [ + "erased-serde", + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float 2.10.1", + "serde", +] + [[package]] name = "serde_bytes" version = "0.11.19" @@ -12289,7 +12764,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12322,7 +12797,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12370,7 +12845,7 @@ dependencies = [ "darling 0.23.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -12784,7 +13259,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13083,7 +13558,7 @@ source = "git+https://github.com/autonomys/polkadot-sdk?rev=962a09e060d0d07dacad dependencies = [ "quote", "sp-crypto-hashing 0.1.0 (git+https://github.com/autonomys/polkadot-sdk?rev=962a09e060d0d07dacadca5cd3a2864fb456d610)", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13102,7 +13577,7 @@ source = "git+https://github.com/autonomys/polkadot-sdk?rev=962a09e060d0d07dacad dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13525,7 +14000,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13733,7 +14208,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13786,6 +14261,61 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spirv" +version = "0.4.0+sdk-1.4.341.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9571ea910ebd84c86af4b3ed27f9dbdc6ad06f17c5f96146b2b671e2976744f" +dependencies = [ + "bitflags 2.11.0", + "serde", +] + +[[package]] +name = "spirv-builder" +version = "0.10.0-alpha.1" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=bd49568baf08301107c9f698494735724980dee0#bd49568baf08301107c9f698494735724980dee0" +dependencies = [ + "cargo_metadata 0.21.0", + "log", + "memchr", + "raw-string", + "rustc_codegen_spirv-types", + "semver 1.0.27", + "serde", + "serde_json", + "thiserror 2.0.18", +] + +[[package]] +name = "spirv-std" +version = "0.10.0-alpha.1" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=bd49568baf08301107c9f698494735724980dee0#bd49568baf08301107c9f698494735724980dee0" +dependencies = [ + "bitflags 1.3.2", + "glam", + "libm", + "num-traits", + "spirv-std-macros", + "spirv-std-types", +] + +[[package]] +name = "spirv-std-macros" +version = "0.10.0-alpha.1" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=bd49568baf08301107c9f698494735724980dee0#bd49568baf08301107c9f698494735724980dee0" +dependencies = [ + "proc-macro2", + "quote", + "spirv-std-types", + "syn 2.0.118", +] + +[[package]] +name = "spirv-std-types" +version = "0.10.0-alpha.1" +source = "git+https://github.com/Rust-GPU/rust-gpu?rev=bd49568baf08301107c9f698494735724980dee0#bd49568baf08301107c9f698494735724980dee0" + [[package]] name = "spki" version = "0.7.3" @@ -13796,15 +14326,6 @@ dependencies = [ "der", ] -[[package]] -name = "sppark" -version = "0.1.8" -source = "git+https://github.com/autonomys/sppark?rev=b2a181eb99c8200f1a604f04122551ea39fbf63f#b2a181eb99c8200f1a604f04122551ea39fbf63f" -dependencies = [ - "cc", - "which", -] - [[package]] name = "ss58-registry" version = "1.51.0" @@ -13978,7 +14499,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -13990,7 +14511,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -14154,7 +14675,7 @@ dependencies = [ "subspace-networking", "subspace-process", "subspace-proof-of-space", - "subspace-proof-of-space-gpu", + "subspace-proof-of-space-wgpu", "subspace-rpc-primitives", "subspace-verification", "substrate-bip39", @@ -14194,6 +14715,7 @@ dependencies = [ "subspace-erasure-coding", "subspace-kzg", "subspace-proof-of-space", + "subspace-proof-of-space-wgpu", "subspace-verification", "thiserror 2.0.18", "tokio", @@ -14476,8 +14998,9 @@ name = "subspace-proof-of-space" version = "0.1.0" dependencies = [ "ab-chacha8", + "ab-proof-of-space", "blake3", - "chacha20 0.10.0", + "chacha20 0.10.1", "criterion", "derive_more 2.1.1", "hex", @@ -14488,18 +15011,13 @@ dependencies = [ ] [[package]] -name = "subspace-proof-of-space-gpu" +name = "subspace-proof-of-space-wgpu" version = "0.1.0" dependencies = [ - "blst", - "cc", - "rust-kzg-blst", - "sppark", + "ab-proof-of-space-gpu", "subspace-core-primitives", "subspace-erasure-coding", - "subspace-farmer-components", "subspace-kzg", - "subspace-proof-of-space", ] [[package]] @@ -14965,7 +15483,7 @@ version = "31.1.0" source = "git+https://github.com/autonomys/polkadot-sdk?rev=962a09e060d0d07dacadca5cd3a2864fb456d610#962a09e060d0d07dacadca5cd3a2864fb456d610" dependencies = [ "build-helper", - "cargo_metadata", + "cargo_metadata 0.15.4", "console", "filetime", "jobserver", @@ -15040,7 +15558,7 @@ dependencies = [ "scale-info", "scale-typegen", "subxt-metadata", - "syn 2.0.117", + "syn 2.0.118", "thiserror 2.0.18", ] @@ -15105,7 +15623,7 @@ dependencies = [ "subxt-codegen", "subxt-metadata", "subxt-utils-fetchmetadata", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15209,9 +15727,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.117" +version = "2.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" dependencies = [ "proc-macro2", "quote", @@ -15247,7 +15765,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15379,7 +15897,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15390,7 +15908,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15552,7 +16070,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15804,7 +16322,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15847,7 +16365,7 @@ dependencies = [ "proc-macro-crate 3.5.0", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -15988,6 +16506,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.19.0" @@ -16380,7 +16904,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "wasm-bindgen-shared", ] @@ -16780,7 +17304,7 @@ checksum = "a9cdb9c2e3965ee15629d067203cb800e9822664d04335dadc6fe1788d4fc335" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -16857,15 +17381,148 @@ dependencies = [ ] [[package]] -name = "which" -version = "4.4.2" +name = "wgpu" +version = "29.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +checksum = "76e8840e1ba2881d4cbb18d2147627a56af426ff064c0401eb0c8410c6325d07" dependencies = [ - "either", - "home", + "arrayvec 0.7.6", + "bitflags 2.11.0", + "bytemuck", + "cfg-if", + "cfg_aliases 0.2.1", + "document-features", + "hashbrown 0.16.1", + "js-sys", + "log", + "naga", + "parking_lot 0.12.5", + "portable-atomic", + "profiling", + "raw-window-handle", + "smallvec", + "static_assertions", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "wgpu-core", + "wgpu-hal", + "wgpu-types", +] + +[[package]] +name = "wgpu-core" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f519832254e56965a9940c4af57dcb75f702b6f6fa4a0b172f685395843a4d7" +dependencies = [ + "arrayvec 0.7.6", + "bit-set 0.9.1", + "bit-vec 0.9.1", + "bitflags 2.11.0", + "bytemuck", + "cfg_aliases 0.2.1", + "document-features", + "hashbrown 0.16.1", + "indexmap", + "log", + "naga", "once_cell", - "rustix 0.38.44", + "parking_lot 0.12.5", + "portable-atomic", + "profiling", + "raw-window-handle", + "rustc-hash 1.1.0", + "smallvec", + "thiserror 2.0.18", + "wgpu-core-deps-apple", + "wgpu-core-deps-windows-linux-android", + "wgpu-hal", + "wgpu-naga-bridge", + "wgpu-types", +] + +[[package]] +name = "wgpu-core-deps-apple" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5e39e26c4c0e07589e67d18546cf79ff45383659fc72fca4dd293358a0347f3" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-core-deps-windows-linux-android" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e592c1bbef6ad047647ae6e666ebd8cee7a32bb4544d9700ec96cbf73230257" +dependencies = [ + "wgpu-hal", +] + +[[package]] +name = "wgpu-hal" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ace1c17727311c22a46e4e3faf56ea6de81af99dcc839bdfb54857b94d448d" +dependencies = [ + "android_system_properties", + "arrayvec 0.7.6", + "ash", + "bitflags 2.11.0", + "block2", + "bytemuck", + "cfg-if", + "cfg_aliases 0.2.1", + "gpu-allocator", + "gpu-descriptor", + "hashbrown 0.16.1", + "libc", + "libloading", + "log", + "naga", + "objc2", + "objc2-core-foundation", + "objc2-foundation", + "objc2-metal", + "objc2-quartz-core", + "ordered-float 5.3.0", + "parking_lot 0.12.5", + "portable-atomic", + "portable-atomic-util", + "profiling", + "raw-window-handle", + "raw-window-metal", + "renderdoc-sys", + "smallvec", + "thiserror 2.0.18", + "wgpu-naga-bridge", + "wgpu-types", + "windows 0.62.2", +] + +[[package]] +name = "wgpu-naga-bridge" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95226013f547544b223281cd16a4fb549aa9dcb562adbda0faae4c73ffbbc161" +dependencies = [ + "naga", + "wgpu-types", +] + +[[package]] +name = "wgpu-types" +version = "29.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bf84cd9ca8ca45e2b223a3868f1adf9bfc0c66aeac212e76ee7e40fdadf8f5" +dependencies = [ + "bitflags 2.11.0", + "bytemuck", + "js-sys", + "log", + "raw-window-handle", + "web-sys", ] [[package]] @@ -17007,7 +17664,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -17018,7 +17675,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -17420,7 +18077,7 @@ dependencies = [ "heck 0.5.0", "indexmap", "prettyplease", - "syn 2.0.117", + "syn 2.0.118", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -17436,7 +18093,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -17558,7 +18215,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -17636,9 +18293,9 @@ checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" [[package]] name = "yoke" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -17647,13 +18304,13 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure 0.13.2", ] @@ -17674,7 +18331,7 @@ checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -17694,7 +18351,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", "synstructure 0.13.2", ] @@ -17715,7 +18372,7 @@ checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] @@ -17748,7 +18405,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 2.0.118", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4019fa4feee..aee7f9ecc44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,10 @@ members = [ ] [workspace.dependencies] +ab-blake3 = "0.1.0" ab-chacha8 = "0.1.0" +ab-proof-of-space = { path = "shared/ab-proof-of-space", default-features = false } +ab-proof-of-space-gpu = { path = "shared/ab-proof-of-space-gpu", default-features = false } actix-web = { version = "4.9.0", default-features = false } aes = "0.9.0" anyhow = "1.0.89" @@ -50,8 +53,9 @@ blake3 = { version = "1.8.5", default-features = false } blst = "0.3.13" bytes = { version = "1.11.1", default-features = false } bytesize = "2.3.1" +cargo-gpu-install = { version = "=0.10.0-alpha.1", git = "https://github.com/Rust-GPU/rust-gpu", rev = "bd49568baf08301107c9f698494735724980dee0" } cc = "1.2.62" -chacha20 = { version = "0.10.0", default-features = false } +chacha20 = { version = "0.10.1", default-features = false } clap = "4.6.1" console = "0.15.11" core_affinity = "0.8.1" @@ -168,6 +172,7 @@ rand = { version = "0.8.5", default-features = false } rand_chacha = { version = "0.3.1", default-features = false } rand_core = "0.6.4" rayon = "1.12.0" +rclite = "0.4.1" reqwest = { version = "0.12.9", default-features = false } ring = "0.17.8" rlp = "0.6" @@ -260,7 +265,7 @@ sp-trie = { git = "https://github.com/autonomys/polkadot-sdk", rev = "962a09e060 sp-version = { git = "https://github.com/autonomys/polkadot-sdk", rev = "962a09e060d0d07dacadca5cd3a2864fb456d610", default-features = false } sp-weights = { git = "https://github.com/autonomys/polkadot-sdk", rev = "962a09e060d0d07dacadca5cd3a2864fb456d610", default-features = false } spin = "0.12.0" -sppark = { version = "0.1.8", git = "https://github.com/autonomys/sppark", rev = "b2a181eb99c8200f1a604f04122551ea39fbf63f" } +spirv-std = { version = "=0.10.0-alpha.1", git = "https://github.com/Rust-GPU/rust-gpu", rev = "bd49568baf08301107c9f698494735724980dee0" } ss58-registry = "1.51.0" static_assertions = "1.1.0" strum_macros = "0.28.0" @@ -275,7 +280,7 @@ subspace-metrics = { version = "0.1.0", path = "shared/subspace-metrics" } subspace-networking = { version = "0.1.0", path = "crates/subspace-networking" } subspace-process = { version = "0.0.1", path = "shared/subspace-process" } subspace-proof-of-space = { version = "0.1.0", path = "crates/subspace-proof-of-space", default-features = false } -subspace-proof-of-space-gpu = { version = "0.1.0", path = "shared/subspace-proof-of-space-gpu" } +subspace-proof-of-space-wgpu = { version = "0.1.0", path = "shared/subspace-proof-of-space-wgpu" } subspace-proof-of-time = { version = "0.1.0", path = "crates/subspace-proof-of-time" } subspace-rpc-primitives = { version = "0.1.0", path = "crates/subspace-rpc-primitives" } subspace-runtime = { version = "0.1.0", path = "crates/subspace-runtime" } @@ -306,6 +311,7 @@ uint = { version = "0.10.0", default-features = false } ulid = "1.1.3" unsigned-varint = "0.8.0" void = "1.0.2" +wgpu = { version = "29.0.3", default-features = false, features = ["metal", "parking_lot", "spirv", "std", "vulkan"] } x509-parser = "0.18.1" zeroize = "1.8.1" ziggy = { version = "1.7.0", default-features = false } @@ -318,6 +324,7 @@ ziggy = { version = "1.7.0", default-features = false } # # This list is ordered alphabetically. [profile.dev.package] +ab-proof-of-space = { opt-level = 3 } bitvec = { opt-level = 3 } blake2 = { opt-level = 3 } blake3 = { opt-level = 3 } diff --git a/crates/subspace-farmer-components/Cargo.toml b/crates/subspace-farmer-components/Cargo.toml index 1d6afbe3cba..fdfc4fedab7 100644 --- a/crates/subspace-farmer-components/Cargo.toml +++ b/crates/subspace-farmer-components/Cargo.toml @@ -46,10 +46,12 @@ tracing.workspace = true winapi = "0.3.9" [dev-dependencies] +anyhow.workspace = true criterion.workspace = true futures.workspace = true subspace-archiving.workspace = true subspace-proof-of-space.workspace = true +subspace-proof-of-space-wgpu.workspace = true [[bench]] name = "plotting" diff --git a/crates/subspace-farmer-components/benches/auditing.rs b/crates/subspace-farmer-components/benches/auditing.rs index 434821ea18d..675a13b4784 100644 --- a/crates/subspace-farmer-components/benches/auditing.rs +++ b/crates/subspace-farmer-components/benches/auditing.rs @@ -26,9 +26,9 @@ use subspace_farmer_components::sector::{ }; use subspace_kzg::Kzg; use subspace_proof_of_space::Table; -use subspace_proof_of_space::chia::ChiaTable; +use subspace_proof_of_space::chia_v2::ChiaV2Table; -type PosTable = ChiaTable; +type PosTable = ChiaV2Table; const MAX_PIECES_IN_SECTOR: u16 = 1000; diff --git a/crates/subspace-farmer-components/benches/plotting.rs b/crates/subspace-farmer-components/benches/plotting.rs index 63f828b7d4d..b7a203098d3 100644 --- a/crates/subspace-farmer-components/benches/plotting.rs +++ b/crates/subspace-farmer-components/benches/plotting.rs @@ -14,9 +14,9 @@ use subspace_farmer_components::plotting::{CpuRecordsEncoder, PlotSectorOptions, use subspace_farmer_components::sector::sector_size; use subspace_kzg::Kzg; use subspace_proof_of_space::Table; -use subspace_proof_of_space::chia::ChiaTable; +use subspace_proof_of_space::chia_v2::ChiaV2Table; -type PosTable = ChiaTable; +type PosTable = ChiaV2Table; const MAX_PIECES_IN_SECTOR: u16 = 1000; diff --git a/crates/subspace-farmer-components/benches/proving.rs b/crates/subspace-farmer-components/benches/proving.rs index 00e31e23924..14c0e169863 100644 --- a/crates/subspace-farmer-components/benches/proving.rs +++ b/crates/subspace-farmer-components/benches/proving.rs @@ -31,10 +31,10 @@ use subspace_farmer_components::sector::{ SectorContentsMap, SectorMetadata, SectorMetadataChecksummed, sector_size, }; use subspace_kzg::Kzg; -use subspace_proof_of_space::chia::ChiaTable; +use subspace_proof_of_space::chia_v2::ChiaV2Table; use subspace_proof_of_space::{Table, TableGenerator}; -type PosTable = ChiaTable; +type PosTable = ChiaV2Table; const MAX_PIECES_IN_SECTOR: u16 = 1000; diff --git a/crates/subspace-farmer-components/benches/reading.rs b/crates/subspace-farmer-components/benches/reading.rs index fe660d5de3a..55f639e6883 100644 --- a/crates/subspace-farmer-components/benches/reading.rs +++ b/crates/subspace-farmer-components/benches/reading.rs @@ -25,9 +25,9 @@ use subspace_farmer_components::sector::{ use subspace_farmer_components::{FarmerProtocolInfo, ReadAt, ReadAtSync}; use subspace_kzg::Kzg; use subspace_proof_of_space::Table; -use subspace_proof_of_space::chia::ChiaTable; +use subspace_proof_of_space::chia_v2::ChiaV2Table; -type PosTable = ChiaTable; +type PosTable = ChiaV2Table; const MAX_PIECES_IN_SECTOR: u16 = 1000; diff --git a/crates/subspace-farmer-components/src/proving.rs b/crates/subspace-farmer-components/src/proving.rs index 1a8400f26ae..8b88a9c88ab 100644 --- a/crates/subspace-farmer-components/src/proving.rs +++ b/crates/subspace-farmer-components/src/proving.rs @@ -17,6 +17,7 @@ use std::io; use subspace_core_primitives::pieces::{PieceOffset, Record}; use subspace_core_primitives::pos::PosSeed; use subspace_core_primitives::sectors::{SBucket, SectorId}; +use subspace_core_primitives::segments::HistorySize; use subspace_core_primitives::solutions::{ChunkWitness, Solution, SolutionRange}; use subspace_core_primitives::{PublicKey, ScalarBytes}; use subspace_erasure_coding::ErasureCoding; @@ -159,6 +160,11 @@ where self.chunk_candidates.is_empty() } + /// History size of the sector these candidates belong to. + pub fn history_size(&self) -> HistorySize { + self.sector_metadata.history_size + } + /// Turn solution candidates into actual solutions pub fn into_solutions( self, diff --git a/crates/subspace-farmer-components/tests/plot_read_roundtrip.rs b/crates/subspace-farmer-components/tests/plot_read_roundtrip.rs new file mode 100644 index 00000000000..2f69dbd18ea --- /dev/null +++ b/crates/subspace-farmer-components/tests/plot_read_roundtrip.rs @@ -0,0 +1,402 @@ +//! Plot sectors with the old and abundance-backed tables and read pieces back, including under +//! the farmer's per-sector cutover dispatch. + +use futures::executor::block_on; +use rand::prelude::*; +use std::num::{NonZeroU8, NonZeroU64, NonZeroUsize}; +use std::slice; +use std::sync::atomic::AtomicBool; +use subspace_archiving::archiver::Archiver; +use subspace_core_primitives::PublicKey; +use subspace_core_primitives::pieces::{PieceOffset, Record}; +use subspace_core_primitives::sectors::SectorId; +use subspace_core_primitives::segments::{HistorySize, RecordedHistorySegment}; +use subspace_data_retrieval::piece_getter::PieceGetter; +use subspace_erasure_coding::ErasureCoding; +use subspace_farmer_components::plotting::{ + CpuRecordsEncoder, PlotSectorOptions, RecordsEncoder, plot_sector, +}; +use subspace_farmer_components::reading::{ReadSectorRecordChunksMode, read_piece}; +use subspace_farmer_components::sector::SectorContentsMap; +use subspace_farmer_components::{FarmerProtocolInfo, ReadAt}; +use subspace_kzg::Kzg; +use subspace_proof_of_space::chia::ChiaTable; +use subspace_proof_of_space::chia_v2::ChiaV2Table; +use subspace_proof_of_space::{PosTable, Table}; +use subspace_proof_of_space_wgpu::{Device, WgpuDevice}; + +#[test] +fn abundance_plot_read_roundtrip() { + let pieces_in_sector = 10; + let sector_index = 0; + let public_key = PublicKey::default(); + + let mut input = RecordedHistorySegment::new_boxed(); + StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut())); + let kzg = Kzg::new(); + let erasure_coding = ErasureCoding::new( + NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) + .expect("Not zero; qed"), + ) + .unwrap(); + let mut archiver = Archiver::new(kzg.clone(), erasure_coding.clone()); + let archived_history_segment = archiver + .add_block( + AsRef::<[u8]>::as_ref(input.as_ref()).to_vec(), + Default::default(), + true, + ) + .archived_segments + .into_iter() + .next() + .unwrap(); + + let farmer_protocol_info = FarmerProtocolInfo { + history_size: HistorySize::from(NonZeroU64::new(1).unwrap()), + max_pieces_in_sector: pieces_in_sector, + recent_segments: HistorySize::from(NonZeroU64::new(5).unwrap()), + recent_history_fraction: ( + HistorySize::from(NonZeroU64::new(1).unwrap()), + HistorySize::from(NonZeroU64::new(10).unwrap()), + ), + min_sector_lifetime: HistorySize::from(NonZeroU64::new(4).unwrap()), + }; + + let mut table_generator = ChiaV2Table::generator(); + let mut sector = Vec::new(); + let plotted_sector = block_on(plot_sector(PlotSectorOptions { + public_key: &public_key, + sector_index, + piece_getter: &archived_history_segment, + farmer_protocol_info, + kzg: &kzg, + erasure_coding: &erasure_coding, + pieces_in_sector, + sector_output: &mut sector, + downloading_semaphore: None, + encoding_semaphore: None, + records_encoder: &mut CpuRecordsEncoder::::new( + slice::from_mut(&mut table_generator), + &erasure_coding, + &Default::default(), + ), + abort_early: &Default::default(), + })) + .unwrap(); + + let piece_offset = PieceOffset::ZERO; + let piece_index = plotted_sector.piece_indexes[usize::from(piece_offset)]; + let expected = block_on(archived_history_segment.get_piece(piece_index)) + .unwrap() + .unwrap(); + + let read = block_on(read_piece::( + piece_offset, + &plotted_sector.sector_id, + &plotted_sector.sector_metadata, + &ReadAt::from_sync(sector.as_slice()), + &erasure_coding, + ReadSectorRecordChunksMode::ConcurrentChunks, + &mut table_generator, + )) + .unwrap(); + + assert_eq!( + read.record(), + expected.record(), + "record read back from an abundance-plotted sector must match the original" + ); +} + +/// Plot one sector with the old `ChiaTable` and one with the abundance `ChiaV2Table`, then read +/// both back selecting the table per sector via the farmer's cutover dispatch +/// (`PosTable::generator_for(is_post_cutover)`), exactly as an upgraded farm does. +#[test] +fn mixed_old_new_sectors_read_under_cutover_dispatch() { + let pieces_in_sector = 10; + let public_key = PublicKey::default(); + + let mut input = RecordedHistorySegment::new_boxed(); + StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut())); + let kzg = Kzg::new(); + let erasure_coding = ErasureCoding::new( + NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) + .expect("Not zero; qed"), + ) + .unwrap(); + let mut archiver = Archiver::new(kzg.clone(), erasure_coding.clone()); + let archived_history_segment = archiver + .add_block( + AsRef::<[u8]>::as_ref(input.as_ref()).to_vec(), + Default::default(), + true, + ) + .archived_segments + .into_iter() + .next() + .unwrap(); + + let farmer_protocol_info = FarmerProtocolInfo { + history_size: HistorySize::from(NonZeroU64::new(1).unwrap()), + max_pieces_in_sector: pieces_in_sector, + recent_segments: HistorySize::from(NonZeroU64::new(5).unwrap()), + recent_history_fraction: ( + HistorySize::from(NonZeroU64::new(1).unwrap()), + HistorySize::from(NonZeroU64::new(10).unwrap()), + ), + min_sector_lifetime: HistorySize::from(NonZeroU64::new(4).unwrap()), + }; + + // "Old" sector plotted with the old table, "new" sector with the abundance table. + let mut old_sector = Vec::new(); + let mut old_generator = ChiaTable::generator(); + let old_plotted = block_on(plot_sector(PlotSectorOptions { + public_key: &public_key, + sector_index: 0, + piece_getter: &archived_history_segment, + farmer_protocol_info, + kzg: &kzg, + erasure_coding: &erasure_coding, + pieces_in_sector, + sector_output: &mut old_sector, + downloading_semaphore: None, + encoding_semaphore: None, + records_encoder: &mut CpuRecordsEncoder::::new( + slice::from_mut(&mut old_generator), + &erasure_coding, + &Default::default(), + ), + abort_early: &Default::default(), + })) + .unwrap(); + + let mut new_sector = Vec::new(); + let mut new_generator = ChiaV2Table::generator(); + let new_plotted = block_on(plot_sector(PlotSectorOptions { + public_key: &public_key, + sector_index: 1, + piece_getter: &archived_history_segment, + farmer_protocol_info, + kzg: &kzg, + erasure_coding: &erasure_coding, + pieces_in_sector, + sector_output: &mut new_sector, + downloading_semaphore: None, + encoding_semaphore: None, + records_encoder: &mut CpuRecordsEncoder::::new( + slice::from_mut(&mut new_generator), + &erasure_coding, + &Default::default(), + ), + abort_early: &Default::default(), + })) + .unwrap(); + + // Cutover sits between the two sectors: old_history <= cutover < new_history. The farmer's rule + // is `is_post_cutover = history_size > cutover`. + let cutover = 5u64; + let old_is_post_cutover = 3u64 > cutover; // false -> old table + let new_is_post_cutover = 7u64 > cutover; // true -> abundance table + + let piece_offset = PieceOffset::ZERO; + + // Pre-cutover sector must read back via the old table. + let old_expected = block_on( + archived_history_segment.get_piece(old_plotted.piece_indexes[usize::from(piece_offset)]), + ) + .unwrap() + .unwrap(); + let mut old_dispatch = PosTable::generator_for(old_is_post_cutover); + let old_read = block_on(read_piece::( + piece_offset, + &old_plotted.sector_id, + &old_plotted.sector_metadata, + &ReadAt::from_sync(old_sector.as_slice()), + &erasure_coding, + ReadSectorRecordChunksMode::ConcurrentChunks, + &mut old_dispatch, + )) + .unwrap(); + assert_eq!( + old_read.record(), + old_expected.record(), + "pre-cutover sector must read back via the old table under dispatch" + ); + + // Post-cutover sector must read back via the abundance table. + let new_expected = block_on( + archived_history_segment.get_piece(new_plotted.piece_indexes[usize::from(piece_offset)]), + ) + .unwrap() + .unwrap(); + let mut new_dispatch = PosTable::generator_for(new_is_post_cutover); + let new_read = block_on(read_piece::( + piece_offset, + &new_plotted.sector_id, + &new_plotted.sector_metadata, + &ReadAt::from_sync(new_sector.as_slice()), + &erasure_coding, + ReadSectorRecordChunksMode::ConcurrentChunks, + &mut new_dispatch, + )) + .unwrap(); + assert_eq!( + new_read.record(), + new_expected.record(), + "post-cutover sector must read back via the abundance table under dispatch" + ); + + // Misdispatch guard: reading the old sector with the abundance table must not reproduce it, so + // the per-sector dispatch is genuinely load-bearing (not both tables reading everything alike). + let mut wrong_generator = PosTable::generator_for(true); + let old_misread = block_on(read_piece::( + piece_offset, + &old_plotted.sector_id, + &old_plotted.sector_metadata, + &ReadAt::from_sync(old_sector.as_slice()), + &erasure_coding, + ReadSectorRecordChunksMode::ConcurrentChunks, + &mut wrong_generator, + )); + assert!( + old_misread + .map(|read| read.record() != old_expected.record()) + .unwrap_or(true), + "old sector read with the abundance table must not reproduce the original" + ); +} + +/// Minimal single-threaded [`RecordsEncoder`] over the production [`WgpuDevice`], used only to drive +/// `plot_sector` in the byte-identity test (the threaded `WgpuRecordsEncoder` lives in +/// `subspace-farmer`, which this crate cannot depend on). +struct WgpuTestEncoder { + device: WgpuDevice, +} + +impl RecordsEncoder for WgpuTestEncoder { + fn encode_records( + &mut self, + sector_id: &SectorId, + records: &mut [Record], + _abort_early: &AtomicBool, + ) -> anyhow::Result { + let pieces_in_sector = records + .len() + .try_into() + .map_err(|error| anyhow::anyhow!("Failed to convert pieces in sector: {error}"))?; + let mut sector_contents_map = SectorContentsMap::new(pieces_in_sector); + for ((piece_offset, record), mut encoded_chunks_used) in (PieceOffset::ZERO..) + .zip(records.iter_mut()) + .zip(sector_contents_map.iter_record_bitfields_mut()) + { + let pos_seed = sector_id.derive_evaluation_seed(piece_offset); + self.device + .generate_and_encode_pospace(&pos_seed, record, encoded_chunks_used.iter_mut()) + .map_err(|error| anyhow::anyhow!(error))?; + } + Ok(sector_contents_map) + } +} + +/// Byte-identity gate: a sector plotted with wgpu (GPU) must be byte-for-byte identical to one +/// plotted with the abundance CPU `ChiaV2Table`, so new GPU-plotted farms read and verify under the +/// same CPU path. Skips when no GPU is available (e.g. CI runners without a GPU). +#[test] +fn wgpu_plotted_sector_matches_cpu() { + let pieces_in_sector = 10; + let sector_index = 0; + let public_key = PublicKey::default(); + + let mut input = RecordedHistorySegment::new_boxed(); + StdRng::seed_from_u64(42).fill(AsMut::<[u8]>::as_mut(input.as_mut())); + let kzg = Kzg::new(); + let erasure_coding = ErasureCoding::new( + NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) + .expect("Not zero; qed"), + ) + .unwrap(); + let mut archiver = Archiver::new(kzg.clone(), erasure_coding.clone()); + let archived_history_segment = archiver + .add_block( + AsRef::<[u8]>::as_ref(input.as_ref()).to_vec(), + Default::default(), + true, + ) + .archived_segments + .into_iter() + .next() + .unwrap(); + + let farmer_protocol_info = FarmerProtocolInfo { + history_size: HistorySize::from(NonZeroU64::new(1).unwrap()), + max_pieces_in_sector: pieces_in_sector, + recent_segments: HistorySize::from(NonZeroU64::new(5).unwrap()), + recent_history_fraction: ( + HistorySize::from(NonZeroU64::new(1).unwrap()), + HistorySize::from(NonZeroU64::new(10).unwrap()), + ), + min_sector_lifetime: HistorySize::from(NonZeroU64::new(4).unwrap()), + }; + + // One queue is enough for the test; skip entirely when no GPU is present. + let Some(wgpu_device) = block_on(Device::enumerate(|_device_type| { + NonZeroU8::new(1).expect("Not zero; qed") + })) + .into_iter() + .next() else { + eprintln!("No wgpu GPU available; skipping wgpu byte-identity gate"); + return; + }; + let instance = wgpu_device + .create_proofs_encoder_instances() + .into_iter() + .next() + .expect("At least one queue per device; qed"); + let mut wgpu_encoder = WgpuTestEncoder { + device: WgpuDevice::new(instance, erasure_coding.clone()), + }; + + let mut cpu_sector = Vec::new(); + let mut cpu_generator = ChiaV2Table::generator(); + block_on(plot_sector(PlotSectorOptions { + public_key: &public_key, + sector_index, + piece_getter: &archived_history_segment, + farmer_protocol_info, + kzg: &kzg, + erasure_coding: &erasure_coding, + pieces_in_sector, + sector_output: &mut cpu_sector, + downloading_semaphore: None, + encoding_semaphore: None, + records_encoder: &mut CpuRecordsEncoder::::new( + slice::from_mut(&mut cpu_generator), + &erasure_coding, + &Default::default(), + ), + abort_early: &Default::default(), + })) + .unwrap(); + + let mut wgpu_sector = Vec::new(); + block_on(plot_sector(PlotSectorOptions { + public_key: &public_key, + sector_index, + piece_getter: &archived_history_segment, + farmer_protocol_info, + kzg: &kzg, + erasure_coding: &erasure_coding, + pieces_in_sector, + sector_output: &mut wgpu_sector, + downloading_semaphore: None, + encoding_semaphore: None, + records_encoder: &mut wgpu_encoder, + abort_early: &Default::default(), + })) + .unwrap(); + + assert_eq!( + cpu_sector, wgpu_sector, + "wgpu-plotted sector must be byte-identical to the abundance CPU ChiaV2Table plot" + ); +} diff --git a/crates/subspace-farmer/Cargo.toml b/crates/subspace-farmer/Cargo.toml index 3d7d93f02f0..048fa4547a0 100644 --- a/crates/subspace-farmer/Cargo.toml +++ b/crates/subspace-farmer/Cargo.toml @@ -59,6 +59,7 @@ subspace-metrics = { workspace = true, optional = true } subspace-networking.workspace = true subspace-process.workspace = true subspace-proof-of-space.workspace = true +subspace-proof-of-space-wgpu = { workspace = true, optional = true } subspace-rpc-primitives.workspace = true subspace-verification = { workspace = true, features = ["kzg"] } substrate-bip39.workspace = true @@ -71,19 +72,12 @@ tracing.workspace = true ulid = { workspace = true, features = ["serde"] } zeroize.workspace = true -# Avoid an unused dependency on macOS, GPU is not supported there -[target.'cfg(any(target_os = "linux", windows))'.dependencies] -subspace-proof-of-space-gpu = { workspace = true, optional = true } - [features] -default = ["default-library", "binary"] +default = ["default-library", "binary", "wgpu"] cluster = ["dep:async-nats"] numa = ["dep:hwlocality"] -# Only Volta+ architectures are supported (GeForce RTX 16xx consumer GPUs and newer) -cuda = ["_gpu", "subspace-proof-of-space-gpu/cuda"] -# TODO: ROCm can't be enabled at the same time as `cuda` feature at the moment -# Seems to support RDNA 2+, at least on Linux -rocm = ["_gpu", "subspace-proof-of-space-gpu/rocm"] +# Vulkan/Metal plotting via wgpu +wgpu = ["_gpu", "dep:subspace-proof-of-space-wgpu"] # Internal feature, shouldn't be used directly _gpu = [] diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands.rs index 0d6936e9c83..a9966b6262c 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands.rs @@ -7,3 +7,5 @@ mod shared; pub(crate) use info::info; pub(crate) use scrub::scrub; +#[cfg(feature = "wgpu")] +pub(crate) use shared::wgpu::list_gpus; diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs index 8e96fbbc03b..451bf955927 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs @@ -2,7 +2,6 @@ use crate::PosTable; use anyhow::anyhow; use clap::{Parser, Subcommand}; use criterion::{BatchSize, Criterion, Throughput}; -use parking_lot::Mutex; use rayon::{ThreadPool, ThreadPoolBuildError, ThreadPoolBuilder}; use std::collections::HashSet; use std::fs::OpenOptions; @@ -162,9 +161,7 @@ where .expect("Not zero; qed"), ) .map_err(|error| anyhow!("Failed to instantiate erasure coding: {error}"))?; - let table_generator = Mutex::new(PosTable::generator()); - - let sectors_metadata = SingleDiskFarm::read_all_sectors_metadata(&disk_farm) + let (cutover, sectors_metadata) = SingleDiskFarm::read_all_sectors_metadata(&disk_farm) .map_err(|error| anyhow::anyhow!("Failed to read sectors metadata: {error}"))?; let mut criterion = Criterion::default().sample_size(sample_size); @@ -187,7 +184,7 @@ where b.iter_batched( rand::random::<[u8; 32]>, |global_challenge| { - let options = PlotAuditOptions:: { + let options = PlotAuditOptions { public_key: single_disk_farm_info.public_key(), reward_address: single_disk_farm_info.public_key(), slot_info: SlotInfo { @@ -204,10 +201,10 @@ where sectors_being_modified: &HashSet::default(), read_sector_record_chunks_mode: ReadSectorRecordChunksMode::ConcurrentChunks, - table_generator: &table_generator, + cutover, }; - black_box(plot_audit.audit(black_box(options))) + black_box(plot_audit.audit::(black_box(options))) }, BatchSize::SmallInput, ) @@ -224,7 +221,7 @@ where b.iter_batched( rand::random::<[u8; 32]>, |global_challenge| { - let options = PlotAuditOptions:: { + let options = PlotAuditOptions { public_key: single_disk_farm_info.public_key(), reward_address: single_disk_farm_info.public_key(), slot_info: SlotInfo { @@ -241,10 +238,10 @@ where sectors_being_modified: &HashSet::default(), read_sector_record_chunks_mode: ReadSectorRecordChunksMode::ConcurrentChunks, - table_generator: &table_generator, + cutover, }; - black_box(plot_audit.audit(black_box(options))) + black_box(plot_audit.audit::(black_box(options))) }, BatchSize::SmallInput, ) @@ -259,7 +256,7 @@ where b.iter_batched( rand::random::<[u8; 32]>, |global_challenge| { - let options = PlotAuditOptions:: { + let options = PlotAuditOptions { public_key: single_disk_farm_info.public_key(), reward_address: single_disk_farm_info.public_key(), slot_info: SlotInfo { @@ -276,10 +273,10 @@ where sectors_being_modified: &HashSet::default(), read_sector_record_chunks_mode: ReadSectorRecordChunksMode::ConcurrentChunks, - table_generator: &table_generator, + cutover, }; - black_box(plot_audit.audit(black_box(options))) + black_box(plot_audit.audit::(black_box(options))) }, BatchSize::SmallInput, ) @@ -342,9 +339,7 @@ where .expect("Not zero; qed"), ) .map_err(|error| anyhow!("Failed to instantiate erasure coding: {error}"))?; - let table_generator = Mutex::new(PosTable::generator()); - - let mut sectors_metadata = SingleDiskFarm::read_all_sectors_metadata(&disk_farm) + let (cutover, mut sectors_metadata) = SingleDiskFarm::read_all_sectors_metadata(&disk_farm) .map_err(|error| anyhow::anyhow!("Failed to read sectors metadata: {error}"))?; if let Some(limit_sector_count) = limit_sector_count { sectors_metadata.truncate(limit_sector_count); @@ -362,7 +357,7 @@ where .open(disk_farm.join(SingleDiskFarm::PLOT_FILE)) .map_err(|error| anyhow::anyhow!("Failed to open plot: {error}"))?; let plot_audit = PlotAudit::new(&plot); - let mut options = PlotAuditOptions:: { + let mut options = PlotAuditOptions { public_key: single_disk_farm_info.public_key(), reward_address: single_disk_farm_info.public_key(), slot_info: SlotInfo { @@ -378,10 +373,10 @@ where erasure_coding: &erasure_coding, sectors_being_modified: &HashSet::default(), read_sector_record_chunks_mode: ReadSectorRecordChunksMode::ConcurrentChunks, - table_generator: &table_generator, + cutover, }; - let mut audit_results = plot_audit.audit(options).unwrap(); + let mut audit_results = plot_audit.audit::(options).unwrap(); group.bench_function("plot/single/concurrent-chunks", |b| { b.iter_batched( @@ -392,7 +387,7 @@ where options.slot_info.global_challenge = Blake3Hash::from(rand::random::<[u8; 32]>()); - audit_results = plot_audit.audit(options).unwrap(); + audit_results = plot_audit.audit::(options).unwrap(); audit_results.pop().unwrap() }, @@ -406,7 +401,7 @@ where }); options.read_sector_record_chunks_mode = ReadSectorRecordChunksMode::WholeSector; - let mut audit_results = plot_audit.audit(options).unwrap(); + let mut audit_results = plot_audit.audit::(options).unwrap(); group.bench_function("plot/single/whole-sector", |b| { b.iter_batched( @@ -417,7 +412,7 @@ where options.slot_info.global_challenge = Blake3Hash::from(rand::random::<[u8; 32]>()); - audit_results = plot_audit.audit(options).unwrap(); + audit_results = plot_audit.audit::(options).unwrap(); audit_results.pop().unwrap() }, @@ -436,7 +431,7 @@ where }) .map_err(|error| anyhow::anyhow!("Failed to open plot: {error}"))?; let plot_audit = PlotAudit::new(&plot); - let mut options = PlotAuditOptions:: { + let mut options = PlotAuditOptions { public_key: single_disk_farm_info.public_key(), reward_address: single_disk_farm_info.public_key(), slot_info: SlotInfo { @@ -452,10 +447,10 @@ where erasure_coding: &erasure_coding, sectors_being_modified: &HashSet::default(), read_sector_record_chunks_mode: ReadSectorRecordChunksMode::ConcurrentChunks, - table_generator: &table_generator, + cutover, }; - let mut audit_results = plot_audit.audit(options).unwrap(); + let mut audit_results = plot_audit.audit::(options).unwrap(); group.bench_function("plot/rayon/unbuffered/concurrent-chunks", |b| { b.iter_batched( @@ -466,7 +461,7 @@ where options.slot_info.global_challenge = Blake3Hash::from(rand::random::<[u8; 32]>()); - audit_results = plot_audit.audit(options).unwrap(); + audit_results = plot_audit.audit::(options).unwrap(); audit_results.pop().unwrap() }, @@ -480,7 +475,7 @@ where }); options.read_sector_record_chunks_mode = ReadSectorRecordChunksMode::WholeSector; - let mut audit_results = plot_audit.audit(options).unwrap(); + let mut audit_results = plot_audit.audit::(options).unwrap(); group.bench_function("plot/rayon/unbuffered/whole-sector", |b| { b.iter_batched( @@ -491,7 +486,7 @@ where options.slot_info.global_challenge = Blake3Hash::from(rand::random::<[u8; 32]>()); - audit_results = plot_audit.audit(options).unwrap(); + audit_results = plot_audit.audit::(options).unwrap(); audit_results.pop().unwrap() }, @@ -508,7 +503,7 @@ where let plot = RayonFiles::open(disk_farm.join(SingleDiskFarm::PLOT_FILE)) .map_err(|error| anyhow::anyhow!("Failed to open plot: {error}"))?; let plot_audit = PlotAudit::new(&plot); - let mut options = PlotAuditOptions:: { + let mut options = PlotAuditOptions { public_key: single_disk_farm_info.public_key(), reward_address: single_disk_farm_info.public_key(), slot_info: SlotInfo { @@ -524,10 +519,10 @@ where erasure_coding: &erasure_coding, sectors_being_modified: &HashSet::default(), read_sector_record_chunks_mode: ReadSectorRecordChunksMode::ConcurrentChunks, - table_generator: &table_generator, + cutover, }; - let mut audit_results = plot_audit.audit(options).unwrap(); + let mut audit_results = plot_audit.audit::(options).unwrap(); group.bench_function("plot/rayon/regular/concurrent-chunks", |b| { b.iter_batched( @@ -538,7 +533,7 @@ where options.slot_info.global_challenge = Blake3Hash::from(rand::random::<[u8; 32]>()); - audit_results = plot_audit.audit(options).unwrap(); + audit_results = plot_audit.audit::(options).unwrap(); audit_results.pop().unwrap() }, @@ -552,7 +547,7 @@ where }); options.read_sector_record_chunks_mode = ReadSectorRecordChunksMode::WholeSector; - let mut audit_results = plot_audit.audit(options).unwrap(); + let mut audit_results = plot_audit.audit::(options).unwrap(); group.bench_function("plot/rayon/regular/whole-sector", |b| { b.iter_batched( @@ -563,7 +558,7 @@ where options.slot_info.global_challenge = Blake3Hash::from(rand::random::<[u8; 32]>()); - audit_results = plot_audit.audit(options).unwrap(); + audit_results = plot_audit.audit::(options).unwrap(); audit_results.pop().unwrap() }, diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs index 3b44000001f..a886aab64af 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/plotter.rs @@ -1,4 +1,6 @@ use crate::commands::shared::PlottingThreadPriority; +#[cfg(feature = "wgpu")] +use crate::commands::shared::wgpu::{WgpuPlottingOptions, init_wgpu_plotter}; use anyhow::anyhow; use async_lock::{Mutex as AsyncMutex, Semaphore}; use clap::Parser; @@ -16,12 +18,6 @@ use subspace_farmer::cluster::nats_client::NatsClient; use subspace_farmer::cluster::plotter::plotter_service; use subspace_farmer::plotter::Plotter; use subspace_farmer::plotter::cpu::CpuPlotter; -#[cfg(feature = "_gpu")] -use subspace_farmer::plotter::gpu::GpuPlotter; -#[cfg(feature = "cuda")] -use subspace_farmer::plotter::gpu::cuda::CudaRecordsEncoder; -#[cfg(feature = "rocm")] -use subspace_farmer::plotter::gpu::rocm::RocmRecordsEncoder; use subspace_farmer::plotter::pool::PoolPlotter; use subspace_farmer::utils::{ create_plotting_thread_pool_manager, parse_cpu_cores_sets, thread_pool_core_indices, @@ -83,56 +79,16 @@ struct CpuPlottingOptions { cpu_plotting_thread_priority: PlottingThreadPriority, } -#[cfg(feature = "cuda")] -#[derive(Debug, Parser)] -struct CudaPlottingOptions { - /// How many sectors farmer will download concurrently during plotting with CUDA GPUs. - /// Limits memory usage of the plotting process. Defaults to the number of CUDA GPUs * 3, - /// to download future sectors ahead of time. - /// - /// Increasing this value will cause higher memory usage. - #[arg(long)] - cuda_sector_downloading_concurrency: Option, - /// Set the exact GPUs to be used for plotting, instead of using all GPUs (default behavior). - /// - /// GPUs are coma-separated: `--cuda-gpus 0,1,3`. Use an empty string to disable CUDA - /// GPUs. - #[arg(long)] - cuda_gpus: Option, -} - -#[cfg(feature = "rocm")] -#[derive(Debug, Parser)] -struct RocmPlottingOptions { - /// How many sectors farmer will download concurrently during plotting with ROCm GPUs. - /// Limits memory usage of the plotting process. Defaults to the number of ROCm GPUs * 3, - /// to download future sectors ahead of time. - /// - /// Increasing this value will cause higher memory usage. - #[arg(long)] - rocm_sector_downloading_concurrency: Option, - /// Set the exact GPUs to be used for plotting, instead of using all GPUs (default behavior). - /// - /// GPUs are coma-separated: `--rocm-gpus 0,1,3`. Use an empty string to disable ROCm - /// GPUs. - #[arg(long)] - rocm_gpus: Option, -} - /// Arguments for plotter #[derive(Debug, Parser)] pub(super) struct PlotterArgs { /// Plotting options only used by CPU plotter #[clap(flatten)] cpu_plotting_options: CpuPlottingOptions, - /// Plotting options only used by CUDA GPU plotter - #[cfg(feature = "cuda")] - #[clap(flatten)] - cuda_plotting_options: CudaPlottingOptions, - /// Plotting options only used by ROCm GPU plotter - #[cfg(feature = "rocm")] + /// Plotting options only used by wgpu GPU plotter + #[cfg(feature = "wgpu")] #[clap(flatten)] - rocm_plotting_options: RocmPlottingOptions, + wgpu_plotting_options: WgpuPlottingOptions, /// Cache group to use if specified, otherwise all caches are usable by this plotter #[arg(long)] cache_group: Option, @@ -151,10 +107,8 @@ where { let PlotterArgs { cpu_plotting_options, - #[cfg(feature = "cuda")] - cuda_plotting_options, - #[cfg(feature = "rocm")] - rocm_plotting_options, + #[cfg(feature = "wgpu")] + wgpu_plotting_options, cache_group, additional_components: _, } = plotter_args; @@ -171,34 +125,20 @@ where let mut plotters = Vec::>::new(); - #[cfg(feature = "cuda")] + #[cfg(feature = "wgpu")] { - let maybe_cuda_plotter = init_cuda_plotter( - cuda_plotting_options, + let maybe_wgpu_plotter = init_wgpu_plotter( + wgpu_plotting_options, piece_getter.clone(), Arc::clone(&global_mutex), kzg.clone(), erasure_coding.clone(), registry, - )?; - - if let Some(cuda_plotter) = maybe_cuda_plotter { - plotters.push(Box::new(cuda_plotter)); - } - } - #[cfg(feature = "rocm")] - { - let maybe_rocm_plotter = init_rocm_plotter( - rocm_plotting_options, - piece_getter.clone(), - Arc::clone(&global_mutex), - kzg.clone(), - erasure_coding.clone(), - registry, - )?; + ) + .await?; - if let Some(rocm_plotter) = maybe_rocm_plotter { - plotters.push(Box::new(rocm_plotter)); + if let Some(wgpu_plotter) = maybe_wgpu_plotter { + plotters.push(Box::new(wgpu_plotter)); } } { @@ -331,167 +271,3 @@ where Ok(Some(cpu_plotter)) } - -#[cfg(feature = "cuda")] -fn init_cuda_plotter( - cuda_plotting_options: CudaPlottingOptions, - piece_getter: PG, - global_mutex: Arc>, - kzg: Kzg, - erasure_coding: ErasureCoding, - registry: &mut Registry, -) -> anyhow::Result>> -where - PG: PieceGetter + Clone + Send + Sync + 'static, -{ - use std::collections::BTreeSet; - use subspace_proof_of_space_gpu::cuda::cuda_devices; - use tracing::{debug, warn}; - - let CudaPlottingOptions { - cuda_sector_downloading_concurrency, - cuda_gpus, - } = cuda_plotting_options; - - let mut cuda_devices = cuda_devices(); - let mut used_cuda_devices = (0..cuda_devices.len()).collect::>(); - - if let Some(cuda_gpus) = cuda_gpus { - if cuda_gpus.is_empty() { - info!("CUDA GPU plotting was explicitly disabled"); - return Ok(None); - } - - let mut cuda_gpus_to_use = cuda_gpus - .split(',') - .map(|gpu_index| gpu_index.parse()) - .collect::, _>>()?; - - (used_cuda_devices, cuda_devices) = cuda_devices - .into_iter() - .enumerate() - .filter(|(index, _cuda_device)| cuda_gpus_to_use.remove(index)) - .unzip(); - - if !cuda_gpus_to_use.is_empty() { - warn!( - ?cuda_gpus_to_use, - "Some CUDA GPUs were not found on the system" - ); - } - } - - if cuda_devices.is_empty() { - debug!("No CUDA GPU devices found"); - return Ok(None); - } - - info!(?used_cuda_devices, "Using CUDA GPUs"); - - let cuda_downloading_semaphore = Arc::new(Semaphore::new( - cuda_sector_downloading_concurrency - .map(|cuda_sector_downloading_concurrency| cuda_sector_downloading_concurrency.get()) - .unwrap_or(cuda_devices.len() * 3), - )); - - Ok(Some( - GpuPlotter::new( - piece_getter, - cuda_downloading_semaphore, - cuda_devices - .into_iter() - .map(|cuda_device| CudaRecordsEncoder::new(cuda_device, Arc::clone(&global_mutex))) - .collect::>() - .map_err(|error| { - anyhow::anyhow!("Failed to create CUDA records encoder: {error}") - })?, - global_mutex, - kzg, - erasure_coding, - Some(registry), - ) - .map_err(|error| anyhow::anyhow!("Failed to initialize CUDA plotter: {error}"))?, - )) -} - -#[cfg(feature = "rocm")] -fn init_rocm_plotter( - rocm_plotting_options: RocmPlottingOptions, - piece_getter: PG, - global_mutex: Arc>, - kzg: Kzg, - erasure_coding: ErasureCoding, - registry: &mut Registry, -) -> anyhow::Result>> -where - PG: PieceGetter + Clone + Send + Sync + 'static, -{ - use std::collections::BTreeSet; - use subspace_proof_of_space_gpu::rocm::rocm_devices; - use tracing::{debug, warn}; - - let RocmPlottingOptions { - rocm_sector_downloading_concurrency, - rocm_gpus, - } = rocm_plotting_options; - - let mut rocm_devices = rocm_devices(); - let mut used_rocm_devices = (0..rocm_devices.len()).collect::>(); - - if let Some(rocm_gpus) = rocm_gpus { - if rocm_gpus.is_empty() { - info!("ROCm GPU plotting was explicitly disabled"); - return Ok(None); - } - - let mut rocm_gpus_to_use = rocm_gpus - .split(',') - .map(|gpu_index| gpu_index.parse()) - .collect::, _>>()?; - - (used_rocm_devices, rocm_devices) = rocm_devices - .into_iter() - .enumerate() - .filter(|(index, _rocm_device)| rocm_gpus_to_use.remove(index)) - .unzip(); - - if !rocm_gpus_to_use.is_empty() { - warn!( - ?rocm_gpus_to_use, - "Some ROCm GPUs were not found on the system" - ); - } - } - - if rocm_devices.is_empty() { - debug!("No ROCm GPU devices found"); - return Ok(None); - } - - info!(?used_rocm_devices, "Using ROCm GPUs"); - - let rocm_downloading_semaphore = Arc::new(Semaphore::new( - rocm_sector_downloading_concurrency - .map(|rocm_sector_downloading_concurrency| rocm_sector_downloading_concurrency.get()) - .unwrap_or(rocm_devices.len() * 3), - )); - - Ok(Some( - GpuPlotter::new( - piece_getter, - rocm_downloading_semaphore, - rocm_devices - .into_iter() - .map(|rocm_device| RocmRecordsEncoder::new(rocm_device, Arc::clone(&global_mutex))) - .collect::>() - .map_err(|error| { - anyhow::anyhow!("Failed to create ROCm records encoder: {error}") - })?, - global_mutex, - kzg, - erasure_coding, - Some(registry), - ) - .map_err(|error| anyhow::anyhow!("Failed to initialize ROCm plotter: {error}"))?, - )) -} diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs index e06bcab2624..f3b5bc82d3d 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs @@ -1,4 +1,6 @@ use crate::commands::shared::network::{NetworkArgs, configure_network}; +#[cfg(feature = "wgpu")] +use crate::commands::shared::wgpu::{WgpuPlottingOptions, init_wgpu_plotter}; use crate::commands::shared::{DiskFarm, PlottingThreadPriority, derive_libp2p_keypair}; use anyhow::anyhow; use async_lock::{Mutex as AsyncMutex, RwLock as AsyncRwLock, Semaphore}; @@ -31,12 +33,6 @@ use subspace_farmer::node_client::caching_proxy_node_client::CachingProxyNodeCli use subspace_farmer::node_client::rpc_node_client::RpcNodeClient; use subspace_farmer::plotter::Plotter; use subspace_farmer::plotter::cpu::CpuPlotter; -#[cfg(feature = "_gpu")] -use subspace_farmer::plotter::gpu::GpuPlotter; -#[cfg(feature = "cuda")] -use subspace_farmer::plotter::gpu::cuda::CudaRecordsEncoder; -#[cfg(feature = "rocm")] -use subspace_farmer::plotter::gpu::rocm::RocmRecordsEncoder; use subspace_farmer::plotter::pool::PoolPlotter; use subspace_farmer::single_disk_farm::identity::Identity; use subspace_farmer::single_disk_farm::{ @@ -144,42 +140,6 @@ struct CpuPlottingOptions { cpu_plotting_thread_priority: PlottingThreadPriority, } -#[cfg(feature = "cuda")] -#[derive(Debug, Parser)] -struct CudaPlottingOptions { - /// How many sectors farmer will download concurrently during plotting with CUDA GPUs. - /// Limits memory usage of the plotting process. Defaults to the number of CUDA GPUs * 3, - /// to download future sectors ahead of time. - /// - /// Increasing this value will cause higher memory usage. - #[arg(long)] - cuda_sector_downloading_concurrency: Option, - /// Set the exact GPUs to be used for plotting instead of using all GPUs (default behavior). - /// - /// GPUs are coma-separated: `--cuda-gpus 0,1,3`. Use an empty string to disable CUDA - /// GPUs. - #[arg(long)] - cuda_gpus: Option, -} - -#[cfg(feature = "rocm")] -#[derive(Debug, Parser)] -struct RocmPlottingOptions { - /// How many sectors farmer will download concurrently during plotting with ROCm GPUs. - /// Limits memory usage of the plotting process. Defaults to the number of ROCm GPUs * 3, - /// to download future sectors ahead of time. - /// - /// Increasing this value will cause higher memory usage. - #[arg(long)] - rocm_sector_downloading_concurrency: Option, - /// Set the exact GPUs to be used for plotting instead of using all GPUs (default behavior). - /// - /// GPUs are coma-separated: `--rocm-gpus 0,1,3`. Use an empty string to disable ROCm - /// GPUs. - #[arg(long)] - rocm_gpus: Option, -} - /// Arguments for farmer #[derive(Debug, Parser)] pub(crate) struct FarmingArgs { @@ -244,14 +204,10 @@ pub(crate) struct FarmingArgs { /// Plotting options only used by CPU plotter #[clap(flatten)] cpu_plotting_options: CpuPlottingOptions, - /// Plotting options only used by CUDA GPU plotter - #[cfg(feature = "cuda")] + /// Plotting options only used by wgpu GPU plotter + #[cfg(feature = "wgpu")] #[clap(flatten)] - cuda_plotting_options: CudaPlottingOptions, - /// Plotting options only used by ROCm GPU plotter - #[cfg(feature = "rocm")] - #[clap(flatten)] - rocm_plotting_options: RocmPlottingOptions, + wgpu_plotting_options: WgpuPlottingOptions, /// How many sectors a will be plotted concurrently per farm. /// /// Defaults to 2, but can be decreased if there is a large number of farms available to @@ -317,10 +273,8 @@ where prometheus_listen_on, farming_thread_pool_size, cpu_plotting_options, - #[cfg(feature = "cuda")] - cuda_plotting_options, - #[cfg(feature = "rocm")] - rocm_plotting_options, + #[cfg(feature = "wgpu")] + wgpu_plotting_options, max_plotting_sectors_per_farm, plot_cache, disable_farm_locking, @@ -521,34 +475,20 @@ where let mut plotters = Vec::>::new(); - #[cfg(feature = "cuda")] + #[cfg(feature = "wgpu")] { - let maybe_cuda_plotter = init_cuda_plotter( - cuda_plotting_options, + let maybe_wgpu_plotter = init_wgpu_plotter( + wgpu_plotting_options, piece_getter.clone(), Arc::clone(&global_mutex), kzg.clone(), erasure_coding.clone(), &mut registry, - )?; - - if let Some(cuda_plotter) = maybe_cuda_plotter { - plotters.push(Box::new(cuda_plotter)); - } - } - #[cfg(feature = "rocm")] - { - let maybe_rocm_plotter = init_rocm_plotter( - rocm_plotting_options, - piece_getter.clone(), - Arc::clone(&global_mutex), - kzg.clone(), - erasure_coding.clone(), - &mut registry, - )?; + ) + .await?; - if let Some(rocm_plotter) = maybe_rocm_plotter { - plotters.push(Box::new(rocm_plotter)); + if let Some(wgpu_plotter) = maybe_wgpu_plotter { + plotters.push(Box::new(wgpu_plotter)); } } { @@ -1001,167 +941,3 @@ where Ok(Some(cpu_plotter)) } - -#[cfg(feature = "cuda")] -fn init_cuda_plotter( - cuda_plotting_options: CudaPlottingOptions, - piece_getter: PG, - global_mutex: Arc>, - kzg: Kzg, - erasure_coding: ErasureCoding, - registry: &mut Registry, -) -> anyhow::Result>> -where - PG: PieceGetter + Clone + Send + Sync + 'static, -{ - use std::collections::BTreeSet; - use subspace_proof_of_space_gpu::cuda::cuda_devices; - use tracing::debug; - - let CudaPlottingOptions { - cuda_sector_downloading_concurrency, - cuda_gpus, - } = cuda_plotting_options; - - let mut cuda_devices = cuda_devices(); - let mut used_cuda_devices = (0..cuda_devices.len()).collect::>(); - - if let Some(cuda_gpus) = cuda_gpus { - if cuda_gpus.is_empty() { - info!("CUDA GPU plotting was explicitly disabled"); - return Ok(None); - } - - let mut cuda_gpus_to_use = cuda_gpus - .split(',') - .map(|gpu_index| gpu_index.parse()) - .collect::, _>>()?; - - (used_cuda_devices, cuda_devices) = cuda_devices - .into_iter() - .enumerate() - .filter(|(index, _cuda_device)| cuda_gpus_to_use.remove(index)) - .unzip(); - - if !cuda_gpus_to_use.is_empty() { - warn!( - ?cuda_gpus_to_use, - "Some CUDA GPUs were not found on the system" - ); - } - } - - if cuda_devices.is_empty() { - debug!("No CUDA GPU devices found"); - return Ok(None); - } - - info!(?used_cuda_devices, "Using CUDA GPUs"); - - let cuda_downloading_semaphore = Arc::new(Semaphore::new( - cuda_sector_downloading_concurrency - .map(|cuda_sector_downloading_concurrency| cuda_sector_downloading_concurrency.get()) - .unwrap_or(cuda_devices.len() * 3), - )); - - Ok(Some( - GpuPlotter::new( - piece_getter, - cuda_downloading_semaphore, - cuda_devices - .into_iter() - .map(|cuda_device| CudaRecordsEncoder::new(cuda_device, Arc::clone(&global_mutex))) - .collect::>() - .map_err(|error| { - anyhow::anyhow!("Failed to create CUDA records encoder: {error}") - })?, - global_mutex, - kzg, - erasure_coding, - Some(registry), - ) - .map_err(|error| anyhow::anyhow!("Failed to initialize CUDA plotter: {error}"))?, - )) -} - -#[cfg(feature = "rocm")] -fn init_rocm_plotter( - rocm_plotting_options: RocmPlottingOptions, - piece_getter: PG, - global_mutex: Arc>, - kzg: Kzg, - erasure_coding: ErasureCoding, - registry: &mut Registry, -) -> anyhow::Result>> -where - PG: PieceGetter + Clone + Send + Sync + 'static, -{ - use std::collections::BTreeSet; - use subspace_proof_of_space_gpu::rocm::rocm_devices; - use tracing::debug; - - let RocmPlottingOptions { - rocm_sector_downloading_concurrency, - rocm_gpus, - } = rocm_plotting_options; - - let mut rocm_devices = rocm_devices(); - let mut used_rocm_devices = (0..rocm_devices.len()).collect::>(); - - if let Some(rocm_gpus) = rocm_gpus { - if rocm_gpus.is_empty() { - info!("ROCm GPU plotting was explicitly disabled"); - return Ok(None); - } - - let mut rocm_gpus_to_use = rocm_gpus - .split(',') - .map(|gpu_index| gpu_index.parse()) - .collect::, _>>()?; - - (used_rocm_devices, rocm_devices) = rocm_devices - .into_iter() - .enumerate() - .filter(|(index, _rocm_device)| rocm_gpus_to_use.remove(index)) - .unzip(); - - if !rocm_gpus_to_use.is_empty() { - warn!( - ?rocm_gpus_to_use, - "Some ROCm GPUs were not found on the system" - ); - } - } - - if rocm_devices.is_empty() { - debug!("No ROCm GPU devices found"); - return Ok(None); - } - - info!(?used_rocm_devices, "Using ROCm GPUs"); - - let rocm_downloading_semaphore = Arc::new(Semaphore::new( - rocm_sector_downloading_concurrency - .map(|rocm_sector_downloading_concurrency| rocm_sector_downloading_concurrency.get()) - .unwrap_or(rocm_devices.len() + 1), - )); - - Ok(Some( - GpuPlotter::new( - piece_getter, - rocm_downloading_semaphore, - rocm_devices - .into_iter() - .map(|rocm_device| RocmRecordsEncoder::new(rocm_device, Arc::clone(&global_mutex))) - .collect::>() - .map_err(|error| { - anyhow::anyhow!("Failed to create ROCm records encoder: {error}") - })?, - global_mutex, - kzg, - erasure_coding, - Some(registry), - ) - .map_err(|error| anyhow::anyhow!("Failed to initialize ROCm plotter: {error}"))?, - )) -} diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared.rs index 0b3ecdd88e2..5c02576ee2e 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared.rs @@ -1,4 +1,6 @@ pub(super) mod network; +#[cfg(feature = "wgpu")] +pub(super) mod wgpu; use bytesize::ByteSize; use clap::Parser; diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared/wgpu.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared/wgpu.rs new file mode 100644 index 00000000000..272abe6f432 --- /dev/null +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared/wgpu.rs @@ -0,0 +1,329 @@ +//! Shared wgpu GPU plotting options and setup, used by both the `farm` and cluster-plotter commands. + +use async_lock::{Mutex as AsyncMutex, Semaphore}; +use clap::Parser; +use prometheus_client::registry::Registry; +use std::collections::BTreeSet; +use std::num::{NonZeroU8, NonZeroUsize}; +use std::sync::Arc; +use subspace_data_retrieval::piece_getter::PieceGetter; +use subspace_erasure_coding::ErasureCoding; +use subspace_farmer::plotter::gpu::GpuPlotter; +use subspace_farmer::plotter::gpu::wgpu::WgpuRecordsEncoder; +use subspace_kzg::Kzg; +use subspace_proof_of_space_wgpu::{Backend, Device, DeviceType, WgpuDevice}; +use tracing::{debug, info, warn}; + +/// Plotting options for the wgpu GPU plotter. +#[derive(Debug, Parser)] +pub(in super::super) struct WgpuPlottingOptions { + /// How many records the farmer will encode concurrently on the same GPU. + /// + /// Increasing this value will cause higher VRAM usage and will not necessarily improve + /// performance. + /// + /// Defaults to 4 for dGPU and 2 otherwise (iGPU, etc.). + #[arg(long)] + gpu_record_encoding_concurrency: Option, + /// How many sectors farmer will download concurrently during plotting with GPUs. + /// Limits memory usage of the plotting process. Defaults to the number of GPUs * 3, + /// to download future sectors ahead of time. + /// + /// Increasing this value will cause higher memory usage. + #[arg(long)] + gpu_sector_downloading_concurrency: Option, + /// Set the exact GPUs to be used for plotting instead of using recommended GPUs (default + /// behavior). + /// + /// By default, dGPUs are used if available, if not, then iGPUs are used, if neither dGPU nor + /// iGPU is found, a virtual GPU will be used as the last resort. + /// + /// GPUs are comma-separated: `--gpus 0,1,3`. To disable GPU plotting entirely, use + /// `--cpu-only`. + #[arg(long)] + gpus: Option, + /// Plot on the CPU only, skipping GPU plotting even when GPUs are available. + #[arg(long)] + cpu_only: bool, + /// Enable GPU plotting on Metal (Apple GPUs), which is untested. + #[arg(long)] + enable_metal: bool, +} + +/// Print the GPUs wgpu can plot on, with the device ids to pass to `--gpus`. +pub(crate) async fn list_gpus(verbose: bool) { + let mut any_metal = false; + for device in Device::enumerate(|_| NonZeroU8::MIN).await { + let device_type = match device.device_type() { + DeviceType::Other => "other", + DeviceType::IntegratedGpu => "Integrated GPU", + DeviceType::DiscreteGpu => "Discrete GPU", + DeviceType::VirtualGpu => "Virtual GPU", + DeviceType::Cpu => "CPU emulation", + }; + let is_metal = device.backend() == Backend::Metal; + any_metal |= is_metal; + let disabled = if is_metal { + " [untested; enable with --enable-metal]" + } else { + "" + }; + println!( + "{}: {} ({device_type}){disabled}", + device.id(), + device.name() + ); + if verbose { + println!(" Backend: {}", device.backend()); + println!(" Driver: {}", device.driver()); + println!(" Driver info: {}", device.driver_info()); + } + } + if any_metal { + println!(); + println!( + "Metal GPUs are untested and disabled by default; pass --enable-metal to plot with them." + ); + } +} + +/// Choose which enumerated GPU devices to plot on: an explicit `--gpus` set is honored +/// verbatim, otherwise discrete GPUs are preferred, then integrated, then a virtual GPU as a last +/// resort, while unknown and CPU-emulated adapters are never auto-selected. +fn select_gpu_devices( + device_types: &[DeviceType], + explicit_gpus: Option<&BTreeSet>, +) -> Vec { + if let Some(explicit_gpus) = explicit_gpus { + return explicit_gpus + .iter() + .copied() + .filter(|&index| index < device_types.len()) + .collect(); + } + + let has_dgpu = device_types.contains(&DeviceType::DiscreteGpu); + let has_igpu = device_types.contains(&DeviceType::IntegratedGpu); + + device_types + .iter() + .enumerate() + .filter_map(|(index, &device_type)| { + let use_device = match device_type { + DeviceType::DiscreteGpu => true, + DeviceType::IntegratedGpu => !has_dgpu, + DeviceType::VirtualGpu => !has_dgpu && !has_igpu, + DeviceType::Other | DeviceType::Cpu => false, + }; + use_device.then_some(index) + }) + .collect() +} + +pub(in super::super) async fn init_wgpu_plotter( + wgpu_plotting_options: WgpuPlottingOptions, + piece_getter: PG, + global_mutex: Arc>, + kzg: Kzg, + erasure_coding: ErasureCoding, + registry: &mut Registry, +) -> anyhow::Result>> +where + PG: PieceGetter + Clone + Send + Sync + 'static, +{ + let WgpuPlottingOptions { + gpu_record_encoding_concurrency, + gpu_sector_downloading_concurrency, + gpus, + cpu_only, + enable_metal, + } = wgpu_plotting_options; + + if cpu_only { + info!("GPU plotting disabled, plotting on the CPU only"); + return Ok(None); + } + + let number_of_queues = |device_type: DeviceType| { + if let Some(gpu_record_encoding_concurrency) = gpu_record_encoding_concurrency { + return gpu_record_encoding_concurrency; + } + match device_type { + DeviceType::DiscreteGpu => NonZeroU8::new(4).expect("Not zero; qed"), + DeviceType::Other + | DeviceType::IntegratedGpu + | DeviceType::VirtualGpu + | DeviceType::Cpu => NonZeroU8::new(2).expect("Not zero; qed"), + } + }; + let all_gpu_devices = Device::enumerate(number_of_queues).await; + + let all_gpu_devices = if enable_metal { + all_gpu_devices + } else { + let (skipped_metal, all_gpu_devices): (Vec<_>, Vec<_>) = all_gpu_devices + .into_iter() + .partition(|device| device.backend() == Backend::Metal); + if !skipped_metal.is_empty() { + warn!( + count = skipped_metal.len(), + "Skipping Metal GPU(s); GPU plotting on Metal is untested, pass --enable-metal to \ + use them anyway" + ); + } + all_gpu_devices + }; + + let device_types = all_gpu_devices + .iter() + .map(Device::device_type) + .collect::>(); + + let explicit_gpus = match gpus { + Some(gpus) if !gpus.is_empty() => { + let mut gpus_to_use = gpus + .split(',') + .map(str::parse) + .collect::, _>>()?; + + let explicit_gpus = all_gpu_devices + .iter() + .enumerate() + .filter_map(|(index, device)| gpus_to_use.remove(&device.id()).then_some(index)) + .collect::>(); + + if !gpus_to_use.is_empty() { + warn!(?gpus_to_use, "Some GPUs were not found on the system"); + } + + Some(explicit_gpus) + } + _ => None, + }; + + let used_indices = select_gpu_devices(&device_types, explicit_gpus.as_ref()) + .into_iter() + .collect::>(); + + // Explain skipped devices when relying on automatic selection + if explicit_gpus.is_none() { + for (index, device) in all_gpu_devices.iter().enumerate() { + if used_indices.contains(&index) { + continue; + } + match device.device_type() { + DeviceType::Other => debug!(?device, "Skipping an unknown GPU device type"), + DeviceType::IntegratedGpu => debug!(?device, "Skipping iGPU in presence of dGPU"), + DeviceType::VirtualGpu => { + debug!( + ?device, + "Skipping virtualized GPU in presence of iGPU or dGPU" + ) + } + DeviceType::Cpu => debug!(?device, "Skipping GPU device emulated by the CPU"), + DeviceType::DiscreteGpu => {} + } + } + } + + let used_gpu_devices = all_gpu_devices + .into_iter() + .enumerate() + .filter_map(|(index, device)| used_indices.contains(&index).then_some(device)) + .collect::>(); + + if used_gpu_devices.is_empty() { + debug!("No GPU devices were found or used"); + return Ok(None); + } + + info!("Using GPUs:"); + for device in &used_gpu_devices { + let device_type = match device.device_type() { + DeviceType::Other => "other", + DeviceType::IntegratedGpu => "Integrated GPU", + DeviceType::DiscreteGpu => "Discrete GPU", + DeviceType::VirtualGpu => "Virtual GPU", + DeviceType::Cpu => "CPU emulation", + }; + info!("{}: {} ({device_type})", device.id(), device.name()); + } + + let wgpu_downloading_semaphore = Arc::new(Semaphore::new( + gpu_sector_downloading_concurrency + .map(|gpu_sector_downloading_concurrency| gpu_sector_downloading_concurrency.get()) + .unwrap_or(used_gpu_devices.len() * 3), + )); + + Ok(Some( + GpuPlotter::new( + piece_getter, + wgpu_downloading_semaphore, + used_gpu_devices + .into_iter() + .map(|wgpu_device| { + let id = wgpu_device.id(); + let queue_devices = wgpu_device + .create_proofs_encoder_instances() + .into_iter() + .map(|instance| WgpuDevice::new(instance, erasure_coding.clone())) + .collect(); + WgpuRecordsEncoder::new(id, queue_devices, Arc::clone(&global_mutex)) + }) + .collect::>() + .map_err(|error| { + anyhow::anyhow!("Failed to create wgpu records encoder: {error}") + })?, + global_mutex, + kzg, + erasure_coding, + Some(registry), + ) + .map_err(|error| anyhow::anyhow!("Failed to initialize wgpu plotter: {error}"))?, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gpu_device_selection() { + use DeviceType::{Cpu, DiscreteGpu, IntegratedGpu, Other, VirtualGpu}; + + // dGPU wins over iGPU + assert_eq!( + select_gpu_devices(&[DiscreteGpu, IntegratedGpu], None), + vec![0] + ); + // All iGPUs are used when there is no dGPU + assert_eq!( + select_gpu_devices(&[IntegratedGpu, IntegratedGpu], None), + vec![0, 1] + ); + // CPU emulation is never auto-selected + assert_eq!(select_gpu_devices(&[Cpu], None), Vec::::new()); + // All dGPUs are used, iGPU skipped + assert_eq!( + select_gpu_devices(&[DiscreteGpu, DiscreteGpu, IntegratedGpu], None), + vec![0, 1] + ); + // Virtual GPU skipped while an iGPU is present + assert_eq!( + select_gpu_devices(&[IntegratedGpu, VirtualGpu], None), + vec![0] + ); + // Virtual GPU used as the last resort + assert_eq!(select_gpu_devices(&[VirtualGpu], None), vec![0]); + // Virtual GPU still wins over unknown/CPU adapters as the last resort + assert_eq!(select_gpu_devices(&[Cpu, Other, VirtualGpu], None), vec![2]); + // Unknown and CPU adapters alone select nothing + assert_eq!(select_gpu_devices(&[Cpu, Other], None), Vec::::new()); + // Explicit override bypasses the tiering + let explicit = BTreeSet::from([0, 2]); + assert_eq!( + select_gpu_devices(&[Cpu, DiscreteGpu, IntegratedGpu], Some(&explicit)), + vec![0, 2] + ); + } +} diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/main.rs b/crates/subspace-farmer/src/bin/subspace-farmer/main.rs index 2d185e3401b..b38c4657dc0 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/main.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/main.rs @@ -5,13 +5,12 @@ use std::fs; use std::path::PathBuf; use subspace_farmer::single_disk_farm::{ScrubTarget, SingleDiskFarm}; use subspace_process::{init_logger, raise_fd_limit, set_exit_on_panic}; -use subspace_proof_of_space::chia::ChiaTable; use tracing::info; #[global_allocator] static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; -type PosTable = ChiaTable; +type PosTable = subspace_proof_of_space::PosTable; #[allow(clippy::large_enum_variant)] #[derive(Debug, Parser)] @@ -24,6 +23,13 @@ enum Command { /// Run various benchmarks #[clap(subcommand)] Benchmark(commands::benchmark::BenchmarkArgs), + /// List the GPUs available for plotting, with the device ids to pass to `--gpus` + #[cfg(feature = "wgpu")] + ListGpus { + /// Also print each GPU's backend and driver details + #[arg(long)] + verbose: bool, + }, /// Print information about farm and its content Info { /// One or more farm located at specified path. @@ -79,6 +85,10 @@ async fn main() -> anyhow::Result<()> { Command::Benchmark(benchmark_args) => { commands::benchmark::benchmark(benchmark_args)?; } + #[cfg(feature = "wgpu")] + Command::ListGpus { verbose } => { + commands::list_gpus(verbose).await; + } Command::Info { disk_farms } => { if disk_farms.is_empty() { info!("No farm was specified, so there is nothing to do"); diff --git a/crates/subspace-farmer/src/cluster/plotter.rs b/crates/subspace-farmer/src/cluster/plotter.rs index bb6e6161ee1..58f9d207876 100644 --- a/crates/subspace-farmer/src/cluster/plotter.rs +++ b/crates/subspace-farmer/src/cluster/plotter.rs @@ -69,7 +69,7 @@ impl ClusterPlotterId { struct ClusterPlotterFreeInstanceRequest; impl GenericRequest for ClusterPlotterFreeInstanceRequest { - const SUBJECT: &'static str = "subspace.plotter.free-instance"; + const SUBJECT: &'static str = "subspace.plotter.free-instance-v2"; /// Might be `None` if instance had to respond, but turned out it was fully occupied already type Response = Option; } @@ -114,7 +114,7 @@ struct ClusterPlotterPlotSectorRequest { } impl GenericStreamRequest for ClusterPlotterPlotSectorRequest { - const SUBJECT: &'static str = "subspace.plotter.*.plot-sector"; + const SUBJECT: &'static str = "subspace.plotter.*.plot-sector-v2"; type Response = ClusterSectorPlottingProgress; } diff --git a/crates/subspace-farmer/src/plotter/gpu.rs b/crates/subspace-farmer/src/plotter/gpu.rs index 833486700ff..93fde79fd2d 100644 --- a/crates/subspace-farmer/src/plotter/gpu.rs +++ b/crates/subspace-farmer/src/plotter/gpu.rs @@ -1,11 +1,9 @@ //! GPU plotter -#[cfg(feature = "cuda")] -pub mod cuda; mod gpu_encoders_manager; pub mod metrics; -#[cfg(feature = "rocm")] -pub mod rocm; +#[cfg(feature = "wgpu")] +pub mod wgpu; use crate::plotter::gpu::gpu_encoders_manager::GpuRecordsEncoderManager; use crate::plotter::gpu::metrics::GpuPlotterMetrics; diff --git a/crates/subspace-farmer/src/plotter/gpu/rocm.rs b/crates/subspace-farmer/src/plotter/gpu/rocm.rs deleted file mode 100644 index d99a130c5dd..00000000000 --- a/crates/subspace-farmer/src/plotter/gpu/rocm.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! ROCm GPU records encoder - -use crate::plotter::gpu::GpuRecordsEncoder; -use async_lock::Mutex as AsyncMutex; -use parking_lot::Mutex; -use rayon::{ThreadPool, ThreadPoolBuildError, ThreadPoolBuilder, current_thread_index}; -use std::process::exit; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use subspace_core_primitives::pieces::{PieceOffset, Record}; -use subspace_core_primitives::sectors::SectorId; -use subspace_farmer_components::plotting::RecordsEncoder; -use subspace_farmer_components::sector::SectorContentsMap; -use subspace_proof_of_space_gpu::rocm::RocmDevice; - -/// ROCm implementation of [`GpuRecordsEncoder`] -#[derive(Debug)] -pub struct RocmRecordsEncoder { - rocm_device: RocmDevice, - thread_pool: ThreadPool, - global_mutex: Arc>, -} - -impl GpuRecordsEncoder for RocmRecordsEncoder { - const TYPE: &'static str = "rocm"; -} - -impl RecordsEncoder for RocmRecordsEncoder { - fn encode_records( - &mut self, - sector_id: &SectorId, - records: &mut [Record], - abort_early: &AtomicBool, - ) -> anyhow::Result { - let pieces_in_sector = records - .len() - .try_into() - .map_err(|error| anyhow::anyhow!("Failed to convert pieces in sector: {error}"))?; - let mut sector_contents_map = SectorContentsMap::new(pieces_in_sector); - - { - let iter = Mutex::new( - (PieceOffset::ZERO..) - .zip(records.iter_mut()) - .zip(sector_contents_map.iter_record_bitfields_mut()), - ); - let plotting_error = Mutex::new(None::); - - self.thread_pool.scope(|scope| { - scope.spawn_broadcast(|_scope, _ctx| { - loop { - // Take mutex briefly to make sure encoding is allowed right now - self.global_mutex.lock_blocking(); - - // This instead of `while` above because otherwise mutex will be held for the - // duration of the loop and will limit concurrency to 1 record - let Some(((piece_offset, record), mut encoded_chunks_used)) = - iter.lock().next() - else { - return; - }; - let pos_seed = sector_id.derive_evaluation_seed(piece_offset); - - if let Err(error) = self.rocm_device.generate_and_encode_pospace( - &pos_seed, - record, - encoded_chunks_used.iter_mut(), - ) { - plotting_error.lock().replace(error); - return; - } - - if abort_early.load(Ordering::Relaxed) { - return; - } - } - }); - }); - - let plotting_error = plotting_error.lock().take(); - if let Some(error) = plotting_error { - return Err(anyhow::Error::msg(error)); - } - } - - Ok(sector_contents_map) - } -} - -impl RocmRecordsEncoder { - /// Create new instance - pub fn new( - rocm_device: RocmDevice, - global_mutex: Arc>, - ) -> Result { - let id = rocm_device.id(); - let thread_name = move |thread_index| format!("rocm-{id:02}.{thread_index:02}"); - // TODO: remove this panic handler when rayon logs panic_info - // https://github.com/rayon-rs/rayon/issues/1208 - let panic_handler = move |panic_info| { - if let Some(index) = current_thread_index() { - eprintln!("panic on thread {}: {:?}", thread_name(index), panic_info); - } else { - // We want to guarantee exit, rather than panicking in a panic handler. - eprintln!("rayon panic handler called on non-rayon thread: {panic_info:?}"); - } - exit(1); - }; - - let thread_pool = ThreadPoolBuilder::new() - .thread_name(thread_name) - .panic_handler(panic_handler) - // Make sure there is overlap between records, so GPU is almost always busy - .num_threads(2) - .build()?; - - Ok(Self { - rocm_device, - thread_pool, - global_mutex, - }) - } -} diff --git a/crates/subspace-farmer/src/plotter/gpu/cuda.rs b/crates/subspace-farmer/src/plotter/gpu/wgpu.rs similarity index 71% rename from crates/subspace-farmer/src/plotter/gpu/cuda.rs rename to crates/subspace-farmer/src/plotter/gpu/wgpu.rs index ee8c0e316f1..7e0e8f34646 100644 --- a/crates/subspace-farmer/src/plotter/gpu/cuda.rs +++ b/crates/subspace-farmer/src/plotter/gpu/wgpu.rs @@ -1,9 +1,10 @@ -//! CUDA GPU records encoder +//! wgpu GPU records encoder use crate::plotter::gpu::GpuRecordsEncoder; use async_lock::Mutex as AsyncMutex; use parking_lot::Mutex; use rayon::{ThreadPool, ThreadPoolBuildError, ThreadPoolBuilder, current_thread_index}; +use std::fmt; use std::process::exit; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -11,21 +12,26 @@ use subspace_core_primitives::pieces::{PieceOffset, Record}; use subspace_core_primitives::sectors::SectorId; use subspace_farmer_components::plotting::RecordsEncoder; use subspace_farmer_components::sector::SectorContentsMap; -use subspace_proof_of_space_gpu::cuda::CudaDevice; +use subspace_proof_of_space_wgpu::WgpuDevice; -/// CUDA implementation of [`GpuRecordsEncoder`] -#[derive(Debug)] -pub struct CudaRecordsEncoder { - cuda_device: CudaDevice, +/// wgpu implementation of [`GpuRecordsEncoder`] +pub struct WgpuRecordsEncoder { + devices: Vec>, thread_pool: ThreadPool, global_mutex: Arc>, } -impl GpuRecordsEncoder for CudaRecordsEncoder { - const TYPE: &'static str = "cuda"; +impl fmt::Debug for WgpuRecordsEncoder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("WgpuRecordsEncoder").finish_non_exhaustive() + } +} + +impl GpuRecordsEncoder for WgpuRecordsEncoder { + const TYPE: &'static str = "wgpu"; } -impl RecordsEncoder for CudaRecordsEncoder { +impl RecordsEncoder for WgpuRecordsEncoder { fn encode_records( &mut self, sector_id: &SectorId, @@ -48,6 +54,15 @@ impl RecordsEncoder for CudaRecordsEncoder { self.thread_pool.scope(|scope| { scope.spawn_broadcast(|_scope, _ctx| { + // One device (GPU queue) per pool thread, so this lock is always uncontended + let thread_index = current_thread_index().unwrap_or_default(); + let Some(device) = self.devices.get(thread_index) else { + return; + }; + let mut device = device + .try_lock() + .expect("1:1 mapping between threads and devices; qed"); + loop { // Take mutex briefly to make sure encoding is allowed right now self.global_mutex.lock_blocking(); @@ -61,7 +76,7 @@ impl RecordsEncoder for CudaRecordsEncoder { }; let pos_seed = sector_id.derive_evaluation_seed(piece_offset); - if let Err(error) = self.cuda_device.generate_and_encode_pospace( + if let Err(error) = device.generate_and_encode_pospace( &pos_seed, record, encoded_chunks_used.iter_mut(), @@ -87,14 +102,16 @@ impl RecordsEncoder for CudaRecordsEncoder { } } -impl CudaRecordsEncoder { - /// Create new instance +impl WgpuRecordsEncoder { + /// Create new instance. + /// + /// One thread is spawned per device (GPU queue), so records encode concurrently across queues. pub fn new( - cuda_device: CudaDevice, + id: u32, + devices: Vec, global_mutex: Arc>, ) -> Result { - let id = cuda_device.id(); - let thread_name = move |thread_index| format!("cuda-{id:02}.{thread_index:02}"); + let thread_name = move |thread_index| format!("wgpu-{id:02}.{thread_index:02}"); // TODO: remove this panic handler when rayon logs panic_info // https://github.com/rayon-rs/rayon/issues/1208 let panic_handler = move |panic_info| { @@ -110,12 +127,11 @@ impl CudaRecordsEncoder { let thread_pool = ThreadPoolBuilder::new() .thread_name(thread_name) .panic_handler(panic_handler) - // Make sure there is overlap between records, so GPU is almost always busy - .num_threads(2) + .num_threads(devices.len()) .build()?; Ok(Self { - cuda_device, + devices: devices.into_iter().map(Mutex::new).collect(), thread_pool, global_mutex, }) diff --git a/crates/subspace-farmer/src/single_disk_farm.rs b/crates/subspace-farmer/src/single_disk_farm.rs index f3184bddf9c..d294911d803 100644 --- a/crates/subspace-farmer/src/single_disk_farm.rs +++ b/crates/subspace-farmer/src/single_disk_farm.rs @@ -255,24 +255,75 @@ pub enum SingleDiskFarmSummary { }, } -#[derive(Debug, Encode, Decode)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Encode, Decode)] +enum PlotMetadataVersion { + /// Original layout, before the abundance-backed proof of space. + #[codec(index = 0)] + V0, + /// Adds the `cutover`, past which sectors use the abundance-backed proof of space. + #[codec(index = 1)] + V1, +} + +impl PlotMetadataVersion { + /// Latest version this implementation writes. + const LATEST: Self = Self::V1; +} + +#[derive(Debug, Encode)] struct PlotMetadataHeader { - version: u8, + version: PlotMetadataVersion, plotted_sector_count: SectorIndex, + /// History size of the newest sector plotted before the proof-of-space cutover, or `None` for + /// a farm with no pre-cutover sectors. Sectors with `history_size <= cutover` use the old + /// proof-of-space, newer ones the new one. Present only in version 1 and later. + cutover: Option, } impl PlotMetadataHeader { #[inline] fn encoded_size() -> usize { let default = PlotMetadataHeader { - version: 0, + version: PlotMetadataVersion::LATEST, plotted_sector_count: 0, + // `Some` is the larger encoding; size the header buffer for it so a version 1 header + // with a recorded cutover always fits. + cutover: Some(HistorySize::from(SegmentIndex::ZERO)), }; default.encoded_size() } } +// TODO(ved): Drop this manual `Decode` and return to `#[derive(Decode)]` once the `cutover` field +// is removed after the proof-of-space migration completes. It exists only to decode version 0 +// headers that predate the `cutover` field. +impl Decode for PlotMetadataHeader { + fn decode( + input: &mut I, + ) -> Result { + // Rejects unknown (future) versions — this is the old-binary lockout. + let version = PlotMetadataVersion::decode(input)?; + let plotted_sector_count = SectorIndex::decode(input)?; + // `cutover` was added in version 1; version 0 headers do not carry it. + let cutover = match version { + PlotMetadataVersion::V0 => None, + PlotMetadataVersion::V1 => Option::::decode(input)?, + }; + + Ok(Self { + version, + plotted_sector_count, + cutover, + }) + } +} + +/// With no cutover (a fresh farm) every sector is post-cutover. +fn is_post_cutover(cutover: Option, history_size: HistorySize) -> bool { + cutover.is_none_or(|cutover| history_size > cutover) +} + /// Options used to open single disk farm #[derive(Debug)] pub struct SingleDiskFarmOptions<'a, NC> @@ -388,9 +439,6 @@ pub enum SingleDiskFarmError { /// Failed to decode metadata header #[error("Failed to decode metadata header: {0}")] FailedToDecodeMetadataHeader(parity_scale_codec::Error), - /// Unexpected metadata version - #[error("Unexpected metadata version {0}")] - UnexpectedMetadataVersion(u8), /// Allocated space is not enough for one sector #[error( "Allocated space is not enough for one sector. \ @@ -528,9 +576,6 @@ pub enum SingleDiskFarmScrubError { /// Failed to decode metadata header #[error("Failed to decode metadata header: {0}")] FailedToDecodeMetadataHeader(parity_scale_codec::Error), - /// Unexpected metadata version - #[error("Unexpected metadata version {0}")] - UnexpectedMetadataVersion(u8), /// Cache can't be opened #[error("Cache at {file} can't be opened: {error}")] CacheCantBeOpened { @@ -821,7 +866,6 @@ impl SingleDiskFarm { pub const PLOT_FILE: &'static str = "plot.bin"; /// Name of the metadata file pub const METADATA_FILE: &'static str = "metadata.bin"; - const SUPPORTED_PLOT_VERSION: u8 = 0; /// Create new single disk farm instance pub async fn new( @@ -970,6 +1014,9 @@ impl SingleDiskFarm { let (farming_plot, farming_thread_pool) = AsyncJoinOnDrop::new(farming_plot_fut, false).await??; + // The plot's cutover decides which proof-of-space each sector reads and proves with. + let cutover = metadata_header.cutover; + let plotting_join_handle = task::spawn_blocking({ let sectors_metadata = Arc::clone(§ors_metadata); let handlers = Arc::clone(&handlers); @@ -1116,6 +1163,7 @@ impl SingleDiskFarm { read_sector_record_chunks_mode, global_mutex, metrics, + cutover, }; farming::(farming_options).await }; @@ -1159,6 +1207,7 @@ impl SingleDiskFarm { erasure_coding, sectors_being_modified, read_sector_record_chunks_mode, + cutover, global_mutex, ); @@ -1350,10 +1399,11 @@ impl SingleDiskFarm { // Align plot file size for disk sector size let expected_metadata_size = expected_metadata_size.div_ceil(DISK_SECTOR_SIZE as u64) * DISK_SECTOR_SIZE as u64; - let metadata_header = if metadata_size == 0 { + let mut metadata_header = if metadata_size == 0 { let metadata_header = PlotMetadataHeader { - version: SingleDiskFarm::SUPPORTED_PLOT_VERSION, + version: PlotMetadataVersion::LATEST, plotted_sector_count: 0, + cutover: None, }; metadata_file @@ -1380,12 +1430,6 @@ impl SingleDiskFarm { PlotMetadataHeader::decode(&mut metadata_header_bytes.as_ref()) .map_err(SingleDiskFarmError::FailedToDecodeMetadataHeader)?; - if metadata_header.version != SingleDiskFarm::SUPPORTED_PLOT_VERSION { - return Err(SingleDiskFarmError::UnexpectedMetadataVersion( - metadata_header.version, - )); - } - if metadata_header.plotted_sector_count > target_sector_count { metadata_header.plotted_sector_count = target_sector_count; metadata_file.write_all_at(&metadata_header.encode(), 0)?; @@ -1430,6 +1474,14 @@ impl SingleDiskFarm { sectors_metadata.push(sector_metadata); } + // Upgrade a version-0 plot in place: its sectors predate the new proof-of-space, so set + // the cutover from their max history size (not node history, which can trail on sync). + if metadata_header.version < PlotMetadataVersion::LATEST { + metadata_header.cutover = sectors_metadata.iter().map(|m| m.history_size).max(); + metadata_header.version = PlotMetadataVersion::LATEST; + metadata_file.write_all_at(&metadata_header.encode(), 0)?; + } + Arc::new(AsyncRwLock::new(sectors_metadata)) }; @@ -1572,10 +1624,10 @@ impl SingleDiskFarm { Ok(effective_disk_usage) } - /// Read all sectors metadata + /// Read the proof-of-space cutover and all sectors metadata pub fn read_all_sectors_metadata( directory: &Path, - ) -> io::Result> { + ) -> io::Result<(Option, Vec)> { let metadata_file = DirectIoFile::open(directory.join(Self::METADATA_FILE))?; let metadata_size = metadata_file.size()?; @@ -1589,13 +1641,6 @@ impl SingleDiskFarm { io::Error::other(format!("Failed to decode metadata header: {error}")) })?; - if metadata_header.version != SingleDiskFarm::SUPPORTED_PLOT_VERSION { - return Err(io::Error::other(format!( - "Unsupported metadata version {}", - metadata_header.version - ))); - } - let mut sectors_metadata = Vec::::with_capacity( ((metadata_size - RESERVED_PLOT_METADATA) / sector_metadata_size as u64) as usize, ); @@ -1613,7 +1658,14 @@ impl SingleDiskFarm { ); } - Ok(sectors_metadata) + // A version-0 plot has no stored cutover; derive it from its sectors, like the upgrade. + let cutover = if metadata_header.version < PlotMetadataVersion::LATEST { + sectors_metadata.iter().map(|m| m.history_size).max() + } else { + metadata_header.cutover + }; + + Ok((cutover, sectors_metadata)) } /// ID of this farm @@ -1863,12 +1915,6 @@ impl SingleDiskFarm { .map_err(SingleDiskFarmScrubError::FailedToDecodeMetadataHeader)? }; - if metadata_header.version != SingleDiskFarm::SUPPORTED_PLOT_VERSION { - return Err(SingleDiskFarmScrubError::UnexpectedMetadataVersion( - metadata_header.version, - )); - } - let plotted_sector_count = metadata_header.plotted_sector_count; let expected_metadata_size = RESERVED_PLOT_METADATA @@ -2361,3 +2407,52 @@ fn write_dummy_sector_metadata( error, }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plot_metadata_header_version_upgrade() { + // A version 0 header carries no cutover field yet must still decode, with cutover = None. + #[derive(Encode)] + struct V0Header { + version: u8, + plotted_sector_count: SectorIndex, + } + let v0_bytes = V0Header { + version: 0, + plotted_sector_count: 42, + } + .encode(); + let decoded = + PlotMetadataHeader::decode(&mut v0_bytes.as_slice()).expect("version 0 header decodes"); + assert_eq!(decoded.version, PlotMetadataVersion::V0); + assert_eq!(decoded.plotted_sector_count, 42); + assert_eq!(decoded.cutover, None); + + // A current header round-trips with its cutover intact. + let cutover = Some(HistorySize::from(SegmentIndex::ZERO)); + let header = PlotMetadataHeader { + version: PlotMetadataVersion::LATEST, + plotted_sector_count: 7, + cutover, + }; + let decoded = PlotMetadataHeader::decode(&mut header.encode().as_slice()) + .expect("current header round-trips"); + assert_eq!(decoded.version, PlotMetadataVersion::LATEST); + assert_eq!(decoded.plotted_sector_count, 7); + assert_eq!(decoded.cutover, cutover); + } + + #[test] + fn is_post_cutover_boundary() { + let h = |n: u64| HistorySize::from(SegmentIndex::from(n)); + // No cutover: a fresh farm plots everything with the new proof-of-space. + assert!(is_post_cutover(None, h(1))); + // At or below the cutover stays old; strictly above moves to the new proof-of-space. + assert!(!is_post_cutover(Some(h(10)), h(9))); + assert!(!is_post_cutover(Some(h(10)), h(10))); + assert!(is_post_cutover(Some(h(10)), h(11))); + } +} diff --git a/crates/subspace-farmer/src/single_disk_farm/farming.rs b/crates/subspace-farmer/src/single_disk_farm/farming.rs index 73cf9782093..21b5b1abd54 100644 --- a/crates/subspace-farmer/src/single_disk_farm/farming.rs +++ b/crates/subspace-farmer/src/single_disk_farm/farming.rs @@ -14,7 +14,6 @@ use crate::single_disk_farm::metrics::SingleDiskFarmMetrics; use async_lock::{Mutex as AsyncMutex, RwLock as AsyncRwLock}; use futures::StreamExt; use futures::channel::mpsc; -use parking_lot::Mutex; use rayon::ThreadPool; use std::collections::HashSet; use std::sync::Arc; @@ -73,11 +72,8 @@ where } /// Plot audit options -#[derive(Debug)] -pub struct PlotAuditOptions<'a, 'b, PosTable> -where - PosTable: Table, -{ +#[derive(Debug, Clone, Copy)] +pub struct PlotAuditOptions<'a, 'b> { /// Public key of the farm pub public_key: &'a PublicKey, /// Reward address to use for solutions @@ -95,22 +91,11 @@ where pub sectors_being_modified: &'b HashSet, /// Mode of reading chunks during proving pub read_sector_record_chunks_mode: ReadSectorRecordChunksMode, - /// Proof of space table generator - pub table_generator: &'a Mutex, -} - -impl Clone for PlotAuditOptions<'_, '_, PosTable> -where - PosTable: Table, -{ - #[inline] - fn clone(&self) -> Self { - *self - } + /// Proof-of-space cutover: sectors with a history size above this use the new implementation, + /// `None` on a farm with no pre-cutover sectors. + pub cutover: Option, } -impl Copy for PlotAuditOptions<'_, '_, PosTable> where PosTable: Table {} - /// Plot auditing implementation #[derive(Debug)] pub struct PlotAudit(Plot) @@ -130,7 +115,7 @@ where #[allow(clippy::type_complexity)] pub fn audit<'b, PosTable>( &'a self, - options: PlotAuditOptions<'a, 'b, PosTable>, + options: PlotAuditOptions<'a, 'b>, ) -> Result< Vec<( SectorIndex, @@ -151,7 +136,7 @@ where erasure_coding, sectors_being_modified, read_sector_record_chunks_mode: mode, - table_generator, + cutover, } = options; let audit_results = audit_plot_sync( @@ -168,12 +153,17 @@ where .filter_map(|audit_results| { let sector_index = audit_results.sector_index; - let sector_solutions = audit_results.solution_candidates.into_solutions( + let solution_candidates = audit_results.solution_candidates; + let is_post_cutover = + super::is_post_cutover(cutover, solution_candidates.history_size()); + let table_generator = PosTable::generator_for(is_post_cutover); + + let sector_solutions = solution_candidates.into_solutions( reward_address, kzg, erasure_coding, mode, - |seed: &PosSeed| table_generator.lock().generate_parallel(seed), + move |seed: &PosSeed| table_generator.generate_parallel(seed), ); let sector_solutions = match sector_solutions { @@ -214,6 +204,7 @@ pub(super) struct FarmingOptions { pub(super) read_sector_record_chunks_mode: ReadSectorRecordChunksMode, pub(super) global_mutex: Arc>, pub(super) metrics: Option>, + pub(super) cutover: Option, } /// Starts farming process. @@ -243,6 +234,7 @@ where read_sector_record_chunks_mode, global_mutex, metrics, + cutover, } = farming_options; let farmer_app_info = node_client @@ -253,7 +245,6 @@ where // We assume that each slot is one second let farming_timeout = farmer_app_info.farming_timeout; - let table_generator = Arc::new(Mutex::new(PosTable::generator())); let span = Span::current(); let mut non_fatal_errors = 0; @@ -278,7 +269,7 @@ where .install(|| { let _span_guard = span.enter(); - plot_audit.audit(PlotAuditOptions:: { + plot_audit.audit::(PlotAuditOptions { public_key: &public_key, reward_address: &reward_address, slot_info, @@ -287,7 +278,7 @@ where erasure_coding: &erasure_coding, sectors_being_modified, read_sector_record_chunks_mode, - table_generator: &table_generator, + cutover, }) }) .map_err(FarmingError::LowLevelAuditing)? diff --git a/crates/subspace-farmer/src/single_disk_farm/piece_reader.rs b/crates/subspace-farmer/src/single_disk_farm/piece_reader.rs index b67d0de69b2..07c594af5e0 100644 --- a/crates/subspace-farmer/src/single_disk_farm/piece_reader.rs +++ b/crates/subspace-farmer/src/single_disk_farm/piece_reader.rs @@ -12,6 +12,7 @@ use std::sync::Arc; use subspace_core_primitives::PublicKey; use subspace_core_primitives::pieces::{Piece, PieceOffset}; use subspace_core_primitives::sectors::{SectorId, SectorIndex}; +use subspace_core_primitives::segments::HistorySize; use subspace_erasure_coding::ErasureCoding; use subspace_farmer_components::reading::ReadSectorRecordChunksMode; use subspace_farmer_components::sector::{SectorMetadataChecksummed, sector_size}; @@ -58,6 +59,7 @@ impl DiskPieceReader { erasure_coding: ErasureCoding, sectors_being_modified: Arc>>, read_sector_record_chunks_mode: ReadSectorRecordChunksMode, + cutover: Option, global_mutex: Arc>, ) -> (Self, impl Future) where @@ -75,6 +77,7 @@ impl DiskPieceReader { sectors_being_modified, read_piece_receiver, read_sector_record_chunks_mode, + cutover, global_mutex, ) .await @@ -118,12 +121,16 @@ async fn read_pieces( sectors_being_modified: Arc>>, mut read_piece_receiver: mpsc::Receiver, mode: ReadSectorRecordChunksMode, + cutover: Option, global_mutex: Arc>, ) where PosTable: Table, S: ReadAtSync, { - let mut table_generator = PosTable::generator(); + // Keep a warm generator for each proof-of-space so old and new sectors can be read without + // re-allocating table caches per request. + let mut table_generator_old = PosTable::generator_for(false); + let mut table_generator_new = PosTable::generator_for(true); while let Some(read_piece_request) = read_piece_receiver.next().await { let ReadPieceRequest { @@ -194,6 +201,13 @@ async fn read_pieces( // Take mutex briefly to make sure piece reading is allowed right now global_mutex.lock().await; + let is_post_cutover = super::is_post_cutover(cutover, sector_metadata.history_size); + let table_generator = if is_post_cutover { + &mut table_generator_new + } else { + &mut table_generator_old + }; + let maybe_piece = read_piece::( &public_key, piece_offset, @@ -202,7 +216,7 @@ async fn read_pieces( &ReadAt::from_sync(§or), &erasure_coding, mode, - &mut table_generator, + table_generator, ) .await; diff --git a/crates/subspace-farmer/src/single_disk_farm/plotting.rs b/crates/subspace-farmer/src/single_disk_farm/plotting.rs index 9fa6d5e03fa..4ef6435f007 100644 --- a/crates/subspace-farmer/src/single_disk_farm/plotting.rs +++ b/crates/subspace-farmer/src/single_disk_farm/plotting.rs @@ -125,6 +125,7 @@ where } = plotting_options; let sector_plotting_options = §or_plotting_options; + let cutover = metadata_header.cutover; let plotting_semaphore = Semaphore::new(max_plotting_sectors_per_farm.get()); let mut sectors_being_plotted = FuturesOrdered::new(); // Channel size is intentionally unbounded for easier analysis, but it is bounded by plotting @@ -165,6 +166,7 @@ where sectors_metadata, sectors_being_modified, &plotting_semaphore, + cutover, ) .instrument(info_span!("", %sector_index)) .fuse(); @@ -287,6 +289,7 @@ async fn plot_single_sector<'a, NC>( sectors_metadata: &'a AsyncRwLock>, sectors_being_modified: &'a AsyncRwLock>, plotting_semaphore: &'a Semaphore, + cutover: Option, ) -> PlotSingleSectorResult< impl Future, PlottingError>> + 'a, > @@ -314,17 +317,6 @@ where } = sector_to_plot; trace!("Preparing to plot sector"); - // Inform others that this sector is being modified - { - let mut sectors_being_modified = sectors_being_modified.write().await; - if !sectors_being_modified.insert(sector_index) { - debug!("Skipped sector plotting, it is already in progress"); - return PlotSingleSectorResult::Skipped; - } - } - - let plotting_permit = plotting_semaphore.acquire().await; - let maybe_old_sector_metadata = sectors_metadata .read() .await @@ -332,20 +324,9 @@ where .cloned(); let replotting = maybe_old_sector_metadata.is_some(); - if let Some(metrics) = metrics { - metrics.sector_plotting.inc(); - } - let sector_state = SectorUpdate::Plotting(SectorPlottingDetails::Starting { - progress, - replotting, - last_queued, - }); - handlers - .sector_update - .call_simple(&(sector_index, sector_state)); - - let start = Instant::now(); - + // Wait until the sector can be plotted before taking the plotting permit or marking it as being + // modified, so neither is held while waiting. + // // This `loop` is a workaround for edge-case in local setup if expiration is configured to 1. // In that scenario we get replotting notification essentially straight from block import // pipeline of the node, before block is imported. This can result in subsequent request for @@ -384,9 +365,48 @@ where } } + // Wait for history to pass the cutover before plotting, so a new sector isn't stamped at or + // below it and later read back with the old proof-of-space. + if let Some(cutover) = cutover + && farmer_app_info.protocol_info.history_size <= cutover + { + debug!( + current_history_size = %farmer_app_info.protocol_info.history_size, + %cutover, + "History size has not advanced past the proof-of-space cutover yet, waiting" + ); + tokio::time::sleep(FARMER_APP_INFO_RETRY_INTERVAL).await; + continue; + } + break farmer_app_info; }; + // Inform others that this sector is being modified + { + let mut sectors_being_modified = sectors_being_modified.write().await; + if !sectors_being_modified.insert(sector_index) { + debug!("Skipped sector plotting, it is already in progress"); + return PlotSingleSectorResult::Skipped; + } + } + + let plotting_permit = plotting_semaphore.acquire().await; + + if let Some(metrics) = metrics { + metrics.sector_plotting.inc(); + } + let sector_state = SectorUpdate::Plotting(SectorPlottingDetails::Starting { + progress, + replotting, + last_queued, + }); + handlers + .sector_update + .call_simple(&(sector_index, sector_state)); + + let start = Instant::now(); + let (progress_sender, mut progress_receiver) = mpsc::channel(10); // Initiate plotting diff --git a/crates/subspace-proof-of-space/Cargo.toml b/crates/subspace-proof-of-space/Cargo.toml index 7fe5239d108..38975d8215c 100644 --- a/crates/subspace-proof-of-space/Cargo.toml +++ b/crates/subspace-proof-of-space/Cargo.toml @@ -19,6 +19,7 @@ bench = false [dependencies] ab-chacha8.workspace = true +ab-proof-of-space = { workspace = true, optional = true } blake3 = { workspace = true, default-features = false } chacha20 = { workspace = true, features = ["cipher"], optional = true } derive_more = { workspace = true, features = ["full"] } @@ -44,6 +45,8 @@ default = [] alloc = [ "dep:chacha20", "dep:seq-macro", + "dep:ab-proof-of-space", + "ab-proof-of-space/alloc", ] # Enabling this feature exposes quality search on `chiapos` module as well as enables support for K=15..=25 (by default # only K=20 is exposed) @@ -53,4 +56,5 @@ full-chiapos = [ parallel = [ "alloc", "dep:rayon", + "ab-proof-of-space/parallel", ] diff --git a/crates/subspace-proof-of-space/benches/pos.rs b/crates/subspace-proof-of-space/benches/pos.rs index 947fce582dc..3516fd869ff 100644 --- a/crates/subspace-proof-of-space/benches/pos.rs +++ b/crates/subspace-proof-of-space/benches/pos.rs @@ -250,13 +250,14 @@ fn pos_bench( } pub fn criterion_benchmark(c: &mut Criterion) { + #[cfg(feature = "alloc")] { // This challenge index with above seed is known to not have a solution - let challenge_index_without_solution = 1232460437; + let challenge_index_without_solution = 0; // This challenge index with above seed is known to have a solution - let challenge_index_with_solution = 600426542; + let challenge_index_with_solution = 3; - pos_bench::( + pos_bench::( c, "chia", challenge_index_without_solution, diff --git a/crates/subspace-proof-of-space/src/chia_v2.rs b/crates/subspace-proof-of-space/src/chia_v2.rs new file mode 100644 index 00000000000..ea328d9bd72 --- /dev/null +++ b/crates/subspace-proof-of-space/src/chia_v2.rs @@ -0,0 +1,75 @@ +//! Chia proof of space backed by abundance's `ab-proof-of-space`. +//! +//! Proofs are looked up under Subspace's s-bucket convention, so they verify under the existing +//! [`ChiaTable`] verifier. + +use crate::chia::ChiaTable; +use crate::{PosTableType, Table, TableGenerator}; +use ab_proof_of_space::chiapos::{Proofs, Tables, TablesCache}; +use alloc::boxed::Box; +use core::fmt; +use subspace_core_primitives::pos::{PosProof, PosSeed}; +use subspace_core_primitives::sectors::SBucket; +use subspace_core_primitives::solutions::SolutionPotVerifier; + +const K: u8 = PosProof::K; + +/// Proof of space table generator. +/// +/// Chia implementation. +#[derive(Debug, Default, Clone)] +pub struct ChiaV2TableGenerator { + tables_cache: TablesCache, +} + +impl TableGenerator for ChiaV2TableGenerator { + fn generate(&self, seed: &PosSeed) -> ChiaV2Table { + ChiaV2Table { + proofs: Tables::::create_proofs((*seed).into(), &self.tables_cache), + } + } + + #[cfg(feature = "parallel")] + fn generate_parallel(&self, seed: &PosSeed) -> ChiaV2Table { + ChiaV2Table { + proofs: Tables::::create_proofs_parallel((*seed).into(), &self.tables_cache), + } + } +} + +/// Proof of space table. +/// +/// Chia implementation. +pub struct ChiaV2Table { + proofs: Box>, +} + +impl fmt::Debug for ChiaV2Table { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ChiaV2Table").finish_non_exhaustive() + } +} + +impl SolutionPotVerifier for ChiaV2Table { + fn is_proof_valid(seed: &PosSeed, challenge_index: u32, proof: &PosProof) -> bool { + ::is_proof_valid(seed, challenge_index, proof) + } +} + +impl Table for ChiaV2Table { + const TABLE_TYPE: PosTableType = PosTableType::Chia; + type Generator = ChiaV2TableGenerator; + + fn find_proof(&self, challenge_index: u32) -> Option { + self.proofs + .for_s_bucket(SBucket::from(challenge_index as u16)) + .map(PosProof::from) + } + + fn is_proof_valid(seed: &PosSeed, challenge_index: u32, proof: &PosProof) -> bool { + ::is_proof_valid(seed, challenge_index, proof) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/subspace-proof-of-space/src/chia_v2/tests.rs b/crates/subspace-proof-of-space/src/chia_v2/tests.rs new file mode 100644 index 00000000000..8ef345cf23d --- /dev/null +++ b/crates/subspace-proof-of-space/src/chia_v2/tests.rs @@ -0,0 +1,29 @@ +use crate::chia::ChiaTable; +use crate::chia_v2::ChiaV2TableGenerator; +use crate::{Table, TableGenerator}; +use subspace_core_primitives::pos::PosSeed; +use subspace_core_primitives::sectors::SBucket; + +// Every abundance-backed proof must verify under ChiaTable::is_proof_valid. +#[test] +fn proofs_verify_under_consensus() { + let generator = ChiaV2TableGenerator::default(); + + for seed_byte in 0..3u8 { + let seed = PosSeed::from([seed_byte; 32]); + let table = generator.generate(&seed); + + let mut verified = 0u32; + for s_bucket in u16::from(SBucket::ZERO)..=u16::from(SBucket::MAX) { + if let Some(proof) = table.find_proof(u32::from(s_bucket)) { + assert!( + ChiaTable::is_proof_valid(&seed, u32::from(s_bucket), &proof), + "seed {seed_byte}: proof for s-bucket {s_bucket} rejected by the consensus \ + verifier" + ); + verified += 1; + } + } + assert!(verified > 0, "seed {seed_byte}: no proofs produced"); + } +} diff --git a/crates/subspace-proof-of-space/src/lib.rs b/crates/subspace-proof-of-space/src/lib.rs index ed99a725d0b..30c27bc60b7 100644 --- a/crates/subspace-proof-of-space/src/lib.rs +++ b/crates/subspace-proof-of-space/src/lib.rs @@ -8,9 +8,16 @@ #![cfg_attr(feature = "parallel", feature(exact_size_is_empty, sync_unsafe_cell))] pub mod chia; +#[cfg(feature = "alloc")] +pub mod chia_v2; pub mod chiapos; +#[cfg(feature = "alloc")] +pub mod pos_table; pub mod shim; +#[cfg(feature = "alloc")] +pub use pos_table::{PosTable, PosTableGenerator}; + #[cfg(feature = "alloc")] extern crate alloc; @@ -70,4 +77,11 @@ pub trait Table: SolutionPotVerifier + Sized + Send + Sync + 'static { fn generator() -> Self::Generator { Self::Generator::default() } + + /// Table generator for a sector, given whether it was plotted after the cutover. Single-impl + /// tables ignore the argument. + #[cfg(feature = "alloc")] + fn generator_for(_is_post_cutover: bool) -> Self::Generator { + Self::generator() + } } diff --git a/crates/subspace-proof-of-space/src/pos_table.rs b/crates/subspace-proof-of-space/src/pos_table.rs new file mode 100644 index 00000000000..caec6a96f0c --- /dev/null +++ b/crates/subspace-proof-of-space/src/pos_table.rs @@ -0,0 +1,93 @@ +//! Proof of space that dispatches per sector between the old [`ChiaTable`] and the new +//! [`ChiaV2Table`], so one farm can mix them across the migration cutover. + +use crate::chia::ChiaTable; +use crate::chia_v2::ChiaV2Table; +use crate::{PosTableType, Table, TableGenerator}; +use subspace_core_primitives::pos::{PosProof, PosSeed}; +use subspace_core_primitives::solutions::SolutionPotVerifier; + +/// Proof of space table generator dispatching between the old and new Chia implementations. +#[derive(Debug, Clone)] +pub enum PosTableGenerator { + /// Old (pre-cutover) implementation. + V1(::Generator), + /// New (post-cutover) implementation. + V2(::Generator), +} + +impl Default for PosTableGenerator { + #[inline] + fn default() -> Self { + Self::V2(ChiaV2Table::generator()) + } +} + +impl PosTableGenerator { + /// Generator for a sector: the new implementation for post-cutover sectors, the old one + /// otherwise. + #[inline] + pub fn new(is_post_cutover: bool) -> Self { + if is_post_cutover { + Self::V2(ChiaV2Table::generator()) + } else { + Self::V1(ChiaTable::generator()) + } + } +} + +impl TableGenerator for PosTableGenerator { + fn generate(&self, seed: &PosSeed) -> PosTable { + match self { + Self::V1(generator) => PosTable::V1(generator.generate(seed)), + Self::V2(generator) => PosTable::V2(generator.generate(seed)), + } + } + + #[cfg(feature = "parallel")] + fn generate_parallel(&self, seed: &PosSeed) -> PosTable { + match self { + Self::V1(generator) => PosTable::V1(generator.generate_parallel(seed)), + Self::V2(generator) => PosTable::V2(generator.generate_parallel(seed)), + } + } +} + +/// Proof of space table dispatching between the old and new Chia implementations. +#[derive(Debug)] +pub enum PosTable { + /// Old (pre-cutover) implementation. + V1(ChiaTable), + /// New (post-cutover) implementation. + V2(ChiaV2Table), +} + +impl SolutionPotVerifier for PosTable { + fn is_proof_valid(seed: &PosSeed, challenge_index: u32, proof: &PosProof) -> bool { + // Both V1 and V2 proofs verify under ChiaTable. + ::is_proof_valid(seed, challenge_index, proof) + } +} + +impl Table for PosTable { + const TABLE_TYPE: PosTableType = PosTableType::Chia; + type Generator = PosTableGenerator; + + fn generator_for(is_post_cutover: bool) -> Self::Generator { + PosTableGenerator::new(is_post_cutover) + } + + fn find_proof(&self, challenge_index: u32) -> Option { + match self { + Self::V1(table) => table.find_proof(challenge_index), + Self::V2(table) => table.find_proof(challenge_index), + } + } + + fn is_proof_valid(seed: &PosSeed, challenge_index: u32, proof: &PosProof) -> bool { + ::is_proof_valid(seed, challenge_index, proof) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/subspace-proof-of-space/src/pos_table/tests.rs b/crates/subspace-proof-of-space/src/pos_table/tests.rs new file mode 100644 index 00000000000..96527cca084 --- /dev/null +++ b/crates/subspace-proof-of-space/src/pos_table/tests.rs @@ -0,0 +1,18 @@ +use crate::pos_table::PosTableGenerator; + +// New sectors get the abundance-backed table; sectors at or below the cutover keep the old one. +#[test] +fn dispatch_selects_by_cutover() { + assert!(matches!( + PosTableGenerator::new(true), + PosTableGenerator::V2(_) + )); + assert!(matches!( + PosTableGenerator::new(false), + PosTableGenerator::V1(_) + )); + assert!(matches!( + PosTableGenerator::default(), + PosTableGenerator::V2(_) + )); +} diff --git a/docker/farmer.Dockerfile b/docker/farmer.Dockerfile index c69f26c9192..0cea0f75a76 100644 --- a/docker/farmer.Dockerfile +++ b/docker/farmer.Dockerfile @@ -54,49 +54,13 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --de # Up until this line all Rust images in this repo should be the same to share the same layers -# CUDA toolchain, including support for cross-compilation from x86-64 to aarch64, but NOT from aarch64 to x86-64 -RUN \ - if [ "$BUILDARCH/$TARGETARCH" = "amd64/amd64" ] || [ "$BUILDARCH/$TARGETARCH" = "amd64/arm64" ] || [ "$BUILDARCH/$TARGETARCH" = "arm64/arm64" ]; then \ - CUDA_ARCH=$(echo $BUILDARCH | sed "s/amd64/x86_64/g") && \ - curl -OL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/cuda-keyring_1.1-1_all.deb && \ - dpkg -i cuda-keyring_1.1-1_all.deb && \ - echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/ /" | tee /etc/apt/sources.list.d/cuda-ubuntu2204-$CUDA_ARCH.list && \ - curl -OL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/cuda-ubuntu2204.pin && \ - mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cuda-minimal-build-12-4 && \ - echo "/usr/local/cuda/lib64" > /etc/ld.so.conf.d/cuda.conf && \ - if [ "$BUILDARCH/$TARGETARCH" = "amd64/arm64" ]; then \ - CUDA_ARCH="cross-linux-aarch64" && \ - echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/ /" | tee /etc/apt/sources.list.d/cuda-ubuntu2204-$CUDA_ARCH.list && \ - curl -OL https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$CUDA_ARCH/cuda-ubuntu2204.pin && \ - mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600-cross-aarch64 && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cuda-cross-aarch64-12-4 \ - ; fi && \ - ldconfig \ - ; fi - -# ROCm is only used on x86-64 since they don't have other packages -ARG ROCM_VERSION=6.2.2 -RUN \ - if [ "$BUILDARCH/$TARGETARCH" = "amd64/amd64" ]; then \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gpg && \ - mkdir -p --mode=0755 /etc/apt/keyrings && \ - curl -L https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor > /etc/apt/keyrings/rocm.gpg && \ - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION jammy main" > /etc/apt/sources.list.d/rocm.list && \ - echo "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" > /etc/apt/preferences.d/rocm-pin-600 && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends rocm-hip-runtime-dev && \ - echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf && \ - ldconfig \ - ; fi +# GPU plotting uses wgpu, whose shader is built here by cargo-gpu (via the wgpu crate's build.rs), +# so there's no vendor GPU SDK to install. COPY . /code ARG TARGETVARIANT -# ROCm is only used on x86-64 since they don't have other packages RUN \ if [ $BUILDARCH != "arm64" ] && [ $TARGETARCH = "arm64" ]; then \ export RUSTFLAGS="$RUSTFLAGS -C linker=aarch64-linux-gnu-gcc" \ @@ -119,62 +83,19 @@ RUN \ if [ $BUILDARCH != "amd64" ] && [ $TARGETARCH = "amd64" ]; then \ export RUSTFLAGS="$RUSTFLAGS -C linker=x86_64-linux-gnu-gcc" \ ; fi && \ - export PATH=/usr/local/cuda/bin${PATH:+:${PATH}} && \ RUSTC_TARGET_ARCH=$(echo $TARGETARCH | sed "s/amd64/x86_64/g" | sed "s/arm64/aarch64/g" | sed "s/riscv64/riscv64gc/g") && \ - if [ $BUILDARCH = "amd64" ] && [ $TARGETARCH = "amd64" ]; then \ - /root/.cargo/bin/cargo -Zgitoxide -Zgit build \ - --locked \ - -Z build-std \ - --profile $PROFILE \ - --bin subspace-farmer \ - --features rocm \ - --target $RUSTC_TARGET_ARCH-unknown-linux-gnu && \ - mv target/*/*/subspace-farmer subspace-farmer-rocm \ - ; fi && \ - if [ $(which nvcc) ]; then \ - /root/.cargo/bin/cargo -Zgitoxide -Zgit build \ - --locked \ - -Z build-std \ - --profile $PROFILE \ - --bin subspace-farmer \ - --features cuda \ - --target $RUSTC_TARGET_ARCH-unknown-linux-gnu \ - ; else \ - /root/.cargo/bin/cargo -Zgitoxide -Zgit build \ - --locked \ - -Z build-std \ - --profile $PROFILE \ - --bin subspace-farmer \ - --target $RUSTC_TARGET_ARCH-unknown-linux-gnu \ - ; fi && \ + /root/.cargo/bin/cargo -Zgitoxide -Zgit build \ + --locked \ + -Z build-std \ + --profile $PROFILE \ + --bin subspace-farmer \ + --target $RUSTC_TARGET_ARCH-unknown-linux-gnu && \ mv target/*/*/subspace-farmer subspace-farmer && \ rm -rf target FROM ubuntu:22.04 -ARG BUILDARCH -ARG TARGETARCH - -# Next block is for ROCm support -# ROCm is only used on x86-64 since they don't have other packages -ARG ROCM_VERSION=6.2.2 -RUN \ - if [ "$BUILDARCH/$TARGETARCH" = "amd64/amd64" ]; then \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl ca-certificates gpg && \ - mkdir -p --mode=0755 /etc/apt/keyrings && \ - curl -L https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor > /etc/apt/keyrings/rocm.gpg && \ - echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/$ROCM_VERSION jammy main" > /etc/apt/sources.list.d/rocm.list && \ - echo "Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600" > /etc/apt/preferences.d/rocm-pin-600 && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends hip-runtime-amd && \ - DEBIAN_FRONTEND=noninteractive apt-get remove -y --purge --autoremove curl ca-certificates gpg && \ - rm -rf /var/lib/apt/lists/* && \ - echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf && \ - ldconfig \ - ; fi - -COPY --from=0 /code/subspace-farmer* / +COPY --from=0 /code/subspace-farmer / RUN mkdir /var/subspace && chown nobody:nogroup /var/subspace diff --git a/rust-toolchain.toml b/rust-toolchain.toml index d58759e7ea8..9ce5833632c 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "nightly-2026-04-11" +channel = "nightly-2026-05-03" components = ["rust-src"] targets = ["wasm32v1-none"] profile = "default" diff --git a/scripts/find-unused-deps.sh b/scripts/find-unused-deps.sh index f1f7a53c17d..9e2d4fd8b46 100755 --- a/scripts/find-unused-deps.sh +++ b/scripts/find-unused-deps.sh @@ -15,27 +15,14 @@ if [[ ! -d "./crates/pallet-domains" ]] || [[ ! -d "./domains/runtime/evm" ]]; t fi fi -# This feature list is `--all-features` except `rocm`, which is incompatible with `cuda` -# When cargo supports excluding features (or mutually exclusive features), we can use -# `--all-features --exclude-feature rocm` -# -# -BASE_FEATURES="async-trait,binary,cluster,default-library,domain-block-builder,domain-block-preprocessor,frame-benchmarking-cli,frame-system-benchmarking,hex-literal,kzg,numa,pallet-subspace,pallet-timestamp,pallet-utility,parallel,parking_lot,rand,runtime-benchmarks,sc-client-api,sc-executor,schnorrkel,serde,sp-blockchain,sp-core,sp-io,sp-state-machine,sp-std,sp-storage,static_assertions,std,subspace-proof-of-space-gpu,substrate-wasm-builder,testing,wasm-builder,with-tracing,x509-parser,fuzz" -if [[ "$(uname)" == "Darwin" ]]; then - echo "Skipping GPU features because we're on macOS" - EXTRA_FEATURES=("") -else - # We skip non-GPU builds on Linux and Windows, because they have unused GPU deps - EXTRA_FEATURES=("cuda,sppark,_gpu" "rocm,sppark,_gpu") -fi +# This feature list is `--all-features`. wgpu (GPU plotting) is a default feature and builds on all +# platforms, so it is part of the base list rather than a separate per-vendor check. +BASE_FEATURES="async-trait,binary,cluster,default-library,domain-block-builder,domain-block-preprocessor,frame-benchmarking-cli,frame-system-benchmarking,hex-literal,kzg,numa,pallet-subspace,pallet-timestamp,pallet-utility,parallel,parking_lot,rand,runtime-benchmarks,sc-client-api,sc-executor,schnorrkel,serde,sp-blockchain,sp-core,sp-io,sp-state-machine,sp-std,sp-storage,static_assertions,std,substrate-wasm-builder,testing,wasm-builder,wgpu,with-tracing,x509-parser,fuzz" # Show commands before executing them set -x -for EXTRA_FEATURE in "${EXTRA_FEATURES[@]}"; do - echo "Checking for unused dependencies with '$EXTRA_FEATURE' extra features..." - cargo -Zgitoxide -Zgit udeps --workspace --all-targets --locked --features "$BASE_FEATURES,$EXTRA_FEATURE" -done +cargo -Zgitoxide -Zgit udeps --workspace --all-targets --locked --features "$BASE_FEATURES" # Stop showing executed commands set +x @@ -43,8 +30,5 @@ set +x echo echo "============================================" echo "Successfully checked for unused dependencies" -if [[ "$(uname)" == "Darwin" ]]; then - echo "GPU features were not checked, because we're on macOS" -fi echo "============================================" echo diff --git a/shared/ab-proof-of-space-gpu/Cargo.toml b/shared/ab-proof-of-space-gpu/Cargo.toml new file mode 100644 index 00000000000..fee3379c4d7 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "ab-proof-of-space-gpu" +description = "Proof of space plotting utilities for GPU (Vulkan/Metal)" +license = "0BSD" +version = "0.1.0" +authors = ["Nazar Mokrynskyi "] +edition = "2024" +include = [ + "/src", + "/build.rs", + "/Cargo.toml", +] + +[package.metadata.docs.rs] +all-features = true + +# TODO: Benches + +[dependencies] +ab-blake3 = { workspace = true } +ab-chacha8 = { workspace = true } +derive_more = { workspace = true, features = ["from", "into"] } +spirv-std = { workspace = true } + +[target.'cfg(not(target_arch = "spirv"))'.dependencies] +subspace-core-primitives = { workspace = true } +futures = { workspace = true, features = ["std"] } +parking_lot = { workspace = true } +rclite = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +wgpu = { workspace = true } + +# TODO: This will be built in the shader as well, figure out a way to avoid that +[build-dependencies] +cargo-gpu-install = { workspace = true } + +[lints.rust] +# `spirv` is the rust-gpu target this crate compiles to, unknown to rustc's built-in cfg checking +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(target_arch, values("spirv"))'] } diff --git a/shared/ab-proof-of-space-gpu/build.rs b/shared/ab-proof-of-space-gpu/build.rs new file mode 100644 index 00000000000..c296ddb4823 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/build.rs @@ -0,0 +1,83 @@ +use cargo_gpu_install::install::Install; +use cargo_gpu_install::spirv_builder::{Capability, SpirvBuilderError, SpirvMetadata}; +use std::error::Error; +use std::path::PathBuf; +use std::{env, fs}; + +fn main() -> Result<(), Box> { + let target_arch = env::var("CARGO_CFG_TARGET_ARCH").expect("Always set by Cargo; qed"); + + if target_arch == "spirv" { + return Ok(()); + } + + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("Always set by Cargo; qed")); + + // Skip compilation under Miri and rustdoc, it will not be used + if ["MIRI_SYSROOT", "RUSTDOCFLAGS"] + .iter() + .any(|var| env::var(var).is_ok()) + { + let empty_file = out_dir.join("empty.bin"); + fs::write(&empty_file, [])?; + println!("cargo::rustc-env=SHADER_PATH={}", empty_file.display()); + + return Ok(()); + } + + let cargo_manifest_dir = env::var("CARGO_MANIFEST_DIR").expect("Always set by Cargo; qed"); + let profile = env::var("PROFILE").expect("Always set by Cargo; qed"); + + let shader_crate = PathBuf::from(cargo_manifest_dir); + + let backend = Install::from_shader_crate(shader_crate.clone()).run()?; + + // TODO: Workaround for https://github.com/Rust-GPU/rust-gpu/issues/461 + // SAFETY: Single-threaded + unsafe { + env::set_var("RUST_MIN_STACK", "16777216"); + } + + let mut spirv_builder = backend + .to_spirv_builder(shader_crate, "spirv-unknown-vulkan1.2") + .spirv_metadata(if profile == "debug" { + SpirvMetadata::NameVariables + } else { + SpirvMetadata::None + }) + .release(profile != "debug") + // TODO: This should not be needed: https://github.com/Rust-GPU/rust-gpu/issues/386 + .capability(Capability::GroupNonUniformArithmetic) + // TODO: This should not be needed: https://github.com/Rust-GPU/rust-gpu/issues/386 + .capability(Capability::GroupNonUniformBallot) + // TODO: This should not be needed: https://github.com/Rust-GPU/rust-gpu/issues/386 + .capability(Capability::GroupNonUniformShuffle) + // Avoid Cargo deadlock, customize target + .target_dir_path(out_dir.clone()); + spirv_builder.build_script.defaults = true; + spirv_builder + .build_script + .forward_rustc_warnings + .replace(true); + + println!("cargo::rerun-if-env-changed=CLIPPY_ARGS"); + let path_to_spv = if env::var("CLIPPY_ARGS").is_ok() { + match spirv_builder.clippy() { + Ok(compile_result) => compile_result.module.unwrap_single().to_path_buf(), + Err(SpirvBuilderError::NoArtifactProduced { .. }) => { + let empty_file = out_dir.join("empty.bin"); + fs::write(&empty_file, [])?; + empty_file + } + Err(error) => { + return Err(error.into()); + } + } + } else { + spirv_builder.build()?.module.unwrap_single().to_path_buf() + }; + + println!("cargo::rustc-env=SHADER_PATH={}", path_to_spv.display()); + + Ok(()) +} diff --git a/shared/ab-proof-of-space-gpu/src/host.rs b/shared/ab-proof-of-space-gpu/src/host.rs new file mode 100644 index 00000000000..a456442c32f --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/host.rs @@ -0,0 +1,1333 @@ +use crate::shader::constants::{ + MAX_BUCKET_SIZE, MAX_TABLE_SIZE, NUM_BUCKETS, NUM_MATCH_BUCKETS, NUM_S_BUCKETS, + REDUCED_MATCHES_COUNT, +}; +use crate::shader::find_matches_and_compute_f7::{NUM_ELEMENTS_PER_S_BUCKET, ProofTargets}; +use crate::shader::find_proofs::ProofsHost; +use crate::shader::types::{Metadata, Position, PositionR}; +use crate::shader::{compute_f1, find_proofs, select_shader_features_limits}; +use ab_chacha8::{ChaCha8Block, ChaCha8State, block_to_bytes}; +use futures::stream::FuturesOrdered; +use futures::{StreamExt, TryStreamExt}; +use parking_lot::Mutex; +use rclite::Arc; +use std::num::NonZeroU8; +use std::{fmt, iter}; +use subspace_core_primitives::pos::PosSeed; +use tracing::{debug, warn}; +use wgpu::{ + AdapterInfo, Backend, BackendOptions, Backends, BindGroup, BindGroupDescriptor, BindGroupEntry, + BindGroupLayoutDescriptor, BindGroupLayoutEntry, BindingType, Buffer, BufferAddress, + BufferAsyncError, BufferBindingType, BufferDescriptor, BufferUsages, CommandEncoderDescriptor, + ComputePassDescriptor, ComputePipeline, ComputePipelineDescriptor, DeviceDescriptor, + DeviceType, Instance, InstanceDescriptor, InstanceFlags, MapMode, MemoryBudgetThresholds, + PipelineCompilationOptions, PipelineLayoutDescriptor, PollError, PollType, Queue, + RequestDeviceError, ShaderModule, ShaderRuntimeChecks, ShaderStages, +}; + +/// Proof creation error +#[derive(Debug, thiserror::Error)] +pub enum RecordEncodingError { + /// Proof creation failed previously and the device is now considered broken + #[error("Proof creation failed previously and the device is now considered broken")] + DeviceBroken, + /// Failed to map buffer + #[error("Failed to map buffer: {0}")] + BufferMapping(#[from] BufferAsyncError), + /// Poll error + #[error("Poll error: {0}")] + DevicePoll(#[from] PollError), +} + +/// Handle to GPU-produced proofs; keeps the underlying buffer mapped until dropped. +pub struct ProofsHostWrapper<'a> { + proofs: &'a ProofsHost, + proofs_host: &'a Buffer, +} + +impl ProofsHostWrapper<'_> { + /// Proofs produced for a single seed. + pub fn proofs(&self) -> &ProofsHost { + self.proofs + } +} + +impl fmt::Debug for ProofsHostWrapper<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ProofsHostWrapper").finish_non_exhaustive() + } +} + +impl Drop for ProofsHostWrapper<'_> { + fn drop(&mut self) { + self.proofs_host.unmap(); + } +} + +/// Wrapper data structure encapsulating a single compatible device +#[derive(Clone)] +pub struct Device { + id: u32, + devices: Vec<(wgpu::Device, Queue, ShaderModule)>, + adapter_info: AdapterInfo, +} + +impl fmt::Debug for Device { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Device") + .field("id", &self.id) + .field("name", &self.adapter_info.name) + .field("device_type", &self.adapter_info.device_type) + .field("driver", &self.adapter_info.driver) + .field("driver_info", &self.adapter_info.driver_info) + .field("backend", &self.adapter_info.backend) + .finish_non_exhaustive() + } +} + +impl Device { + /// Returns [`Device`] for each available device + pub async fn enumerate(number_of_queues: NOQ) -> Vec + where + NOQ: Fn(DeviceType) -> NonZeroU8, + { + let backends = Backends::from_env().unwrap_or(Backends::METAL | Backends::VULKAN); + let instance = Instance::new(InstanceDescriptor { + backends, + flags: if cfg!(debug_assertions) { + InstanceFlags::debugging().with_env() + } else { + InstanceFlags::from_env_or_default() + }, + memory_budget_thresholds: MemoryBudgetThresholds::default(), + backend_options: BackendOptions::from_env_or_default(), + display: None, + }); + + let adapters = instance.enumerate_adapters(backends).await; + let number_of_queues = &number_of_queues; + + adapters + .into_iter() + .zip(0..) + .map(|(adapter, id)| async move { + let adapter_info = adapter.get_info(); + + let (shader, required_features, required_limits) = + if let Some((shader, required_features, required_limits)) = + select_shader_features_limits(&adapter) + { + debug!( + %id, + adapter_info = ?adapter_info, + "Compatible adapter found" + ); + + (shader, required_features, required_limits) + } else { + debug!( + %id, + adapter_info = ?adapter_info, + "Incompatible adapter found" + ); + + return None; + }; + + // TODO: creation of multiple devices is a workaround for lack of support for + // multiple queues: https://github.com/gfx-rs/wgpu/issues/1066 + let devices = iter::repeat_with(|| async { + let (device, queue) = adapter + .request_device(&DeviceDescriptor { + label: None, + required_features, + required_limits: required_limits.clone(), + ..DeviceDescriptor::default() + }) + .await + .inspect_err(|error| { + warn!(%id, ?adapter_info, %error, "Failed to request the device"); + })?; + let module = if cfg!(debug_assertions) { + device.create_shader_module(shader.clone()) + } else { + // SAFETY: The shader is trusted, individual tests include correctness + // checks and debug builds of this crate too only release version skips + // checks for better runtime performance + unsafe { + device.create_shader_module_trusted( + shader.clone(), + ShaderRuntimeChecks::unchecked(), + ) + } + }; + + Ok::<_, RequestDeviceError>((device, queue, module)) + }) + .take(usize::from( + number_of_queues(adapter_info.device_type).get(), + )) + .collect::>() + .try_collect::>() + .await + .ok()?; + + Some(Self { + id, + devices, + adapter_info, + }) + }) + .collect::>() + .filter_map(|device| async move { device }) + .collect() + .await + } + + /// Gpu ID + pub fn id(&self) -> u32 { + self.id + } + + /// Device name + pub fn name(&self) -> &str { + &self.adapter_info.name + } + + /// Device type + pub fn device_type(&self) -> DeviceType { + self.adapter_info.device_type + } + + /// Driver + pub fn driver(&self) -> &str { + &self.adapter_info.driver + } + + /// Driver info + pub fn driver_info(&self) -> &str { + &self.adapter_info.driver_info + } + + /// Backend + pub fn backend(&self) -> Backend { + self.adapter_info.backend + } + + /// One proof-producing encoder instance per queue. Consumers run their own record encoding + /// (Subspace does its KZG encoding in `subspace-proof-of-space-wgpu`). + pub fn create_proofs_encoder_instances(&self) -> Vec { + self.devices + .clone() + .into_iter() + .map(|(device, queue, module)| GpuRecordsEncoderInstance::new(device, queue, module)) + .collect() + } +} + +pub struct GpuRecordsEncoderInstance { + device: wgpu::Device, + queue: Queue, + mapping_error: Arc>>, + tainted: bool, + initial_state_host: Buffer, + initial_state_gpu: Buffer, + proofs_host: Buffer, + proofs_gpu: Buffer, + bind_group_compute_f1: BindGroup, + compute_pipeline_compute_f1: ComputePipeline, + bind_group_sort_buckets_a: BindGroup, + compute_pipeline_sort_buckets_a: ComputePipeline, + bind_group_sort_buckets_b: BindGroup, + compute_pipeline_sort_buckets_b: ComputePipeline, + bind_group_find_matches_and_compute_f2: BindGroup, + compute_pipeline_find_matches_and_compute_f2: ComputePipeline, + bind_group_find_matches_and_compute_f3: BindGroup, + compute_pipeline_find_matches_and_compute_f3: ComputePipeline, + bind_group_find_matches_and_compute_f4: BindGroup, + compute_pipeline_find_matches_and_compute_f4: ComputePipeline, + bind_group_find_matches_and_compute_f5: BindGroup, + compute_pipeline_find_matches_and_compute_f5: ComputePipeline, + bind_group_find_matches_and_compute_f6: BindGroup, + compute_pipeline_find_matches_and_compute_f6: ComputePipeline, + bind_group_find_matches_and_compute_f7: BindGroup, + compute_pipeline_find_matches_and_compute_f7: ComputePipeline, + bind_group_find_proofs: BindGroup, + compute_pipeline_find_proofs: ComputePipeline, +} + +impl fmt::Debug for GpuRecordsEncoderInstance { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("GpuRecordsEncoderInstance") + .finish_non_exhaustive() + } +} + +impl GpuRecordsEncoderInstance { + fn new(device: wgpu::Device, queue: Queue, module: ShaderModule) -> Self { + let initial_state_host = device.create_buffer(&BufferDescriptor { + label: Some("initial_state_host"), + size: size_of::() as BufferAddress, + usage: BufferUsages::MAP_WRITE | BufferUsages::COPY_SRC, + mapped_at_creation: true, + }); + + let initial_state_gpu = device.create_buffer(&BufferDescriptor { + label: Some("initial_state_gpu"), + size: initial_state_host.size(), + usage: BufferUsages::UNIFORM | BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + let bucket_sizes_gpu_buffer_size = size_of::<[u32; NUM_BUCKETS]>() as BufferAddress; + let table_6_proof_targets_sizes_gpu_buffer_size = + size_of::<[u32; NUM_S_BUCKETS]>() as BufferAddress; + // TODO: Sizes are excessive, for `bucket_sizes_gpu` are less than `u16` and could use SWAR + // approach for storing bucket sizes. Similarly, `table_6_proof_targets_sizes_gpu` sizes + // are less than `u8` and could use SWAR too with even higher compression ratio + let bucket_sizes_gpu = device.create_buffer(&BufferDescriptor { + label: Some("bucket_sizes_gpu"), + size: bucket_sizes_gpu_buffer_size.max(table_6_proof_targets_sizes_gpu_buffer_size), + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + // Reuse the same buffer as `bucket_sizes_gpu`, they are not overlapping in use + let table_6_proof_targets_sizes_gpu = bucket_sizes_gpu.clone(); + + let buckets_a_gpu = device.create_buffer(&BufferDescriptor { + label: Some("buckets_a_gpu"), + size: size_of::<[[PositionR; MAX_BUCKET_SIZE]; NUM_BUCKETS]>() as BufferAddress, + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + let buckets_b_gpu = device.create_buffer(&BufferDescriptor { + label: Some("buckets_b_gpu"), + size: buckets_a_gpu.size(), + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + let positions_f2_gpu = device.create_buffer(&BufferDescriptor { + label: Some("positions_f2_gpu"), + size: size_of::<[[[Position; 2]; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS]>() + as BufferAddress, + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + let positions_f3_gpu = device.create_buffer(&BufferDescriptor { + label: Some("positions_f3_gpu"), + size: positions_f2_gpu.size(), + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + let positions_f4_gpu = device.create_buffer(&BufferDescriptor { + label: Some("positions_f4_gpu"), + size: positions_f2_gpu.size(), + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + let positions_f5_gpu = device.create_buffer(&BufferDescriptor { + label: Some("positions_f5_gpu"), + size: positions_f2_gpu.size(), + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + let positions_f6_gpu = device.create_buffer(&BufferDescriptor { + label: Some("positions_f6_gpu"), + size: positions_f2_gpu.size(), + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + let metadatas_gpu_buffer_size = + size_of::<[[Metadata; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS]>() as BufferAddress; + let table_6_proof_targets_gpu_buffer_size = size_of::< + [[ProofTargets; NUM_ELEMENTS_PER_S_BUCKET]; NUM_S_BUCKETS], + >() as BufferAddress; + let metadatas_a_gpu = device.create_buffer(&BufferDescriptor { + label: Some("metadatas_a_gpu"), + size: metadatas_gpu_buffer_size.max(table_6_proof_targets_gpu_buffer_size), + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + // Reuse the same buffer as `metadatas_a_gpu`, they are not overlapping in use + let table_6_proof_targets_gpu = metadatas_a_gpu.clone(); + + let metadatas_b_gpu = device.create_buffer(&BufferDescriptor { + label: Some("metadatas_b_gpu"), + size: metadatas_gpu_buffer_size, + usage: BufferUsages::STORAGE, + mapped_at_creation: false, + }); + + let proofs_host = device.create_buffer(&BufferDescriptor { + label: Some("proofs_host"), + size: size_of::() as BufferAddress, + usage: BufferUsages::MAP_READ | BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + + let proofs_gpu = device.create_buffer(&BufferDescriptor { + label: Some("proofs_gpu"), + size: proofs_host.size(), + usage: BufferUsages::STORAGE | BufferUsages::COPY_SRC, + mapped_at_creation: false, + }); + + let (bind_group_compute_f1, compute_pipeline_compute_f1) = + bind_group_and_pipeline_compute_f1( + &device, + &module, + &initial_state_gpu, + &bucket_sizes_gpu, + &buckets_a_gpu, + ); + let (bind_group_sort_buckets_a, compute_pipeline_sort_buckets_a) = + bind_group_and_pipeline_sort_buckets( + &device, + &module, + &bucket_sizes_gpu, + &buckets_a_gpu, + ); + + let (bind_group_sort_buckets_b, compute_pipeline_sort_buckets_b) = + bind_group_and_pipeline_sort_buckets( + &device, + &module, + &bucket_sizes_gpu, + &buckets_b_gpu, + ); + + let (bind_group_find_matches_and_compute_f2, compute_pipeline_find_matches_and_compute_f2) = + bind_group_and_pipeline_find_matches_and_compute_f2( + &device, + &module, + &buckets_a_gpu, + &bucket_sizes_gpu, + &buckets_b_gpu, + &positions_f2_gpu, + &metadatas_b_gpu, + ); + + let (bind_group_find_matches_and_compute_f3, compute_pipeline_find_matches_and_compute_f3) = + bind_group_and_pipeline_find_matches_and_compute_fn::<3>( + &device, + &module, + &buckets_b_gpu, + &metadatas_b_gpu, + &bucket_sizes_gpu, + &buckets_a_gpu, + &positions_f3_gpu, + &metadatas_a_gpu, + ); + + let (bind_group_find_matches_and_compute_f4, compute_pipeline_find_matches_and_compute_f4) = + bind_group_and_pipeline_find_matches_and_compute_fn::<4>( + &device, + &module, + &buckets_a_gpu, + &metadatas_a_gpu, + &bucket_sizes_gpu, + &buckets_b_gpu, + &positions_f4_gpu, + &metadatas_b_gpu, + ); + + let (bind_group_find_matches_and_compute_f5, compute_pipeline_find_matches_and_compute_f5) = + bind_group_and_pipeline_find_matches_and_compute_fn::<5>( + &device, + &module, + &buckets_b_gpu, + &metadatas_b_gpu, + &bucket_sizes_gpu, + &buckets_a_gpu, + &positions_f5_gpu, + &metadatas_a_gpu, + ); + + let (bind_group_find_matches_and_compute_f6, compute_pipeline_find_matches_and_compute_f6) = + bind_group_and_pipeline_find_matches_and_compute_fn::<6>( + &device, + &module, + &buckets_a_gpu, + &metadatas_a_gpu, + &bucket_sizes_gpu, + &buckets_b_gpu, + &positions_f6_gpu, + &metadatas_b_gpu, + ); + + let (bind_group_find_matches_and_compute_f7, compute_pipeline_find_matches_and_compute_f7) = + bind_group_and_pipeline_find_matches_and_compute_f7( + &device, + &module, + &buckets_b_gpu, + &metadatas_b_gpu, + &table_6_proof_targets_sizes_gpu, + &table_6_proof_targets_gpu, + ); + + let (bind_group_find_proofs, compute_pipeline_find_proofs) = + bind_group_and_pipeline_find_proofs( + &device, + &module, + &positions_f2_gpu, + &positions_f3_gpu, + &positions_f4_gpu, + &positions_f5_gpu, + &positions_f6_gpu, + &table_6_proof_targets_sizes_gpu, + &table_6_proof_targets_gpu, + &proofs_gpu, + ); + + Self { + device, + queue, + mapping_error: Arc::new(Mutex::new(None)), + tainted: false, + initial_state_host, + initial_state_gpu, + proofs_host, + proofs_gpu, + bind_group_compute_f1, + compute_pipeline_compute_f1, + bind_group_sort_buckets_a, + compute_pipeline_sort_buckets_a, + bind_group_sort_buckets_b, + compute_pipeline_sort_buckets_b, + bind_group_find_matches_and_compute_f2, + compute_pipeline_find_matches_and_compute_f2, + bind_group_find_matches_and_compute_f3, + compute_pipeline_find_matches_and_compute_f3, + bind_group_find_matches_and_compute_f4, + compute_pipeline_find_matches_and_compute_f4, + bind_group_find_matches_and_compute_f5, + compute_pipeline_find_matches_and_compute_f5, + bind_group_find_matches_and_compute_f6, + compute_pipeline_find_matches_and_compute_f6, + bind_group_find_matches_and_compute_f7, + compute_pipeline_find_matches_and_compute_f7, + bind_group_find_proofs, + compute_pipeline_find_proofs, + } + } + + pub fn create_proofs( + &mut self, + seed: &PosSeed, + ) -> Result, RecordEncodingError> { + if self.tainted { + return Err(RecordEncodingError::DeviceBroken); + } + self.tainted = true; + + let mut encoder = self + .device + .create_command_encoder(&CommandEncoderDescriptor { + label: Some("create_proofs"), + }); + + // Mapped initially and re-mapped at the end of the computation + self.initial_state_host + .get_mapped_range_mut(..) + .copy_from_slice(&block_to_bytes( + &ChaCha8State::init(seed, &[0; _]).to_repr(), + )); + self.initial_state_host.unmap(); + + encoder.copy_buffer_to_buffer( + &self.initial_state_host, + 0, + &self.initial_state_gpu, + 0, + self.initial_state_host.size(), + ); + + { + let mut cpass = encoder.begin_compute_pass(&ComputePassDescriptor { + label: Some("create_proofs"), + timestamp_writes: None, + }); + + cpass.set_bind_group(0, &self.bind_group_compute_f1, &[]); + cpass.set_pipeline(&self.compute_pipeline_compute_f1); + cpass.dispatch_workgroups( + MAX_TABLE_SIZE + .div_ceil(compute_f1::WORKGROUP_SIZE * compute_f1::ELEMENTS_PER_INVOCATION), + 1, + 1, + ); + + cpass.set_bind_group(0, &self.bind_group_sort_buckets_a, &[]); + cpass.set_pipeline(&self.compute_pipeline_sort_buckets_a); + cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f2, &[]); + cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f2); + cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_sort_buckets_b, &[]); + cpass.set_pipeline(&self.compute_pipeline_sort_buckets_b); + cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f3, &[]); + cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f3); + cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_sort_buckets_a, &[]); + cpass.set_pipeline(&self.compute_pipeline_sort_buckets_a); + cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f4, &[]); + cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f4); + cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_sort_buckets_b, &[]); + cpass.set_pipeline(&self.compute_pipeline_sort_buckets_b); + cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f5, &[]); + cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f5); + cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_sort_buckets_a, &[]); + cpass.set_pipeline(&self.compute_pipeline_sort_buckets_a); + cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f6, &[]); + cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f6); + cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_sort_buckets_b, &[]); + cpass.set_pipeline(&self.compute_pipeline_sort_buckets_b); + cpass.dispatch_workgroups(NUM_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_find_matches_and_compute_f7, &[]); + cpass.set_pipeline(&self.compute_pipeline_find_matches_and_compute_f7); + cpass.dispatch_workgroups(NUM_MATCH_BUCKETS as u32, 1, 1); + + cpass.set_bind_group(0, &self.bind_group_find_proofs, &[]); + cpass.set_pipeline(&self.compute_pipeline_find_proofs); + cpass.dispatch_workgroups(NUM_S_BUCKETS as u32 / find_proofs::WORKGROUP_SIZE, 1, 1); + } + + encoder.copy_buffer_to_buffer( + &self.proofs_gpu, + 0, + &self.proofs_host, + 0, + self.proofs_host.size(), + ); + + // Map initial state for writes for the next iteration + encoder.map_buffer_on_submit(&self.initial_state_host, MapMode::Write, .., { + let mapping_error = Arc::clone(&self.mapping_error); + + move |r| { + if let Err(error) = r { + mapping_error.lock().replace(error); + } + } + }); + encoder.map_buffer_on_submit(&self.proofs_host, MapMode::Read, .., { + let mapping_error = Arc::clone(&self.mapping_error); + + move |r| { + if let Err(error) = r { + mapping_error.lock().replace(error); + } + } + }); + + let submission_index = self.queue.submit([encoder.finish()]); + + self.device.poll(PollType::Wait { + submission_index: Some(submission_index), + timeout: None, + })?; + + if let Some(error) = self.mapping_error.lock().take() { + return Err(RecordEncodingError::BufferMapping(error)); + } + + let proofs = { + let proofs_host_ptr = self + .proofs_host + .get_mapped_range(..) + .as_ptr() + .cast::(); + // SAFETY: Initialized on the GPU + unsafe { &*proofs_host_ptr } + }; + + self.tainted = false; + + Ok(ProofsHostWrapper { + proofs, + proofs_host: &self.proofs_host, + }) + } +} + +fn bind_group_and_pipeline_compute_f1( + device: &wgpu::Device, + module: &ShaderModule, + initial_state_gpu: &Buffer, + bucket_sizes_gpu: &Buffer, + buckets_gpu: &Buffer, +) -> (BindGroup, ComputePipeline) { + let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor { + label: Some("compute_f1"), + entries: &[ + BindGroupLayoutEntry { + binding: 0, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Uniform, + }, + }, + BindGroupLayoutEntry { + binding: 1, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 2, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor { + label: Some("compute_f1"), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + + let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor { + compilation_options: PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: false, + }, + cache: None, + label: Some("compute_f1"), + layout: Some(&pipeline_layout), + module, + entry_point: Some("compute_f1"), + }); + + let bind_group = device.create_bind_group(&BindGroupDescriptor { + label: Some("compute_f1"), + layout: &bind_group_layout, + entries: &[ + BindGroupEntry { + binding: 0, + resource: initial_state_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 1, + resource: bucket_sizes_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 2, + resource: buckets_gpu.as_entire_binding(), + }, + ], + }); + + (bind_group, compute_pipeline) +} + +fn bind_group_and_pipeline_sort_buckets( + device: &wgpu::Device, + module: &ShaderModule, + bucket_sizes_gpu: &Buffer, + buckets_gpu: &Buffer, +) -> (BindGroup, ComputePipeline) { + let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor { + label: Some("sort_buckets"), + entries: &[ + BindGroupLayoutEntry { + binding: 0, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 1, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor { + label: Some("sort_buckets"), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + + let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor { + compilation_options: PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: false, + }, + cache: None, + label: Some("sort_buckets"), + layout: Some(&pipeline_layout), + module, + entry_point: Some("sort_buckets"), + }); + + let bind_group = device.create_bind_group(&BindGroupDescriptor { + label: Some("sort_buckets"), + layout: &bind_group_layout, + entries: &[ + BindGroupEntry { + binding: 0, + resource: bucket_sizes_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 1, + resource: buckets_gpu.as_entire_binding(), + }, + ], + }); + + (bind_group, compute_pipeline) +} + +fn bind_group_and_pipeline_find_matches_and_compute_f2( + device: &wgpu::Device, + module: &ShaderModule, + parent_buckets_gpu: &Buffer, + bucket_sizes_gpu: &Buffer, + buckets_gpu: &Buffer, + positions_gpu: &Buffer, + metadatas_gpu: &Buffer, +) -> (BindGroup, ComputePipeline) { + let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor { + label: Some("find_matches_and_compute_f2"), + entries: &[ + BindGroupLayoutEntry { + binding: 0, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 1, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 2, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 3, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 4, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor { + label: Some("find_matches_and_compute_f2"), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + + let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor { + compilation_options: PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: true, + }, + cache: None, + label: Some("find_matches_and_compute_f2"), + layout: Some(&pipeline_layout), + module, + entry_point: Some("find_matches_and_compute_f2"), + }); + + let bind_group = device.create_bind_group(&BindGroupDescriptor { + label: Some("find_matches_and_compute_f2"), + layout: &bind_group_layout, + entries: &[ + BindGroupEntry { + binding: 0, + resource: parent_buckets_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 1, + resource: bucket_sizes_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 2, + resource: buckets_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 3, + resource: positions_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 4, + resource: metadatas_gpu.as_entire_binding(), + }, + ], + }); + + (bind_group, compute_pipeline) +} + +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +fn bind_group_and_pipeline_find_matches_and_compute_fn( + device: &wgpu::Device, + module: &ShaderModule, + parent_buckets_gpu: &Buffer, + parent_metadatas_gpu: &Buffer, + bucket_sizes_gpu: &Buffer, + buckets_gpu: &Buffer, + positions_gpu: &Buffer, + metadatas_gpu: &Buffer, +) -> (BindGroup, ComputePipeline) { + let label = format!("find_matches_and_compute_f{TABLE_NUMBER}"); + let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor { + label: Some(&label), + entries: &[ + BindGroupLayoutEntry { + binding: 0, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 1, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 2, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 3, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 4, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 5, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor { + label: Some(&label), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + + let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor { + compilation_options: PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: true, + }, + cache: None, + label: Some(&label), + layout: Some(&pipeline_layout), + module, + entry_point: Some(&format!("find_matches_and_compute_f{TABLE_NUMBER}")), + }); + + let bind_group = device.create_bind_group(&BindGroupDescriptor { + label: Some(&label), + layout: &bind_group_layout, + entries: &[ + BindGroupEntry { + binding: 0, + resource: parent_buckets_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 1, + resource: parent_metadatas_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 2, + resource: bucket_sizes_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 3, + resource: buckets_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 4, + resource: positions_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 5, + resource: metadatas_gpu.as_entire_binding(), + }, + ], + }); + + (bind_group, compute_pipeline) +} + +fn bind_group_and_pipeline_find_matches_and_compute_f7( + device: &wgpu::Device, + module: &ShaderModule, + parent_buckets_gpu: &Buffer, + parent_metadatas_gpu: &Buffer, + table_6_proof_targets_sizes_gpu: &Buffer, + table_6_proof_targets_gpu: &Buffer, +) -> (BindGroup, ComputePipeline) { + let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor { + label: Some("find_matches_and_compute_f7"), + entries: &[ + BindGroupLayoutEntry { + binding: 0, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 1, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 2, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 3, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor { + label: Some("find_matches_and_compute_f7"), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + + let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor { + compilation_options: PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: true, + }, + cache: None, + label: Some("find_matches_and_compute_f7"), + layout: Some(&pipeline_layout), + module, + entry_point: Some("find_matches_and_compute_f7"), + }); + + let bind_group = device.create_bind_group(&BindGroupDescriptor { + label: Some("find_matches_and_compute_f7"), + layout: &bind_group_layout, + entries: &[ + BindGroupEntry { + binding: 0, + resource: parent_buckets_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 1, + resource: parent_metadatas_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 2, + resource: table_6_proof_targets_sizes_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 3, + resource: table_6_proof_targets_gpu.as_entire_binding(), + }, + ], + }); + + (bind_group, compute_pipeline) +} + +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +fn bind_group_and_pipeline_find_proofs( + device: &wgpu::Device, + module: &ShaderModule, + table_2_positions_gpu: &Buffer, + table_3_positions_gpu: &Buffer, + table_4_positions_gpu: &Buffer, + table_5_positions_gpu: &Buffer, + table_6_positions_gpu: &Buffer, + bucket_sizes_gpu: &Buffer, + buckets_gpu: &Buffer, + proofs_gpu: &Buffer, +) -> (BindGroup, ComputePipeline) { + let bind_group_layout = device.create_bind_group_layout(&BindGroupLayoutDescriptor { + label: Some("find_proofs"), + entries: &[ + BindGroupLayoutEntry { + binding: 0, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 1, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 2, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 3, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 4, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 5, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + BindGroupLayoutEntry { + binding: 6, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: true }, + }, + }, + BindGroupLayoutEntry { + binding: 7, + count: None, + visibility: ShaderStages::COMPUTE, + ty: BindingType::Buffer { + has_dynamic_offset: false, + min_binding_size: None, + ty: BufferBindingType::Storage { read_only: false }, + }, + }, + ], + }); + + let pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor { + label: Some("find_proofs"), + bind_group_layouts: &[Some(&bind_group_layout)], + immediate_size: 0, + }); + + let compute_pipeline = device.create_compute_pipeline(&ComputePipelineDescriptor { + compilation_options: PipelineCompilationOptions { + constants: &[], + zero_initialize_workgroup_memory: false, + }, + cache: None, + label: Some("find_proofs"), + layout: Some(&pipeline_layout), + module, + entry_point: Some("find_proofs"), + }); + + let bind_group = device.create_bind_group(&BindGroupDescriptor { + label: Some("find_proofs"), + layout: &bind_group_layout, + entries: &[ + BindGroupEntry { + binding: 0, + resource: table_2_positions_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 1, + resource: table_3_positions_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 2, + resource: table_4_positions_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 3, + resource: table_5_positions_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 4, + resource: table_6_positions_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 5, + resource: bucket_sizes_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 6, + resource: buckets_gpu.as_entire_binding(), + }, + BindGroupEntry { + binding: 7, + resource: proofs_gpu.as_entire_binding(), + }, + ], + }); + + (bind_group, compute_pipeline) +} diff --git a/shared/ab-proof-of-space-gpu/src/lib.rs b/shared/ab-proof-of-space-gpu/src/lib.rs new file mode 100644 index 00000000000..d19d79312d5 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/lib.rs @@ -0,0 +1,30 @@ +//! Proof of space plotting utilities for GPU (Vulkan/Metal). +//! +//! Similarly to `ab-proof-of-space`, max supported `K` within range `15..=24` due to internal data +//! structures used (`ab-proof-of-space` also supports `K=25`, but this crate doesn't for now). + +#![cfg_attr(target_arch = "spirv", no_std)] +#![feature(generic_const_exprs, step_trait)] +#![expect(incomplete_features, reason = "generic_const_exprs")] + +#[cfg(not(target_arch = "spirv"))] +mod host; +// This is used for benchmarks of isolated shaders externally, not for general use +#[doc(hidden)] +pub mod shader; + +// TODO: Remove gate after https://github.com/Rust-GPU/rust-gpu/pull/249 +#[cfg(not(target_arch = "spirv"))] +pub use crate::shader::find_proofs::ProofsHost; +#[cfg(not(target_arch = "spirv"))] +pub use host::{Device, GpuRecordsEncoderInstance, ProofsHostWrapper, RecordEncodingError}; +#[cfg(not(target_arch = "spirv"))] +use subspace_core_primitives::pos::PosProof; +#[cfg(not(target_arch = "spirv"))] +pub use wgpu::{Backend, DeviceType}; + +// TODO: Remove gate after https://github.com/Rust-GPU/rust-gpu/pull/249 +#[cfg(not(target_arch = "spirv"))] +const _: () = { + assert!(PosProof::K >= 15 && PosProof::K <= 24); +}; diff --git a/shared/ab-proof-of-space-gpu/src/shader.rs b/shared/ab-proof-of-space-gpu/src/shader.rs new file mode 100644 index 00000000000..ab2c98bc08a --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader.rs @@ -0,0 +1,59 @@ +pub mod compute_f1; +pub mod compute_fn; +// TODO: Reuse constants from `ab-proof-of-space` once it compiles with `rust-gpu` +pub mod constants; +pub mod find_matches_and_compute_f2; +pub mod find_matches_and_compute_f7; +pub mod find_matches_and_compute_fn; +pub mod find_matches_in_buckets; +pub mod find_proofs; +mod polyfills; +#[cfg(not(target_arch = "spirv"))] +mod shader_bytes; +pub mod sort_buckets; +// TODO: Reuse types from `ab-proof-of-space` once it compiles with `rust-gpu` +pub mod types; +mod u32n; + +#[cfg(not(target_arch = "spirv"))] +use wgpu::{Adapter, Features, Limits}; + +#[cfg(not(target_endian = "little"))] +compile_error!("Only little-endian platforms are supported"); + +/// This should be more than any usable implementation has. +/// +/// There are assertions elsewhere ensuring this is large enough at compile time. +const MIN_SUBGROUP_SIZE: u32 = 4; +/// Compiled SPIR-V shader +#[cfg(not(target_arch = "spirv"))] +const SHADER: wgpu::ShaderModuleDescriptor<'static> = { + use crate::shader::shader_bytes::ShaderBytes; + + const SHADER_BYTES_INTERNAL: &ShaderBytes<[u8]> = + &ShaderBytes(*include_bytes!(env!("SHADER_PATH"))); + + SHADER_BYTES_INTERNAL.to_module() +}; + +/// For a given set of adapter features and limits, this function returns the shader itself, +/// required features, and required limits. +/// +/// Returns `None` for unsupported adapter. +#[cfg(not(target_arch = "spirv"))] +pub fn select_shader_features_limits( + adapter: &Adapter, +) -> Option<(wgpu::ShaderModuleDescriptor<'static>, Features, Limits)> { + const SHADER_BASELINE_FEATURES: Features = Features::SUBGROUP; + + let adapter_features = adapter.features(); + let adapter_limits = adapter.limits(); + + if adapter_features.contains(SHADER_BASELINE_FEATURES) + && adapter.get_info().subgroup_min_size >= MIN_SUBGROUP_SIZE + { + Some((SHADER, SHADER_BASELINE_FEATURES, adapter_limits)) + } else { + None + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/compute_f1.rs b/shared/ab-proof-of-space-gpu/src/shader/compute_f1.rs new file mode 100644 index 00000000000..ebb9a2797c2 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/compute_f1.rs @@ -0,0 +1,200 @@ +use crate::shader::constants::{K, MAX_BUCKET_SIZE, MAX_TABLE_SIZE, NUM_BUCKETS, PARAM_EXT}; +#[cfg(target_arch = "spirv")] +use crate::shader::polyfills::ArrayIndexingPolyfill; +use crate::shader::types::{Position, PositionExt, PositionR, X, Y}; +use crate::shader::u32n::U32N; +use ab_chacha8::{ChaCha8Block, ChaCha8State}; +use core::mem::MaybeUninit; +use spirv_std::arch::atomic_i_increment; +use spirv_std::glam::{UVec3, UVec4}; +use spirv_std::memory::{Scope, Semantics}; +use spirv_std::spirv; + +const fn gcd(mut a: u32, mut b: u32) -> u32 { + while b != 0 { + let t = b; + b = a % b; + a = t; + } + a +} + +const fn lcm(a: u32, b: u32) -> u32 { + let g = gcd(a, b); + (a / g) * b +} + +// TODO: Same number as hardcoded in `#[spirv(compute(threads(..)))]` below, can be removed once +// https://github.com/Rust-GPU/rust-gpu/discussions/287 is resolved +pub const WORKGROUP_SIZE: u32 = 256; +const CHACHA8_BLOCK_BITS: u32 = size_of::() as u32 * u8::BITS; +const CHACHA8_BLOCK_WORDS: usize = size_of::() / size_of::(); +// This number is both a multiple of `K` (bits per element) and ChaCha8 block (bits per block) +const BITS_PER_INVOCATION: u32 = lcm(CHACHA8_BLOCK_BITS, K as u32); +pub const ELEMENTS_PER_INVOCATION: u32 = BITS_PER_INVOCATION / K as u32; +const BLOCKS_PER_INVOCATION: u32 = BITS_PER_INVOCATION / CHACHA8_BLOCK_BITS; +// `+1` is needed due to the way `compute_fn_impl` does slightly outside what it, strictly speaking, +// needs (for efficiency purposes) +const INVOCATION_KEYSTREAM_WORDS: usize = BLOCKS_PER_INVOCATION as usize * CHACHA8_BLOCK_WORDS + 1; + +#[derive(Debug, Copy, Clone)] +pub struct UniformChaCha8Block([UVec4; 4]); + +impl From for UniformChaCha8Block { + #[inline(always)] + fn from(value: ChaCha8Block) -> Self { + Self([ + UVec4 { + x: value[0], + y: value[1], + z: value[2], + w: value[3], + }, + UVec4 { + x: value[4], + y: value[5], + z: value[6], + w: value[7], + }, + UVec4 { + x: value[8], + y: value[9], + z: value[10], + w: value[11], + }, + UVec4 { + x: value[12], + y: value[13], + z: value[14], + w: value[15], + }, + ]) + } +} + +impl From for ChaCha8Block { + #[inline(always)] + fn from(value: UniformChaCha8Block) -> Self { + [ + value.0[0].x, + value.0[0].y, + value.0[0].z, + value.0[0].w, + value.0[1].x, + value.0[1].y, + value.0[1].z, + value.0[1].w, + value.0[2].x, + value.0[2].y, + value.0[2].z, + value.0[2].w, + value.0[3].x, + value.0[3].y, + value.0[3].z, + value.0[3].w, + ] + } +} + +// TODO: Make unsafe and avoid bounds check +// TODO: Reuse code from `ab-proof-of-space` after https://github.com/Rust-GPU/rust-gpu/pull/249 and +// https://github.com/Rust-GPU/rust-gpu/discussions/301 +/// `partial_y_offset` is in bits within `partial_y` +#[inline(always)] +fn compute_f1_impl(x: X, chacha8_keystream: &[u32; INVOCATION_KEYSTREAM_WORDS]) -> Y { + let skip_bits = (u32::from(K) * u32::from(x)) % BITS_PER_INVOCATION; + let skip_u32s = skip_bits / u32::BITS; + let partial_y_offset = skip_bits % u32::BITS; + + let high = chacha8_keystream[skip_u32s as usize].to_be(); + let low = chacha8_keystream[skip_u32s as usize + 1].to_be(); + let partial_y = U32N::from_low_high(low, high); + + let pre_y = partial_y >> (u64::BITS - u32::from(K + PARAM_EXT) - partial_y_offset); + let pre_y = pre_y.as_u32(); + // Mask for clearing the rest of bits of `pre_y`. + let pre_y_mask = (u32::MAX << PARAM_EXT) & (u32::MAX >> (u32::BITS - u32::from(K + PARAM_EXT))); + + // Extract `PARAM_EXT` most significant bits from `x` and store in the final offset of + // eventual `y` with the rest of bits being zero (`x` is `0..2^K`) + let pre_ext = u32::from(x) >> (K - PARAM_EXT); + + // Combine all of the bits together: + // [padding zero bits][`K` bits from `partial_y`][`PARAM_EXT` bits from `x`] + Y::from((pre_y & pre_y_mask) | pre_ext) +} + +/// Compute Chia's `f1()` function for the whole table using the initial state of ChaCha8 cipher +/// straight into buckets of the first table. +/// +/// Buckets need to be sorted by position afterward due to concurrent writes that do not have +/// deterministic order. Content of the bucket beyond the size specified in `bucket_sizes` is +/// undefined. +/// +/// # Safety +/// `bucket_sizes` must be zero-initialized, which is the case by default in `wgpu`. +#[spirv(compute(threads(256), entry_point_name = "compute_f1"))] +pub unsafe fn compute_f1( + #[spirv(global_invocation_id)] global_invocation_id: UVec3, + #[spirv(uniform, descriptor_set = 0, binding = 0)] initial_state: &UniformChaCha8Block, + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] bucket_sizes: &mut [u32; NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 2)] buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; + NUM_BUCKETS], +) { + // TODO: Make a single input bounds check and use unsafe to avoid bounds check later + let global_invocation_id = global_invocation_id.x; + + let initial_state = ChaCha8State::from_repr(ChaCha8Block::from(*initial_state)); + + let x_start = global_invocation_id * ELEMENTS_PER_INVOCATION; + + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + // let chacha8_keystream = [MaybeUninit; BLOCKS_PER_INVOCATION as usize]; + let mut chacha8_keystream = [0u32; INVOCATION_KEYSTREAM_WORDS]; + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + let first_block_counter = x_start * u32::from(K) / CHACHA8_BLOCK_BITS; + for block_index in 0..BLOCKS_PER_INVOCATION { + let counter = first_block_counter + block_index; + + let block = initial_state.compute_block(counter); + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + let block_write_offset = block_index as usize * CHACHA8_BLOCK_WORDS; + #[expect(clippy::manual_memcpy, reason = "Doesn't compile under rust-gpu")] + for offset in 0..CHACHA8_BLOCK_WORDS { + chacha8_keystream[block_write_offset + offset] = block[offset]; + } + } + + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + for x in x_start..(x_start + ELEMENTS_PER_INVOCATION).min(MAX_TABLE_SIZE) { + let y = compute_f1_impl(X::from(x), &chacha8_keystream); + + let (bucket_index, r) = y.into_bucket_index_and_r(); + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition + let bucket_size = unsafe { bucket_sizes.get_unchecked_mut(bucket_index as usize) }; + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + let bucket_offset = unsafe { + atomic_i_increment::<_, { Scope::QueueFamily as u32 }, { Semantics::NONE.bits() }>( + bucket_size, + ) + }; + + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition. Bucket + // size upper bound is known statically to be [`MAX_BUCKET_SIZE`], so `bucket_offset` is + // also always within bounds. + unsafe { + buckets + .get_unchecked_mut(bucket_index as usize) + .get_unchecked_mut(bucket_offset as usize) + } + .write(PositionR { + position: Position::from_u32(x), + r, + }); + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/compute_fn.rs b/shared/ab-proof-of-space-gpu/src/shader/compute_fn.rs new file mode 100644 index 00000000000..de7a9591963 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/compute_fn.rs @@ -0,0 +1,161 @@ +use crate::shader::constants::{K, PARAM_EXT}; +use crate::shader::types::{Metadata, Y}; +use crate::shader::u32n::U32N; + +// TODO: Reuse code from `ab-proof-of-space` after https://github.com/Rust-GPU/rust-gpu/pull/249 and +// https://github.com/Rust-GPU/rust-gpu/discussions/301 +/// Compute the size of `y` in bits +const fn y_size_bits(k: u8) -> u32 { + k as u32 + PARAM_EXT as u32 +} + +// TODO: Reuse code from `ab-proof-of-space` after https://github.com/Rust-GPU/rust-gpu/pull/249 and +// https://github.com/Rust-GPU/rust-gpu/discussions/301 +/// Metadata size in bits +const fn metadata_size_bits(k: u8, table_number: u8) -> u32 { + k as u32 + * match table_number { + 1 => 1, + 2 => 2, + 3 | 4 => 4, + 5 => 3, + 6 => 2, + 7 => 0, + _ => unreachable!(), + } +} + +// TODO: Make unsafe and avoid bounds check +// TODO: Reuse code from `ab-proof-of-space` after https://github.com/Rust-GPU/rust-gpu/pull/249 and +// https://github.com/Rust-GPU/rust-gpu/discussions/301 +#[inline(always)] +pub(super) fn compute_fn_impl( + y: Y, + left_metadata: Metadata, + right_metadata: Metadata, +) -> (Y, Metadata) { + let left_metadata = U32N::<4>::from(left_metadata); + let right_metadata = U32N::<4>::from(right_metadata); + + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + let parent_metadata_bits = const { metadata_size_bits(K, PARENT_TABLE_NUMBER) }; + + // Only supports `K` from 15 to 25 (otherwise math will not be correct when concatenating y, + // left metadata and right metadata) + let mut input_words = [0; _]; + let byte_length = { + // Take only bytes where bits were set + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + let num_bytes_with_data = + (const { y_size_bits(K) } + parent_metadata_bits * 2).div_ceil(u8::BITS); + + // Collect `K` most significant bits of `y` at the final offset of eventual `input_a` + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + let y_bits = U32N::<4>::from(y) << (U32N::<4>::BITS - const { y_size_bits(K) }); + + // Move bits of `left_metadata` at the final offset of eventual `input_a` + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + let left_metadata_bits = + left_metadata << (U32N::<4>::BITS - parent_metadata_bits - const { y_size_bits(K) }); + + // Part of the `right_bits` at the final offset of eventual `input_a` + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + let y_and_left_bits = const { y_size_bits(K) } + parent_metadata_bits; + let right_bits_start_offset = U32N::<4>::BITS - parent_metadata_bits; + + // If `right_metadata` bits start to the left of the desired position in `input_a` move + // bits right, else move left + if right_bits_start_offset < y_and_left_bits { + let right_bits_pushed_into_input_b = y_and_left_bits - right_bits_start_offset; + // Collect bits of `right_metadata` that will fit into `input_a` at the final offset in + // eventual `input_a` + let right_bits_a = right_metadata >> right_bits_pushed_into_input_b; + let input_a = y_bits | left_metadata_bits | right_bits_a; + // Collect bits of `right_metadata` that will spill over into `input_b` + let input_b = right_metadata << (U32N::<4>::BITS - right_bits_pushed_into_input_b); + + let input_a_words = input_a.as_be_bytes_to_le_u32_words(); + // TODO: Manually indexing elements and constructing an array is a workaround for + // rust-gpu to compile + // input_words[..input_a_words.len()].copy_from_slice(&input_a_words); + input_words[0] = input_a_words[0]; + input_words[1] = input_a_words[1]; + input_words[2] = input_a_words[2]; + input_words[3] = input_a_words[3]; + let input_b_words = input_b.as_be_bytes_to_le_u32_words(); + // TODO: Manually indexing elements and constructing an array is a workaround for + // rust-gpu to compile + // input_words[input_a_words.len()..].copy_from_slice(&input_b_words); + input_words[4] = input_b_words[0]; + input_words[5] = input_b_words[1]; + input_words[6] = input_b_words[2]; + input_words[7] = input_b_words[3]; + + size_of::>() as u32 + right_bits_pushed_into_input_b.div_ceil(u8::BITS) + } else { + let right_bits_a = right_metadata << (right_bits_start_offset - y_and_left_bits); + let input_a = y_bits | left_metadata_bits | right_bits_a; + let input_a_words = input_a.as_be_bytes_to_le_u32_words(); + // TODO: Manually indexing elements and constructing an array is a workaround for + // rust-gpu to compile + // input_words[..input_a_words.len()].copy_from_slice(&input_a_words); + input_words[0] = input_a_words[0]; + input_words[1] = input_a_words[1]; + input_words[2] = input_a_words[2]; + input_words[3] = input_a_words[3]; + + num_bytes_with_data + } + }; + let hash = ab_blake3::single_block_hash_portable_words(&input_words, byte_length); + + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + let y_output = Y::from(hash[0].to_be() >> (u32::BITS - const { y_size_bits(K) })); + + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + let metadata_size_bits = const { metadata_size_bits(K, TABLE_NUMBER) }; + + let metadata = if TABLE_NUMBER < 4 { + Metadata::from( + (left_metadata.cast::<3>() << parent_metadata_bits) | right_metadata.cast::<3>(), + ) + } else if metadata_size_bits > 0 { + // For K up to 24 it is guaranteed that metadata + bit offset will always fit into 4 `u32` + // words (equivalent to `u128` size). For K=25 it'll be necessary to have fifth word, which + // will become more cumbersome to handle. We collect bytes necessary, potentially with extra + // bits at the start and end of the bytes that will be taken care of later. + // TODO: Manually indexing elements and constructing an array is a workaround for rust-gpu + // to compile + // let metadata = U128::from_le_u32_words_as_be_bytes( + // hash[(y_size_bits(K) / u32::BITS) as usize..][..size_of::() / size_of::()] + // .try_into() + // .expect("Always enough bits for any K; qed"), + // ); + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + let first_element = (const { y_size_bits(K) } / u32::BITS) as usize; + let metadata = U32N::<4>::from_le_u32_words_as_be_bytes(&[ + hash[first_element], + hash[first_element + 1], + hash[first_element + 2], + hash[first_element + 3], + ]); + // Remove extra bits at the beginning + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + let metadata = metadata << (const { y_size_bits(K) } % u32::BITS); + // Move bits into the correct location + Metadata::from((metadata >> (U32N::<4>::BITS - metadata_size_bits)).cast()) + } else { + Metadata::default() + }; + + (y_output, metadata) +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/constants.rs b/shared/ab-proof-of-space-gpu/src/shader/constants.rs new file mode 100644 index 00000000000..bdd3b717cfc --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/constants.rs @@ -0,0 +1,52 @@ +// TODO: Replace this constant with usage of `PosProof::K` after +// https://github.com/Rust-GPU/rust-gpu/pull/249 is merged +pub(super) const K: u8 = 20; +#[cfg(not(target_arch = "spirv"))] +const _: () = { + assert!(K == subspace_core_primitives::pos::PosProof::K); +}; +// TODO: Replace this constant with usage of `Record::NUM_S_BUCKETS` after +// https://github.com/Rust-GPU/rust-gpu/pull/249 is merged +pub const NUM_S_BUCKETS: usize = 2usize.pow(16); +#[cfg(not(target_arch = "spirv"))] +const _: () = { + assert!(NUM_S_BUCKETS == subspace_core_primitives::pieces::Record::NUM_S_BUCKETS); +}; +pub const MAX_BUCKET_SIZE: usize = 512; +/// Reducing bucket size for better performance. +/// +/// The number should be sufficient to produce enough proofs for sector encoding with high +/// probability. +// TODO: Statistical analysis if possible, confirming there will be enough proofs +pub(super) const REDUCED_BUCKET_SIZE: usize = 272; +/// Reducing matches count for better performance. +/// +/// The number should be sufficient to produce enough proofs for sector encoding with high +/// probability. +// TODO: Statistical analysis if possible, confirming there will be enough proofs +pub const REDUCED_MATCHES_COUNT: usize = 288; +/// PRNG extension parameter to avoid collisions +pub(super) const PARAM_EXT: u8 = 6; +pub(super) const PARAM_M: u16 = 1 << PARAM_EXT; +pub(super) const PARAM_B: u16 = 119; +pub(super) const PARAM_C: u16 = 127; +pub(super) const PARAM_BC: u16 = PARAM_B * PARAM_C; +pub(super) const NUM_TABLES: u8 = 7; +/// Size of the first table and max size for other tables +pub const MAX_TABLE_SIZE: u32 = 1 << K; + +/// Compute the size of `y` in bits +pub(super) const fn y_size_bits(k: u8) -> usize { + k as usize + PARAM_EXT as usize +} + +/// Number of buckets for a given `k` +const fn num_buckets(k: u8) -> usize { + 2_usize + .pow(y_size_bits(k) as u32) + .div_ceil(PARAM_BC as usize) +} + +pub const NUM_BUCKETS: usize = num_buckets(K); +// Buckets are matched with a sliding window of `2`, hence one less bucket exists +pub const NUM_MATCH_BUCKETS: usize = NUM_BUCKETS - 1; diff --git a/shared/ab-proof-of-space-gpu/src/shader/find_matches_and_compute_f2.rs b/shared/ab-proof-of-space-gpu/src/shader/find_matches_and_compute_f2.rs new file mode 100644 index 00000000000..c58bfea7050 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/find_matches_and_compute_f2.rs @@ -0,0 +1,275 @@ +use crate::shader::compute_fn::compute_fn_impl; +use crate::shader::constants::{ + MAX_BUCKET_SIZE, NUM_BUCKETS, NUM_MATCH_BUCKETS, PARAM_BC, REDUCED_BUCKET_SIZE, + REDUCED_MATCHES_COUNT, +}; +use crate::shader::find_matches_in_buckets::{FindMatchesShared, find_matches_in_buckets_impl}; +#[cfg(target_arch = "spirv")] +use crate::shader::polyfills::ArrayIndexingPolyfill; +use crate::shader::types::{Match, Metadata, Position, PositionExt, PositionR, Y}; +use core::fmt; +use core::mem::MaybeUninit; +use spirv_std::arch::{atomic_i_increment, workgroup_memory_barrier_with_group_sync}; +use spirv_std::glam::UVec3; +use spirv_std::memory::{Scope, Semantics}; +use spirv_std::spirv; + +// TODO: Same number as hardcoded in `#[spirv(compute(threads(..)))]` below, can be removed once +// https://github.com/Rust-GPU/rust-gpu/discussions/287 is resolved +pub const WORKGROUP_SIZE: u32 = 256; +const TABLE_NUMBER: u8 = 2; +const PARENT_TABLE_NUMBER: u8 = 1; + +const _: () = { + assert!(crate::shader::find_matches_in_buckets::WORKGROUP_SIZE == WORKGROUP_SIZE); +}; + +// TODO: Should be union, but it currently doesn't compile: +// https://github.com/Rust-GPU/rust-gpu/issues/241 +#[derive(Copy, Clone)] +pub struct FindMatchesAndComputeF2Shared { + find_matches_shared: FindMatchesShared, + bucket_scratch: [PositionR; REDUCED_BUCKET_SIZE], +} + +impl fmt::Debug for FindMatchesAndComputeF2Shared { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FindMatchesAndComputeF2Shared") + .finish_non_exhaustive() + } +} + +/// # Safety +/// `bucket_index` must be within range `0..REDUCED_MATCHES_COUNT`. `matches_count` elements in +/// `matches` must be initialized. `matches` must have valid pointers into left/right buckets and +/// `parent_metadatas`. +#[inline(always)] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +unsafe fn compute_f2_into_buckets_inner( + index: u32, + left_bucket_base: u32, + metadatas_offset: u32, + left_bucket: &[PositionR; MAX_BUCKET_SIZE], + // TODO: `&[Match]` would have been nicer, but it currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + matches: &[MaybeUninit; MAX_BUCKET_SIZE], + bucket_sizes: &mut [u32; NUM_BUCKETS], + buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; NUM_BUCKETS], + positions: &mut [MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT], + metadatas: &mut [MaybeUninit; REDUCED_MATCHES_COUNT], + bucket_scratch: &[PositionR; REDUCED_BUCKET_SIZE], +) { + // SAFETY: Guaranteed by function contract + let (bucket_offset, r_target, positions_offset) = + unsafe { matches.get_unchecked(index as usize).assume_init() }.split(); + + // SAFETY: Guaranteed by function contract + let left_position_r = *unsafe { left_bucket.get_unchecked(bucket_offset as usize) }; + let left_position = left_position_r.position; + let left_r = left_position_r.r.get(); + + // Repurpose variable for two purposes to save on registers + let mut right_position_or_skip = positions_offset; + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + #[expect( + clippy::needless_range_loop, + reason = "Intentional workaround for rust-gpu" + )] + for offset in 0..REDUCED_BUCKET_SIZE { + let position_r = bucket_scratch[offset]; + if position_r.r.get() == r_target { + if right_position_or_skip == 0 { + right_position_or_skip = position_r.position; + break; + } + + right_position_or_skip -= 1; + } + } + let right_position = right_position_or_skip; + + let left_metadata = Metadata::from(left_position); + let right_metadata = Metadata::from(right_position); + + let (y, metadata) = compute_fn_impl::( + Y::from(left_bucket_base + left_r), + left_metadata, + right_metadata, + ); + + let (bucket_index, r) = y.into_bucket_index_and_r(); + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition + let bucket_size = unsafe { bucket_sizes.get_unchecked_mut(bucket_index as usize) }; + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + let bucket_offset = unsafe { + atomic_i_increment::<_, { Scope::QueueFamily as u32 }, { Semantics::NONE.bits() }>( + bucket_size, + ) + }; + + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition. Bucket + // size upper bound is known statically to be [`MAX_BUCKET_SIZE`], so `bucket_offset` + // is also always within bounds. + unsafe { + buckets + .get_unchecked_mut(bucket_index as usize) + .get_unchecked_mut(bucket_offset as usize) + } + .write(PositionR { + position: Position::from_u32(metadatas_offset + index), + r, + }); + + positions[index as usize].write([left_position, right_position]); + + metadatas[index as usize].write(metadata); +} + +/// # Safety +/// `bucket_index` must be within range `0..REDUCED_MATCHES_COUNT`. `matches_count` elements in +/// `matches` must be initialized. `matches` must have valid pointers into left/right buckets and +/// `parent_metadatas`. +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +unsafe fn compute_f2_into_buckets( + local_invocation_id: u32, + left_bucket_index: u32, + left_bucket: &[PositionR; MAX_BUCKET_SIZE], + right_bucket: &[PositionR; MAX_BUCKET_SIZE], + matches_count: usize, + // TODO: `&[Match]` would have been nicer, but it currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + matches: &[MaybeUninit; MAX_BUCKET_SIZE], + bucket_sizes: &mut [u32; NUM_BUCKETS], + buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; NUM_BUCKETS], + positions: &mut [MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT], + metadatas: &mut [MaybeUninit; REDUCED_MATCHES_COUNT], + bucket_scratch: &mut [PositionR; REDUCED_BUCKET_SIZE], +) { + // Load the right bucket into shared memory for faster access + for bucket_offset in + (local_invocation_id as usize..REDUCED_BUCKET_SIZE).step_by(WORKGROUP_SIZE as usize) + { + bucket_scratch[bucket_offset] = right_bucket[bucket_offset]; + } + + workgroup_memory_barrier_with_group_sync(); + + let left_bucket_base = left_bucket_index * u32::from(PARAM_BC); + let metadatas_offset = left_bucket_index * REDUCED_MATCHES_COUNT as u32; + + const { + assert!(MAX_BUCKET_SIZE == WORKGROUP_SIZE as usize * 2); + } + // TODO: This should have been a loop, but register usage is too high, see: + // https://github.com/Rust-GPU/rust-gpu/issues/462 + // SAFETY: Guaranteed by function contract + unsafe { + if (local_invocation_id as usize) < matches_count { + compute_f2_into_buckets_inner( + local_invocation_id, + left_bucket_base, + metadatas_offset, + left_bucket, + matches, + bucket_sizes, + buckets, + positions, + metadatas, + bucket_scratch, + ); + } + if ((local_invocation_id + WORKGROUP_SIZE) as usize) < matches_count { + compute_f2_into_buckets_inner( + local_invocation_id + WORKGROUP_SIZE, + left_bucket_base, + metadatas_offset, + left_bucket, + matches, + bucket_sizes, + buckets, + positions, + metadatas, + bucket_scratch, + ); + } + } +} + +/// This is similar to `find_matches_and_compute_fn`, but it doesn't use any parent table metadata. +/// +/// Buckets need to be sorted by position afterward due to concurrent writes that do not have +/// deterministic order. Content of the bucket beyond the size specified in `bucket_sizes` is +/// undefined. +/// +/// # Safety +/// Must be called from [`WORKGROUP_SIZE`] threads. All buckets must contain valid positions. +#[spirv(compute(threads(256), entry_point_name = "find_matches_and_compute_f2"))] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +pub unsafe fn find_matches_and_compute_f2( + #[spirv(local_invocation_id)] local_invocation_id: UVec3, + #[spirv(workgroup_id)] workgroup_id: UVec3, + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] parent_buckets: &[[PositionR; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] bucket_sizes: &mut [u32; NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 2)] buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 3)] positions: &mut [[MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 4)] metadatas: &mut [[MaybeUninit; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(workgroup)] matches: &mut [MaybeUninit; MAX_BUCKET_SIZE], + #[spirv(workgroup)] shared: &mut FindMatchesAndComputeF2Shared, +) { + let local_invocation_id = local_invocation_id.x; + let workgroup_id = workgroup_id.x; + + let left_bucket_index = workgroup_id as usize; + let left_bucket = &parent_buckets[left_bucket_index]; + let right_bucket = &parent_buckets[left_bucket_index + 1]; + let positions = &mut positions[left_bucket_index]; + let metadatas = &mut metadatas[left_bucket_index]; + let left_bucket_index = left_bucket_index as u32; + + // TODO: Truncate buckets to reduced size here once it compiles: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + // SAFETY: Guaranteed by function contract + let matches_count = unsafe { + find_matches_in_buckets_impl( + local_invocation_id, + left_bucket_index, + left_bucket, + right_bucket, + matches, + &mut shared.find_matches_shared, + ) + }; + + // SAFETY: Guaranteed by function contract and call to `find_matches_in_buckets_impl` + unsafe { + compute_f2_into_buckets( + local_invocation_id, + left_bucket_index, + left_bucket, + right_bucket, + matches_count as usize, + matches, + bucket_sizes, + buckets, + positions, + metadatas, + &mut shared.bucket_scratch, + ); + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/find_matches_and_compute_f7.rs b/shared/ab-proof-of-space-gpu/src/shader/find_matches_and_compute_f7.rs new file mode 100644 index 00000000000..1a1fc311a43 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/find_matches_and_compute_f7.rs @@ -0,0 +1,369 @@ +use crate::shader::compute_fn::compute_fn_impl; +use crate::shader::constants::{ + K, MAX_BUCKET_SIZE, NUM_BUCKETS, NUM_MATCH_BUCKETS, NUM_S_BUCKETS, PARAM_BC, + REDUCED_BUCKET_SIZE, REDUCED_MATCHES_COUNT, +}; +use crate::shader::find_matches_in_buckets::{FindMatchesShared, find_matches_in_buckets_impl}; +#[cfg(target_arch = "spirv")] +use crate::shader::polyfills::ArrayIndexingPolyfill; +use crate::shader::types::{Match, Metadata, Position, PositionR, Y}; +use core::fmt; +use core::mem::MaybeUninit; +use spirv_std::arch::{atomic_i_increment, workgroup_memory_barrier_with_group_sync}; +use spirv_std::glam::UVec3; +use spirv_std::memory::{Scope, Semantics}; +use spirv_std::spirv; + +// TODO: Same number as hardcoded in `#[spirv(compute(threads(..)))]` below, can be removed once +// https://github.com/Rust-GPU/rust-gpu/discussions/287 is resolved +pub const WORKGROUP_SIZE: u32 = 256; +const TABLE_NUMBER: u8 = 7; +const PARENT_TABLE_NUMBER: u8 = 6; + +const _: () = { + assert!(crate::shader::find_matches_in_buckets::WORKGROUP_SIZE == WORKGROUP_SIZE); +}; + +const PROOFS_BUCKET_SIZE_UPPER_BOUND_SECURITY_BITS: u8 = 128; +/// Upper-bound estimation of the number of matched elements per s-bucket +pub const NUM_ELEMENTS_PER_S_BUCKET: usize = + proofs_bucket_upper_bound(PROOFS_BUCKET_SIZE_UPPER_BOUND_SECURITY_BITS) as usize; + +// TODO: This data structure is quite large, could be compressed to a single `u64`, saving 1/3 of +// space +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[repr(C)] +pub struct ProofTargets { + // Absolute position derived from bucket and match index + pub absolute_position: u32, + /// Left and right positions + pub positions: [Position; 2], +} + +/// Upper-bound estimation of the number of matched elements per s-bucket. +/// +/// Buckets are defined by the lower `NUM_S_BUCKETS.ilog2()` bits of the values. This is based on a +/// Chernoff bound for the Poisson distribution with mean `lambda = 1`, ensuring the probability +/// that any bucket exceeds the bound is less than `2^{-security_bits}`. The bound is +/// `lambda + ceil(sqrt(3 * lambda * (NUM_S_BUCKETS.ilog2() + security_bits) * ln(2)))`. +/// Accounts for the filter to values in `0..NUM_S_BUCKETS-1` by using the expected number of +/// remaining elements ~`NUM_S_BUCKETS`, distributed uniformly across all `NUM_S_BUCKETS` buckets. +const fn proofs_bucket_upper_bound(security_bits: u8) -> u64 { + // Lambda is the expected number of entries in a bucket: + // ~`NUM_S_BUCKETS / NUM_S_BUCKETS = 1` + const LAMBDA: u64 = 1; + // Approximation of ln(2) as a fraction: `ln(2) ≈ LN2_NUM / LN2_DEN`. + // This allows integer-only computation of the square root term involving ln(2). + const LN2_NUM: u128 = 693_147; + const LN2_DEN: u128 = 1_000_000; + + // `log2(NUM_S_BUCKETS) + security_bits` for the union bound over `NUM_S_BUCKETS` buckets + let ks = NUM_S_BUCKETS.ilog2() as u128 + security_bits as u128; + // Compute numerator for the expression under the square root: + // `3 * lambda * ks * LN2_NUM` + let num = 3u128 * LAMBDA as u128 * ks * LN2_NUM; + // Denominator for ln(2): `LN2_DEN` + let den = LN2_DEN; + + let ceil_div = num.div_ceil(den); + + // Binary search to find the smallest `x` such that `x * x >= ceil_div`, + // which computes `ceil(sqrt(num / den))` without floating-point. + // We use a custom binary search over `u64` range because binary search in the standard library + // operates on sorted slices, not directly on integer ranges for solving inequalities like this. + let mut low = 0u64; + let mut high = u64::MAX; + while low < high { + let mid = low + (high - low) / 2; + let left = (mid as u128) * (mid as u128); + if left >= ceil_div { + high = mid; + } else { + low = mid + 1; + } + } + let add_term = low; + + LAMBDA + add_term +} + +// TODO: Should be union, but it currently doesn't compile: +// https://github.com/Rust-GPU/rust-gpu/issues/241 +#[derive(Copy, Clone)] +pub struct FindMatchesAndComputeF7Shared { + find_matches_shared: FindMatchesShared, + bucket_scratch: [PositionR; REDUCED_BUCKET_SIZE], +} + +impl fmt::Debug for FindMatchesAndComputeF7Shared { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FindMatchesAndComputeF7Shared") + .finish_non_exhaustive() + } +} + +/// Subspace's little-endian s-bucket convention (duplicated in the CPU `ab-proof-of-space` chiapos): +/// maps a table-7 entry's `first_k_bits` to its s-bucket, returning a value `>= NUM_S_BUCKETS` for +/// entries whose low `K - 16` bits are set (the caller discards those). Consensus verification +/// derives the challenge from the s-bucket with the same little-endian byte layout, so this +/// convention is fixed. +#[inline(always)] +fn little_endian_s_bucket(first_k_bits: u32) -> u32 { + let low_bits = K as u32 - 16; + if first_k_bits & ((1 << low_bits) - 1) != 0 { + return u32::MAX; + } + let cs_lo = (first_k_bits >> (K as u32 - 8)) & 0xff; + let cs_hi = (first_k_bits >> low_bits) & 0xff; + cs_lo | (cs_hi << 8) +} + +/// # Safety +/// `bucket_index` must be within range `0..REDUCED_MATCHES_COUNT`. `matches_count` elements in +/// `matches` must be initialized, `matches` must have valid pointers into left/right buckets and +/// `parent_metadatas`. +#[inline(always)] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +unsafe fn compute_f7_into_buckets_inner( + index: u32, + left_bucket_base: u32, + absolute_position_base: u32, + left_bucket: &[PositionR; MAX_BUCKET_SIZE], + // TODO: `&[Match]` would have been nicer, but it currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + matches: &[MaybeUninit; MAX_BUCKET_SIZE], + // TODO: This should have been `&[[Metadata; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS]`, but + // it currently doesn't compile if flattened: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + table_6_proof_targets_sizes: &mut [u32; NUM_S_BUCKETS], + table_6_proof_targets: &mut [[MaybeUninit; NUM_ELEMENTS_PER_S_BUCKET]; + NUM_S_BUCKETS], + bucket_scratch: &[PositionR; REDUCED_BUCKET_SIZE], +) { + // SAFETY: Guaranteed by function contract + let (bucket_offset, r_target, positions_offset) = + unsafe { matches.get_unchecked(index as usize).assume_init() }.split(); + + // SAFETY: Guaranteed by function contract + let left_position_r = *unsafe { left_bucket.get_unchecked(bucket_offset as usize) }; + let left_position = left_position_r.position; + let left_r = left_position_r.r.get(); + + // Repurpose variable for two purposes to save on registers + let mut right_position_or_skip = positions_offset; + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + #[expect( + clippy::needless_range_loop, + reason = "Intentional workaround for rust-gpu" + )] + for offset in 0..REDUCED_BUCKET_SIZE { + let position_r = bucket_scratch[offset]; + if position_r.r.get() == r_target { + if right_position_or_skip == 0 { + right_position_or_skip = position_r.position; + break; + } + + right_position_or_skip -= 1; + } + } + let right_position = right_position_or_skip; + + // TODO: Correct version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + // let left_metadata = parent_metadatas[usize::from(left_position)]; + // let right_metadata = parent_metadatas[usize::from(right_position)]; + // SAFETY: Guaranteed by function contract + let left_metadata = *unsafe { parent_metadatas.get_unchecked(left_position as usize) }; + // SAFETY: Guaranteed by function contract + let right_metadata = *unsafe { parent_metadatas.get_unchecked(right_position as usize) }; + + let (y, _) = compute_fn_impl::( + Y::from(left_bucket_base + left_r), + left_metadata, + right_metadata, + ); + + let s_bucket = little_endian_s_bucket(y.first_k_bits()) as usize; + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + // let Some(bucket_count) = bucket_sizes.get_mut(s_bucket) else { + // continue; + // }; + if s_bucket >= NUM_S_BUCKETS { + return; + } + let bucket_size = &mut table_6_proof_targets_sizes[s_bucket]; + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + let bucket_offset = unsafe { + atomic_i_increment::<_, { Scope::QueueFamily as u32 }, { Semantics::NONE.bits() }>( + bucket_size, + ) + }; + + // TODO: Maybe store `absolute_position` separately from more densely populated positions, + // then per-s-bucket `atomic_u_min()` can be done to store the minimum pointer. This all + // will use less memory overall + // SAFETY: `s_bucket` is checked above to be correct. Bucket size upper bound is known + // statically to be [`NUM_ELEMENTS_PER_S_BUCKET`], so `bucket_offset` is also always within + // bounds. + unsafe { + table_6_proof_targets + .get_unchecked_mut(s_bucket) + .get_unchecked_mut(bucket_offset as usize) + } + .write(ProofTargets { + absolute_position: absolute_position_base + index, + positions: [left_position, right_position], + }); +} + +/// # Safety +/// `bucket_index` must be within range `0..REDUCED_MATCHES_COUNT`. `matches_count` elements in +/// `matches` must be initialized, `matches` must have valid pointers into left/right buckets and +/// `parent_metadatas`. +#[inline(always)] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +unsafe fn compute_f7_into_buckets( + local_invocation_id: u32, + left_bucket_index: u32, + left_bucket: &[PositionR; MAX_BUCKET_SIZE], + right_bucket: &[PositionR; MAX_BUCKET_SIZE], + matches_count: usize, + // TODO: `&[Match]` would have been nicer, but it currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + matches: &[MaybeUninit; MAX_BUCKET_SIZE], + // TODO: This should have been `&[[Metadata; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS]`, but + // it currently doesn't compile if flattened: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + table_6_proof_targets_sizes: &mut [u32; NUM_S_BUCKETS], + table_6_proof_targets: &mut [[MaybeUninit; NUM_ELEMENTS_PER_S_BUCKET]; + NUM_S_BUCKETS], + bucket_scratch: &mut [PositionR; REDUCED_BUCKET_SIZE], +) { + // Load the right bucket into shared memory for faster access + for bucket_offset in + (local_invocation_id as usize..REDUCED_BUCKET_SIZE).step_by(WORKGROUP_SIZE as usize) + { + bucket_scratch[bucket_offset] = right_bucket[bucket_offset]; + } + + workgroup_memory_barrier_with_group_sync(); + + let left_bucket_base = left_bucket_index * u32::from(PARAM_BC); + let absolute_position_base = left_bucket_index * REDUCED_MATCHES_COUNT as u32; + + const { + assert!(MAX_BUCKET_SIZE == WORKGROUP_SIZE as usize * 2); + } + // TODO: This should have been a loop, but register usage is too high, see: + // https://github.com/Rust-GPU/rust-gpu/issues/462 + // SAFETY: Guaranteed by function contract + unsafe { + if (local_invocation_id as usize) < matches_count { + compute_f7_into_buckets_inner( + local_invocation_id, + left_bucket_base, + absolute_position_base, + left_bucket, + matches, + parent_metadatas, + table_6_proof_targets_sizes, + table_6_proof_targets, + bucket_scratch, + ); + } + if ((local_invocation_id + WORKGROUP_SIZE) as usize) < matches_count { + compute_f7_into_buckets_inner( + local_invocation_id + WORKGROUP_SIZE, + left_bucket_base, + absolute_position_base, + left_bucket, + matches, + parent_metadatas, + table_6_proof_targets_sizes, + table_6_proof_targets, + bucket_scratch, + ); + } + } +} + +/// This is similar to `find_matches_and_compute_fn`, but it stores results in buckets grouped by +/// s-buckets, which is how proofs can later be found efficiently. +/// +/// Buckets need to be sorted by position afterward due to concurrent writes that do not have +/// deterministic order. Content of the bucket beyond the size specified in `bucket_sizes` is +/// undefined. +/// +/// # Safety +/// Must be called from [`WORKGROUP_SIZE`] threads. All buckets must contain valid positions. +#[spirv(compute(threads(256), entry_point_name = "find_matches_and_compute_f7"))] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +pub unsafe fn find_matches_and_compute_f7( + #[spirv(local_invocation_id)] local_invocation_id: UVec3, + #[spirv(workgroup_id)] workgroup_id: UVec3, + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] parent_buckets: &[[PositionR; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 2)] + table_6_proof_targets_sizes: &mut [u32; NUM_S_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 3)] + table_6_proof_targets: &mut [[MaybeUninit; NUM_ELEMENTS_PER_S_BUCKET]; + NUM_S_BUCKETS], + #[spirv(workgroup)] matches: &mut [MaybeUninit; MAX_BUCKET_SIZE], + #[spirv(workgroup)] shared: &mut FindMatchesAndComputeF7Shared, +) { + let local_invocation_id = local_invocation_id.x; + let workgroup_id = workgroup_id.x; + + let left_bucket_index = workgroup_id as usize; + let left_bucket = &parent_buckets[left_bucket_index]; + let right_bucket = &parent_buckets[left_bucket_index + 1]; + let left_bucket_index = left_bucket_index as u32; + + // TODO: Truncate buckets to reduced size here once it compiles: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + // SAFETY: Guaranteed by function contract + let matches_count = unsafe { + find_matches_in_buckets_impl( + local_invocation_id, + left_bucket_index, + left_bucket, + right_bucket, + matches, + &mut shared.find_matches_shared, + ) + }; + + // SAFETY: Guaranteed by function contract and call to `find_matches_in_buckets_impl` + unsafe { + compute_f7_into_buckets( + local_invocation_id, + left_bucket_index, + left_bucket, + right_bucket, + matches_count as usize, + matches, + parent_metadatas, + table_6_proof_targets_sizes, + table_6_proof_targets, + &mut shared.bucket_scratch, + ); + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/find_matches_and_compute_fn.rs b/shared/ab-proof-of-space-gpu/src/shader/find_matches_and_compute_fn.rs new file mode 100644 index 00000000000..62c8e6e630c --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/find_matches_and_compute_fn.rs @@ -0,0 +1,464 @@ +use crate::shader::compute_fn::compute_fn_impl; +use crate::shader::constants::{ + MAX_BUCKET_SIZE, NUM_BUCKETS, NUM_MATCH_BUCKETS, PARAM_BC, REDUCED_BUCKET_SIZE, + REDUCED_MATCHES_COUNT, +}; +use crate::shader::find_matches_in_buckets::{FindMatchesShared, find_matches_in_buckets_impl}; +#[cfg(target_arch = "spirv")] +use crate::shader::polyfills::ArrayIndexingPolyfill; +use crate::shader::types::{Match, Metadata, Position, PositionExt, PositionR, Y}; +use core::fmt; +use core::mem::MaybeUninit; +use spirv_std::arch::{atomic_i_increment, workgroup_memory_barrier_with_group_sync}; +use spirv_std::glam::UVec3; +use spirv_std::memory::{Scope, Semantics}; +use spirv_std::spirv; + +// TODO: Same number as hardcoded in `#[spirv(compute(threads(..)))]` below, can be removed once +// https://github.com/Rust-GPU/rust-gpu/discussions/287 is resolved +pub const WORKGROUP_SIZE: u32 = 256; + +const _: () = { + assert!(crate::shader::find_matches_in_buckets::WORKGROUP_SIZE == WORKGROUP_SIZE); +}; + +// TODO: Should be union, but it currently doesn't compile: +// https://github.com/Rust-GPU/rust-gpu/issues/241 +#[derive(Copy, Clone)] +pub struct FindMatchesAndComputeFnShared { + find_matches_shared: FindMatchesShared, + bucket_scratch: [PositionR; REDUCED_BUCKET_SIZE], +} + +impl fmt::Debug for FindMatchesAndComputeFnShared { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FindMatchesAndComputeFnShared") + .finish_non_exhaustive() + } +} + +/// # Safety +/// `bucket_index` must be within range `0..REDUCED_MATCHES_COUNT`. `matches_count` elements in +/// `matches` must be initialized, `matches` must have valid pointers into left/right buckets and +/// `parent_metadatas`. +#[inline(always)] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +unsafe fn compute_fn_into_buckets_inner( + index: u32, + left_bucket_base: u32, + metadatas_offset: u32, + left_bucket: &[PositionR; MAX_BUCKET_SIZE], + // TODO: `&[Match]` would have been nicer, but it currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + matches: &[MaybeUninit; MAX_BUCKET_SIZE], + // TODO: This should have been `&[[Metadata; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS]`, but + // it currently doesn't compile if flattened: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + bucket_sizes: &mut [u32; NUM_BUCKETS], + buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; NUM_BUCKETS], + positions: &mut [MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT], + metadatas: &mut [MaybeUninit; REDUCED_MATCHES_COUNT], + bucket_scratch: &[PositionR; REDUCED_BUCKET_SIZE], +) { + // SAFETY: Guaranteed by function contract + let (bucket_offset, r_target, positions_offset) = + unsafe { matches.get_unchecked(index as usize).assume_init() }.split(); + + // SAFETY: Guaranteed by function contract + let left_position_r = *unsafe { left_bucket.get_unchecked(bucket_offset as usize) }; + let left_position = left_position_r.position; + let left_r = left_position_r.r.get(); + + // Repurpose variable for two purposes to save on registers + let mut right_position_or_skip = positions_offset; + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + #[expect( + clippy::needless_range_loop, + reason = "Intentional workaround for rust-gpu" + )] + for offset in 0..REDUCED_BUCKET_SIZE { + let position_r = bucket_scratch[offset]; + if position_r.r.get() == r_target { + if right_position_or_skip == 0 { + right_position_or_skip = position_r.position; + break; + } + + right_position_or_skip -= 1; + } + } + let right_position = right_position_or_skip; + + // TODO: Correct version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + // let left_metadata = parent_metadatas[usize::from(left_position)]; + // let right_metadata = parent_metadatas[usize::from(right_position)]; + // SAFETY: Guaranteed by function contract + let left_metadata = *unsafe { parent_metadatas.get_unchecked(left_position as usize) }; + // SAFETY: Guaranteed by function contract + let right_metadata = *unsafe { parent_metadatas.get_unchecked(right_position as usize) }; + + let (y, metadata) = compute_fn_impl::( + Y::from(left_bucket_base + left_r), + left_metadata, + right_metadata, + ); + + let (bucket_index, r) = y.into_bucket_index_and_r(); + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition + let bucket_size = unsafe { bucket_sizes.get_unchecked_mut(bucket_index as usize) }; + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + let bucket_offset = unsafe { + atomic_i_increment::<_, { Scope::QueueFamily as u32 }, { Semantics::NONE.bits() }>( + bucket_size, + ) + }; + + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition. Bucket + // size upper bound is known statically to be [`MAX_BUCKET_SIZE`], so `bucket_offset` + // is also always within bounds. + unsafe { + buckets + .get_unchecked_mut(bucket_index as usize) + .get_unchecked_mut(bucket_offset as usize) + } + .write(PositionR { + position: Position::from_u32(metadatas_offset + index), + r, + }); + + positions[index as usize].write([left_position, right_position]); + + // The last table doesn't have any metadata + if TABLE_NUMBER < 7 { + metadatas[index as usize].write(metadata); + } +} + +/// # Safety +/// `bucket_index` must be within range `0..REDUCED_MATCHES_COUNT`. `matches_count` elements in +/// `matches` must be initialized, `matches` must have valid pointers into left/right buckets and +/// `parent_metadatas`. +#[inline(always)] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +unsafe fn compute_fn_into_buckets( + local_invocation_id: u32, + left_bucket_index: u32, + left_bucket: &[PositionR; MAX_BUCKET_SIZE], + right_bucket: &[PositionR; MAX_BUCKET_SIZE], + matches_count: usize, + // TODO: `&[Match]` would have been nicer, but it currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + matches: &[MaybeUninit; MAX_BUCKET_SIZE], + // TODO: This should have been `&[[Metadata; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS]`, but + // it currently doesn't compile if flattened: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + bucket_sizes: &mut [u32; NUM_BUCKETS], + buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; NUM_BUCKETS], + positions: &mut [MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT], + metadatas: &mut [MaybeUninit; REDUCED_MATCHES_COUNT], + bucket_scratch: &mut [PositionR; REDUCED_BUCKET_SIZE], +) { + // Load the right bucket into shared memory for faster access + for bucket_offset in + (local_invocation_id as usize..REDUCED_BUCKET_SIZE).step_by(WORKGROUP_SIZE as usize) + { + bucket_scratch[bucket_offset] = right_bucket[bucket_offset]; + } + + workgroup_memory_barrier_with_group_sync(); + + let left_bucket_base = left_bucket_index * u32::from(PARAM_BC); + let metadatas_offset = left_bucket_index * REDUCED_MATCHES_COUNT as u32; + + const { + assert!(MAX_BUCKET_SIZE == WORKGROUP_SIZE as usize * 2); + } + // TODO: This should have been a loop, but register usage is too high, see: + // https://github.com/Rust-GPU/rust-gpu/issues/462 + // SAFETY: Guaranteed by function contract + unsafe { + if (local_invocation_id as usize) < matches_count { + compute_fn_into_buckets_inner::( + local_invocation_id, + left_bucket_base, + metadatas_offset, + left_bucket, + matches, + parent_metadatas, + bucket_sizes, + buckets, + positions, + metadatas, + bucket_scratch, + ); + } + if ((local_invocation_id + WORKGROUP_SIZE) as usize) < matches_count { + compute_fn_into_buckets_inner::( + local_invocation_id + WORKGROUP_SIZE, + left_bucket_base, + metadatas_offset, + left_bucket, + matches, + parent_metadatas, + bucket_sizes, + buckets, + positions, + metadatas, + bucket_scratch, + ); + } + } +} + +/// # Safety +/// Must be called from [`WORKGROUP_SIZE`] threads. All buckets must contain valid positions. +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +pub unsafe fn find_matches_and_compute_fn( + local_invocation_id: UVec3, + workgroup_id: UVec3, + parent_buckets: &[[PositionR; MAX_BUCKET_SIZE]; NUM_BUCKETS], + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + bucket_sizes: &mut [u32; NUM_BUCKETS], + buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; NUM_BUCKETS], + positions: &mut [[MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS], + metadatas: &mut [[MaybeUninit; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS], + matches: &mut [MaybeUninit; MAX_BUCKET_SIZE], + shared: &mut FindMatchesAndComputeFnShared, +) { + let local_invocation_id = local_invocation_id.x; + let workgroup_id = workgroup_id.x; + + let left_bucket_index = workgroup_id as usize; + let left_bucket = &parent_buckets[left_bucket_index]; + let right_bucket = &parent_buckets[left_bucket_index + 1]; + let positions = &mut positions[left_bucket_index]; + let metadatas = &mut metadatas[left_bucket_index]; + let left_bucket_index = left_bucket_index as u32; + + // TODO: Truncate buckets to reduced size here once it compiles: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + // SAFETY: Guaranteed by function contract + let matches_count = unsafe { + find_matches_in_buckets_impl( + local_invocation_id, + left_bucket_index, + left_bucket, + right_bucket, + matches, + &mut shared.find_matches_shared, + ) + }; + + // SAFETY: Guaranteed by function contract and call to `find_matches_in_buckets_impl` + unsafe { + compute_fn_into_buckets::( + local_invocation_id, + left_bucket_index, + left_bucket, + right_bucket, + matches_count as usize, + matches, + parent_metadatas, + bucket_sizes, + buckets, + positions, + metadatas, + &mut shared.bucket_scratch, + ); + } +} + +/// Buckets need to be sorted by position afterward due to concurrent writes that do not have +/// deterministic order. Content of the bucket beyond the size specified in `bucket_sizes` is +/// undefined. +/// +/// # Safety +/// Must be called from [`WORKGROUP_SIZE`] threads. +#[spirv(compute(threads(256), entry_point_name = "find_matches_and_compute_f3"))] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +pub unsafe fn find_matches_and_compute_f3( + #[spirv(local_invocation_id)] local_invocation_id: UVec3, + #[spirv(workgroup_id)] workgroup_id: UVec3, + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] parent_buckets: &[[PositionR; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 2)] bucket_sizes: &mut [u32; NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 3)] buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 4)] positions: &mut [[MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 5)] metadatas: &mut [[MaybeUninit; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(workgroup)] matches: &mut [MaybeUninit; MAX_BUCKET_SIZE], + #[spirv(workgroup)] shared: &mut FindMatchesAndComputeFnShared, +) { + // SAFETY: Guaranteed by function contract + unsafe { + find_matches_and_compute_fn::<3, 2>( + local_invocation_id, + workgroup_id, + parent_buckets, + parent_metadatas, + bucket_sizes, + buckets, + positions, + metadatas, + matches, + shared, + ); + } +} + +/// Buckets need to be sorted by position afterward due to concurrent writes that do not have +/// deterministic order. Content of the bucket beyond the size specified in `bucket_sizes` is +/// undefined. +/// +/// # Safety +/// Must be called from [`WORKGROUP_SIZE`] threads. +#[spirv(compute(threads(256), entry_point_name = "find_matches_and_compute_f4"))] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +pub unsafe fn find_matches_and_compute_f4( + #[spirv(local_invocation_id)] local_invocation_id: UVec3, + #[spirv(workgroup_id)] workgroup_id: UVec3, + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] parent_buckets: &[[PositionR; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 2)] bucket_sizes: &mut [u32; NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 3)] buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 4)] positions: &mut [[MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 5)] metadatas: &mut [[MaybeUninit; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(workgroup)] matches: &mut [MaybeUninit; MAX_BUCKET_SIZE], + #[spirv(workgroup)] shared: &mut FindMatchesAndComputeFnShared, +) { + // SAFETY: Guaranteed by function contract + unsafe { + find_matches_and_compute_fn::<4, 3>( + local_invocation_id, + workgroup_id, + parent_buckets, + parent_metadatas, + bucket_sizes, + buckets, + positions, + metadatas, + matches, + shared, + ); + } +} + +/// Buckets need to be sorted by position afterward due to concurrent writes that do not have +/// deterministic order. Content of the bucket beyond the size specified in `bucket_sizes` is +/// undefined. +/// +/// # Safety +/// Must be called from [`WORKGROUP_SIZE`] threads. +#[spirv(compute(threads(256), entry_point_name = "find_matches_and_compute_f5"))] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +pub unsafe fn find_matches_and_compute_f5( + #[spirv(local_invocation_id)] local_invocation_id: UVec3, + #[spirv(workgroup_id)] workgroup_id: UVec3, + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] parent_buckets: &[[PositionR; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 2)] bucket_sizes: &mut [u32; NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 3)] buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 4)] positions: &mut [[MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 5)] metadatas: &mut [[MaybeUninit; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(workgroup)] matches: &mut [MaybeUninit; MAX_BUCKET_SIZE], + #[spirv(workgroup)] shared: &mut FindMatchesAndComputeFnShared, +) { + // SAFETY: Guaranteed by function contract + unsafe { + find_matches_and_compute_fn::<5, 4>( + local_invocation_id, + workgroup_id, + parent_buckets, + parent_metadatas, + bucket_sizes, + buckets, + positions, + metadatas, + matches, + shared, + ); + } +} + +/// Buckets need to be sorted by position afterward due to concurrent writes that do not have +/// deterministic order. Content of the bucket beyond the size specified in `bucket_sizes` is +/// undefined. +/// +/// # Safety +/// Must be called from [`WORKGROUP_SIZE`] threads. +#[spirv(compute(threads(256), entry_point_name = "find_matches_and_compute_f6"))] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +pub unsafe fn find_matches_and_compute_f6( + #[spirv(local_invocation_id)] local_invocation_id: UVec3, + #[spirv(workgroup_id)] workgroup_id: UVec3, + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] parent_buckets: &[[PositionR; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] + parent_metadatas: &[Metadata; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 2)] bucket_sizes: &mut [u32; NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 3)] buckets: &mut [[MaybeUninit; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 4)] positions: &mut [[MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 5)] metadatas: &mut [[MaybeUninit; REDUCED_MATCHES_COUNT]; + NUM_MATCH_BUCKETS], + #[spirv(workgroup)] matches: &mut [MaybeUninit; MAX_BUCKET_SIZE], + #[spirv(workgroup)] shared: &mut FindMatchesAndComputeFnShared, +) { + // SAFETY: Guaranteed by function contract + unsafe { + find_matches_and_compute_fn::<6, 5>( + local_invocation_id, + workgroup_id, + parent_buckets, + parent_metadatas, + bucket_sizes, + buckets, + positions, + metadatas, + matches, + shared, + ); + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/find_matches_in_buckets.rs b/shared/ab-proof-of-space-gpu/src/shader/find_matches_in_buckets.rs new file mode 100644 index 00000000000..516597b7bec --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/find_matches_in_buckets.rs @@ -0,0 +1,194 @@ +pub mod rmap; + +use crate::shader::constants::{ + MAX_BUCKET_SIZE, PARAM_B, PARAM_C, PARAM_M, REDUCED_BUCKET_SIZE, REDUCED_MATCHES_COUNT, +}; +use crate::shader::find_matches_in_buckets::rmap::Rmap; +use crate::shader::sort_buckets::sort_shared_bucket; +use crate::shader::types::{Match, Position, PositionExt, PositionR, R}; +use core::mem::MaybeUninit; +use spirv_std::arch::{atomic_i_add, workgroup_memory_barrier_with_group_sync}; +use spirv_std::glam::UVec3; +use spirv_std::memory::{Scope, Semantics}; +use spirv_std::spirv; + +// TODO: Same number as hardcoded in `#[spirv(compute(threads(..)))]` below, can be removed once +// https://github.com/Rust-GPU/rust-gpu/discussions/287 is resolved +pub const WORKGROUP_SIZE: u32 = 256; + +fn calculate_left_target_on_demand(parity: u32, r: u32, m: u32) -> u32 { + let param_b = u32::from(PARAM_B); + let param_c = u32::from(PARAM_C); + + ((r / param_c + m) % param_b) * param_c + (((2 * m + parity) * (2 * m + parity) + r) % param_c) +} + +#[derive(Debug, Copy, Clone)] +pub struct FindMatchesShared { + rmap: Rmap, + matches_counter: u32, +} + +// TODO: Reuse code from `ab-proof-of-space` after https://github.com/Rust-GPU/rust-gpu/pull/249 and +// https://github.com/Rust-GPU/rust-gpu/discussions/301 +/// Returns the number of matches found. +/// +/// # Safety +/// Must be called from [`WORKGROUP_SIZE`] threads with `local_invocation_id` corresponding to the +/// thread index. All buckets must contain valid positions. +// TODO: Try to reduce the `matches` size further by processing `left_bucket` in chunks (like halves +// for example) +#[inline(always)] +pub(super) unsafe fn find_matches_in_buckets_impl( + local_invocation_id: u32, + left_bucket_index: u32, + // TODO: These should use `REDUCED_BUCKET_SIZE`, but it currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + left_bucket: &[PositionR; MAX_BUCKET_SIZE], + right_bucket: &[PositionR; MAX_BUCKET_SIZE], + matches: &mut [MaybeUninit; MAX_BUCKET_SIZE], + shared: &mut FindMatchesShared, +) -> u32 { + let FindMatchesShared { + rmap, + matches_counter, + } = shared; + + for index in + (local_invocation_id as usize..REDUCED_BUCKET_SIZE).step_by(WORKGROUP_SIZE as usize) + { + let PositionR { position, r } = right_bucket[index]; + + // TODO: Wouldn't it make more sense to check the size here instead of sentinel? + if position == Position::SENTINEL { + break; + } + + rmap.add_with_data_parallel(r); + } + + workgroup_memory_barrier_with_group_sync(); + + let parity = left_bucket_index % 2; + + const CHUNK_SIZE: usize = WORKGROUP_SIZE as usize / PARAM_M as usize; + const { + // `CHUNK_SIZE` with `PARAM_M` must cover workgroup exactly + assert!(CHUNK_SIZE as u32 * PARAM_M as u32 == WORKGROUP_SIZE); + // The bucket size should be possible to iterate in exact chunks + assert!(REDUCED_BUCKET_SIZE.is_multiple_of(CHUNK_SIZE)); + } + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + for chunk_index in 0..REDUCED_BUCKET_SIZE / CHUNK_SIZE { + // First `PARAM_M` invocations in a workgroup process the first chunk index, next + // `PARAM_M` process the second chunk index and so on, with each chunk index corresponding + // to `PARAM_M` `r_target` values + let index_within_chunk = local_invocation_id as usize / PARAM_M as usize; + let bucket_offset = chunk_index * CHUNK_SIZE + index_within_chunk; + let PositionR { position, r } = left_bucket[bucket_offset]; + let left_r = r.get(); + + // TODO: Wouldn't it make more sense to check the size here instead of sentinel? + // Check if reached the end of the bucket + let (m, local_matches_count) = if position == Position::SENTINEL { + (Match::SENTINEL, 0) + } else { + let m = local_invocation_id % u32::from(PARAM_M); + let r_target = calculate_left_target_on_demand(parity, left_r, m); + + // SAFETY: Right targets are guaranteed to be within `0..PARAM_BC` range + let local_matches_count = rmap.num_r_items(unsafe { R::new(r_target) }); + + let m = if local_matches_count == 0 { + Match::SENTINEL + } else { + // SAFETY: `bucket_offset` is guaranteed to be within `0..MAX_BUCKET_SIZE` range, + // `m` is guaranteed to be within `0..PARAM_M` range, `r_target` is guaranteed to be + // within `0..PARAM_BC` range + + unsafe { Match::new(bucket_offset as u32, m, r_target) } + }; + + (m, local_matches_count) + }; + + if local_matches_count >= 1 { + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + let local_matches_offset = unsafe { + atomic_i_add::<_, { Scope::Workgroup as u32 }, { Semantics::NONE.bits() }>( + matches_counter, + local_matches_count, + ) + }; + matches[local_matches_offset as usize].write(m); + + if local_matches_count == 2 { + matches[local_matches_offset as usize + 1].write(m.second_second_position()); + } + } + } + + workgroup_memory_barrier_with_group_sync(); + + let matches_counter = *matches_counter; + + for index in ((matches_counter + local_invocation_id) as usize..MAX_BUCKET_SIZE) + .step_by(WORKGROUP_SIZE as usize) + { + matches[index].write(Match::SENTINEL); + } + + workgroup_memory_barrier_with_group_sync(); + + sort_shared_bucket(local_invocation_id, matches, |a, b| { + // SAFETY: Initialized above + unsafe { a.assume_init() }.cmp_key() <= unsafe { b.assume_init() }.cmp_key() + }); + + matches_counter.min(REDUCED_MATCHES_COUNT as u32) +} + +/// # Safety +/// Must be called from [`WORKGROUP_SIZE`] threads. All buckets must contain valid positions. +#[spirv(compute(threads(256), entry_point_name = "find_matches_in_buckets"))] +pub unsafe fn find_matches_in_buckets( + #[spirv(local_invocation_id)] local_invocation_id: UVec3, + #[spirv(workgroup_id)] workgroup_id: UVec3, + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] buckets: &[[PositionR; + MAX_BUCKET_SIZE]], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] matches: &mut [[MaybeUninit; + MAX_BUCKET_SIZE]], + #[spirv(storage_buffer, descriptor_set = 0, binding = 2)] matches_counts: &mut [MaybeUninit< + u32, + >], + #[spirv(workgroup)] shared: &mut FindMatchesShared, +) { + let local_invocation_id = local_invocation_id.x; + let workgroup_id = workgroup_id.x; + + let left_bucket_index = workgroup_id as usize; + if left_bucket_index >= buckets.len() - 1 { + return; + } + + let left_bucket = &buckets[left_bucket_index]; + let right_bucket = &buckets[left_bucket_index + 1]; + let matches = &mut matches[left_bucket_index]; + let matches_count = &mut matches_counts[left_bucket_index]; + + // TODO: Truncate buckets to reduced size here once it compiles: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + // SAFETY: Guaranteed by function contract + matches_count.write(unsafe { + find_matches_in_buckets_impl( + local_invocation_id, + left_bucket_index as u32, + left_bucket, + right_bucket, + matches, + shared, + ) + }); +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/find_matches_in_buckets/rmap.rs b/shared/ab-proof-of-space-gpu/src/shader/find_matches_in_buckets/rmap.rs new file mode 100644 index 00000000000..0cfc26491e8 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/find_matches_in_buckets/rmap.rs @@ -0,0 +1,67 @@ +use crate::shader::constants::PARAM_BC; +#[cfg(target_arch = "spirv")] +use crate::shader::polyfills::ArrayIndexingPolyfill; +use crate::shader::types::R; +use spirv_std::arch::atomic_or; +use spirv_std::memory::{Scope, Semantics}; + +const PRESENCE_FLAGS_WORDS: usize = (PARAM_BC as usize * 2).div_ceil(u32::BITS as usize); + +#[derive(Debug, Copy, Clone)] +pub(super) struct Rmap { + /// Store two bits per target, indicating whether matching elements are present (two at most) + presence_flags: [u32; PRESENCE_FLAGS_WORDS], +} + +impl Rmap { + /// Returns `0`, `1` or `2` depending on whether `r` was present and whether there were + /// duplicates. + /// + /// `0` means not present, `1` means exactly one value. `2` means two or more values, but + /// doesn't track how many exactly. + pub(super) fn num_r_items(&self, r_target: R) -> u32 { + let bit_position = r_target.get() * 2; + let word_offset = (bit_position / u32::BITS) as usize; + let bit_offset = bit_position % u32::BITS; + // SAFETY: `bit_position` is within bounds of presence flags + let word = *unsafe { self.presence_flags.get_unchecked(word_offset) }; + + ((word >> bit_offset) & 0b11).count_ones() + } + + pub(super) fn add_with_data_parallel(&mut self, r: R) { + let bit_position = r.get() * 2; + let word_offset = (bit_position / u32::BITS) as usize; + let bit_offset = bit_position % u32::BITS; + // SAFETY: `bit_position` is within bounds of presence flags + let word = unsafe { self.presence_flags.get_unchecked_mut(word_offset) }; + let mask = 1 << bit_offset; + + if cfg!(target_arch = "spirv") { + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + let prev_word_value = unsafe { + atomic_or::<_, { Scope::Workgroup as u32 }, { Semantics::NONE.bits() }>(word, mask) + }; + + if prev_word_value & mask != 0 { + // Bit was already set, so this is not the first such `r` value, set a flag + // indicating there was a duplicate + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + unsafe { + atomic_or::<_, { Scope::Workgroup as u32 }, { Semantics::NONE.bits() }>( + word, + mask << 1u8, + ); + } + } + } else if *word & mask != 0 { + // Bit was already set, so this is not the first such `r` value, set a flag + // indicating there was a duplicate + *word |= mask << 1u8; + } else { + *word |= mask; + } + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/find_proofs.rs b/shared/ab-proof-of-space-gpu/src/shader/find_proofs.rs new file mode 100644 index 00000000000..f3bc8c3af33 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/find_proofs.rs @@ -0,0 +1,687 @@ +use crate::shader::MIN_SUBGROUP_SIZE; +use crate::shader::constants::{ + K, NUM_MATCH_BUCKETS, NUM_S_BUCKETS, NUM_TABLES, REDUCED_MATCHES_COUNT, +}; +use crate::shader::find_matches_and_compute_f7::{NUM_ELEMENTS_PER_S_BUCKET, ProofTargets}; +use crate::shader::types::{Position, PositionExt}; +use core::mem::MaybeUninit; +use spirv_std::arch::{ + atomic_or, subgroup_ballot, subgroup_memory_barrier, subgroup_shuffle, subgroup_u_min, + workgroup_memory_barrier_with_group_sync, +}; +use spirv_std::glam::UVec3; +use spirv_std::memory::{Scope, Semantics}; +use spirv_std::spirv; +#[cfg(not(target_arch = "spirv"))] +use subspace_core_primitives::pos::PosProof; + +// TODO: Same number as hardcoded in `#[spirv(compute(threads(..)))]` below, can be removed once +// https://github.com/Rust-GPU/rust-gpu/discussions/287 is resolved +pub const WORKGROUP_SIZE: u32 = 256; +const PROOF_X_SOURCES: usize = 2usize.pow(NUM_TABLES as u32 - 1); +const PROOF_BITS: usize = PROOF_X_SOURCES * K as usize; +const PROOF_BYTES: usize = PROOF_BITS.div_ceil(u8::BITS as usize); +pub const PROOF_U32_WORDS: usize = PROOF_BYTES.div_ceil(size_of::()); +pub const FOUND_PROOFS_U32_WORDS: usize = { + assert!(NUM_S_BUCKETS.is_multiple_of(u32::BITS as usize)); + + NUM_S_BUCKETS / u32::BITS as usize +}; + +#[derive(Debug, Copy, Clone)] +#[repr(C)] +pub struct Proofs { + found_proofs: [MaybeUninit; FOUND_PROOFS_U32_WORDS], + // TODO: Calculate bit mask for proofs found upfront and reduce the size here to just + // `NUM_CHUNKS` + proofs: [MaybeUninit; PROOF_U32_WORDS * NUM_S_BUCKETS], +} + +// This is equivalent to the above but used for interpretation by the host +#[derive(Debug, Copy, Clone)] +#[cfg(not(target_arch = "spirv"))] +#[repr(C)] +pub struct ProofsHost { + // TODO: Would have been nice to avoid filtering-out on the host + /// S-buckets at which proofs were found, there will be more than `Record::NUM_CHUNKS` proofs + /// here, needs to be filtered-out by the host + pub found_proofs: [u8; NUM_S_BUCKETS / u8::BITS as usize], + // TODO: Calculate bit mask for proofs found upfront and reduce the size here to just + // `NUM_CHUNKS` + /// All proofs, those that correspond to set bits of `found_proofs` exist + pub proofs: [PosProof; NUM_S_BUCKETS], +} + +#[cfg(not(target_arch = "spirv"))] +const _: () = { + assert!(size_of::() == size_of::()); +}; + +// TODO: Optimize this for various cases like when all buckets fit into subgroup size, when subgroup +// size is large enough to process multiple buckets at once (especially since buckets often are +// less than 16 elements, meaning AMD GPUs can process 4 at once) with clustered subgroup +// operations, etc. +// TODO: Make unsafe and avoid bounds check +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +#[inline(always)] +fn find_local_proof_targets( + local_invocation_id: u32, + subgroup_id: u32, + subgroup_local_invocation_id: u32, + positions_group_index: u32, + bucket_sizes: &mut [u32; NUM_S_BUCKETS], + buckets: &[[ProofTargets; NUM_ELEMENTS_PER_S_BUCKET]; NUM_S_BUCKETS], + found_proofs: &mut [MaybeUninit; FOUND_PROOFS_U32_WORDS], + found_proofs_scratch: &mut [MaybeUninit; (WORKGROUP_SIZE / u32::BITS) as usize], +) -> [Position; 2] { + let local_invocation_id = local_invocation_id as usize; + let base = positions_group_index * SUBGROUP_SIZE; + + let mut min = [Position::SENTINEL; 2]; + + let local_bucket_size = { + let bucket_id = base + subgroup_local_invocation_id; + + let local_bucket_size = bucket_sizes[bucket_id as usize]; + bucket_sizes[bucket_id as usize] = 0; + + local_bucket_size + }; + + for local_bucket_id in 0..SUBGROUP_SIZE { + let bucket_id = (base + local_bucket_id) as usize; + let bucket_size = subgroup_shuffle(local_bucket_size, local_bucket_id); + let bucket = &buckets[bucket_id]; + + // TODO: Can't use the struct due to this issue in Naga: + // https://github.com/gfx-rs/wgpu/issues/8389#issuecomment-3430788603 + // let mut local_min = ProofTargets { + // absolute_position: u32::MAX, + // positions: [Position::SENTINEL; 2], + // }; + let mut local_min_absolute_position = [u32::MAX]; + let mut local_min_positions = [Position::SENTINEL; 2]; + + for index in (subgroup_local_invocation_id..bucket_size).step_by(SUBGROUP_SIZE as usize) { + let proof_targets = bucket[index as usize]; + if proof_targets.absolute_position < local_min_absolute_position[0] { + local_min_absolute_position[0] = proof_targets.absolute_position; + local_min_positions = proof_targets.positions; + } + } + + let min_absolute_position = subgroup_u_min(local_min_absolute_position[0]); + let source_lane_mask = + subgroup_ballot(local_min_absolute_position[0] == min_absolute_position); + // TODO: This intrinsic is not supported by `wgpu` yet: + // https://github.com/gfx-rs/wgpu/issues/8403 + // let source_lane = subgroup_ballot_find_lsb(source_lane_mask); + let source_lane_mask = source_lane_mask.to_array(); + let mut source_lane = u32::MAX; + for i in 0..SUBGROUP_SIZE.div_ceil(u32::BITS) { + let word = source_lane_mask[i as usize]; + if word != 0 { + source_lane = word.trailing_zeros() + i * u32::BITS; + break; + } + } + let local_min = subgroup_shuffle(local_min_positions, source_lane); + + if subgroup_local_invocation_id == local_bucket_id { + min = local_min; + } + } + + let has_proof = local_bucket_size > 0; + let found_proofs_words = subgroup_ballot(has_proof); + + if SUBGROUP_SIZE >= u32::BITS { + // For subgroup sizes that are multiple of `u32` words, results can be written directly into + // global memory + // TODO: should have been `subgroup_elect()`, but it is not implemented in `wgpu` yet: + // https://github.com/gfx-rs/wgpu/issues/5555 + if subgroup_local_invocation_id == 0 { + let start_word = (base / u32::BITS) as usize; + found_proofs[start_word].write(found_proofs_words.x); + + if SUBGROUP_SIZE >= 2 * u32::BITS { + found_proofs[start_word + 1].write(found_proofs_words.y); + + if SUBGROUP_SIZE >= 4 * u32::BITS { + found_proofs[start_word + 2].write(found_proofs_words.z); + found_proofs[start_word + 3].write(found_proofs_words.w); + } + } + } + } else { + if local_invocation_id < found_proofs_scratch.len() { + found_proofs_scratch[local_invocation_id].write(0); + } + + workgroup_memory_barrier_with_group_sync(); + + // For subgroups of smaller sizes aggregate results in shared memory first, then write to + // global memory + // TODO: should have been `subgroup_elect()`, but it is not implemented in `wgpu` yet: + // https://github.com/gfx-rs/wgpu/issues/5555 + if subgroup_local_invocation_id == 0 { + let local_start_bit = subgroup_id * SUBGROUP_SIZE; + let local_word_index = (local_start_bit / u32::BITS) as usize; + let local_word_shift = local_start_bit % u32::BITS; + + // SAFETY: Initialized above + let found_proofs_word = + unsafe { found_proofs_scratch[local_word_index].assume_init_mut() }; + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + unsafe { + atomic_or::<_, { Scope::Workgroup as u32 }, { Semantics::NONE.bits() }>( + found_proofs_word, + found_proofs_words.x << local_word_shift, + ); + } + } + + workgroup_memory_barrier_with_group_sync(); + + if local_invocation_id < found_proofs_scratch.len() { + let workgroup_base_group_index = positions_group_index - subgroup_id; + let workgroup_start_bucket = workgroup_base_group_index * SUBGROUP_SIZE; + let global_start_word = (workgroup_start_bucket / u32::BITS) as usize; + + // SAFETY: Initialized above + let found_proofs_word = + unsafe { found_proofs_scratch[local_invocation_id].assume_init() }; + found_proofs[global_start_word + local_invocation_id].write(found_proofs_word); + } + } + + min +} + +// TODO: Make unsafe and avoid bounds check +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +fn find_proofs_impl( + local_invocation_id: u32, + subgroup_id: u32, + subgroup_local_invocation_id: u32, + positions_group_index: u32, + // TODO: This should have been `&[[[Position; 2]; REDUCED_MATCHES_COUNT]; NUM_MATCH_BUCKETS]`, + // but it currently doesn't compile if flattened: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + table_2_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + table_3_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + table_4_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + table_5_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + table_6_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + bucket_sizes: &mut [u32; NUM_S_BUCKETS], + buckets: &[[ProofTargets; NUM_ELEMENTS_PER_S_BUCKET]; NUM_S_BUCKETS], + found_proofs: &mut [MaybeUninit; FOUND_PROOFS_U32_WORDS], + // TODO: This should have been `&mut [[MaybeUninit; PROOF_U32_WORDS]; NUM_S_BUCKETS]`, + // but it currently doesn't compile if flattened: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + proofs: &mut [MaybeUninit; PROOF_U32_WORDS * NUM_S_BUCKETS], + found_proofs_scratch: &mut [MaybeUninit; (WORKGROUP_SIZE / u32::BITS) as usize], +) where + [(); PROOF_X_SOURCES.div_ceil(SUBGROUP_SIZE as usize)]:, + [(); PROOF_U32_WORDS.div_ceil(SUBGROUP_SIZE as usize)]:, +{ + let table_6_proof_targets = find_local_proof_targets::( + local_invocation_id, + subgroup_id, + subgroup_local_invocation_id, + positions_group_index, + bucket_sizes, + buckets, + found_proofs, + found_proofs_scratch, + ); + + // TODO: This proof zeroing will not be needed once proofs are assembled in registers and no + // longer use atomic writes + // Zero the proofs range with coalesced writes + { + let positions_group_words = SUBGROUP_SIZE as usize * PROOF_U32_WORDS; + let base_word = positions_group_index as usize * positions_group_words; + + for offset in (subgroup_local_invocation_id as usize..positions_group_words) + .step_by(SUBGROUP_SIZE as usize) + { + proofs[base_word + offset].write(0); + } + + subgroup_memory_barrier(); + } + + // `0` for left `1` for right + let left_right = (subgroup_local_invocation_id % 2) as usize; + // Otherwise `left_right` will not work as expected + const { + assert!(MIN_SUBGROUP_SIZE >= 2); + } + + // `chunk_index` is used to emulate `for _ in 0..2` loops, while using a single variable for + // tracking the progress instead of a separate variable for each loop + let mut chunk_index = 0u32; + // Reading positions from table 6 + loop { + let table_6_proof_targets = subgroup_shuffle( + table_6_proof_targets, + SUBGROUP_SIZE / 2 * (chunk_index & 1) + subgroup_local_invocation_id / 2, + ); + let table_6_proof_target = table_6_proof_targets[left_right]; + + let table_5_proof_targets = if table_6_proof_target == Position::SENTINEL { + [Position::SENTINEL; 2] + } else { + table_6_positions[table_6_proof_target as usize] + }; + + // Reading positions from table 5 + chunk_index <<= 1u8; + loop { + let table_5_proof_targets = subgroup_shuffle( + table_5_proof_targets, + SUBGROUP_SIZE / 2 * (chunk_index & 1) + subgroup_local_invocation_id / 2, + ); + let table_5_proof_target = table_5_proof_targets[left_right]; + + let table_4_proof_targets = if table_5_proof_target == Position::SENTINEL { + [Position::SENTINEL; 2] + } else { + table_5_positions[table_5_proof_target as usize] + }; + + // Reading positions from table 4 + chunk_index <<= 1u8; + loop { + let table_4_proof_targets = subgroup_shuffle( + table_4_proof_targets, + SUBGROUP_SIZE / 2 * (chunk_index & 1) + subgroup_local_invocation_id / 2, + ); + let table_4_proof_target = table_4_proof_targets[left_right]; + + let table_3_proof_targets = if table_4_proof_target == Position::SENTINEL { + [Position::SENTINEL; 2] + } else { + table_4_positions[table_4_proof_target as usize] + }; + + // Reading positions from table 3 + chunk_index <<= 1u8; + loop { + let table_3_proof_targets = subgroup_shuffle( + table_3_proof_targets, + SUBGROUP_SIZE / 2 * (chunk_index & 1) + subgroup_local_invocation_id / 2, + ); + let table_3_proof_target = table_3_proof_targets[left_right]; + + let table_2_proof_targets = if table_3_proof_target == Position::SENTINEL { + [Position::SENTINEL; 2] + } else { + table_3_positions[table_3_proof_target as usize] + }; + + // Reading positions from table 2 + chunk_index <<= 1u8; + loop { + let table_2_proof_targets = subgroup_shuffle( + table_2_proof_targets, + SUBGROUP_SIZE / 2 * (chunk_index & 1) + + subgroup_local_invocation_id / 2, + ); + let table_2_proof_target = table_2_proof_targets[left_right]; + + let [x_left, x_right] = if table_2_proof_target == Position::SENTINEL { + [Position::SENTINEL; 2] + } else { + table_2_positions[table_2_proof_target as usize] + }; + + let global_x_left_offset = + subgroup_local_invocation_id * 2 + chunk_index * SUBGROUP_SIZE * 2; + let group_proof_index = global_x_left_offset / PROOF_X_SOURCES as u32; + let x_left_offset = global_x_left_offset % PROOF_X_SOURCES as u32; + let global_proof_index = + positions_group_index * SUBGROUP_SIZE + group_proof_index; + + let proof_base = global_proof_index as usize * PROOF_U32_WORDS; + let first_proof_word_index = + ((u32::from(K) * x_left_offset) / u32::BITS) as usize; + + // TODO: Writes below can be optimized by building the full proof into the + // registers first and only write final result without atomics to global + // memory + + let mut local_proof_words = [0u32; 3]; + let x_left_offset_in_bits = u32::from(K) * x_left_offset; + { + let bit_offset = x_left_offset_in_bits % u32::BITS; + let x_shifted_to_start = x_left << (u32::BITS - u32::from(K)); + + let first_word = x_shifted_to_start >> bit_offset; + let second_word = + x_shifted_to_start.unbounded_shl(u32::BITS - bit_offset); + + local_proof_words[0] = first_word; + local_proof_words[1] = second_word; + } + + let max_local_proof_word_index = { + let x_right_offset_in_bits = x_left_offset_in_bits + u32::from(K); + let local_proof_words_index = (x_right_offset_in_bits / u32::BITS) + as usize + - first_proof_word_index; + let bit_offset = x_right_offset_in_bits % u32::BITS; + let x_shifted_to_start = x_right << (u32::BITS - u32::from(K)); + + let first_word = x_shifted_to_start >> bit_offset; + let second_word = + x_shifted_to_start.unbounded_shl(u32::BITS - bit_offset); + + local_proof_words[local_proof_words_index] |= first_word; + local_proof_words[local_proof_words_index + 1] = second_word; + + local_proof_words_index + usize::from(second_word != 0) + }; + + // The first word is written unconditionally + { + // SAFETY: The whole proof is initialized at the beginning of the + // function + let word = unsafe { + proofs[proof_base + first_proof_word_index].assume_init_mut() + }; + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + unsafe { + atomic_or::< + _, + { Scope::Subgroup as u32 }, + { Semantics::NONE.bits() }, + >( + word, local_proof_words[0].to_be() + ); + } + } + // Process remaining words, the loop is unrolled to save vector registers + if max_local_proof_word_index > 0 { + // SAFETY: The whole proof is initialized at the beginning of the + // function + let word = unsafe { + proofs[proof_base + first_proof_word_index + 1].assume_init_mut() + }; + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + unsafe { + atomic_or::< + _, + { Scope::Subgroup as u32 }, + { Semantics::NONE.bits() }, + >( + word, local_proof_words[1].to_be() + ); + } + } + if max_local_proof_word_index > 1 { + // SAFETY: The whole proof is initialized at the beginning of the + // function + let word = unsafe { + proofs[proof_base + first_proof_word_index + 2].assume_init_mut() + }; + // SAFETY: TODO: Probably should not be unsafe to begin with: + // https://github.com/Rust-GPU/rust-gpu/pull/394#issuecomment-3316594485 + unsafe { + atomic_or::< + _, + { Scope::Subgroup as u32 }, + { Semantics::NONE.bits() }, + >( + word, local_proof_words[2].to_be() + ); + } + } + + if chunk_index & 1 == 1 { + break; + } + chunk_index += 1; + } + chunk_index >>= 1u8; + + if chunk_index & 1 == 1 { + break; + } + chunk_index += 1; + } + chunk_index >>= 1u8; + + if chunk_index & 1 == 1 { + break; + } + chunk_index += 1; + } + chunk_index >>= 1u8; + + if chunk_index & 1 == 1 { + break; + } + chunk_index += 1; + } + chunk_index >>= 1u8; + + if chunk_index & 1 == 1 { + break; + } + chunk_index += 1; + } +} + +/// NOTE: bucket sizes are zeroed after use +// TODO: Maybe split `found_proofs` and `proofs` searching into separate shaders, such that less +// compute is wasted on searching proofs overall (right now up to half of the compute is wasted +// when computing proofs). It'll also be easier to add hashing after proof computation that way. +#[spirv(compute(threads(256), entry_point_name = "find_proofs"))] +#[expect( + clippy::too_many_arguments, + reason = "Both I/O and Vulkan stuff together take a lot of arguments" +)] +pub fn find_proofs( + #[spirv(workgroup_id)] workgroup_id: UVec3, + #[spirv(local_invocation_id)] local_invocation_id: UVec3, + #[spirv(subgroup_id)] subgroup_id: u32, + #[spirv(subgroup_size)] subgroup_size: u32, + #[spirv(num_subgroups)] num_subgroups: u32, + #[spirv(subgroup_local_invocation_id)] subgroup_local_invocation_id: u32, + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] + table_2_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] + table_3_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 2)] + table_4_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 3)] + table_5_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 4)] + table_6_positions: &[[Position; 2]; REDUCED_MATCHES_COUNT * NUM_MATCH_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 5)] bucket_sizes: &mut [u32; + NUM_S_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 6)] buckets: &[[ProofTargets; NUM_ELEMENTS_PER_S_BUCKET]; + NUM_S_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 7)] proofs: &mut Proofs, + #[spirv(workgroup)] found_proofs_scratch: &mut [MaybeUninit; + (WORKGROUP_SIZE / u32::BITS) as usize], +) { + let local_invocation_id = local_invocation_id.x; + let workgroup_id = workgroup_id.x; + + let global_subgroup_id = workgroup_id * num_subgroups + subgroup_id; + + let positions_group_index = global_subgroup_id; + // Specify some common subgroup sizes so the driver can easily eliminate dead code. This is + // important because `local_words` inside the function is generic and impacts the number of + // registers used, so we want to minimize them. + match subgroup_size { + // Hypothetically possible + 1 => { + find_proofs_impl::<1>( + local_invocation_id, + subgroup_id, + subgroup_local_invocation_id, + positions_group_index, + table_2_positions, + table_3_positions, + table_4_positions, + table_5_positions, + table_6_positions, + bucket_sizes, + buckets, + &mut proofs.found_proofs, + &mut proofs.proofs, + found_proofs_scratch, + ); + } + // Hypothetically possible + 2 => { + find_proofs_impl::<2>( + local_invocation_id, + subgroup_id, + subgroup_local_invocation_id, + positions_group_index, + table_2_positions, + table_3_positions, + table_4_positions, + table_5_positions, + table_6_positions, + bucket_sizes, + buckets, + &mut proofs.found_proofs, + &mut proofs.proofs, + found_proofs_scratch, + ); + } + // LLVMpipe (Mesa 24, SSE) + 4 => { + find_proofs_impl::<4>( + local_invocation_id, + subgroup_id, + subgroup_local_invocation_id, + positions_group_index, + table_2_positions, + table_3_positions, + table_4_positions, + table_5_positions, + table_6_positions, + bucket_sizes, + buckets, + &mut proofs.found_proofs, + &mut proofs.proofs, + found_proofs_scratch, + ); + } + // LLVMpipe (Mesa 25, AVX/AVX2) + 8 => { + find_proofs_impl::<8>( + local_invocation_id, + subgroup_id, + subgroup_local_invocation_id, + positions_group_index, + table_2_positions, + table_3_positions, + table_4_positions, + table_5_positions, + table_6_positions, + bucket_sizes, + buckets, + &mut proofs.found_proofs, + &mut proofs.proofs, + found_proofs_scratch, + ); + } + // Raspberry PI 5 + 16 => { + find_proofs_impl::<16>( + local_invocation_id, + subgroup_id, + subgroup_local_invocation_id, + positions_group_index, + table_2_positions, + table_3_positions, + table_4_positions, + table_5_positions, + table_6_positions, + bucket_sizes, + buckets, + &mut proofs.found_proofs, + &mut proofs.proofs, + found_proofs_scratch, + ); + } + // Intel/Nvidia + 32 => { + find_proofs_impl::<32>( + local_invocation_id, + subgroup_id, + subgroup_local_invocation_id, + positions_group_index, + table_2_positions, + table_3_positions, + table_4_positions, + table_5_positions, + table_6_positions, + bucket_sizes, + buckets, + &mut proofs.found_proofs, + &mut proofs.proofs, + found_proofs_scratch, + ); + } + // AMD + 64 => { + find_proofs_impl::<64>( + local_invocation_id, + subgroup_id, + subgroup_local_invocation_id, + positions_group_index, + table_2_positions, + table_3_positions, + table_4_positions, + table_5_positions, + table_6_positions, + bucket_sizes, + buckets, + &mut proofs.found_proofs, + &mut proofs.proofs, + found_proofs_scratch, + ); + } + // Hypothetically possible + 128 => { + find_proofs_impl::<128>( + local_invocation_id, + subgroup_id, + subgroup_local_invocation_id, + positions_group_index, + table_2_positions, + table_3_positions, + table_4_positions, + table_5_positions, + table_6_positions, + bucket_sizes, + buckets, + &mut proofs.found_proofs, + &mut proofs.proofs, + found_proofs_scratch, + ); + } + _ => { + // https://registry.khronos.org/vulkan/specs/latest/man/html/SubgroupSize.html + unreachable!("All Vulkan targets use power of two and subgroup size <= 128") + } + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/polyfills.rs b/shared/ab-proof-of-space-gpu/src/shader/polyfills.rs new file mode 100644 index 00000000000..896e8678d9d --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/polyfills.rs @@ -0,0 +1,27 @@ +#[cfg(target_arch = "spirv")] +use spirv_std::arch::IndexUnchecked; + +// TODO: This is a polyfill to work around for this issue: +// https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 +#[cfg(target_arch = "spirv")] +pub(super) trait ArrayIndexingPolyfill { + /// The same as [`<[T]>::get_unchecked()`] + unsafe fn get_unchecked(&self, index: usize) -> &T; + /// The same as [`<[T]>::get_unchecked_mut()`] + unsafe fn get_unchecked_mut(&mut self, index: usize) -> &mut T; +} + +#[cfg(target_arch = "spirv")] +impl ArrayIndexingPolyfill for [T; N] { + #[inline(always)] + unsafe fn get_unchecked(&self, index: usize) -> &T { + // SAFETY: Explicitly unchecked + unsafe { self.index_unchecked(index) } + } + + #[inline(always)] + unsafe fn get_unchecked_mut(&mut self, index: usize) -> &mut T { + // SAFETY: Explicitly unchecked + unsafe { self.index_unchecked_mut(index) } + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/shader_bytes.rs b/shared/ab-proof-of-space-gpu/src/shader/shader_bytes.rs new file mode 100644 index 00000000000..5bc6885c264 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/shader_bytes.rs @@ -0,0 +1,30 @@ +use std::borrow::Cow; +use std::slice; +use wgpu::{ShaderModuleDescriptor, ShaderSource}; + +#[repr(align(4))] +pub(super) struct ShaderBytes(pub(super) T) +where + T: ?Sized; + +#[cfg(not(target_arch = "spirv"))] +impl ShaderBytes<[u8]> { + pub(super) const fn to_module(&self) -> ShaderModuleDescriptor<'_> { + assert!(align_of_val(self) == align_of::()); + let shader_bytes = &self.0; + + // SAFETY: Correctly aligned, all bit patterns are valid, lifetime is static before and + // after + let shader_bytes = unsafe { + slice::from_raw_parts( + shader_bytes.as_ptr().cast::(), + shader_bytes.len() / size_of::(), + ) + }; + + ShaderModuleDescriptor { + label: Some(env!("CARGO_PKG_NAME")), + source: ShaderSource::SpirV(Cow::Borrowed(shader_bytes)), + } + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/sort_buckets.rs b/shared/ab-proof-of-space-gpu/src/shader/sort_buckets.rs new file mode 100644 index 00000000000..218de81b330 --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/sort_buckets.rs @@ -0,0 +1,156 @@ +use crate::shader::constants::{MAX_BUCKET_SIZE, NUM_BUCKETS}; +use crate::shader::types::PositionR; +use spirv_std::arch::workgroup_memory_barrier_with_group_sync; +use spirv_std::glam::UVec3; +use spirv_std::spirv; + +// TODO: Same number as hardcoded in `#[spirv(compute(threads(..)))]` below, can be removed once +// https://github.com/Rust-GPU/rust-gpu/discussions/287 is resolved +pub const WORKGROUP_SIZE: u32 = 256; + +const _: () = { + // This implementation assumes the following is true + assert!(MAX_BUCKET_SIZE == WORKGROUP_SIZE as usize * 2); +}; + +#[inline(always)] +fn perform_compare_swap( + local_invocation_id: u32, + bit_position: u32, + block_size: usize, + shared_bucket: &mut [T; MAX_BUCKET_SIZE], + // TODO: Should have been just `fn()`, but https://github.com/Rust-GPU/rust-gpu/issues/452 + less_or_equal: LessOrEqual, +) where + T: Copy, + LessOrEqual: Fn(&T, &T) -> bool, +{ + // Take a pair `(a_offset, b_offset)` where indices differ only at `bit_position` and swaps them + let pair_id = local_invocation_id as usize; + // Bits above `bit_position` + let high = (pair_id >> bit_position) << (bit_position + 1); + // Bits below `bit_position` + let low = pair_id & (u32::MAX as usize).unbounded_shr(u32::BITS - bit_position); + let a_offset = high | low; + let b_offset = a_offset | (1 << bit_position); + + // Determine the sort direction: ascending if `a_offset`'s bit at `block_size` is `0`. + // This alternates direction in bitonic merges to create sorted sequences. + let ascending = (a_offset & block_size) == 0; + + let a = shared_bucket[a_offset]; + let b = shared_bucket[b_offset]; + + // Only update when necessary + if less_or_equal(&a, &b) != ascending { + shared_bucket[a_offset] = b; + shared_bucket[b_offset] = a; + } +} + +#[inline(always)] +pub(super) fn sort_shared_bucket( + local_invocation_id: u32, + shared_bucket: &mut [T; MAX_BUCKET_SIZE], + // TODO: Should have been just `fn()`, but https://github.com/Rust-GPU/rust-gpu/issues/452 + less_or_equal: LessOrEqual, +) where + T: Copy, + LessOrEqual: Fn(&T, &T) -> bool, +{ + // Iterate over merger stages, doubling block_size each time + for merger_stage in 1..=MAX_BUCKET_SIZE.ilog2() { + let block_size = 1 << merger_stage; + // For each stage, process bit positions in reverse for bitonic comparisons + for bit_position in (0..merger_stage).rev() { + perform_compare_swap( + local_invocation_id, + bit_position, + block_size, + shared_bucket, + &less_or_equal, + ); + + workgroup_memory_barrier_with_group_sync(); + } + } +} + +#[inline(always)] +fn load_into_shared_bucket( + local_invocation_id: u32, + bucket_size: u32, + bucket: &[PositionR; MAX_BUCKET_SIZE], + shared_bucket: &mut [PositionR; MAX_BUCKET_SIZE], +) { + for bucket_offset in + (local_invocation_id as usize..MAX_BUCKET_SIZE).step_by(WORKGROUP_SIZE as usize) + { + shared_bucket[bucket_offset] = if bucket_offset < bucket_size as usize { + bucket[bucket_offset] + } else { + PositionR::SENTINEL + }; + } +} + +#[inline(always)] +fn store_from_shared_bucket( + local_invocation_id: u32, + bucket: &mut [PositionR; MAX_BUCKET_SIZE], + shared_bucket: &[PositionR; MAX_BUCKET_SIZE], +) { + for bucket_offset in + (local_invocation_id as usize..MAX_BUCKET_SIZE).step_by(WORKGROUP_SIZE as usize) + { + bucket[bucket_offset] = shared_bucket[bucket_offset]; + } +} + +// TODO: Make unsafe and avoid bounds check +/// Sort a bucket using bitonic sort +fn sort_bucket_impl( + local_invocation_id: u32, + bucket_size: u32, + bucket: &mut [PositionR; MAX_BUCKET_SIZE], + shared_bucket: &mut [PositionR; MAX_BUCKET_SIZE], +) { + load_into_shared_bucket(local_invocation_id, bucket_size, bucket, shared_bucket); + + workgroup_memory_barrier_with_group_sync(); + + sort_shared_bucket( + local_invocation_id, + shared_bucket, + #[inline(always)] + |a, b| a.position <= b.position, + ); + + store_from_shared_bucket(local_invocation_id, bucket, shared_bucket); +} + +/// NOTE: bucket sizes are zeroed after use +#[spirv(compute(threads(256), entry_point_name = "sort_buckets"))] +pub fn sort_buckets( + #[spirv(workgroup_id)] workgroup_id: UVec3, + #[spirv(local_invocation_id)] local_invocation_id: UVec3, + #[spirv(storage_buffer, descriptor_set = 0, binding = 0)] bucket_sizes: &mut [u32; NUM_BUCKETS], + #[spirv(storage_buffer, descriptor_set = 0, binding = 1)] buckets: &mut [[PositionR; MAX_BUCKET_SIZE]; + NUM_BUCKETS], + #[spirv(workgroup)] shared_bucket: &mut [PositionR; MAX_BUCKET_SIZE], +) { + let local_invocation_id = local_invocation_id.x; + let workgroup_id = workgroup_id.x; + + // Process one bucket per workgroup + // TODO: More idiomatic version currently doesn't compile: + // https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 + let bucket_size = bucket_sizes[workgroup_id as usize]; + let bucket = &mut buckets[workgroup_id as usize]; + + sort_bucket_impl(local_invocation_id, bucket_size, bucket, shared_bucket); + + if local_invocation_id == 0 { + bucket_sizes[workgroup_id as usize] = 0; + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/types.rs b/shared/ab-proof-of-space-gpu/src/shader/types.rs new file mode 100644 index 00000000000..0ec8db4060b --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/types.rs @@ -0,0 +1,252 @@ +use crate::shader::constants::{MAX_BUCKET_SIZE, NUM_BUCKETS, PARAM_BC, PARAM_EXT, PARAM_M}; +use crate::shader::u32n::U32N; +use core::fmt; +use core::iter::Step; +use derive_more::{From, Into}; + +/// Stores data in lower bits +#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, From, Into)] +#[repr(C)] +pub(super) struct X(u32); + +impl Step for X { + #[inline(always)] + fn steps_between(start: &Self, end: &Self) -> (usize, Option) { + u32::steps_between(&start.0, &end.0) + } + + #[inline(always)] + fn forward_checked(start: Self, count: usize) -> Option { + u32::forward_checked(start.0, count).map(Self) + } + + #[inline(always)] + fn backward_checked(start: Self, count: usize) -> Option { + u32::backward_checked(start.0, count).map(Self) + } +} + +/// Stores data in lower bits +#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, From, Into)] +#[repr(C)] +pub struct Y(u32); + +impl From for U32N<4> { + #[inline(always)] + fn from(value: Y) -> Self { + Self::from(value.0) + } +} + +impl Y { + /// Get the first `K` bits + #[inline(always)] + pub(in super::super) const fn first_k_bits(self) -> u32 { + self.0 >> PARAM_EXT + } + + pub(super) fn into_bucket_index_and_r(self) -> (u32, R) { + let bucket_index = self.0 / u32::from(PARAM_BC); + let r = self.0 % u32::from(PARAM_BC); + // SAFETY: `r` is within `0..PARAM_BC` range + let r = unsafe { R::new(r) }; + (bucket_index, r) + } +} + +// TODO: The struct in this form currently doesn't compile: +// https://github.com/Rust-GPU/rust-gpu/issues/241#issuecomment-3005693043 +// #[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, From, Into)] +// #[repr(C)] +// pub struct Position(u32); +// +// impl From for usize { +// #[inline(always)] +// fn from(value: Position) -> Self { +// value.0 as Self +// } +// } +// +// impl Position { +// /// Position that can't exist +// pub(super) const SENTINEL: Self = +// Self(u32::MAX >> (u32::BITS - (NUM_BUCKETS * MAX_BUCKET_SIZE).bit_width())); +// } + +pub type Position = u32; + +// TODO: Remove once normal `Position` struct can be used +pub(super) trait PositionExt: Sized { + /// Position that can't exist + const SENTINEL: Self; + + // TODO: This is just `Position::from()` usually + fn from_u32(value: u32) -> Self; +} + +impl PositionExt for Position { + const SENTINEL: Self = u32::MAX >> (u32::BITS - (NUM_BUCKETS * MAX_BUCKET_SIZE).bit_width()); + + #[inline(always)] + fn from_u32(value: u32) -> Self { + value + } +} + +/// Stores data in lower bits +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[repr(C)] +pub struct Metadata(U32N<3>); + +impl Default for Metadata { + #[inline(always)] + fn default() -> Self { + Self(U32N::ZERO) + } +} + +impl From for U32N<3> { + #[inline(always)] + fn from(value: Metadata) -> Self { + value.0 + } +} + +impl From for U32N<4> { + #[inline(always)] + fn from(value: Metadata) -> Self { + value.0.cast() + } +} + +impl From> for Metadata { + #[inline(always)] + fn from(value: U32N<3>) -> Self { + Self(value) + } +} + +impl From for Metadata { + #[inline(always)] + fn from(value: Position) -> Self { + Self(U32N::from(value)) + } +} + +#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq)] +#[repr(C)] +pub struct R(u32); + +impl R { + /// R that can't exist + pub(super) const SENTINEL: Self = Self(u32::MAX); + /// Number of bits used to fully represent `R` + pub(super) const BITS: u32 = (PARAM_BC - 1).bit_width(); + + /// Create new `R` from provided value. + /// + /// # Safety + /// `r` value must be within `0..PARAM_BC` range. + #[inline(always)] + pub(super) unsafe fn new(r: u32) -> Self { + Self(r) + } + + /// Get the inner stored value + #[inline(always)] + pub(super) fn get(self) -> u32 { + self.0 + } +} + +/// A tuple of [`Position`] and [`Y`] with guaranteed memory layout +#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq)] +#[repr(C)] +pub struct PositionR { + /// Position + pub position: Position, + /// R + pub r: R, +} + +impl PositionR { + /// PositionR that can't exist + pub(super) const SENTINEL: Self = Self { + position: Position::SENTINEL, + r: R::SENTINEL, + }; +} + +#[derive(Copy, Clone, Eq, PartialEq)] +#[repr(C)] +pub struct Match(u32); + +impl fmt::Debug for Match { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let (bucket_offset, r_target, positions_offset) = self.split(); + f.debug_struct("Match") + .field("bucket_offset", &bucket_offset) + .field("r_target", &r_target) + .field("positions_offset", &positions_offset) + .finish() + } +} + +impl Match { + /// Match that can't exist + pub(super) const SENTINEL: Self = Self(u32::MAX); + + /// NOTE: `m` is only present here to ensure correct sorting order, it is not used for anything + /// else. + /// + /// # Safety + /// `bucket_offset` value must be within `0..MAX_BUCKET_SIZE` range, `m` must be within + /// `0..PARAM_M` range and `r_target` must be within `0..PARAM_BC` range + #[inline(always)] + pub(super) unsafe fn new(bucket_offset: u32, m: u32, r_target: u32) -> Self { + #[expect(clippy::int_plus_one, reason = "Better explains the underlying logic")] + const { + assert!( + (MAX_BUCKET_SIZE - 1).bit_width() + (PARAM_M - 1).bit_width() + R::BITS + 1 + <= u32::BITS + ); + } + + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + Self( + (bucket_offset << const { (PARAM_M - 1).bit_width() + R::BITS + 1 }) + | (m << const { R::BITS + 1 }) + | (r_target << 1), + ) + } + + /// Initially, `Match` assumes the first position is used. This allows changing it to the + /// second. + /// + /// # Safety + /// Must only be called once on any `Match` instance. + #[inline(always)] + pub(super) fn second_second_position(mut self) -> Self { + self.0 += 1; + self + } + + /// Returns `(bucket_offset, r_target, positions_offset)` + #[inline(always)] + pub(super) fn split(self) -> (u32, u32, u32) { + // TODO: `const {}` is a workaround for https://github.com/Rust-GPU/rust-gpu/issues/322 and + // shouldn't be necessary otherwise + ( + self.0 >> const { (PARAM_M - 1).bit_width() + R::BITS + 1 }, + (self.0 >> 1) & const { u32::MAX >> (u32::BITS - R::BITS) }, + self.0 & 1, + ) + } + + /// Extracts a key that can be used for sorting matches + #[inline(always)] + pub(super) fn cmp_key(self) -> u32 { + self.0 + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/u32n.rs b/shared/ab-proof-of-space-gpu/src/shader/u32n.rs new file mode 100644 index 00000000000..0bc6660ecca --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/u32n.rs @@ -0,0 +1,681 @@ +#[cfg(test)] +mod tests; + +use core::cmp::{Eq, PartialEq}; +use core::ops::{ + Add, AddAssign, BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, BitXorAssign, Shl, ShlAssign, + Shr, ShrAssign, Sub, SubAssign, +}; + +/// Generalized unsigned integer as an array of u32 words, least significant word first +#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] +#[repr(C)] +pub struct U32N([u32; N]); + +impl U32N { + pub const ZERO: Self = Self([0; N]); + pub const BITS: u32 = u32::BITS * N as u32; + pub const WORD_BITS: u32 = u32::BITS; + + #[cfg(test)] + #[inline(always)] + pub(super) fn to_be_bytes(self) -> [u8; N * 4] { + let mut bytes = [0u8; _]; + let mut idx = 0; + for &word in self.0.iter().rev() { + let b = word.to_be_bytes(); + bytes[idx..idx + 4].copy_from_slice(&b); + idx += 4; + } + bytes + } + + #[cfg(test)] + #[inline(always)] + pub(super) fn from_be_bytes(bytes: [u8; N * 4]) -> Self { + let mut words = [0u32; _]; + let mut idx = 0; + for i in (0..N).rev() { + let b = [bytes[idx], bytes[idx + 1], bytes[idx + 2], bytes[idx + 3]]; + words[i] = u32::from_be_bytes(b); + idx += 4; + } + Self(words) + } + + #[inline(always)] + pub(super) fn as_u32(&self) -> u32 { + self.0[0] + } + + pub(super) fn cast(self) -> U32N { + let mut output = [0u32; O]; + if N.min(O) > 0 { + output[0] = self.0[0]; + } + if N.min(O) > 1 { + output[1] = self.0[1]; + } + if N.min(O) > 2 { + output[2] = self.0[2]; + } + if N.min(O) > 3 { + output[3] = self.0[3]; + } + if N.min(O) > 4 { + unimplemented!(); + } + U32N(output) + } +} + +impl U32N<2> { + #[inline(always)] + pub(super) fn from_low_high(low: u32, high: u32) -> Self { + Self([low, high]) + } +} + +impl From for U32N<2> { + #[inline(always)] + fn from(n: u32) -> Self { + Self([n, 0]) + } +} + +impl Add for U32N<2> { + type Output = Self; + + #[inline(always)] + fn add(self, rhs: Self) -> Self { + let mut result = [0u32; 2]; + let mut carry = false; + (result[0], carry) = self.0[0].carrying_add(rhs.0[0], carry); + (result[1], _) = self.0[1].carrying_add(rhs.0[1], carry); + Self(result) + } +} + +impl AddAssign for U32N<2> { + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } +} + +impl Sub for U32N<2> { + type Output = Self; + + #[inline(always)] + fn sub(self, rhs: Self) -> Self { + let mut result = [0u32; 2]; + let mut borrow = false; + (result[0], borrow) = self.0[0].borrowing_sub(rhs.0[0], borrow); + (result[1], _) = self.0[1].borrowing_sub(rhs.0[1], borrow); + Self(result) + } +} + +impl SubAssign for U32N<2> { + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; + } +} + +impl BitAnd for U32N<2> { + type Output = Self; + + #[inline(always)] + fn bitand(self, rhs: Self) -> Self { + Self([self.0[0] & rhs.0[0], self.0[1] & rhs.0[1]]) + } +} + +impl BitAndAssign for U32N<2> { + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = *self & rhs; + } +} + +impl BitXor for U32N<2> { + type Output = Self; + + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + Self([self.0[0] ^ rhs.0[0], self.0[1] ^ rhs.0[1]]) + } +} + +impl BitXorAssign for U32N<2> { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = *self ^ rhs; + } +} + +impl BitOr for U32N<2> { + type Output = Self; + + #[inline(always)] + fn bitor(self, rhs: Self) -> Self { + Self([self.0[0] | rhs.0[0], self.0[1] | rhs.0[1]]) + } +} + +impl BitOrAssign for U32N<2> { + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = *self | rhs; + } +} + +impl Shl for U32N<2> { + type Output = Self; + + #[inline(always)] + fn shl(mut self, rhs: u32) -> Self { + if rhs == 0 { + return self; + } + + let bit_shift = rhs % u32::BITS; + match rhs / u32::BITS { + 0 => { + self.0[1] = + (self.0[1] << bit_shift) | (self.0[0].unbounded_shr(u32::BITS - bit_shift)); + self.0[0] <<= bit_shift; + } + 1 => { + self.0[1] = self.0[0] << bit_shift; + self.0[0] = 0; + } + _ => unreachable!(), + } + + self + } +} + +impl ShlAssign for U32N<2> { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = *self << rhs; + } +} + +impl Shr for U32N<2> { + type Output = Self; + + #[inline(always)] + fn shr(mut self, rhs: u32) -> Self { + if rhs == 0 { + return self; + } + + let bit_shift = rhs % u32::BITS; + match rhs / u32::BITS { + 0 => { + self.0[0] = + (self.0[0] >> bit_shift) | (self.0[1].unbounded_shl(u32::BITS - bit_shift)); + self.0[1] >>= bit_shift; + } + 1 => { + self.0[0] = self.0[1] >> bit_shift; + self.0[1] = 0; + } + _ => unreachable!(), + } + + self + } +} + +impl ShrAssign for U32N<2> { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = *self >> rhs; + } +} + +impl From for U32N<3> { + #[inline(always)] + fn from(n: u32) -> Self { + Self([n, 0, 0]) + } +} + +impl Add for U32N<3> { + type Output = Self; + + #[inline(always)] + fn add(self, rhs: Self) -> Self { + let mut result = [0u32; 3]; + let mut carry = false; + (result[0], carry) = self.0[0].carrying_add(rhs.0[0], carry); + (result[1], carry) = self.0[1].carrying_add(rhs.0[1], carry); + (result[2], _) = self.0[2].carrying_add(rhs.0[2], carry); + Self(result) + } +} + +impl AddAssign for U32N<3> { + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } +} + +impl Sub for U32N<3> { + type Output = Self; + + #[inline(always)] + fn sub(self, rhs: Self) -> Self { + let mut result = [0u32; 3]; + let mut borrow = false; + (result[0], borrow) = self.0[0].borrowing_sub(rhs.0[0], borrow); + (result[1], borrow) = self.0[1].borrowing_sub(rhs.0[1], borrow); + (result[2], _) = self.0[2].borrowing_sub(rhs.0[2], borrow); + Self(result) + } +} + +impl SubAssign for U32N<3> { + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; + } +} + +impl BitAnd for U32N<3> { + type Output = Self; + + #[inline(always)] + fn bitand(self, rhs: Self) -> Self { + Self([ + self.0[0] & rhs.0[0], + self.0[1] & rhs.0[1], + self.0[2] & rhs.0[2], + ]) + } +} + +impl BitAndAssign for U32N<3> { + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = *self & rhs; + } +} + +impl BitXor for U32N<3> { + type Output = Self; + + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + Self([ + self.0[0] ^ rhs.0[0], + self.0[1] ^ rhs.0[1], + self.0[2] ^ rhs.0[2], + ]) + } +} + +impl BitXorAssign for U32N<3> { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = *self ^ rhs; + } +} + +impl BitOr for U32N<3> { + type Output = Self; + + #[inline(always)] + fn bitor(self, rhs: Self) -> Self { + Self([ + self.0[0] | rhs.0[0], + self.0[1] | rhs.0[1], + self.0[2] | rhs.0[2], + ]) + } +} + +impl BitOrAssign for U32N<3> { + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = *self | rhs; + } +} + +impl Shl for U32N<3> { + type Output = Self; + + #[inline(always)] + fn shl(mut self, rhs: u32) -> Self { + if rhs == 0 { + return self; + } + + let bit_shift = rhs % u32::BITS; + match rhs / u32::BITS { + 0 => { + self.0[2] = + (self.0[2] << bit_shift) | (self.0[1].unbounded_shr(u32::BITS - bit_shift)); + self.0[1] = + (self.0[1] << bit_shift) | (self.0[0].unbounded_shr(u32::BITS - bit_shift)); + self.0[0] <<= bit_shift; + } + 1 => { + self.0[2] = + (self.0[1] << bit_shift) | (self.0[0].unbounded_shr(u32::BITS - bit_shift)); + self.0[1] = self.0[0] << bit_shift; + self.0[0] = 0; + } + 2 => { + self.0[2] = self.0[0] << bit_shift; + self.0[1] = 0; + self.0[0] = 0; + } + _ => unreachable!(), + } + + self + } +} + +impl ShlAssign for U32N<3> { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = *self << rhs; + } +} + +impl Shr for U32N<3> { + type Output = Self; + + #[inline(always)] + fn shr(mut self, rhs: u32) -> Self { + if rhs == 0 { + return self; + } + + let bit_shift = rhs % u32::BITS; + match rhs / u32::BITS { + 0 => { + self.0[0] = + (self.0[0] >> bit_shift) | (self.0[1].unbounded_shl(u32::BITS - bit_shift)); + self.0[1] = + (self.0[1] >> bit_shift) | (self.0[2].unbounded_shl(u32::BITS - bit_shift)); + self.0[2] >>= bit_shift; + } + 1 => { + self.0[0] = + (self.0[1] >> bit_shift) | (self.0[2].unbounded_shl(u32::BITS - bit_shift)); + self.0[1] = self.0[2] >> bit_shift; + self.0[2] = 0; + } + 2 => { + self.0[0] = self.0[2] >> bit_shift; + self.0[1] = 0; + self.0[2] = 0; + } + _ => unreachable!(), + } + + self + } +} + +impl ShrAssign for U32N<3> { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = *self >> rhs; + } +} + +impl U32N<4> { + #[inline(always)] + pub(super) fn as_be_bytes_to_le_u32_words(&self) -> [u32; 4] { + [ + self.0[3].swap_bytes(), + self.0[2].swap_bytes(), + self.0[1].swap_bytes(), + self.0[0].swap_bytes(), + ] + } + + #[inline(always)] + pub(super) fn from_le_u32_words_as_be_bytes(words: &[u32; 4]) -> Self { + Self([ + words[3].swap_bytes(), + words[2].swap_bytes(), + words[1].swap_bytes(), + words[0].swap_bytes(), + ]) + } +} + +impl From for U32N<4> { + #[inline(always)] + fn from(n: u32) -> Self { + Self([n, 0, 0, 0]) + } +} + +impl Add for U32N<4> { + type Output = Self; + + #[inline(always)] + fn add(self, rhs: Self) -> Self { + let mut result = [0u32; 4]; + let mut carry = false; + (result[0], carry) = self.0[0].carrying_add(rhs.0[0], carry); + (result[1], carry) = self.0[1].carrying_add(rhs.0[1], carry); + (result[2], carry) = self.0[2].carrying_add(rhs.0[2], carry); + (result[3], _) = self.0[3].carrying_add(rhs.0[3], carry); + Self(result) + } +} + +impl AddAssign for U32N<4> { + #[inline(always)] + fn add_assign(&mut self, rhs: Self) { + *self = *self + rhs; + } +} + +impl Sub for U32N<4> { + type Output = Self; + + #[inline(always)] + fn sub(self, rhs: Self) -> Self { + let mut result = [0u32; 4]; + let mut borrow = false; + (result[0], borrow) = self.0[0].borrowing_sub(rhs.0[0], borrow); + (result[1], borrow) = self.0[1].borrowing_sub(rhs.0[1], borrow); + (result[2], borrow) = self.0[2].borrowing_sub(rhs.0[2], borrow); + (result[3], _) = self.0[3].borrowing_sub(rhs.0[3], borrow); + Self(result) + } +} + +impl SubAssign for U32N<4> { + #[inline(always)] + fn sub_assign(&mut self, rhs: Self) { + *self = *self - rhs; + } +} + +impl BitAnd for U32N<4> { + type Output = Self; + + #[inline(always)] + fn bitand(self, rhs: Self) -> Self { + Self([ + self.0[0] & rhs.0[0], + self.0[1] & rhs.0[1], + self.0[2] & rhs.0[2], + self.0[3] & rhs.0[3], + ]) + } +} + +impl BitAndAssign for U32N<4> { + #[inline(always)] + fn bitand_assign(&mut self, rhs: Self) { + *self = *self & rhs; + } +} + +impl BitXor for U32N<4> { + type Output = Self; + + #[inline(always)] + fn bitxor(self, rhs: Self) -> Self { + Self([ + self.0[0] ^ rhs.0[0], + self.0[1] ^ rhs.0[1], + self.0[2] ^ rhs.0[2], + self.0[3] ^ rhs.0[3], + ]) + } +} + +impl BitXorAssign for U32N<4> { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + *self = *self ^ rhs; + } +} + +impl BitOr for U32N<4> { + type Output = Self; + + #[inline(always)] + fn bitor(self, rhs: Self) -> Self { + Self([ + self.0[0] | rhs.0[0], + self.0[1] | rhs.0[1], + self.0[2] | rhs.0[2], + self.0[3] | rhs.0[3], + ]) + } +} + +impl BitOrAssign for U32N<4> { + #[inline(always)] + fn bitor_assign(&mut self, rhs: Self) { + *self = *self | rhs; + } +} + +impl Shl for U32N<4> { + type Output = Self; + + #[inline(always)] + fn shl(mut self, rhs: u32) -> Self { + if rhs == 0 { + return self; + } + + let bit_shift = rhs % u32::BITS; + match rhs / u32::BITS { + 0 => { + self.0[3] = + (self.0[3] << bit_shift) | (self.0[2].unbounded_shr(u32::BITS - bit_shift)); + self.0[2] = + (self.0[2] << bit_shift) | (self.0[1].unbounded_shr(u32::BITS - bit_shift)); + self.0[1] = + (self.0[1] << bit_shift) | (self.0[0].unbounded_shr(u32::BITS - bit_shift)); + self.0[0] <<= bit_shift; + } + 1 => { + self.0[3] = + (self.0[2] << bit_shift) | (self.0[1].unbounded_shr(u32::BITS - bit_shift)); + self.0[2] = + (self.0[1] << bit_shift) | (self.0[0].unbounded_shr(u32::BITS - bit_shift)); + self.0[1] = self.0[0] << bit_shift; + self.0[0] = 0; + } + 2 => { + self.0[3] = + (self.0[1] << bit_shift) | (self.0[0].unbounded_shr(u32::BITS - bit_shift)); + self.0[2] = self.0[0] << bit_shift; + self.0[1] = 0; + self.0[0] = 0; + } + 3 => { + self.0[3] = self.0[0] << bit_shift; + self.0[2] = 0; + self.0[1] = 0; + self.0[0] = 0; + } + _ => unreachable!(), + } + + self + } +} + +impl ShlAssign for U32N<4> { + #[inline(always)] + fn shl_assign(&mut self, rhs: u32) { + *self = *self << rhs; + } +} + +impl Shr for U32N<4> { + type Output = Self; + + #[inline(always)] + fn shr(mut self, rhs: u32) -> Self { + if rhs == 0 { + return self; + } + + let bit_shift = rhs % u32::BITS; + match rhs / u32::BITS { + 0 => { + self.0[0] = + (self.0[0] >> bit_shift) | (self.0[1].unbounded_shl(u32::BITS - bit_shift)); + self.0[1] = + (self.0[1] >> bit_shift) | (self.0[2].unbounded_shl(u32::BITS - bit_shift)); + self.0[2] = + (self.0[2] >> bit_shift) | (self.0[3].unbounded_shl(u32::BITS - bit_shift)); + self.0[3] >>= bit_shift; + } + 1 => { + self.0[0] = + (self.0[1] >> bit_shift) | (self.0[2].unbounded_shl(u32::BITS - bit_shift)); + self.0[1] = + (self.0[2] >> bit_shift) | (self.0[3].unbounded_shl(u32::BITS - bit_shift)); + self.0[2] = self.0[3] >> bit_shift; + self.0[3] = 0; + } + 2 => { + self.0[0] = + (self.0[2] >> bit_shift) | (self.0[3].unbounded_shl(u32::BITS - bit_shift)); + self.0[1] = self.0[3] >> bit_shift; + self.0[2] = 0; + self.0[3] = 0; + } + 3 => { + self.0[0] = self.0[3] >> bit_shift; + self.0[1] = 0; + self.0[2] = 0; + self.0[3] = 0; + } + _ => unreachable!(), + } + + self + } +} + +impl ShrAssign for U32N<4> { + #[inline(always)] + fn shr_assign(&mut self, rhs: u32) { + *self = *self >> rhs; + } +} diff --git a/shared/ab-proof-of-space-gpu/src/shader/u32n/tests.rs b/shared/ab-proof-of-space-gpu/src/shader/u32n/tests.rs new file mode 100644 index 00000000000..fff181a83fa --- /dev/null +++ b/shared/ab-proof-of-space-gpu/src/shader/u32n/tests.rs @@ -0,0 +1,201 @@ +use super::*; + +#[test] +fn test_from_low_high() { + for num in [0u32, 1, 42, 0x7FFF_FFFF, u32::MAX] { + let low = num; + let high = 42u32; + let correct = (u64::from(high) << u32::BITS) | u64::from(low); + let u32n = U32N::<2>::from_low_high(low, high); + assert_eq!(u32n.to_be_bytes(), correct.to_be_bytes()); + } +} + +#[test] +fn test_from_u32() { + for num in [0u32, 1, 42, 0x7FFF_FFFF, u32::MAX] { + let u32n = U32N::<2>::from(num); + assert_eq!(u32n.to_be_bytes(), u64::from(num).to_be_bytes()); + } +} + +#[test] +fn test_as_u32() { + for num in [0u32, 1, 42, 0x7FFF_FFFF, u32::MAX] { + let u32n = U32N::<2>::from_be_bytes(u64::from(num).to_be_bytes()); + assert_eq!(u32n.as_u32(), num); + } +} + +#[test] +fn test_u64_add() { + let cases = [ + (0u64, 0u64), + (1, 2), + (u64::from(u32::MAX), 1), + (u64::MAX - 1, 1), + ]; + for &(a, b) in &cases { + let a_u = U32N::<2>::from_be_bytes(a.to_be_bytes()); + let b_u = U32N::<2>::from_be_bytes(b.to_be_bytes()); + let sum = a + b; + let sum_u = a_u + b_u; + assert_eq!(sum.to_be_bytes(), sum_u.to_be_bytes()); + } +} + +#[test] +fn test_u64_sub() { + let cases = [(2u64, 1u64), (u64::MAX, 1), (1, 0)]; + for &(a, b) in &cases { + let a_u = U32N::<2>::from_be_bytes(a.to_be_bytes()); + let b_u = U32N::<2>::from_be_bytes(b.to_be_bytes()); + let diff = a - b; + let diff_u = a_u - b_u; + assert_eq!(diff.to_be_bytes(), diff_u.to_be_bytes()); + } +} + +#[test] +fn test_u64_bitwise() { + let a = 0xFF00_FF00_FF00_FF00_u64; + let b = 0x00FF_00FF_00FF_00FF_u64; + + let a_u = U32N::<2>::from_be_bytes(a.to_be_bytes()); + let b_u = U32N::<2>::from_be_bytes(b.to_be_bytes()); + + assert_eq!((a & b).to_be_bytes(), (a_u & b_u).to_be_bytes()); + assert_eq!((a | b).to_be_bytes(), (a_u | b_u).to_be_bytes()); + assert_eq!((a ^ b).to_be_bytes(), (a_u ^ b_u).to_be_bytes()); +} + +#[test] +fn test_u64_shifts() { + let val = 0x0123_4567_89AB_CDEF_u64; + let val_u = U32N::<2>::from_be_bytes(val.to_be_bytes()); + + for shift in 0..64_u32 { + assert_eq!((val << shift).to_be_bytes(), (val_u << shift).to_be_bytes()); + assert_eq!((val >> shift).to_be_bytes(), (val_u >> shift).to_be_bytes()); + } +} + +#[test] +fn test_u64_roundtrip_bytes() { + let values = [0u64, 1u64, u64::from(u32::MAX), u64::MAX]; + for v in values { + let bytes = v.to_be_bytes(); + let u = U32N::<2>::from_be_bytes(bytes); + assert_eq!(bytes, u.to_be_bytes()); + } +} + +#[test] +fn test_u128_add() { + let cases = [ + (0u128, 0u128), + (1, 2), + (u128::from(u64::MAX), 1), + (u128::MAX - 1, 1), + ]; + for &(a, b) in &cases { + let a_u = U32N::<4>::from_be_bytes(a.to_be_bytes()); + let b_u = U32N::<4>::from_be_bytes(b.to_be_bytes()); + let sum = a + b; + let sum_u = a_u + b_u; + assert_eq!(sum.to_be_bytes(), sum_u.to_be_bytes()); + } +} + +#[test] +fn test_u128_sub() { + let cases = [(2u128, 1u128), (u128::MAX, 1), (1, 0)]; + for &(a, b) in &cases { + let a_u = U32N::<4>::from_be_bytes(a.to_be_bytes()); + let b_u = U32N::<4>::from_be_bytes(b.to_be_bytes()); + let diff = a - b; + let diff_u = a_u - b_u; + assert_eq!(diff.to_be_bytes(), diff_u.to_be_bytes()); + } +} + +#[test] +fn test_u128_bitwise() { + let a = 0xFF00_FF00_FF00_FF00_FF00_FF00_FF00_FF00_u128; + let b = 0x00FF_00FF_00FF_00FF_00FF_00FF_00FF_00FF_u128; + + let a_u = U32N::<4>::from_be_bytes(a.to_be_bytes()); + let b_u = U32N::<4>::from_be_bytes(b.to_be_bytes()); + + assert_eq!((a & b).to_be_bytes(), (a_u & b_u).to_be_bytes()); + assert_eq!((a | b).to_be_bytes(), (a_u | b_u).to_be_bytes()); + assert_eq!((a ^ b).to_be_bytes(), (a_u ^ b_u).to_be_bytes()); +} + +#[test] +fn test_u128_shifts() { + let val = 0x0123_4567_89AB_CDEF_0123_4567_89AB_CDEF_u128; + let val_u = U32N::<4>::from_be_bytes(val.to_be_bytes()); + + for shift in 0..128_u32 { + assert_eq!((val << shift).to_be_bytes(), (val_u << shift).to_be_bytes()); + assert_eq!((val >> shift).to_be_bytes(), (val_u >> shift).to_be_bytes()); + } +} + +#[test] +fn test_u128_roundtrip_bytes() { + let values = [0u128, 1u128, u128::from(u64::MAX), u128::MAX]; + for v in values { + let bytes = v.to_be_bytes(); + let u = U32N::<4>::from_be_bytes(bytes); + assert_eq!(bytes, u.to_be_bytes()); + } +} + +#[test] +fn test_as_be_bytes_to_le_u32_words() { + let values = [ + 0u128, + 1, + u128::from(u32::MAX), + u128::from(u64::MAX), + u128::MAX, + 0x0011_2233_4455_6677_8899_aabb_ccdd_eeff, + ]; + + for v in values { + let u = U32N::<4>::from_be_bytes(v.to_be_bytes()); + let words = u.as_be_bytes_to_le_u32_words(); + + let bytes = v.to_be_bytes(); + let expected = [ + u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]), + u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]), + u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]), + u32::from_le_bytes([bytes[12], bytes[13], bytes[14], bytes[15]]), + ]; + + assert_eq!(words, expected, "v={v}"); + } +} + +#[test] +fn test_from_le_u32_words_as_be_bytes() { + let values = [ + 0u128, + 1, + u128::from(u32::MAX), + u128::from(u64::MAX), + u128::MAX, + 0x0011_2233_4455_6677_8899_aabb_ccdd_eeff, + ]; + + for &v in &values { + let u = U32N::<4>::from_be_bytes(v.to_be_bytes()); + let words = u.as_be_bytes_to_le_u32_words(); + let reconstructed = U32N::<4>::from_le_u32_words_as_be_bytes(&words); + + assert_eq!(u.to_be_bytes(), reconstructed.to_be_bytes(), "v={v}"); + } +} diff --git a/shared/ab-proof-of-space/Cargo.toml b/shared/ab-proof-of-space/Cargo.toml new file mode 100644 index 00000000000..646d2528d81 --- /dev/null +++ b/shared/ab-proof-of-space/Cargo.toml @@ -0,0 +1,54 @@ +[package] +name = "ab-proof-of-space" +description = "Proof of space implementation based" +license = "0BSD" +version = "0.1.0" +authors = ["Nazar Mokrynskyi "] +edition = "2024" +include = [ + "/src", + "/Cargo.toml", +] + +[package.metadata.docs.rs] +all-features = true + +[lib] +bench = false + +[dependencies] +ab-blake3 = { workspace = true } +ab-chacha8 = { workspace = true } +subspace-core-primitives = { workspace = true } +chacha20 = { workspace = true, features = ["cipher"], optional = true } +derive_more = { workspace = true, features = ["full"] } +rayon = { workspace = true, optional = true } +rclite = { workspace = true, optional = true } +seq-macro = { workspace = true, optional = true } +sha2 = { workspace = true, optional = true } + +[dev-dependencies] +chacha20 = { workspace = true, features = ["cipher"] } +rayon = { workspace = true } +seq-macro = { workspace = true } +sha2 = { workspace = true } + +[features] +default = [] +alloc = [ + "dep:chacha20", + "dep:rclite", + "dep:seq-macro", +] +# Enabling this feature exposes quality search on `chiapos` module as well as enables support for K=15..=25 (by default +# only K=20 is exposed) +full-chiapos = [ + "dep:sha2", +] +parallel = [ + "alloc", + "dep:rayon", +] + +[lints] +workspace = true diff --git a/shared/ab-proof-of-space/src/chiapos.rs b/shared/ab-proof-of-space/src/chiapos.rs new file mode 100644 index 00000000000..a08e77c14a8 --- /dev/null +++ b/shared/ab-proof-of-space/src/chiapos.rs @@ -0,0 +1,702 @@ +//! Chia proof of space reimplementation in Rust + +mod constants; +mod table; +#[cfg(all(feature = "alloc", test))] +mod tests; +mod utils; + +#[cfg(feature = "alloc")] +use crate::PosProofs; +use crate::chiapos::constants::NUM_TABLES; +#[cfg(feature = "alloc")] +pub use crate::chiapos::table::TablesCache; +#[cfg(feature = "alloc")] +use crate::chiapos::table::types::Position; +use crate::chiapos::table::types::{Metadata, X, Y}; +use crate::chiapos::table::{ + COMPUTE_F1_SIMD_FACTOR, compute_f1, compute_fn, has_match, metadata_size_bytes, num_buckets, +}; +#[cfg(feature = "alloc")] +use crate::chiapos::table::{PrunedTable, Table}; +use crate::chiapos::utils::EvaluatableUsize; +#[cfg(feature = "alloc")] +use alloc::boxed::Box; +use core::array; +use core::mem::MaybeUninit; +#[cfg(feature = "alloc")] +use core::mem::offset_of; +#[cfg(any(feature = "full-chiapos", test))] +use sha2::{Digest, Sha256}; +#[cfg(feature = "alloc")] +use subspace_core_primitives::pieces::Record; +#[cfg(feature = "alloc")] +use subspace_core_primitives::pos::PosProof; +#[cfg(feature = "alloc")] +use subspace_core_primitives::sectors::SBucket; + +mod private { + pub trait Supported {} +} + +/// Proof-of-space proofs +#[derive(Debug)] +#[cfg(feature = "alloc")] +#[repr(C)] +pub struct Proofs +where + [(); 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize]:, +{ + /// S-buckets at which proofs were found. + /// + /// S-buckets are grouped by 8, within each `u8` bits right to left (LSB) indicate the presence + /// of a proof for corresponding s-bucket, so that the whole array of bytes can be thought as a + /// large set of bits. + /// + /// There will be at most [`Record::NUM_CHUNKS`] proofs produced/bits set to `1`. + pub found_proofs: [u8; Record::NUM_S_BUCKETS / u8::BITS as usize], + /// [`Record::NUM_CHUNKS`] proofs, corresponding to set bits of `found_proofs`. + pub proofs: [[u8; 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize]; + Record::NUM_CHUNKS], +} + +#[cfg(feature = "alloc")] +impl From>> for Box { + fn from(proofs: Box>) -> Self { + // Statically ensure types are the same + const { + // TODO: Latest nightly can't understand `{ PosProof::K }` for some reason, hence a + // separate assert + assert!(PosProof::K == 20); + assert!(size_of::>() == size_of::()); + assert!(align_of::>() == align_of::()); + assert!(offset_of!(Proofs<20>, found_proofs) == offset_of!(PosProofs, found_proofs)); + assert!(offset_of!(Proofs<20>, proofs) == offset_of!(PosProofs, proofs)); + } + // SAFETY: Both structs have an identical layout with `#[repr(C)]` internals + unsafe { Box::from_raw(Box::into_raw(proofs).cast()) } + } +} + +#[cfg(feature = "alloc")] +impl Proofs +where + [(); 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize]:, +{ + /// Get proof for specified s-bucket (if exists). + /// + /// Note that this is not the most efficient API possible, so prefer using the `proofs` field + /// directly if the use case allows. + #[inline] + pub fn for_s_bucket( + &self, + s_bucket: SBucket, + ) -> Option<[u8; 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize]> + { + let proof_index = PosProofs::proof_index_for_s_bucket(&self.found_proofs, s_bucket)?; + + Some(self.proofs[proof_index]) + } +} + +type Seed = [u8; 32]; +#[cfg(any(feature = "full-chiapos", test))] +type Challenge = [u8; 32]; +#[cfg(any(feature = "full-chiapos", test))] +type Quality = [u8; 32]; + +/// Pick position in `table_number` based on challenge bits +#[cfg(all(feature = "alloc", any(feature = "full-chiapos", test)))] +const fn pick_position( + [left_position, right_position]: [Position; 2], + last_5_challenge_bits: u8, + table_number: u8, +) -> Position { + if ((last_5_challenge_bits >> (table_number - 2)) & 1) == 0 { + left_position + } else { + right_position + } +} + +/// Collection of Chia tables +#[derive(Debug)] +pub struct Tables +where + Self: private::Supported, + EvaluatableUsize<{ metadata_size_bytes(K, 7) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + #[cfg(feature = "alloc")] + table_2: PrunedTable, + #[cfg(feature = "alloc")] + table_3: PrunedTable, + #[cfg(feature = "alloc")] + table_4: PrunedTable, + #[cfg(feature = "alloc")] + table_5: PrunedTable, + #[cfg(feature = "alloc")] + table_6: PrunedTable, + #[cfg(feature = "alloc")] + table_7: Table, +} + +impl Tables +where + Self: private::Supported, + EvaluatableUsize<{ metadata_size_bytes(K, 1) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, 2) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, 3) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, 4) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, 5) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, 6) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, 7) }>: Sized, + EvaluatableUsize<{ usize::from(K) * COMPUTE_F1_SIMD_FACTOR / u8::BITS as usize }>: Sized, + EvaluatableUsize< + { 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize }, + >: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + /// Create Chia proof of space tables. + /// + /// There is also `Self::create_parallel()` that can achieve higher performance and lower + /// latency at the cost of lower CPU efficiency and higher memory usage. + #[cfg(all(feature = "alloc", any(feature = "full-chiapos", test)))] + pub fn create(seed: Seed, cache: &TablesCache) -> Self { + let table_1 = Table::::create(seed); + let (table_2, _) = Table::::create(table_1, cache); + let (table_3, table_2) = Table::::create(table_2, cache); + let (table_4, table_3) = Table::::create(table_3, cache); + let (table_5, table_4) = Table::::create(table_4, cache); + let (table_6, table_5) = Table::::create(table_5, cache); + let (table_7, table_6) = Table::::create(table_6, cache); + + Self { + table_2, + table_3, + table_4, + table_5, + table_6, + table_7, + } + } + + /// Create proofs. + /// + /// This is an optimized combination of `Self::create()` and `Self::find_proof()`. + /// + /// There is also `Self::create_proofs_parallel()` that can achieve higher performance and lower + /// latency at the cost of lower CPU efficiency and higher memory usage. + #[cfg(feature = "alloc")] + pub fn create_proofs(seed: Seed, cache: &TablesCache) -> Box> + where + [(); 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize]:, + { + let table_1 = Table::::create(seed); + let (table_2, _) = Table::::create(table_1, cache); + let (table_3, table_2) = Table::::create(table_2, cache); + let (table_4, table_3) = Table::::create(table_3, cache); + let (table_5, table_4) = Table::::create(table_4, cache); + let (table_6, table_5) = Table::::create(table_5, cache); + let (table_6_proof_targets, table_6) = Table::::create_proof_targets(table_6, cache); + + // TODO: Rewrite this more efficiently + let mut proofs = Box::>::new_uninit(); + { + let proofs_ptr = proofs.as_mut().as_mut_ptr(); + // SAFETY: This is the correct way to access uninit reference to the inner field + let found_proofs = unsafe { + (&raw mut (*proofs_ptr).found_proofs) + .as_uninit_mut() + .expect("Not null; qed") + }; + let found_proofs = found_proofs.write([0; _]); + // SAFETY: This is the correct way to access uninit reference to the inner field + let proofs = unsafe { + (&raw mut (*proofs_ptr).proofs) + .cast::<[MaybeUninit<_>; Record::NUM_CHUNKS]>() + .as_mut_unchecked() + }; + + let mut num_found_proofs = 0_usize; + 'outer: for (table_6_proof_targets, found_proofs) in table_6_proof_targets + .as_chunks::<{ u8::BITS as usize }>() + .0 + .iter() + .zip(found_proofs) + { + // TODO: Find proofs with SIMD + for (proof_offset, table_6_proof_targets) in + table_6_proof_targets.iter().enumerate() + { + if table_6_proof_targets != &[Position::ZERO; 2] { + let proof = Self::find_proof_raw_internal( + &table_2, + &table_3, + &table_4, + &table_5, + &table_6, + *table_6_proof_targets, + ); + + *found_proofs |= 1 << proof_offset; + + proofs[num_found_proofs].write(proof); + num_found_proofs += 1; + + if num_found_proofs == Record::NUM_CHUNKS { + break 'outer; + } + } + } + } + + // It is statically known to be the case, and there is a test that checks the lower + // bound + debug_assert_eq!(num_found_proofs, Record::NUM_CHUNKS); + } + + // SAFETY: Fully initialized above + unsafe { proofs.assume_init() } + } + + /// Almost the same as [`Self::create()`], but uses parallelism internally for better + /// performance and lower latency at the cost of lower CPU efficiency and higher memory usage + #[cfg(all(feature = "parallel", any(feature = "full-chiapos", test)))] + pub fn create_parallel(seed: Seed, cache: &TablesCache) -> Self { + let table_1 = Table::::create_parallel(seed); + let (table_2, _) = Table::::create_parallel(table_1, cache); + let (table_3, table_2) = Table::::create_parallel(table_2, cache); + let (table_4, table_3) = Table::::create_parallel(table_3, cache); + let (table_5, table_4) = Table::::create_parallel(table_4, cache); + let (table_6, table_5) = Table::::create_parallel(table_5, cache); + let (table_7, table_6) = Table::::create_parallel(table_6, cache); + + Self { + table_2, + table_3, + table_4, + table_5, + table_6, + table_7, + } + } + + /// Almost the same as [`Self::create_proofs()`], but uses parallelism internally for better + /// performance and lower latency at the cost of lower CPU efficiency and higher memory usage + #[cfg(feature = "parallel")] + pub fn create_proofs_parallel(seed: Seed, cache: &TablesCache) -> Box> + where + [(); 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize]:, + { + let table_1 = Table::::create_parallel(seed); + let (table_2, _) = Table::::create_parallel(table_1, cache); + let (table_3, table_2) = Table::::create_parallel(table_2, cache); + let (table_4, table_3) = Table::::create_parallel(table_3, cache); + let (table_5, table_4) = Table::::create_parallel(table_4, cache); + let (table_6, table_5) = Table::::create_parallel(table_5, cache); + let (table_6_proof_targets, table_6) = + Table::::create_proof_targets_parallel(table_6, cache); + + // TODO: Rewrite this more efficiently + let mut proofs = Box::>::new_uninit(); + { + let proofs_ptr = proofs.as_mut().as_mut_ptr(); + // SAFETY: This is the correct way to access uninit reference to the inner field + let found_proofs = unsafe { + (&raw mut (*proofs_ptr).found_proofs) + .as_uninit_mut() + .expect("Not null; qed") + }; + let found_proofs = found_proofs.write([0; _]); + // SAFETY: This is the correct way to access uninit reference to the inner field + let proofs = unsafe { + (&raw mut (*proofs_ptr).proofs) + .cast::<[MaybeUninit<_>; Record::NUM_CHUNKS]>() + .as_mut_unchecked() + }; + + let mut num_found_proofs = 0_usize; + 'outer: for (table_6_proof_targets, found_proofs) in table_6_proof_targets + .as_chunks::<{ u8::BITS as usize }>() + .0 + .iter() + .zip(found_proofs) + { + // TODO: Find proofs with SIMD + for (proof_offset, table_6_proof_targets) in + table_6_proof_targets.iter().enumerate() + { + if table_6_proof_targets != &[Position::ZERO; 2] { + let proof = Self::find_proof_raw_internal( + &table_2, + &table_3, + &table_4, + &table_5, + &table_6, + *table_6_proof_targets, + ); + + *found_proofs |= 1 << proof_offset; + + proofs[num_found_proofs].write(proof); + num_found_proofs += 1; + + if num_found_proofs == Record::NUM_CHUNKS { + break 'outer; + } + } + } + } + + // It is statically known to be the case, and there is a test that checks the lower + // bound + debug_assert_eq!(num_found_proofs, Record::NUM_CHUNKS); + } + + // SAFETY: Fully initialized above + unsafe { proofs.assume_init() } + } + + /// Find proof of space quality for a given challenge + #[cfg(all(feature = "alloc", any(feature = "full-chiapos", test)))] + pub fn find_quality<'a>( + &'a self, + challenge: &'a Challenge, + ) -> impl Iterator + 'a { + let last_5_challenge_bits = challenge[challenge.len() - 1] & 0b0001_1111; + + let first_k_challenge_bits = u32::from_be_bytes( + challenge[..size_of::()] + .try_into() + .expect("Challenge is known to statically have enough bytes; qed"), + ) >> (u32::BITS as usize - usize::from(K)); + + // Iterate just over elements that are matching `first_k_challenge_bits` prefix + self.table_7.buckets()[Y::bucket_range_from_first_k_bits(first_k_challenge_bits)] + .iter() + .flat_map(move |positions| { + positions + .iter() + .take_while(|&&(position, _y)| position != Position::SENTINEL) + .filter(move |&&(_position, y)| y.first_k_bits() == first_k_challenge_bits) + }) + .map(move |&(position, _y)| { + // SAFETY: Internally generated positions that come from the parent table + let positions = unsafe { self.table_7.position(position) }; + // SAFETY: Internally generated positions that come from the parent table + let positions = unsafe { + self.table_6 + .position(pick_position(positions, last_5_challenge_bits, 6)) + }; + // SAFETY: Internally generated positions that come from the parent table + let positions = unsafe { + self.table_5 + .position(pick_position(positions, last_5_challenge_bits, 5)) + }; + // SAFETY: Internally generated positions that come from the parent table + let positions = unsafe { + self.table_4 + .position(pick_position(positions, last_5_challenge_bits, 4)) + }; + // SAFETY: Internally generated positions that come from the parent table + let positions = unsafe { + self.table_3 + .position(pick_position(positions, last_5_challenge_bits, 3)) + }; + // SAFETY: Internally generated positions that come from the parent table + let [left_position, right_position] = unsafe { + self.table_2 + .position(pick_position(positions, last_5_challenge_bits, 2)) + }; + + // X matches position + let left_x = X::from(u32::from(left_position)); + let right_x = X::from(u32::from(right_position)); + + let mut hasher = Sha256::new(); + hasher.update(challenge); + let left_right_xs = (u64::from(left_x) << (u64::BITS as usize - usize::from(K))) + | (u64::from(right_x) << (u64::BITS as usize - usize::from(K * 2))); + hasher.update( + &left_right_xs.to_be_bytes() + [..(usize::from(K) * 2).div_ceil(u8::BITS as usize)], + ); + hasher.finalize().into() + }) + } + + /// Similar to `Self::find_proof()`, but takes the first `k` challenge bits in the least + /// significant bits of `u32` as a challenge instead + #[cfg(feature = "alloc")] + pub fn find_proof_raw( + &self, + first_k_challenge_bits: u32, + ) -> impl Iterator< + Item = [u8; 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize], + > + '_ { + // Iterate just over elements that are matching `first_k_challenge_bits` prefix + self.table_7.buckets()[Y::bucket_range_from_first_k_bits(first_k_challenge_bits)] + .iter() + .flat_map(move |positions| { + positions + .iter() + .take_while(|&&(position, _y)| position != Position::SENTINEL) + .filter(move |&&(_position, y)| y.first_k_bits() == first_k_challenge_bits) + }) + .map(move |&(position, _y)| { + // SAFETY: Internally generated positions that come from the parent table + let table_6_proof_targets = unsafe { self.table_7.position(position) }; + + Self::find_proof_raw_internal( + &self.table_2, + &self.table_3, + &self.table_4, + &self.table_5, + &self.table_6, + table_6_proof_targets, + ) + }) + } + + #[cfg(feature = "alloc")] + #[inline(always)] + fn find_proof_raw_internal( + table_2: &PrunedTable, + table_3: &PrunedTable, + table_4: &PrunedTable, + table_5: &PrunedTable, + table_6: &PrunedTable, + table_6_proof_targets: [Position; 2], + ) -> [u8; 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize] { + let mut proof = + [0u8; 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize]; + + // TODO: Optimize with SIMD + table_6_proof_targets + .into_iter() + .flat_map(|position| { + // SAFETY: Internally generated positions that come from the parent table + unsafe { table_6.position(position) } + }) + .flat_map(|position| { + // SAFETY: Internally generated positions that come from the parent table + unsafe { table_5.position(position) } + }) + .flat_map(|position| { + // SAFETY: Internally generated positions that come from the parent table + unsafe { table_4.position(position) } + }) + .flat_map(|position| { + // SAFETY: Internally generated positions that come from the parent table + unsafe { table_3.position(position) } + }) + .flat_map(|position| { + // SAFETY: Internally generated positions that come from the parent table + unsafe { table_2.position(position) } + }) + .map(|position| { + // X matches position + X::from(u32::from(position)) + }) + .enumerate() + .for_each(|(offset, x)| { + let x_offset_in_bits = usize::from(K) * offset; + // Collect bytes where bits of `x` will be written + let proof_bytes = &mut proof[x_offset_in_bits / u8::BITS as usize..] + [..(x_offset_in_bits % u8::BITS as usize + usize::from(K)) + .div_ceil(u8::BITS as usize)]; + + // Bits of `x` already shifted to the correct location as they will appear + // in `proof` + let x_shifted = u32::from(x) + << (u32::BITS as usize + - (usize::from(K) + x_offset_in_bits % u8::BITS as usize)); + + // TODO: Store proofs in words, like GPU version does + // Copy `x` bits into proof + x_shifted + .to_be_bytes() + .iter() + .zip(proof_bytes) + .for_each(|(from, to)| { + *to |= from; + }); + }); + + proof + } + + /// Find proof of space for a given challenge + #[cfg(all(feature = "alloc", any(feature = "full-chiapos", test)))] + pub fn find_proof( + &self, + first_challenge_bytes: [u8; 4], + ) -> impl Iterator< + Item = [u8; 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) / u8::BITS as usize], + > + '_ { + let first_k_challenge_bits = + u32::from_be_bytes(first_challenge_bytes) >> (u32::BITS as usize - usize::from(K)); + + self.find_proof_raw(first_k_challenge_bits) + } + + /// Similar to `Self::verify()`, but takes the first `k` challenge bits in the least significant + /// bits of `u32` as a challenge instead and doesn't compute quality + pub fn verify_only_raw( + seed: &Seed, + first_k_challenge_bits: u32, + proof_of_space: &[u8; 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) + / u8::BITS as usize], + ) -> bool { + let ys_and_metadata = array::from_fn::<_, 64, _>(|offset| { + let mut pre_x_bytes = 0u64.to_be_bytes(); + let offset_in_bits = usize::from(K) * offset; + let bytes_to_copy = + (offset_in_bits % u8::BITS as usize + usize::from(K)).div_ceil(u8::BITS as usize); + // Copy full bytes that contain bits of `x` + pre_x_bytes[..bytes_to_copy].copy_from_slice( + &proof_of_space[offset_in_bits / u8::BITS as usize..][..bytes_to_copy], + ); + // Extract `pre_x` whose last `K` bits start with `x` + let pre_x = u64::from_be_bytes(pre_x_bytes) + >> (u64::BITS as usize - (usize::from(K) + offset_in_bits % u8::BITS as usize)); + // Convert to the desired type and clear extra bits + let x = X::from(pre_x as u32 & (u32::MAX >> (u32::BITS as usize - usize::from(K)))); + + let y = compute_f1::(x, seed); + + (y, Metadata::from(x)) + }); + + let mut next_ys_and_metadata = [MaybeUninit::uninit(); _]; + let ys_and_metadata = + Self::collect_ys_and_metadata::<2, 1, 64>(&ys_and_metadata, &mut next_ys_and_metadata); + let mut next_ys_and_metadata = [MaybeUninit::uninit(); _]; + let ys_and_metadata = + Self::collect_ys_and_metadata::<3, 2, 32>(ys_and_metadata, &mut next_ys_and_metadata); + let mut next_ys_and_metadata = [MaybeUninit::uninit(); _]; + let ys_and_metadata = + Self::collect_ys_and_metadata::<4, 3, 16>(ys_and_metadata, &mut next_ys_and_metadata); + let mut next_ys_and_metadata = [MaybeUninit::uninit(); _]; + let ys_and_metadata = + Self::collect_ys_and_metadata::<5, 4, 8>(ys_and_metadata, &mut next_ys_and_metadata); + let mut next_ys_and_metadata = [MaybeUninit::uninit(); _]; + let ys_and_metadata = + Self::collect_ys_and_metadata::<6, 5, 4>(ys_and_metadata, &mut next_ys_and_metadata); + let mut next_ys_and_metadata = [MaybeUninit::uninit(); _]; + let ys_and_metadata = + Self::collect_ys_and_metadata::<7, 6, 2>(ys_and_metadata, &mut next_ys_and_metadata); + + let Some((y, _metadata)) = ys_and_metadata.first() else { + return false; + }; + + // Check if the first K bits of `y` match + y.first_k_bits() == first_k_challenge_bits + } + + /// Verify proof of space for a given seed and challenge + #[cfg(any(feature = "full-chiapos", test))] + pub fn verify( + seed: &Seed, + challenge: &Challenge, + proof_of_space: &[u8; 2usize.pow(u32::from(NUM_TABLES - 1)) * usize::from(K) + / u8::BITS as usize], + ) -> Option + where + EvaluatableUsize<{ (usize::from(K) * 2).div_ceil(u8::BITS as usize) }>: Sized, + { + let first_k_challenge_bits = + u32::from_be_bytes([challenge[0], challenge[1], challenge[2], challenge[3]]) + >> (u32::BITS as usize - usize::from(K)); + + if !Self::verify_only_raw(seed, first_k_challenge_bits, proof_of_space) { + return None; + } + + let last_5_challenge_bits = challenge[challenge.len() - 1] & 0b0001_1111; + + let mut quality_index = 0_usize.to_be_bytes(); + quality_index[0] = last_5_challenge_bits; + let quality_index = usize::from_be_bytes(quality_index); + + // NOTE: this works correctly but may overflow if `quality_index` is changed to + // not be zero-initialized anymore + let left_right_xs_bit_offset = quality_index * usize::from(K * 2); + // Collect `left_x` and `right_x` bits, potentially with extra bits at the beginning + // and the end + let left_right_xs_bytes = &proof_of_space[left_right_xs_bit_offset / u8::BITS as usize..] + [..(left_right_xs_bit_offset % u8::BITS as usize + usize::from(K * 2)) + .div_ceil(u8::BITS as usize)]; + + let mut left_right_xs = 0u64.to_be_bytes(); + left_right_xs[..left_right_xs_bytes.len()].copy_from_slice(left_right_xs_bytes); + // Move `left_x` and `right_x` bits to most significant bits + let left_right_xs = + u64::from_be_bytes(left_right_xs) << (left_right_xs_bit_offset % u8::BITS as usize); + // Clear extra bits + let left_right_xs_mask = u64::MAX << (u64::BITS as usize - usize::from(K * 2)); + let left_right_xs = left_right_xs & left_right_xs_mask; + + let mut hasher = Sha256::new(); + hasher.update(challenge); + hasher + .update(&left_right_xs.to_be_bytes()[..usize::from(K * 2).div_ceil(u8::BITS as usize)]); + Some(hasher.finalize().into()) + } + + fn collect_ys_and_metadata< + 'a, + const TABLE_NUMBER: u8, + const PARENT_TABLE_NUMBER: u8, + const N: usize, + >( + ys_and_metadata: &[(Y, Metadata)], + next_ys_and_metadata: &'a mut [MaybeUninit<(Y, Metadata)>; N], + ) -> &'a [(Y, Metadata)] + where + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, PARENT_TABLE_NUMBER) }>: Sized, + { + let mut next_offset = 0_usize; + for &[(left_y, left_metadata), (right_y, right_metadata)] in + ys_and_metadata.as_chunks::<2>().0 + { + if !has_match(left_y, right_y) { + continue; + } + + next_ys_and_metadata[next_offset].write(compute_fn::< + K, + TABLE_NUMBER, + PARENT_TABLE_NUMBER, + >( + left_y, left_metadata, right_metadata + )); + next_offset += 1; + } + + // SAFETY: Initialized `next_offset` elements + unsafe { next_ys_and_metadata[..next_offset].assume_init_ref() } + } +} + +macro_rules! impl_supported { + ($($k: expr$(,)? )*) => { + $( +impl private::Supported for Tables<$k> {} + )* + } +} + +// Only these k values are supported by the current implementation +#[cfg(feature = "full-chiapos")] +impl_supported!(15, 16, 18, 19, 21, 22, 23, 24, 25); +#[cfg(any(feature = "full-chiapos", test))] +impl_supported!(17); +impl_supported!(20); diff --git a/shared/ab-proof-of-space/src/chiapos/constants.rs b/shared/ab-proof-of-space/src/chiapos/constants.rs new file mode 100644 index 00000000000..abb4838ff3b --- /dev/null +++ b/shared/ab-proof-of-space/src/chiapos/constants.rs @@ -0,0 +1,7 @@ +/// PRNG extension parameter to avoid collisions +pub(super) const PARAM_EXT: u8 = 6; +pub(super) const PARAM_M: u16 = 1 << PARAM_EXT; +pub(super) const PARAM_B: u16 = 119; +pub(super) const PARAM_C: u16 = 127; +pub(super) const PARAM_BC: u16 = PARAM_B * PARAM_C; +pub(super) const NUM_TABLES: u8 = 7; diff --git a/shared/ab-proof-of-space/src/chiapos/table.rs b/shared/ab-proof-of-space/src/chiapos/table.rs new file mode 100644 index 00000000000..6b8afb591ae --- /dev/null +++ b/shared/ab-proof-of-space/src/chiapos/table.rs @@ -0,0 +1,1785 @@ +#[cfg(any(feature = "alloc", test))] +mod rmap; +#[cfg(test)] +mod tests; +pub(super) mod types; + +use crate::chiapos::Seed; +use crate::chiapos::constants::{PARAM_B, PARAM_BC, PARAM_C, PARAM_EXT, PARAM_M}; +#[cfg(feature = "alloc")] +use crate::chiapos::table::rmap::Rmap; +use crate::chiapos::table::types::{Metadata, X, Y}; +#[cfg(feature = "alloc")] +use crate::chiapos::table::types::{Position, R}; +use crate::chiapos::utils::EvaluatableUsize; +use ab_chacha8::{ChaCha8Block, ChaCha8State}; +#[cfg(feature = "alloc")] +use alloc::boxed::Box; +#[cfg(feature = "alloc")] +use alloc::vec; +#[cfg(feature = "alloc")] +use alloc::vec::Vec; +#[cfg(feature = "alloc")] +use chacha20::cipher::{Iv, KeyIvInit, StreamCipher}; +#[cfg(feature = "alloc")] +use chacha20::{ChaCha8, Key}; +use core::array; +#[cfg(feature = "parallel")] +use core::cell::SyncUnsafeCell; +#[cfg(feature = "alloc")] +use core::mem; +#[cfg(feature = "alloc")] +use core::mem::MaybeUninit; +#[cfg(any(feature = "alloc", test))] +use core::simd::prelude::*; +#[cfg(feature = "parallel")] +use core::sync::atomic::{AtomicUsize, Ordering}; +#[cfg(feature = "alloc")] +use derive_more::Deref; +#[cfg(feature = "alloc")] +use rclite::Arc; +#[cfg(any(feature = "alloc", test))] +use seq_macro::seq; +#[cfg(feature = "alloc")] +use subspace_core_primitives::pieces::Record; + +pub(super) const COMPUTE_F1_SIMD_FACTOR: usize = 8; +#[cfg(any(feature = "alloc", test))] +const COMPUTE_FN_SIMD_FACTOR: usize = 16; +const MAX_BUCKET_SIZE: usize = 512; +#[cfg(any(feature = "alloc", test))] +const BUCKET_SIZE_UPPER_BOUND_SECURITY_BITS: u8 = 128; +/// Reducing bucket size for better performance. +/// +/// The number should be sufficient to produce enough proofs for sector encoding with high +/// probability. +const REDUCED_BUCKET_SIZE: usize = 272; +/// Reducing matches count for better performance. +/// +/// The number should be sufficient to produce enough proofs for sector encoding with high +/// probability. +const REDUCED_MATCHES_COUNT: usize = 288; +#[cfg(feature = "parallel")] +const CACHE_LINE_SIZE: usize = 64; + +const { + debug_assert!(REDUCED_BUCKET_SIZE <= MAX_BUCKET_SIZE); + debug_assert!(REDUCED_MATCHES_COUNT <= MAX_BUCKET_SIZE); +} + +/// Compute the size of `y` in bits +pub(super) const fn y_size_bits(k: u8) -> usize { + k as usize + PARAM_EXT as usize +} + +/// Metadata size in bytes +pub const fn metadata_size_bytes(k: u8, table_number: u8) -> usize { + metadata_size_bits(k, table_number).div_ceil(u8::BITS as usize) +} + +/// Metadata size in bits +pub(super) const fn metadata_size_bits(k: u8, table_number: u8) -> usize { + k as usize + * match table_number { + 1 => 1, + 2 => 2, + 3 | 4 => 4, + 5 => 3, + 6 => 2, + 7 => 0, + _ => unreachable!(), + } +} + +/// Number of buckets for a given `k` +pub const fn num_buckets(k: u8) -> usize { + 2_usize + .pow(y_size_bits(k) as u32) + .div_ceil(usize::from(PARAM_BC)) +} + +#[cfg(feature = "parallel")] +#[inline(always)] +fn strip_sync_unsafe_cell(value: Box<[SyncUnsafeCell; N]>) -> Box<[T; N]> { + // SAFETY: `SyncUnsafeCell` has the same layout as `T` + unsafe { Box::from_raw(Box::into_raw(value).cast()) } +} + +/// ChaCha8 [`Vec`] sufficient for the whole first table for [`K`]. +/// Prefer [`partial_y`] if you need partial y just for a single `x`. +#[cfg(feature = "alloc")] +fn partial_ys(seed: Seed) -> Vec { + let output_len_bits = usize::from(K) * (1 << K); + let mut output = vec![0; output_len_bits.div_ceil(u8::BITS as usize)]; + + let key = Key::from(seed); + let iv = Iv::::default(); + + let mut cipher = ChaCha8::new(&key, &iv); + + cipher.write_keystream(&mut output); + + output +} + +/// Calculate a probabilistic upper bound on the Chia bucket size for a given `k` and +/// `security_bits` (security level). +/// +/// This is based on a Chernoff bound for the Poisson distribution with mean +/// `lambda = PARAM_BC / 2^PARAM_EXT`, ensuring the probability that any bucket exceeds the bound is +/// less than `2^{-security_bits}`. +/// The bound is lambda + ceil(sqrt(3 * lambda * (k + security_bits) * ln(2))). +#[cfg(feature = "alloc")] +const fn bucket_size_upper_bound(k: u8, security_bits: u8) -> usize { + // Lambda is the expected number of entries in a bucket, approximated as + // `PARAM_BC / 2^PARAM_EXT`. It is independent of `k`. + const LAMBDA: u64 = PARAM_BC as u64 / 2u64.pow(PARAM_EXT as u32); + // Approximation of ln(2) as a fraction: ln(2) ≈ LN2_NUM / LN2_DEN. + // This allows integer-only computation of the square root term involving ln(2). + const LN2_NUM: u128 = 693_147; + const LN2_DEN: u128 = 1_000_000; + + // `k + security_bits` for the union bound over ~2^k intervals + let ks = k as u128 + security_bits as u128; + // Compute numerator for the expression under the square root: + // `3 * lambda * (k + security_bits) * LN2_NUM` + let num = 3u128 * LAMBDA as u128 * ks * LN2_NUM; + // Denominator for ln(2): `LN2_DEN` + let den = LN2_DEN; + + let ceil_div = num.div_ceil(den); + + // Binary search to find the smallest `x` such that `x * x * den >= num`, + // which computes `ceil(sqrt(num / den))` without floating-point. + // We use a custom binary search over `u64` range because binary search in the standard library + // operates on sorted slices, not directly on integer ranges for solving inequalities like this. + let mut low = 0u64; + let mut high = u64::MAX; + while low < high { + let mid = low + (high - low) / 2; + let left = (mid as u128) * (mid as u128); + if left >= ceil_div { + high = mid; + } else { + low = mid + 1; + } + } + let add_term = low; + + (LAMBDA + add_term) as usize +} + +#[cfg(feature = "alloc")] +fn group_by_buckets( + ys: &[Y], +) -> Box<[[(Position, Y); REDUCED_BUCKET_SIZE]; num_buckets(K)]> +where + [(); num_buckets(K)]:, +{ + let mut bucket_offsets = [0_u16; num_buckets(K)]; + // SAFETY: Contents is `MaybeUninit` + let mut buckets = unsafe { + Box::<[[MaybeUninit<(Position, Y)>; REDUCED_BUCKET_SIZE]; num_buckets(K)]>::new_uninit() + .assume_init() + }; + + for (&y, position) in ys.iter().zip(Position::ZERO..) { + let bucket_index = (u32::from(y) / u32::from(PARAM_BC)) as usize; + + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition + let bucket_offset = unsafe { bucket_offsets.get_unchecked_mut(bucket_index) }; + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition + let bucket = unsafe { buckets.get_unchecked_mut(bucket_index) }; + + if *bucket_offset < REDUCED_BUCKET_SIZE as u16 { + bucket[*bucket_offset as usize].write((position, y)); + *bucket_offset += 1; + } + } + + for (bucket, initialized) in buckets.iter_mut().zip(bucket_offsets) { + bucket[usize::from(initialized)..].write_filled((Position::SENTINEL, Y::SENTINEL)); + } + + // SAFETY: All entries are initialized + unsafe { Box::from_raw(Box::into_raw(buckets).cast()) } +} + +/// Similar to [`group_by_buckets()`], but processes buckets instead of a flat list of `y`s. +/// +/// # Safety +/// Iterator item is a list of potentially uninitialized `y`s and a number of initialized `y`s. The +/// number of initialized `y`s must be correct. +#[cfg(feature = "parallel")] +unsafe fn group_by_buckets_from_buckets<'a, const K: u8, I>( + iter: I, +) -> Box<[[(Position, Y); REDUCED_BUCKET_SIZE]; num_buckets(K)]> +where + I: Iterator; REDUCED_MATCHES_COUNT], usize)> + 'a, + [(); num_buckets(K)]:, +{ + let mut bucket_offsets = [0_u16; num_buckets(K)]; + // SAFETY: Contents is `MaybeUninit` + let mut buckets = unsafe { + Box::<[[MaybeUninit<(Position, Y)>; REDUCED_BUCKET_SIZE]; num_buckets(K)]>::new_uninit() + .assume_init() + }; + + for ((ys, count), batch_start) in iter.zip((Position::ZERO..).step_by(REDUCED_MATCHES_COUNT)) { + // SAFETY: Function contract guarantees that `y`s are initialized + let ys = unsafe { ys[..count].assume_init_ref() }; + for (&y, position) in ys.iter().zip(batch_start..) { + let bucket_index = (u32::from(y) / u32::from(PARAM_BC)) as usize; + + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition + let bucket_offset = unsafe { bucket_offsets.get_unchecked_mut(bucket_index) }; + // SAFETY: Bucket is obtained using division by `PARAM_BC` and fits by definition + let bucket = unsafe { buckets.get_unchecked_mut(bucket_index) }; + + if *bucket_offset < REDUCED_BUCKET_SIZE as u16 { + bucket[*bucket_offset as usize].write((position, y)); + *bucket_offset += 1; + } + } + } + + for (bucket, initialized) in buckets.iter_mut().zip(bucket_offsets) { + bucket[usize::from(initialized)..].write_filled((Position::SENTINEL, Y::SENTINEL)); + } + + // SAFETY: All entries are initialized + unsafe { Box::from_raw(Box::into_raw(buckets).cast()) } +} + +#[cfg(feature = "alloc")] +#[derive(Debug, Copy, Clone, Deref)] +#[repr(align(64))] +struct CacheLineAligned(T); + +/// Mapping from `parity` to `r` to `m` +#[cfg(feature = "alloc")] +type LeftTargets = [[CacheLineAligned<[R; PARAM_M as usize]>; PARAM_BC as usize]; 2]; + +#[cfg(feature = "alloc")] +fn calculate_left_targets() -> Arc { + let mut left_targets = Arc::::new_uninit(); + // SAFETY: Same layout and uninitialized in both cases + let left_targets_slice = unsafe { + mem::transmute::< + &mut MaybeUninit<[[CacheLineAligned<[R; PARAM_M as usize]>; PARAM_BC as usize]; 2]>, + &mut [[MaybeUninit>; PARAM_BC as usize]; 2], + >(Arc::get_mut_unchecked(&mut left_targets)) + }; + + for parity in 0..=1 { + for r in 0..PARAM_BC { + let c = r / PARAM_C; + + let arr = array::from_fn(|m| { + let m = m as u16; + R::from( + ((c + m) % PARAM_B) * PARAM_C + + (((2 * m + parity) * (2 * m + parity) + r) % PARAM_C), + ) + }); + left_targets_slice[parity as usize][r as usize].write(CacheLineAligned(arr)); + } + } + + // SAFETY: Initialized all entries + unsafe { left_targets.assume_init() } +} + +fn calculate_left_target_on_demand(parity: u32, r: u32, m: u32) -> u32 { + let param_b = u32::from(PARAM_B); + let param_c = u32::from(PARAM_C); + + ((r / param_c + m) % param_b) * param_c + (((2 * m + parity) * (2 * m + parity) + r) % param_c) +} + +/// Caches that can be used to optimize the creation of multiple [`Tables`](super::Tables). +#[cfg(feature = "alloc")] +#[derive(Debug, Clone)] +pub struct TablesCache { + left_targets: Arc, +} + +#[cfg(feature = "alloc")] +impl Default for TablesCache { + /// Create a new instance + fn default() -> Self { + Self { + left_targets: calculate_left_targets(), + } + } +} + +#[cfg(feature = "alloc")] +#[derive(Debug, Copy, Clone)] +struct Match { + left_position: Position, + left_y: Y, + right_position: Position, +} + +/// `partial_y_offset` is in bits within `partial_y` +pub(super) fn compute_f1(x: X, seed: &Seed) -> Y { + let skip_bits = u32::from(K) * u32::from(x); + let skip_u32s = skip_bits / u32::BITS; + let partial_y_offset = skip_bits % u32::BITS; + + const U32S_PER_BLOCK: usize = size_of::() / size_of::(); + + let initial_state = ChaCha8State::init(seed, &[0; _]); + let first_block_counter = skip_u32s / U32S_PER_BLOCK as u32; + let u32_in_first_block = skip_u32s as usize % U32S_PER_BLOCK; + + let first_block = initial_state.compute_block(first_block_counter); + let hi = first_block[u32_in_first_block].to_be(); + + // TODO: Is SIMD version of `compute_block()` that produces two blocks at once possible? + let lo = if u32_in_first_block + 1 == U32S_PER_BLOCK { + // Spilled over into the second block + let second_block = initial_state.compute_block(first_block_counter + 1); + second_block[0].to_be() + } else { + first_block[u32_in_first_block + 1].to_be() + }; + + let partial_y = (u64::from(hi) << u32::BITS) | u64::from(lo); + + let pre_y = partial_y >> (u64::BITS - u32::from(K + PARAM_EXT) - partial_y_offset); + let pre_y = pre_y as u32; + // Mask for clearing the rest of bits of `pre_y`. + let pre_y_mask = (u32::MAX << PARAM_EXT) & (u32::MAX >> (u32::BITS - u32::from(K + PARAM_EXT))); + + // Extract `PARAM_EXT` most significant bits from `x` and store in the final offset of + // eventual `y` with the rest of bits being zero (`x` is `0..2^K`) + let pre_ext = u32::from(x) >> (K - PARAM_EXT); + + // Combine all of the bits together: + // [padding zero bits][`K` bits rom `partial_y`][`PARAM_EXT` bits from `x`] + Y::from((pre_y & pre_y_mask) | pre_ext) +} + +#[cfg(any(feature = "alloc", test))] +pub(super) fn compute_f1_simd( + xs: Simd, + partial_ys: &[u8; usize::from(K) * COMPUTE_F1_SIMD_FACTOR / u8::BITS as usize], +) -> [Y; COMPUTE_F1_SIMD_FACTOR] { + // Each element contains `K` desired bits of `partial_ys` in the final offset of eventual `ys` + // with the rest of bits being in undefined state + let pre_ys_bytes = array::from_fn(|i| { + let partial_y_offset = i * usize::from(K); + let partial_y_length = + (partial_y_offset % u8::BITS as usize + usize::from(K)).div_ceil(u8::BITS as usize); + let mut pre_y_bytes = 0u64.to_be_bytes(); + pre_y_bytes[..partial_y_length].copy_from_slice( + &partial_ys[partial_y_offset / u8::BITS as usize..][..partial_y_length], + ); + + u64::from_be_bytes(pre_y_bytes) + }); + let pre_ys_right_offset = array::from_fn(|i| { + let partial_y_offset = i as u32 * u32::from(K); + u64::from(u64::BITS - u32::from(K + PARAM_EXT) - partial_y_offset % u8::BITS) + }); + let pre_ys = Simd::from_array(pre_ys_bytes) >> Simd::from_array(pre_ys_right_offset); + + // Mask for clearing the rest of bits of `pre_ys`. + let pre_ys_mask = Simd::splat( + (u32::MAX << usize::from(PARAM_EXT)) + & (u32::MAX >> (u32::BITS as usize - usize::from(K + PARAM_EXT))), + ); + + // Extract `PARAM_EXT` most significant bits from `xs` and store in the final offset of + // eventual `ys` with the rest of bits being in undefined state. + let pre_exts = xs >> Simd::splat(u32::from(K - PARAM_EXT)); + + // Combine all of the bits together: + // [padding zero bits][`K` bits rom `partial_y`][`PARAM_EXT` bits from `x`] + let ys = (pre_ys.cast() & pre_ys_mask) | pre_exts; + + Y::array_from_repr(ys.to_array()) +} + +/// For verification use [`has_match`] instead. +/// +/// # Safety +/// Left and right bucket positions must correspond to the parent table. +// TODO: Try to reduce the `matches` size further by processing `left_bucket` in chunks (like halves +// for example) +#[cfg(feature = "alloc")] +unsafe fn find_matches_in_buckets<'a>( + left_bucket_index: u32, + left_bucket: &[(Position, Y); REDUCED_BUCKET_SIZE], + right_bucket: &[(Position, Y); REDUCED_BUCKET_SIZE], + // `PARAM_M as usize * 2` corresponds to the upper bound number of matches a single `y` in the + // left bucket might have here + matches: &'a mut [MaybeUninit; REDUCED_MATCHES_COUNT + PARAM_M as usize * 2], + left_targets: &LeftTargets, +) -> &'a [Match] { + let left_base = left_bucket_index * u32::from(PARAM_BC); + let right_base = left_base + u32::from(PARAM_BC); + + let mut rmap = Rmap::new(); + for &(right_position, y) in right_bucket { + // TODO: Wouldn't it make more sense to check the size here instead of sentinel? + if right_position == Position::SENTINEL { + break; + } + let r = R::from((u32::from(y) - right_base) as u16); + // SAFETY: `r` is within `0..PARAM_BC` range by definition, the right bucket is limited to + // `REDUCED_BUCKETS_SIZE` + unsafe { + rmap.add(r, right_position); + } + } + + let parity = left_base % 2; + let left_targets_parity = &left_targets[parity as usize]; + let mut next_match_index = 0; + + // TODO: Simd read for left bucket? It might be more efficient in terms of memory access to + // process chunks of the left bucket against one right value for each at a time + for &(left_position, y) in left_bucket { + // TODO: Wouldn't it make more sense to check the size here instead of sentinel? + // `next_match_index >= REDUCED_MATCHES_COUNT` is crucial to make sure + if left_position == Position::SENTINEL || next_match_index >= REDUCED_MATCHES_COUNT { + // Sentinel values are padded to the end of the bucket + break; + } + + let r = R::from((u32::from(y) - left_base) as u16); + // SAFETY: `r` is within a bucket and exists by definition + let left_targets_r = unsafe { left_targets_parity.get_unchecked(usize::from(r)) }; + + for &r_target in left_targets_r.iter() { + // SAFETY: Targets are always limited to `PARAM_BC` + let [right_position_a, right_position_b] = unsafe { rmap.get(r_target) }; + + if right_position_a != Position::SENTINEL { + // SAFETY: Iteration will stop before `REDUCED_MATCHES_COUNT + PARAM_M * 2` + // elements is inserted + unsafe { matches.get_unchecked_mut(next_match_index) }.write(Match { + left_position, + left_y: y, + right_position: right_position_a, + }); + next_match_index += 1; + + if right_position_b != Position::SENTINEL { + // SAFETY: Iteration will stop before + // `REDUCED_MATCHES_COUNT + PARAM_M * 2` elements is inserted + unsafe { matches.get_unchecked_mut(next_match_index) }.write(Match { + left_position, + left_y: y, + right_position: right_position_b, + }); + next_match_index += 1; + } + } + } + } + + // SAFETY: Initialized this many matches + unsafe { matches[..next_match_index].assume_init_ref() } +} + +/// Simplified version of [`find_matches_in_buckets`] for verification purposes. +pub(super) fn has_match(left_y: Y, right_y: Y) -> bool { + let right_r = u32::from(right_y) % u32::from(PARAM_BC); + let parity = (u32::from(left_y) / u32::from(PARAM_BC)) % 2; + let left_r = u32::from(left_y) % u32::from(PARAM_BC); + + let r_targets = array::from_fn::<_, { PARAM_M as usize }, _>(|i| { + calculate_left_target_on_demand(parity, left_r, i as u32) + }); + + r_targets.contains(&right_r) +} + +#[inline(always)] +pub(super) fn compute_fn( + y: Y, + left_metadata: Metadata, + right_metadata: Metadata, +) -> (Y, Metadata) +where + EvaluatableUsize<{ metadata_size_bytes(K, PARENT_TABLE_NUMBER) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, +{ + let left_metadata = u128::from(left_metadata); + let right_metadata = u128::from(right_metadata); + + let parent_metadata_bits = metadata_size_bits(K, PARENT_TABLE_NUMBER); + + // Part of the `right_bits` at the final offset of eventual `input_a` + let y_and_left_bits = y_size_bits(K) + parent_metadata_bits; + let right_bits_start_offset = u128::BITS as usize - parent_metadata_bits; + + // Take only bytes where bits were set + let num_bytes_with_data = + (y_size_bits(K) + parent_metadata_bits * 2).div_ceil(u8::BITS as usize); + + // Only supports `K` from 15 to 25 (otherwise math will not be correct when concatenating y, + // left metadata and right metadata) + let hash = { + // Collect `K` most significant bits of `y` at the final offset of eventual `input_a` + let y_bits = u128::from(y) << (u128::BITS as usize - y_size_bits(K)); + + // Move bits of `left_metadata` at the final offset of eventual `input_a` + let left_metadata_bits = + left_metadata << (u128::BITS as usize - parent_metadata_bits - y_size_bits(K)); + + // If `right_metadata` bits start to the left of the desired position in `input_a` move + // bits right, else move left + if right_bits_start_offset < y_and_left_bits { + let right_bits_pushed_into_input_b = y_and_left_bits - right_bits_start_offset; + // Collect bits of `right_metadata` that will fit into `input_a` at the final offset in + // eventual `input_a` + let right_bits_a = right_metadata >> right_bits_pushed_into_input_b; + let input_a = y_bits | left_metadata_bits | right_bits_a; + // Collect bits of `right_metadata` that will spill over into `input_b` + let input_b = right_metadata << (u128::BITS as usize - right_bits_pushed_into_input_b); + + let input = [input_a.to_be_bytes(), input_b.to_be_bytes()]; + let input_len = + size_of::() + right_bits_pushed_into_input_b.div_ceil(u8::BITS as usize); + ab_blake3::single_block_hash(&input.as_flattened()[..input_len]) + .expect("Exactly a single block worth of bytes; qed") + } else { + let right_bits_a = right_metadata << (right_bits_start_offset - y_and_left_bits); + let input_a = y_bits | left_metadata_bits | right_bits_a; + + ab_blake3::single_block_hash(&input_a.to_be_bytes()[..num_bytes_with_data]) + .expect("Less than a single block worth of bytes; qed") + } + }; + + let y_output = Y::from( + u32::from_be_bytes([hash[0], hash[1], hash[2], hash[3]]) + >> (u32::BITS as usize - y_size_bits(K)), + ); + + let metadata_size_bits = metadata_size_bits(K, TABLE_NUMBER); + + let metadata = if TABLE_NUMBER < 4 { + (left_metadata << parent_metadata_bits) | right_metadata + } else if metadata_size_bits > 0 { + // For K up to 25 it is guaranteed that metadata + bit offset will always fit into u128. + // We collect the bytes necessary, potentially with extra bits at the start and end of the + // bytes that will be taken care of later. + let metadata = u128::from_be_bytes( + hash[y_size_bits(K) / u8::BITS as usize..][..size_of::()] + .try_into() + .expect("Always enough bits for any K; qed"), + ); + // Remove extra bits at the beginning + let metadata = metadata << (y_size_bits(K) % u8::BITS as usize); + // Move bits into the correct location + metadata >> (u128::BITS as usize - metadata_size_bits) + } else { + 0 + }; + + (y_output, Metadata::from(metadata)) +} + +// TODO: This is actually using only pipelining rather than real SIMD (at least explicitly) due to: +// * https://github.com/rust-lang/portable-simd/issues/108 +// * https://github.com/BLAKE3-team/BLAKE3/issues/478#issuecomment-3200106103 +#[cfg(any(feature = "alloc", test))] +fn compute_fn_simd( + left_ys: [Y; COMPUTE_FN_SIMD_FACTOR], + left_metadatas: [Metadata; COMPUTE_FN_SIMD_FACTOR], + right_metadatas: [Metadata; COMPUTE_FN_SIMD_FACTOR], +) -> ( + Simd, + [Metadata; COMPUTE_FN_SIMD_FACTOR], +) +where + EvaluatableUsize<{ metadata_size_bytes(K, PARENT_TABLE_NUMBER) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, +{ + let parent_metadata_bits = metadata_size_bits(K, PARENT_TABLE_NUMBER); + let metadata_size_bits = metadata_size_bits(K, TABLE_NUMBER); + + // TODO: `u128` is not supported as SIMD element yet, see + // https://github.com/rust-lang/portable-simd/issues/108 + let left_metadatas: [u128; COMPUTE_FN_SIMD_FACTOR] = seq!(N in 0..16 { + [ + #( + u128::from(left_metadatas[N]), + )* + ] + }); + let right_metadatas: [u128; COMPUTE_FN_SIMD_FACTOR] = seq!(N in 0..16 { + [ + #( + u128::from(right_metadatas[N]), + )* + ] + }); + + // Part of the `right_bits` at the final offset of eventual `input_a` + let y_and_left_bits = y_size_bits(K) + parent_metadata_bits; + let right_bits_start_offset = u128::BITS as usize - parent_metadata_bits; + + // Take only bytes where bits were set + let num_bytes_with_data = + (y_size_bits(K) + parent_metadata_bits * 2).div_ceil(u8::BITS as usize); + + // Only supports `K` from 15 to 25 (otherwise math will not be correct when concatenating y, + // left metadata and right metadata) + // TODO: SIMD hashing once this is possible: + // https://github.com/BLAKE3-team/BLAKE3/issues/478#issuecomment-3200106103 + let hashes: [_; COMPUTE_FN_SIMD_FACTOR] = seq!(N in 0..16 { + [ + #( + { + let y = left_ys[N]; + let left_metadata = left_metadatas[N]; + let right_metadata = right_metadatas[N]; + + // Collect `K` most significant bits of `y` at the final offset of eventual + // `input_a` + let y_bits = u128::from(y) << (u128::BITS as usize - y_size_bits(K)); + + // Move bits of `left_metadata` at the final offset of eventual `input_a` + let left_metadata_bits = + left_metadata << (u128::BITS as usize - parent_metadata_bits - y_size_bits(K)); + + // If `right_metadata` bits start to the left of the desired position in `input_a` move + // bits right, else move left + if right_bits_start_offset < y_and_left_bits { + let right_bits_pushed_into_input_b = y_and_left_bits - right_bits_start_offset; + // Collect bits of `right_metadata` that will fit into `input_a` at the final offset + // in eventual `input_a` + let right_bits_a = right_metadata >> right_bits_pushed_into_input_b; + let input_a = y_bits | left_metadata_bits | right_bits_a; + // Collect bits of `right_metadata` that will spill over into `input_b` + let input_b = right_metadata << (u128::BITS as usize - right_bits_pushed_into_input_b); + + let input = [input_a.to_be_bytes(), input_b.to_be_bytes()]; + let input_len = + size_of::() + right_bits_pushed_into_input_b.div_ceil(u8::BITS as usize); + ab_blake3::single_block_hash(&input.as_flattened()[..input_len]) + .expect("Exactly a single block worth of bytes; qed") + } else { + let right_bits_a = right_metadata << (right_bits_start_offset - y_and_left_bits); + let input_a = y_bits | left_metadata_bits | right_bits_a; + + ab_blake3::single_block_hash(&input_a.to_be_bytes()[..num_bytes_with_data]) + .expect("Exactly a single block worth of bytes; qed") + } + }, + )* + ] + }); + + let y_outputs = Simd::from_array( + hashes.map(|hash| u32::from_be_bytes([hash[0], hash[1], hash[2], hash[3]])), + ) >> (u32::BITS - y_size_bits(K) as u32); + + let metadatas = if TABLE_NUMBER < 4 { + seq!(N in 0..16 { + [ + #( + Metadata::from((left_metadatas[N] << parent_metadata_bits) | right_metadatas[N]), + )* + ] + }) + } else if metadata_size_bits > 0 { + // For K up to 25 it is guaranteed that metadata + bit offset will always fit into u128. + // We collect the bytes necessary, potentially with extra bits at the start and end of the + // bytes that will be taken care of later. + seq!(N in 0..16 { + [ + #( + { + let metadata = u128::from_be_bytes( + hashes[N][y_size_bits(K) / u8::BITS as usize..][..size_of::()] + .try_into() + .expect("Always enough bits for any K; qed"), + ); + // Remove extra bits at the beginning + let metadata = metadata << (y_size_bits(K) % u8::BITS as usize); + // Move bits into the correct location + Metadata::from(metadata >> (u128::BITS as usize - metadata_size_bits)) + }, + )* + ] + }) + } else { + [Metadata::default(); _] + }; + + (y_outputs, metadatas) +} + +/// # Safety +/// `m` must contain positions that correspond to the parent table +#[cfg(feature = "alloc")] +#[inline(always)] +unsafe fn match_to_result( + parent_table: &Table, + m: &Match, +) -> (Y, [Position; 2], Metadata) +where + Table: private::NotLastTable, + EvaluatableUsize<{ metadata_size_bytes(K, PARENT_TABLE_NUMBER) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + // SAFETY: Guaranteed by function contract + let left_metadata = unsafe { parent_table.metadata(m.left_position) }; + // SAFETY: Guaranteed by function contract + let right_metadata = unsafe { parent_table.metadata(m.right_position) }; + + let (y, metadata) = + compute_fn::(m.left_y, left_metadata, right_metadata); + + (y, [m.left_position, m.right_position], metadata) +} + +/// # Safety +/// `matches` must contain positions that correspond to the parent table +#[cfg(feature = "alloc")] +#[inline(always)] +unsafe fn match_to_result_simd( + parent_table: &Table, + matches: &[Match; COMPUTE_FN_SIMD_FACTOR], +) -> ( + Simd, + [[Position; 2]; COMPUTE_FN_SIMD_FACTOR], + [Metadata; COMPUTE_FN_SIMD_FACTOR], +) +where + Table: private::NotLastTable, + EvaluatableUsize<{ metadata_size_bytes(K, PARENT_TABLE_NUMBER) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + let left_ys: [_; COMPUTE_FN_SIMD_FACTOR] = seq!(N in 0..16 { + [ + #( + matches[N].left_y, + )* + ] + }); + // SAFETY: Guaranteed by function contract + let left_metadatas: [_; COMPUTE_FN_SIMD_FACTOR] = unsafe { + seq!(N in 0..16 { + [ + #( + parent_table.metadata(matches[N].left_position), + )* + ] + }) + }; + // SAFETY: Guaranteed by function contract + let right_metadatas: [_; COMPUTE_FN_SIMD_FACTOR] = unsafe { + seq!(N in 0..16 { + [ + #( + parent_table.metadata(matches[N].right_position), + )* + ] + }) + }; + + let (y_outputs, metadatas) = compute_fn_simd::( + left_ys, + left_metadatas, + right_metadatas, + ); + + let positions = seq!(N in 0..16 { + [ + #( + [ + matches[N].left_position, + matches[N].right_position, + ], + )* + ] + }); + + (y_outputs, positions, metadatas) +} + +/// # Safety +/// `matches` must contain positions that correspond to the parent table. `ys`, `position` and +/// `metadatas` length must be at least the length of `matches` +#[cfg(feature = "alloc")] +#[inline(always)] +unsafe fn matches_to_results( + parent_table: &Table, + matches: &[Match], + ys: &mut [MaybeUninit], + positions: &mut [MaybeUninit<[Position; 2]>], + metadatas: &mut [MaybeUninit>], +) where + Table: private::NotLastTable, + EvaluatableUsize<{ metadata_size_bytes(K, PARENT_TABLE_NUMBER) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + let (grouped_matches, other_matches) = matches.as_chunks::(); + let (grouped_ys, other_ys) = ys.split_at_mut(grouped_matches.as_flattened().len()); + let grouped_ys = grouped_ys.as_chunks_mut::().0; + let (grouped_positions, other_positions) = + positions.split_at_mut(grouped_matches.as_flattened().len()); + let grouped_positions = grouped_positions + .as_chunks_mut::() + .0; + let (grouped_metadatas, other_metadatas) = + metadatas.split_at_mut(grouped_matches.as_flattened().len()); + let grouped_metadatas = grouped_metadatas + .as_chunks_mut::() + .0; + + for (((grouped_matches, grouped_ys), grouped_positions), grouped_metadatas) in grouped_matches + .iter() + .zip(grouped_ys) + .zip(grouped_positions) + .zip(grouped_metadatas) + { + // SAFETY: Guaranteed by function contract + let (ys_group, positions_group, metadatas_group) = + unsafe { match_to_result_simd(parent_table, grouped_matches) }; + let ys_group = Y::array_from_repr(ys_group.to_array()); + grouped_ys.write_copy_of_slice(&ys_group); + grouped_positions.write_copy_of_slice(&positions_group); + + // The last table doesn't have metadata + if metadata_size_bits(K, TABLE_NUMBER) > 0 { + grouped_metadatas.write_copy_of_slice(&metadatas_group); + } + } + for (((other_match, other_y), other_positions), other_metadata) in other_matches + .iter() + .zip(other_ys) + .zip(other_positions) + .zip(other_metadatas) + { + // SAFETY: Guaranteed by function contract + let (y, p, metadata) = unsafe { match_to_result(parent_table, other_match) }; + other_y.write(y); + other_positions.write(p); + // The last table doesn't have metadata + if metadata_size_bits(K, TABLE_NUMBER) > 0 { + other_metadata.write(metadata); + } + } +} + +/// Similar to [`Table`], but smaller size for later processing stages +#[cfg(feature = "alloc")] +#[derive(Debug)] +pub(super) enum PrunedTable +where + [(); 1 << K]:, + [(); num_buckets(K) - 1]:, +{ + First, + /// Other tables + Other { + /// Left and right entry positions in a previous table encoded into bits + positions: Box<[MaybeUninit<[Position; 2]>; 1 << K]>, + }, + /// Other tables + #[cfg(feature = "parallel")] + OtherBuckets { + /// Left and right entry positions in a previous table encoded into bits. + /// + /// Only positions from the `buckets` field are guaranteed to be initialized. + positions: Box<[[MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT]; num_buckets(K) - 1]>, + }, +} + +#[cfg(feature = "alloc")] +impl PrunedTable +where + [(); 1 << K]:, + [(); num_buckets(K) - 1]:, +{ + /// Get `[left_position, right_position]` of a previous table for a specified position in a + /// current table. + /// + /// # Safety + /// `position` must come from [`Table::buckets()`] or [`Self::position()`] and not be a sentinel + /// value. + #[inline(always)] + pub(super) unsafe fn position(&self, position: Position) -> [Position; 2] { + match self { + Self::First => { + unreachable!("Not the first table"); + } + Self::Other { positions, .. } => { + // SAFETY: All non-sentinel positions returned by [`Self::buckets()`] are valid + unsafe { positions.get_unchecked(usize::from(position)).assume_init() } + } + #[cfg(feature = "parallel")] + Self::OtherBuckets { positions, .. } => { + // SAFETY: All non-sentinel positions returned by [`Self::buckets()`] are valid + unsafe { + positions + .as_flattened() + .get_unchecked(usize::from(position)) + .assume_init() + } + } + } + } +} + +#[cfg(feature = "alloc")] +#[derive(Debug)] +pub(super) enum Table +where + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + /// First table + First { + /// Each bucket contains positions of `Y` values that belong to it and corresponding `y`. + /// + /// Buckets are padded with sentinel values to `REDUCED_BUCKETS_SIZE`. + buckets: Box<[[(Position, Y); REDUCED_BUCKET_SIZE]; num_buckets(K)]>, + }, + /// Other tables + Other { + /// Left and right entry positions in a previous table encoded into bits + positions: Box<[MaybeUninit<[Position; 2]>; 1 << K]>, + /// Metadata corresponding to each entry + metadatas: Box<[MaybeUninit>; 1 << K]>, + /// Each bucket contains positions of `Y` values that belong to it and corresponding `y`. + /// + /// Buckets are padded with sentinel values to `REDUCED_BUCKETS_SIZE`. + buckets: Box<[[(Position, Y); REDUCED_BUCKET_SIZE]; num_buckets(K)]>, + }, + /// Other tables + #[cfg(feature = "parallel")] + OtherBuckets { + /// Left and right entry positions in a previous table encoded into bits. + /// + /// Only positions from the `buckets` field are guaranteed to be initialized. + positions: Box<[[MaybeUninit<[Position; 2]>; REDUCED_MATCHES_COUNT]; num_buckets(K) - 1]>, + /// Metadata corresponding to each entry. + /// + /// Only positions from the `buckets` field are guaranteed to be initialized. + metadatas: Box< + [[MaybeUninit>; REDUCED_MATCHES_COUNT]; num_buckets(K) - 1], + >, + /// Each bucket contains positions of `Y` values that belong to it and corresponding `y`. + /// + /// Buckets are padded with sentinel values to `REDUCED_BUCKETS_SIZE`. + buckets: Box<[[(Position, Y); REDUCED_BUCKET_SIZE]; num_buckets(K)]>, + }, +} + +#[cfg(feature = "alloc")] +impl Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 1) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + /// Create the table + pub(super) fn create(seed: Seed) -> Self + where + EvaluatableUsize<{ usize::from(K) * COMPUTE_F1_SIMD_FACTOR / u8::BITS as usize }>: Sized, + { + // `MAX_BUCKET_SIZE` is not actively used, but is an upper-bound reference for the other + // parameters + debug_assert!( + MAX_BUCKET_SIZE >= bucket_size_upper_bound(K, BUCKET_SIZE_UPPER_BOUND_SECURITY_BITS), + "Max bucket size is not sufficiently large" + ); + + let partial_ys = partial_ys::(seed); + + // SAFETY: Contents is `MaybeUninit` + let mut ys = unsafe { Box::<[MaybeUninit; 1 << K]>::new_uninit().assume_init() }; + + for ((ys, xs_batch_start), partial_ys) in ys + .as_chunks_mut::() + .0 + .iter_mut() + .zip((X::ZERO..).step_by(COMPUTE_F1_SIMD_FACTOR)) + .zip( + partial_ys + .as_chunks::<{ usize::from(K) * COMPUTE_F1_SIMD_FACTOR / u8::BITS as usize }>() + .0, + ) + { + let xs = Simd::splat(u32::from(xs_batch_start)) + + Simd::from_array(array::from_fn(|i| i as u32)); + let ys_batch = compute_f1_simd::(xs, partial_ys); + + ys.write_copy_of_slice(&ys_batch); + } + + // SAFETY: All elements were initialized + let ys = unsafe { ys.assume_init_ref() }; + + // TODO: Try to group buckets in the process of collecting `y`s + let buckets = group_by_buckets::(ys); + + Self::First { buckets } + } + + /// Create the table, leverages available parallelism + #[cfg(feature = "parallel")] + pub(super) fn create_parallel(seed: Seed) -> Self + where + EvaluatableUsize<{ usize::from(K) * COMPUTE_F1_SIMD_FACTOR / u8::BITS as usize }>: Sized, + { + // `MAX_BUCKET_SIZE` is not actively used, but is an upper-bound reference for the other + // parameters + debug_assert!( + MAX_BUCKET_SIZE >= bucket_size_upper_bound(K, BUCKET_SIZE_UPPER_BOUND_SECURITY_BITS), + "Max bucket size is not sufficiently large" + ); + + let partial_ys = partial_ys::(seed); + + // SAFETY: Contents is `MaybeUninit` + let mut ys = unsafe { Box::<[MaybeUninit; 1 << K]>::new_uninit().assume_init() }; + + // TODO: Try parallelism here? + for ((ys, xs_batch_start), partial_ys) in ys + .as_chunks_mut::() + .0 + .iter_mut() + .zip((X::ZERO..).step_by(COMPUTE_F1_SIMD_FACTOR)) + .zip( + partial_ys + .as_chunks::<{ usize::from(K) * COMPUTE_F1_SIMD_FACTOR / u8::BITS as usize }>() + .0, + ) + { + let xs = Simd::splat(u32::from(xs_batch_start)) + + Simd::from_array(array::from_fn(|i| i as u32)); + let ys_batch = compute_f1_simd::(xs, partial_ys); + + ys.write_copy_of_slice(&ys_batch); + } + + // SAFETY: All elements were initialized + let ys = unsafe { ys.assume_init_ref() }; + + // TODO: Try to group buckets in the process of collecting `y`s + let buckets = group_by_buckets::(ys); + + Self::First { buckets } + } +} + +#[cfg(feature = "alloc")] +mod private { + pub(in super::super) trait SupportedOtherTables {} + pub(in super::super) trait NotLastTable {} +} + +#[cfg(feature = "alloc")] +impl private::SupportedOtherTables for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 2) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::SupportedOtherTables for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 3) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::SupportedOtherTables for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 4) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::SupportedOtherTables for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 5) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::SupportedOtherTables for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 6) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::SupportedOtherTables for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 7) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} + +#[cfg(feature = "alloc")] +impl private::NotLastTable for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 1) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::NotLastTable for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 2) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::NotLastTable for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 3) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::NotLastTable for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 4) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::NotLastTable for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 5) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} +#[cfg(feature = "alloc")] +impl private::NotLastTable for Table +where + EvaluatableUsize<{ metadata_size_bytes(K, 6) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ +} + +#[cfg(feature = "alloc")] +impl Table +where + Self: private::SupportedOtherTables, + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + /// Creates a new [`TABLE_NUMBER`] table. There also exists [`Self::create_parallel()`] that + /// trades CPU efficiency and memory usage for lower latency and with multiple parallel calls, + /// better overall performance. + pub(super) fn create( + parent_table: Table, + cache: &TablesCache, + ) -> (Self, PrunedTable) + where + Table: private::NotLastTable, + EvaluatableUsize<{ metadata_size_bytes(K, PARENT_TABLE_NUMBER) }>: Sized, + { + let left_targets = &*cache.left_targets; + let mut initialized_elements = 0_usize; + // SAFETY: Contents is `MaybeUninit` + let mut ys = unsafe { Box::<[MaybeUninit; 1 << K]>::new_uninit().assume_init() }; + // SAFETY: Contents is `MaybeUninit` + let mut positions = + unsafe { Box::<[MaybeUninit<[Position; 2]>; 1 << K]>::new_uninit().assume_init() }; + // SAFETY: Contents is `MaybeUninit` + let mut metadatas = unsafe { + Box::<[MaybeUninit>; 1 << K]>::new_uninit().assume_init() + }; + + for ([left_bucket, right_bucket], left_bucket_index) in + parent_table.buckets().array_windows().zip(0..) + { + let mut matches = [MaybeUninit::uninit(); _]; + // SAFETY: Positions are taken from `Table::buckets()` and correspond to initialized + // values + let matches = unsafe { + find_matches_in_buckets( + left_bucket_index, + left_bucket, + right_bucket, + &mut matches, + left_targets, + ) + }; + // Throw away some successful matches that are not that necessary + let matches = &matches[..matches.len().min(REDUCED_MATCHES_COUNT)]; + // SAFETY: Already initialized this many elements + let (ys, positions, metadatas) = unsafe { + ( + ys.get_unchecked_mut(initialized_elements..), + positions.get_unchecked_mut(initialized_elements..), + metadatas.get_unchecked_mut(initialized_elements..), + ) + }; + + // SAFETY: Preallocated length is an upper bound and is always sufficient + let (ys, positions, metadatas) = unsafe { + ( + ys.get_unchecked_mut(..matches.len()), + positions.get_unchecked_mut(..matches.len()), + metadatas.get_unchecked_mut(..matches.len()), + ) + }; + + // SAFETY: Matches come from the parent table and the size of `ys`, `positions` + // and `metadatas` is the same as the number of matches + unsafe { + matches_to_results(&parent_table, matches, ys, positions, metadatas); + } + + initialized_elements += matches.len(); + } + + let parent_table = parent_table.prune(); + + // SAFETY: Converting a boxed array to a vector of the same size, which has the same memory + // layout, the number of elements matches the number of elements that were initialized + let ys = unsafe { + let ys_len = ys.len(); + let ys = Box::into_raw(ys); + Vec::from_raw_parts(ys.cast(), initialized_elements, ys_len) + }; + + // TODO: Try to group buckets in the process of collecting `y`s + let buckets = group_by_buckets::(&ys); + + let table = Self::Other { + positions, + metadatas, + buckets, + }; + + (table, parent_table) + } + + /// Almost the same as [`Self::create()`], but uses parallelism internally for better + /// performance (though not efficiency of CPU and memory usage), if you create multiple tables + /// in parallel, prefer this method for better overall performance. + #[cfg(feature = "parallel")] + pub(super) fn create_parallel( + parent_table: Table, + cache: &TablesCache, + ) -> (Self, PrunedTable) + where + Table: private::NotLastTable, + EvaluatableUsize<{ metadata_size_bytes(K, PARENT_TABLE_NUMBER) }>: Sized, + { + // SAFETY: Contents is `MaybeUninit` + let ys = unsafe { + Box::<[SyncUnsafeCell<[MaybeUninit<_>; REDUCED_MATCHES_COUNT]>; num_buckets(K) - 1]>::new_uninit().assume_init() + }; + // SAFETY: Contents is `MaybeUninit` + let positions = unsafe { + Box::<[SyncUnsafeCell<[MaybeUninit<_>; REDUCED_MATCHES_COUNT]>; num_buckets(K) - 1]>::new_uninit().assume_init() + }; + // SAFETY: Contents is `MaybeUninit` + let metadatas = unsafe { + Box::<[SyncUnsafeCell<[MaybeUninit<_>; REDUCED_MATCHES_COUNT]>; num_buckets(K) - 1]>::new_uninit().assume_init() + }; + let global_results_counts = + array::from_fn::<_, { num_buckets(K) - 1 }, _>(|_| SyncUnsafeCell::new(0u16)); + + let left_targets = &*cache.left_targets; + + let buckets = parent_table.buckets(); + // Iterate over buckets in batches, such that a cache line worth of bytes is taken from + // `global_results_counts` each time to avoid unnecessary false sharing + let bucket_batch_size = CACHE_LINE_SIZE / size_of::(); + let bucket_batch_index = AtomicUsize::new(0); + + rayon::broadcast(|_ctx| { + loop { + let bucket_batch_index = bucket_batch_index.fetch_add(1, Ordering::Relaxed); + + let buckets_batch = buckets + .array_windows() + .enumerate() + .skip(bucket_batch_index * bucket_batch_size) + .take(bucket_batch_size); + + if buckets_batch.is_empty() { + break; + } + + for (left_bucket_index, [left_bucket, right_bucket]) in buckets_batch { + let mut matches = [MaybeUninit::uninit(); _]; + // SAFETY: Positions are taken from `Table::buckets()` and correspond to + // initialized values + let matches = unsafe { + find_matches_in_buckets( + left_bucket_index as u32, + left_bucket, + right_bucket, + &mut matches, + left_targets, + ) + }; + // Throw away some successful matches that are not that necessary + let matches = &matches[..matches.len().min(REDUCED_MATCHES_COUNT)]; + + // SAFETY: This is the only place where `left_bucket_index`'s entry is accessed + // at this time, and it is guaranteed to be in range + let ys = unsafe { &mut *ys.get_unchecked(left_bucket_index).get() }; + // SAFETY: This is the only place where `left_bucket_index`'s entry is accessed + // at this time, and it is guaranteed to be in range + let positions = + unsafe { &mut *positions.get_unchecked(left_bucket_index).get() }; + // SAFETY: This is the only place where `left_bucket_index`'s entry is accessed + // at this time, and it is guaranteed to be in range + let metadatas = + unsafe { &mut *metadatas.get_unchecked(left_bucket_index).get() }; + // SAFETY: This is the only place where `left_bucket_index`'s entry is accessed + // at this time, and it is guaranteed to be in range + let count = unsafe { + &mut *global_results_counts.get_unchecked(left_bucket_index).get() + }; + + // SAFETY: Matches come from the parent table and the size of `ys`, `positions` + // and `metadatas` is larger or equal to the number of matches + unsafe { + matches_to_results::<_, TABLE_NUMBER, _>( + &parent_table, + matches, + ys, + positions, + metadatas, + ); + } + *count = matches.len() as u16; + } + } + }); + + let parent_table = parent_table.prune(); + + let ys = strip_sync_unsafe_cell(ys); + let positions = strip_sync_unsafe_cell(positions); + let metadatas = strip_sync_unsafe_cell(metadatas); + + // TODO: Try to group buckets in the process of collecting `y`s + // SAFETY: `global_results_counts` corresponds to the number of initialized `ys` + let buckets = unsafe { + group_by_buckets_from_buckets::( + ys.iter().zip( + global_results_counts + .into_iter() + .map(|count| usize::from(count.into_inner())), + ), + ) + }; + + let table = Self::OtherBuckets { + positions, + metadatas, + buckets, + }; + + (table, parent_table) + } + + /// Get `[left_position, right_position]` of a previous table for a specified position in a + /// current table. + /// + /// # Safety + /// `position` must come from [`Self::buckets()`] or [`Self::position()`] or + /// [`PrunedTable::position()`] and not be a sentinel value. + #[inline(always)] + pub(super) unsafe fn position(&self, position: Position) -> [Position; 2] { + match self { + Self::First { .. } => { + unreachable!("Not the first table"); + } + Self::Other { positions, .. } => { + // SAFETY: All non-sentinel positions returned by [`Self::buckets()`] are valid + unsafe { positions.get_unchecked(usize::from(position)).assume_init() } + } + #[cfg(feature = "parallel")] + Self::OtherBuckets { positions, .. } => { + // SAFETY: All non-sentinel positions returned by [`Self::buckets()`] are valid + unsafe { + positions + .as_flattened() + .get_unchecked(usize::from(position)) + .assume_init() + } + } + } + } +} + +/// Subspace's little-endian s-bucket convention: maps a table-7 entry's `first_k_bits` to its +/// s-bucket, returning a value `>= Record::NUM_S_BUCKETS` for entries whose low `K - 16` bits are +/// set (those are discarded). Consensus verification derives the challenge from the s-bucket with +/// the same little-endian byte layout, so this convention is fixed. +#[cfg(feature = "alloc")] +#[inline(always)] +fn little_endian_s_bucket(first_k_bits: u32, k: u8) -> u32 { + let low_bits = u32::from(k) - 16; + if first_k_bits & ((1 << low_bits) - 1) != 0 { + return u32::MAX; + } + let cs_lo = (first_k_bits >> (u32::from(k) - 8)) & 0xff; + let cs_hi = (first_k_bits >> low_bits) & 0xff; + cs_lo | (cs_hi << 8) +} + +#[cfg(feature = "alloc")] +impl Table +where + Self: private::SupportedOtherTables, + EvaluatableUsize<{ metadata_size_bytes(K, 7) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + /// Proof targets from the last table into the previous table, one for each + /// [`Record::NUM_S_BUCKETS`]. + pub(super) fn create_proof_targets( + parent_table: Table, + cache: &TablesCache, + ) -> ( + Box<[[Position; 2]; Record::NUM_S_BUCKETS]>, + PrunedTable, + ) + where + Table: private::NotLastTable, + EvaluatableUsize<{ metadata_size_bytes(K, 6) }>: Sized, + { + let left_targets = &*cache.left_targets; + // SAFETY: Data structure filled with zeroes is a valid invariant + let mut table_6_proof_targets = + unsafe { Box::<[[Position; 2]; Record::NUM_S_BUCKETS]>::new_zeroed().assume_init() }; + + for ([left_bucket, right_bucket], left_bucket_index) in + parent_table.buckets().array_windows().zip(0..) + { + let mut matches = [MaybeUninit::uninit(); _]; + // SAFETY: Positions are taken from `Table::buckets()` and correspond to initialized + // values + let matches = unsafe { + find_matches_in_buckets( + left_bucket_index, + left_bucket, + right_bucket, + &mut matches, + left_targets, + ) + }; + // Throw away some successful matches that are not that necessary + let matches = &matches[..matches.len().min(REDUCED_MATCHES_COUNT)]; + + let (grouped_matches, other_matches) = matches.as_chunks::(); + + for grouped_matches in grouped_matches { + // SAFETY: Guaranteed by function contract + let (ys_group, positions_group, _) = + unsafe { match_to_result_simd::<_, 7, _>(&parent_table, grouped_matches) }; + + let s_buckets = ys_group >> Simd::splat(u32::from(PARAM_EXT)); + + for (s_bucket, p) in s_buckets.to_array().into_iter().zip(positions_group) { + const { + assert!(Record::NUM_S_BUCKETS == (u16::MAX as usize) + 1); + } + let s_bucket = little_endian_s_bucket(s_bucket, K); + if s_bucket >= Record::NUM_S_BUCKETS as u32 { + continue; + } + let s_bucket = s_bucket as u16; + let positions = &mut table_6_proof_targets[usize::from(s_bucket)]; + if positions == &[Position::ZERO; 2] { + *positions = p; + } + } + } + for other_match in other_matches { + // SAFETY: Guaranteed by function contract + let (y, p, _) = unsafe { match_to_result::<_, 7, _>(&parent_table, other_match) }; + + let s_bucket = y.first_k_bits(); + + const { + assert!(Record::NUM_S_BUCKETS == (u16::MAX as usize) + 1); + } + let s_bucket = little_endian_s_bucket(s_bucket, K); + if s_bucket >= Record::NUM_S_BUCKETS as u32 { + continue; + } + let s_bucket = s_bucket as u16; + + let positions = &mut table_6_proof_targets[usize::from(s_bucket)]; + if positions == &[Position::ZERO; 2] { + *positions = p; + } + } + } + + let parent_table = parent_table.prune(); + + (table_6_proof_targets, parent_table) + } + + /// Almost the same as [`Self::create_proof_targets()`], but uses parallelism internally for + /// better performance (though not efficiency of CPU and memory usage), if you create multiple + /// tables in parallel, prefer this method for better overall performance. + #[cfg(feature = "parallel")] + pub(super) fn create_proof_targets_parallel( + parent_table: Table, + cache: &TablesCache, + ) -> ( + Box<[[Position; 2]; Record::NUM_S_BUCKETS]>, + PrunedTable, + ) + where + Table: private::NotLastTable, + EvaluatableUsize<{ metadata_size_bytes(K, 6) }>: Sized, + { + // SAFETY: Contents is `MaybeUninit` + let buckets_positions = unsafe { + Box::<[SyncUnsafeCell<[MaybeUninit<_>; REDUCED_MATCHES_COUNT]>; num_buckets(K) - 1]>::new_uninit().assume_init() + }; + let global_results_counts = + array::from_fn::<_, { num_buckets(K) - 1 }, _>(|_| SyncUnsafeCell::new(0u16)); + + let left_targets = &*cache.left_targets; + + let buckets = parent_table.buckets(); + // Iterate over buckets in batches, such that a cache line worth of bytes is taken from + // `global_results_counts` each time to avoid unnecessary false sharing + let bucket_batch_size = CACHE_LINE_SIZE / size_of::(); + let bucket_batch_index = AtomicUsize::new(0); + + rayon::broadcast(|_ctx| { + loop { + let bucket_batch_index = bucket_batch_index.fetch_add(1, Ordering::Relaxed); + + let buckets_batch = buckets + .array_windows() + .enumerate() + .skip(bucket_batch_index * bucket_batch_size) + .take(bucket_batch_size); + + if buckets_batch.is_empty() { + break; + } + + for (left_bucket_index, [left_bucket, right_bucket]) in buckets_batch { + let mut matches = [MaybeUninit::uninit(); _]; + // SAFETY: Positions are taken from `Table::buckets()` and correspond to + // initialized values + let matches = unsafe { + find_matches_in_buckets( + left_bucket_index as u32, + left_bucket, + right_bucket, + &mut matches, + left_targets, + ) + }; + // Throw away some successful matches that are not that necessary + let matches = &matches[..matches.len().min(REDUCED_MATCHES_COUNT)]; + + // SAFETY: This is the only place where `left_bucket_index`'s entry is accessed + // at this time, and it is guaranteed to be in range + let buckets_positions = + unsafe { &mut *buckets_positions.get_unchecked(left_bucket_index).get() }; + // SAFETY: This is the only place where `left_bucket_index`'s entry is accessed + // at this time, and it is guaranteed to be in range + let count = unsafe { + &mut *global_results_counts.get_unchecked(left_bucket_index).get() + }; + + let (grouped_matches, other_matches) = + matches.as_chunks::(); + + let mut reduced_count = 0_usize; + for grouped_matches in grouped_matches { + // SAFETY: Guaranteed by function contract + let (ys_group, positions_group, _) = unsafe { + match_to_result_simd::<_, 7, _>(&parent_table, grouped_matches) + }; + + let s_buckets = ys_group >> Simd::splat(u32::from(PARAM_EXT)); + let s_buckets = s_buckets.to_array(); + + for (s_bucket, p) in s_buckets.into_iter().zip(positions_group) { + const { + assert!(Record::NUM_S_BUCKETS == (u16::MAX as usize) + 1); + } + let s_bucket = little_endian_s_bucket(s_bucket, K); + if s_bucket >= Record::NUM_S_BUCKETS as u32 { + continue; + } + let s_bucket = s_bucket as u16; + + buckets_positions[reduced_count].write((s_bucket, p)); + reduced_count += 1; + } + } + for other_match in other_matches { + // SAFETY: Guaranteed by function contract + let (y, p, _) = + unsafe { match_to_result::<_, 7, _>(&parent_table, other_match) }; + + let s_bucket = y.first_k_bits(); + + const { + assert!(Record::NUM_S_BUCKETS == (u16::MAX as usize) + 1); + } + let s_bucket = little_endian_s_bucket(s_bucket, K); + if s_bucket >= Record::NUM_S_BUCKETS as u32 { + continue; + } + let s_bucket = s_bucket as u16; + + buckets_positions[reduced_count].write((s_bucket, p)); + reduced_count += 1; + } + + *count = reduced_count as u16; + } + } + }); + + let parent_table = parent_table.prune(); + + let buckets_positions = strip_sync_unsafe_cell(buckets_positions); + + // SAFETY: Data structure filled with zeroes is a valid invariant + let mut table_6_proof_targets = + unsafe { Box::<[[Position; 2]; Record::NUM_S_BUCKETS]>::new_zeroed().assume_init() }; + + for (bucket, results_count) in buckets_positions.iter().zip( + global_results_counts + .into_iter() + .map(|count| usize::from(count.into_inner())), + ) { + // SAFETY: `results_count` corresponds to the number of initialized `bucket` elements + for &(s_bucket, p) in unsafe { bucket[..results_count].assume_init_ref() } { + let positions = &mut table_6_proof_targets[usize::from(s_bucket)]; + if positions == &[Position::ZERO; 2] { + *positions = p; + } + } + } + + (table_6_proof_targets, parent_table) + } +} + +#[cfg(feature = "alloc")] +impl Table +where + Self: private::NotLastTable, + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + /// Returns `None` for an invalid position or for table number 7. + /// + /// # Safety + /// `position` must come from [`Self::buckets()`] and not be a sentinel value. + #[inline(always)] + unsafe fn metadata(&self, position: Position) -> Metadata { + match self { + Self::First { .. } => { + // X matches position + Metadata::from(X::from(u32::from(position))) + } + Self::Other { metadatas, .. } => { + // SAFETY: All non-sentinel positions returned by [`Self::buckets()`] are valid + unsafe { metadatas.get_unchecked(usize::from(position)).assume_init() } + } + #[cfg(feature = "parallel")] + Self::OtherBuckets { metadatas, .. } => { + // SAFETY: All non-sentinel positions returned by [`Self::buckets()`] are valid + unsafe { + metadatas + .as_flattened() + .get_unchecked(usize::from(position)) + .assume_init() + } + } + } + } +} + +#[cfg(feature = "alloc")] +impl Table +where + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, + [(); 1 << K]:, + [(); num_buckets(K)]:, + [(); num_buckets(K) - 1]:, +{ + #[inline(always)] + fn prune(self) -> PrunedTable { + match self { + Self::First { .. } => PrunedTable::First, + Self::Other { positions, .. } => PrunedTable::Other { positions }, + #[cfg(feature = "parallel")] + Self::OtherBuckets { positions, .. } => PrunedTable::OtherBuckets { positions }, + } + } + + /// Positions of `y`s grouped by the bucket they belong to + #[inline(always)] + pub(super) fn buckets(&self) -> &[[(Position, Y); REDUCED_BUCKET_SIZE]; num_buckets(K)] { + match self { + Self::First { buckets, .. } => buckets, + Self::Other { buckets, .. } => buckets, + #[cfg(feature = "parallel")] + Self::OtherBuckets { buckets, .. } => buckets, + } + } +} diff --git a/shared/ab-proof-of-space/src/chiapos/table/rmap.rs b/shared/ab-proof-of-space/src/chiapos/table/rmap.rs new file mode 100644 index 00000000000..eece990e86f --- /dev/null +++ b/shared/ab-proof-of-space/src/chiapos/table/rmap.rs @@ -0,0 +1,81 @@ +#[cfg(test)] +mod tests; + +use crate::chiapos::constants::PARAM_BC; +use crate::chiapos::table::REDUCED_BUCKET_SIZE; +use crate::chiapos::table::types::{Position, R}; + +pub(super) struct Rmap { + /// `0` is a sentinel value indicating no virtual pointer is stored yet. + /// + /// Physical pointer must be increased by `1` to get a virtual pointer before storing. Virtual + /// pointer must be decreased by `1` before reading to get a physical pointer. + virtual_pointers: [u16; PARAM_BC as usize], + positions: [[Position; 2]; REDUCED_BUCKET_SIZE], + next_physical_pointer: u16, +} + +impl Rmap { + #[inline(always)] + pub(super) fn new() -> Self { + Self { + virtual_pointers: [0; _], + positions: [[Position::SENTINEL; 2]; _], + next_physical_pointer: 0, + } + } + + /// # Safety + /// `r` must be in the range `0..PARAM_BC`, there must be at most [`REDUCED_BUCKET_SIZE`] items + /// inserted + #[inline(always)] + unsafe fn insertion_item(&mut self, r: R) -> &mut [Position; 2] { + // SAFETY: Guaranteed by function contract + let virtual_pointer = unsafe { self.virtual_pointers.get_unchecked_mut(usize::from(r)) }; + + if let Some(physical_pointer) = virtual_pointer.checked_sub(1) { + // SAFETY: Internal pointers are always valid + return unsafe { self.positions.get_unchecked_mut(physical_pointer as usize) }; + } + + let physical_pointer = self.next_physical_pointer; + self.next_physical_pointer += 1; + *virtual_pointer = physical_pointer + 1; + + // SAFETY: It is guaranteed by the function contract that the number of added elements will + // never exceed `REDUCED_BUCKETS_SIZE`, hence allocated pointers will always be within + // bounds + unsafe { self.positions.get_unchecked_mut(physical_pointer as usize) } + } + + /// # Safety + /// `r` must be in the range `0..PARAM_BC`, there must be at most [`REDUCED_BUCKET_SIZE`] items + /// inserted + #[inline(always)] + pub(super) unsafe fn add(&mut self, r: R, position: Position) { + // SAFETY: Guaranteed by function contract + let rmap_item = unsafe { self.insertion_item(r) }; + + // The same `r` can appear in the table multiple times, one duplicate is supported here + if rmap_item[0] == Position::SENTINEL { + rmap_item[0] = position; + } else if rmap_item[1] == Position::SENTINEL { + rmap_item[1] = position; + } + } + + /// # Safety + /// `r` must be in the range `0..PARAM_BC` + #[inline(always)] + pub(super) unsafe fn get(&self, r: R) -> [Position; 2] { + // SAFETY: Guaranteed by function contract + let virtual_pointer = *unsafe { self.virtual_pointers.get_unchecked(usize::from(r)) }; + + if let Some(physical_pointer) = virtual_pointer.checked_sub(1) { + // SAFETY: Internal pointers are always valid + *unsafe { self.positions.get_unchecked(physical_pointer as usize) } + } else { + [Position::SENTINEL; 2] + } + } +} diff --git a/shared/ab-proof-of-space/src/chiapos/table/rmap/tests.rs b/shared/ab-proof-of-space/src/chiapos/table/rmap/tests.rs new file mode 100644 index 00000000000..74fac4a6f4e --- /dev/null +++ b/shared/ab-proof-of-space/src/chiapos/table/rmap/tests.rs @@ -0,0 +1,57 @@ +use crate::chiapos::table::rmap::Rmap; +use crate::chiapos::table::types::{Position, R}; + +#[test] +fn test_rmap_basic() { + let mut rmap = Rmap::new(); + + // SAFETY: All `r` values are in `0..PARAM_BC` range + unsafe { + assert_eq!( + rmap.get(R::from(0)), + [Position::SENTINEL, Position::SENTINEL] + ); + + rmap.add(R::from(0), Position::from(100)); + assert_eq!( + rmap.get(R::from(0)), + [Position::from(100), Position::SENTINEL] + ); + + rmap.add(R::from(0), Position::from(101)); + assert_eq!( + rmap.get(R::from(0)), + [Position::from(100), Position::from(101)] + ); + + // Ignored as duplicate `r` + rmap.add(R::from(0), Position::from(102)); + assert_eq!( + rmap.get(R::from(0)), + [Position::from(100), Position::from(101)] + ); + + rmap.add(R::from(1), Position::from(200)); + assert_eq!( + rmap.get(R::from(1)), + [Position::from(200), Position::SENTINEL] + ); + } +} + +#[test] +fn test_rmap_zero_when_full() { + let mut rmap = Rmap::new(); + + // SAFETY: All `r` values are in `0..PARAM_BC` range + unsafe { + rmap.add(R::from(3), Position::from(500)); + rmap.add(R::from(3), Position::from(501)); + // Ignored as duplicate `r` + rmap.add(R::from(3), Position::from(0)); + assert_eq!( + rmap.get(R::from(3)), + [Position::from(500), Position::from(501)] + ); + } +} diff --git a/shared/ab-proof-of-space/src/chiapos/table/tests.rs b/shared/ab-proof-of-space/src/chiapos/table/tests.rs new file mode 100644 index 00000000000..88dbb56d42c --- /dev/null +++ b/shared/ab-proof-of-space/src/chiapos/table/tests.rs @@ -0,0 +1,399 @@ +//! Tests translated into Rust from +//! https://github.com/Chia-Network/chiapos/blob/a2049c5367fe60930533a995f7ffded538f04dc4/tests/test.cpp + +use crate::chiapos::Seed; +use crate::chiapos::constants::{NUM_TABLES, PARAM_BC, PARAM_EXT}; +#[cfg(feature = "alloc")] +use crate::chiapos::constants::{PARAM_B, PARAM_C}; +#[cfg(feature = "alloc")] +use crate::chiapos::table::types::Position; +use crate::chiapos::table::types::{Metadata, X, Y}; +use crate::chiapos::table::{ + BUCKET_SIZE_UPPER_BOUND_SECURITY_BITS, COMPUTE_F1_SIMD_FACTOR, REDUCED_BUCKET_SIZE, + REDUCED_MATCHES_COUNT, compute_f1, compute_f1_simd, compute_fn, compute_fn_simd, + metadata_size_bytes, +}; +#[cfg(feature = "alloc")] +use crate::chiapos::table::{calculate_left_targets, find_matches_in_buckets}; +use crate::chiapos::utils::EvaluatableUsize; +#[cfg(feature = "alloc")] +use alloc::collections::BTreeMap; +#[cfg(feature = "alloc")] +use alloc::vec::Vec; +use core::f64::consts::{LN_2, PI, SQRT_2}; +#[cfg(feature = "alloc")] +use core::mem::MaybeUninit; +use core::simd::prelude::*; +use subspace_core_primitives::pieces::Record; + +/// Chia does this for some reason 🤷 +fn to_chia_seed(seed: &Seed) -> Seed { + let mut chia_seed = [1u8; 32]; + chia_seed[1..].copy_from_slice(&seed[..31]); + chia_seed +} + +#[test] +fn test_compute_f1_k25() { + const K: u8 = 25; + let seed = to_chia_seed(&[ + 0, 2, 3, 4, 5, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 41, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 11, 15, 16, + ]); + + let xs = [525, 526, 625_u32]; + let expected_ys = [2_016_650_816, 2_063_162_112, 1_930_299_520_u32]; + + for (x, expected_y) in xs.into_iter().zip(expected_ys) { + let x = X::from(x); + let y = compute_f1::(x, &seed); + assert_eq!(y, Y::from(expected_y)); + + // Make sure SIMD matches non-SIMD version + let mut partial_ys = [0; K as usize * COMPUTE_F1_SIMD_FACTOR / u8::BITS as usize]; + let starts_with_partial_y_bits = y.first_k_bits() << (u32::BITS - u32::from(K)); + partial_ys[..size_of::()].copy_from_slice(&starts_with_partial_y_bits.to_be_bytes()); + let y = compute_f1_simd::(Simd::splat(x.into()), &partial_ys); + assert_eq!(y[0], Y::from(expected_y)); + } +} + +#[test] +fn test_compute_f1_k22() { + const K: u8 = 22; + let seed = to_chia_seed(&[ + 0, 2, 3, 4, 5, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 41, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 11, 15, 16, + ]); + + let xs = [1_837_491, 1_837_491 + 1, 1_837_491 + 2, 1_837_491 + 255_u32]; + let expected_ys = [105_738_140, 192_213_404, 64_977_628, 91_711_644_u32]; + + for (x, expected_y) in xs.into_iter().zip(expected_ys) { + let x = X::from(x); + let y = compute_f1::(x, &seed); + assert_eq!(y, Y::from(expected_y)); + + // Make sure SIMD matches non-SIMD version + let mut partial_ys = [0; K as usize * COMPUTE_F1_SIMD_FACTOR / u8::BITS as usize]; + let starts_with_partial_y_bits = y.first_k_bits() << (u32::BITS - u32::from(K)); + partial_ys[..size_of::()].copy_from_slice(&starts_with_partial_y_bits.to_be_bytes()); + let y = compute_f1_simd::(Simd::splat(x.into()), &partial_ys); + assert_eq!(y[0], Y::from(expected_y)); + } +} + +#[cfg(feature = "alloc")] +fn check_match(yl: u32, yr: u32) -> bool { + let param_b = u64::from(PARAM_B); + let param_c = u64::from(PARAM_C); + let param_bc = u64::from(PARAM_BC); + let yl = u64::from(yl); + let yr = u64::from(yr); + + let bl = yl / param_bc; + let br = yr / param_bc; + if bl + 1 != br { + // Buckets don't match + return false; + } + + let lp = (yl % param_bc) / param_c; + let rp = (yr % param_bc) / param_c; + let lc = (yl % param_bc) % param_c; + let rc = (yr % param_bc) % param_c; + + for m in 0..(1 << PARAM_EXT) { + if rp % param_b == (lp + m) % param_b { + let mut c_diff = 2 * m + bl % 2; + c_diff *= c_diff; + + if rc == (lc + c_diff) % param_c { + return true; + } + } + } + + false +} + +// TODO: This test should be rewritten into something more readable, currently it is more or less +// direct translation from C++ +#[cfg(feature = "alloc")] +#[test] +#[cfg_attr(miri, ignore)] +fn test_matches() { + const K: u8 = 12; + let seed = to_chia_seed(&[ + 20, 2, 5, 4, 51, 52, 23, 84, 91, 10, 111, 12, 13, 24, 151, 16, 228, 211, 254, 45, 92, 198, + 204, 10, 9, 10, 11, 129, 139, 171, 15, 18, + ]); + + let mut bucket_ys = BTreeMap::>::new(); + let mut x = X::from(0); + for _ in 0..=1u32 << (K - 4) { + for _ in 0..16u8 { + let y = compute_f1::(x, &seed); + let bucket_index = usize::from(y) / usize::from(PARAM_BC); + + bucket_ys.entry(bucket_index).or_default().push(y); + + if x + X::from(1) > X::from((1 << K) - 1) { + break; + } + + x += X::from(1); + } + + if x + X::from(1) > X::from((1 << K) - 1) { + break; + } + } + + let left_targets = calculate_left_targets(); + let bucket_ys = bucket_ys.into_values().collect::>(); + let mut total_matches = 0_usize; + for (left_bucket_index, [left_bucket_ys, right_bucket_ys]) in + bucket_ys.array_windows::<2>().enumerate() + { + let mut left_bucket = [(Position::SENTINEL, Y::SENTINEL); _]; + assert!(left_bucket_ys.len() <= left_bucket.len()); + for ((output, &y), index) in left_bucket + .iter_mut() + .zip(left_bucket_ys) + .zip(0..left_bucket_ys.len()) + { + let position = Position::from(index as u32); + *output = (position, y); + } + let mut right_bucket = [(Position::SENTINEL, Y::SENTINEL); _]; + assert!(right_bucket_ys.len() <= right_bucket.len()); + for ((output, &y), index) in right_bucket + .iter_mut() + .zip(right_bucket_ys) + .zip((left_bucket_ys.len()..).take(right_bucket_ys.len())) + { + let position = Position::from(index as u32); + *output = (position, y); + } + let parent_table_ys = left_bucket_ys + .iter() + .copied() + .chain(right_bucket_ys.iter().copied()) + .collect::>(); + + let mut matches = [MaybeUninit::uninit(); _]; + // SAFETY: Positions correspond to `y`s + let matches = unsafe { + find_matches_in_buckets( + left_bucket_index as u32, + &left_bucket, + &right_bucket, + &mut matches, + &left_targets, + ) + }; + for m in matches { + let yl = u32::from(parent_table_ys[usize::from(m.left_position)]); + let yr = u32::from(parent_table_ys[usize::from(m.right_position)]); + + assert!(check_match(yl, yr)); + total_matches += 1; + } + } + + assert!( + total_matches > (1 << K) / 2, + "total_matches {total_matches}" + ); + assert!( + total_matches < (1 << K) * 2, + "total_matches {total_matches}" + ); +} + +fn verify_fn( + left_metadata: u128, + right_metadata: u128, + y: u32, + y_output_expected: u32, + metadata_expected: u128, +) where + EvaluatableUsize<{ metadata_size_bytes(K, PARENT_TABLE_NUMBER) }>: Sized, + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, +{ + let (y_output, metadata) = compute_fn::( + Y::from(y), + Metadata::from(left_metadata), + Metadata::from(right_metadata), + ); + assert_eq!(y_output, Y::from(y_output_expected)); + if metadata_expected != 0 { + assert_eq!(metadata, Metadata::from(metadata_expected)); + } + + let (y_outputs, metadatas) = compute_fn_simd::( + [Y::from(y); _], + [Metadata::from(left_metadata); _], + [Metadata::from(right_metadata); _], + ); + let y_outputs = Y::array_from_repr(y_outputs.to_array()); + assert_eq!([y_output; _], y_outputs); + if metadata_expected != 0 { + assert_eq!([metadata; _], metadatas); + } +} + +#[test] +fn test_verify_fn() { + const K: u8 = 16; + + verify_fn::(0x44cb, 0x204f, 0x20_a61a, 0x2a_f546, 0x44cb_204f); + verify_fn::(0x3c5f, 0xfda9, 0x39_88ec, 0x15_293b, 0x3c5f_fda9); + verify_fn::( + 0x35bf_992d, + 0x7ce4_2c82, + 0x31_e541, + 0xf_73b3, + 0x35bf_992d_7ce4_2c82, + ); + verify_fn::( + 0x7204_e52d, + 0xf1fd_42a2, + 0x28_a188, + 0x3f_b0b5, + 0x7204_e52d_f1fd_42a2, + ); + verify_fn::( + 0x5b6_e6e3_07d4_bedc, + 0x8a9a_021e_a648_a7dd, + 0x30_cb4c, + 0x1_1ad5, + 0xd4bd_0b14_4fc2_6138, + ); + verify_fn::( + 0xb9d1_79e0_6c0f_d4f5, + 0xf06d_3fef_7019_66a0, + 0x1d_d5b6, + 0xe_69a2, + 0xd021_15f5_1200_9d4d, + ); + verify_fn::( + 0xc2cd_789a_3802_08a9, + 0x1999_9e3f_a46d_6753, + 0x25_f01e, + 0x1f_22bd, + 0xabe4_2304_0a33, + ); + verify_fn::( + 0xbe3e_dc0a_1ef2_a4f0, + 0x4da9_8f1d_3099_fdf5, + 0x3f_eb18, + 0x31_501e, + 0x7300_a3a0_3ac5, + ); + verify_fn::( + 0xc965_815a_47c5, + 0xf5e0_08d6_af57, + 0x1f_121a, + 0x1c_abbe, + 0xc8cc_6947, + ); + verify_fn::( + 0xd420_677f_6cbd, + 0x5894_aa2c_a1af, + 0x2e_fde9, + 0xc_2121, + 0x421b_b8ec, + ); + verify_fn::(0x5fec_898f, 0x8228_3d15, 0x14_f410, 0x24_c3c2, 0x0); + verify_fn::(0x64ac_5db9, 0x792_3986, 0x5_90fd, 0x1c_74a2, 0x0); +} + +#[test] +fn test_proofs_lower_bound() { + /// Calculates a probabilistic lower bound on the number of challenges (out of + /// [`Record::NUM_S_BUCKETS`]) that will have at least one proof found, accounting for + /// truncations in matches and bucket sizes. + /// + /// This is based on modeling the entry propagation rate over 6 steps (for 7 tables), assuming + /// normal-distributed matches and bucket sizes with mean `lambda = PARAM_BC / 2^PARAM_EXT`. + /// A variance factor is applied to account for higher variance observed in practice for small + /// `k` due to clustering/non-uniformity. + /// + /// The bound ensures the probability that the actual number is below it is less than + /// `2^{-security_bits}`, using Chernoff on the lower tail of the binomial distribution for + /// non-empty challenge buckets. + /// Uses floating-point for precision in tail loss calculation (exact normal formula for + /// truncation rate with Abramowitz and Stegun erf approximation). + fn proofs_lower_bound( + security_bits: u8, + reduced_matches_count: usize, + reduced_bucket_size: usize, + ) -> u64 { + // Empirical variance factor to match observed higher variance/clustering for small `k` + const V_FACTOR: f64 = 9.0; + + // Lambda is the expected number per bucket/pair, independent of `k`. + let lambda = f64::from(PARAM_BC) / f64::from(2u32.pow(u32::from(PARAM_EXT))); + + // Rate for match truncation + let match_truncation_rate = + normal_rate_approx(lambda, reduced_matches_count as f64, V_FACTOR); + // Rate for bucket truncation + let bucket_truncation_rate = + normal_rate_approx(lambda, reduced_bucket_size as f64, V_FACTOR); + + let step_rate = match_truncation_rate * bucket_truncation_rate; + let overall_rate = step_rate.powi(i32::from(NUM_TABLES - 1)); + // Final lambda for each challenge (density after losses) + let final_lambda_per_challenge = overall_rate; + + let num_challenges = Record::NUM_S_BUCKETS as f64; + let prob_non_empty = 1.0 - (-final_lambda_per_challenge).exp(); + let expected_non_empty_count = num_challenges * prob_non_empty; + + // Chernoff lower tail: solve for delta where exp(-mu * delta^2 / 2) < 2^{-security_bits} + let chernoff_inner = 2.0 * f64::from(security_bits) * LN_2 / expected_non_empty_count; + let relative_deviation = chernoff_inner.sqrt(); + let lower_bound = expected_non_empty_count * (1.0 - relative_deviation); + + (lower_bound as i64).cast_unsigned() + } + + /// Propagation rate for a truncation cap using the exact normal tail loss formula. + /// Returns the fraction of entries retained after truncating at the cap, assuming a normal + /// distribution. + fn normal_rate_approx(mean: f64, cap: f64, variance_factor: f64) -> f64 { + if cap >= mean + 10.0 * (variance_factor * mean).sqrt() { + // Negligible loss for caps far above mean + 1.0 + } else if cap <= 0.0 { + 0.0 + } else { + let sigma = (variance_factor * mean).sqrt(); + let z_score = (cap - mean) / sigma; + if z_score <= -10.0 { + // Cap far below mean: simple ratio + cap / mean + } else { + let tail_prob = (z_score / SQRT_2).erfc() / 2.0; + let density_at_z = (1.0 / (2.0 * PI).sqrt()) * (-z_score * z_score / 2.0).exp(); + let tail_loss = ((mean - cap) * tail_prob + sigma * density_at_z).max(0.0); + 1.0 - tail_loss / mean + } + } + } + + // Ensure there are enough proofs found with overwhelming probability even for truncated + // bucket size and number of matches. + // TODO: LLM generated lower bound calculation formula, it may not be 100% correct, needs + // improvements. + assert!( + proofs_lower_bound( + BUCKET_SIZE_UPPER_BOUND_SECURITY_BITS, + REDUCED_MATCHES_COUNT, + REDUCED_BUCKET_SIZE + ) >= Record::NUM_CHUNKS as u64 + ); +} diff --git a/shared/ab-proof-of-space/src/chiapos/table/types.rs b/shared/ab-proof-of-space/src/chiapos/table/types.rs new file mode 100644 index 00000000000..12195bb7739 --- /dev/null +++ b/shared/ab-proof-of-space/src/chiapos/table/types.rs @@ -0,0 +1,207 @@ +#[cfg(feature = "alloc")] +use crate::chiapos::constants::PARAM_BC; +use crate::chiapos::constants::PARAM_EXT; +use crate::chiapos::table::metadata_size_bytes; +use crate::chiapos::utils::EvaluatableUsize; +use core::iter::Step; +#[cfg(any(feature = "alloc", test))] +use core::mem; +#[cfg(feature = "alloc")] +use core::ops::RangeInclusive; +use derive_more::{Add, AddAssign, From, Into}; + +/// Stores data in lower bits +#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, From, Into, Add, AddAssign)] +#[repr(C)] +pub(in super::super) struct X(u32); + +impl Step for X { + #[inline(always)] + fn steps_between(start: &Self, end: &Self) -> (usize, Option) { + u32::steps_between(&start.0, &end.0) + } + + #[inline(always)] + fn forward_checked(start: Self, count: usize) -> Option { + u32::forward_checked(start.0, count).map(Self) + } + + #[inline(always)] + fn backward_checked(start: Self, count: usize) -> Option { + u32::backward_checked(start.0, count).map(Self) + } +} + +impl From for u64 { + #[inline(always)] + fn from(value: X) -> Self { + Self::from(value.0) + } +} + +impl From for u128 { + #[inline(always)] + fn from(value: X) -> Self { + Self::from(value.0) + } +} + +impl X { + #[cfg(feature = "alloc")] + pub(in super::super) const ZERO: Self = Self(0); +} + +/// Stores data in lower bits +#[derive(Debug, Copy, Clone, Eq, PartialEq, From, Into)] +#[repr(C)] +pub(in super::super) struct Y(u32); + +impl From for u128 { + #[inline(always)] + fn from(value: Y) -> Self { + Self::from(value.0) + } +} + +impl From for usize { + #[inline(always)] + fn from(value: Y) -> Self { + value.0 as Self + } +} + +impl Y { + /// Y that can't exist + #[cfg(feature = "alloc")] + pub(in super::super) const SENTINEL: Self = Self(u32::MAX); + + /// The range of buckets where `Y`s with the provided first `K` bits are located + #[cfg(feature = "alloc")] + #[inline(always)] + pub(in super::super) fn bucket_range_from_first_k_bits(value: u32) -> RangeInclusive { + let from = value << PARAM_EXT; + let to = from | (u32::MAX >> (u32::BITS - u32::from(PARAM_EXT))); + from as usize / usize::from(PARAM_BC)..=to as usize / usize::from(PARAM_BC) + } + + /// Get the first `K` bits + #[inline(always)] + pub(in super::super) const fn first_k_bits(self) -> u32 { + self.0 >> PARAM_EXT + } + + #[cfg(any(feature = "alloc", test))] + #[inline(always)] + pub(super) const fn array_from_repr(array: [u32; N]) -> [Self; N] { + // TODO: Should have been transmute, but https://github.com/rust-lang/rust/issues/152507 + // SAFETY: `Y` is `#[repr(C)]` and guaranteed to have the same memory layout + unsafe { mem::transmute_copy(&array) } + } +} + +#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, From, Into)] +#[repr(C)] +pub(in super::super) struct Position(u32); + +impl Step for Position { + #[inline(always)] + fn steps_between(start: &Self, end: &Self) -> (usize, Option) { + u32::steps_between(&start.0, &end.0) + } + + #[inline(always)] + fn forward_checked(start: Self, count: usize) -> Option { + u32::forward_checked(start.0, count).map(Self) + } + + #[inline(always)] + fn backward_checked(start: Self, count: usize) -> Option { + u32::backward_checked(start.0, count).map(Self) + } +} + +impl From for usize { + #[inline(always)] + fn from(value: Position) -> Self { + value.0 as Self + } +} + +impl Position { + #[cfg(feature = "alloc")] + pub(in super::super) const ZERO: Self = Self(0); + /// Position that can't exist + #[cfg(any(feature = "alloc", test))] + pub(in super::super) const SENTINEL: Self = Self(u32::MAX); +} + +/// Stores data in lower bits +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +#[repr(C)] +pub(in super::super) struct Metadata( + [u8; metadata_size_bytes(K, TABLE_NUMBER)], +) +where + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized; + +impl Default for Metadata +where + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, +{ + #[inline(always)] + fn default() -> Self { + Self([0; metadata_size_bytes(K, TABLE_NUMBER)]) + } +} + +impl From> for u128 +where + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, +{ + #[inline(always)] + fn from(value: Metadata) -> Self { + // `*_be_bytes()` is used such that `Ord`/`PartialOrd` impl works as expected + let mut output = 0u128.to_be_bytes(); + output[size_of::() - value.0.len()..].copy_from_slice(&value.0); + + Self::from_be_bytes(output) + } +} + +impl From for Metadata +where + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, +{ + /// If used incorrectly, will truncate information, it is up to implementation to ensure `u128` + /// only contains data in lower bits and fits into internal byte array of `Metadata` + #[inline(always)] + fn from(value: u128) -> Self { + Self( + value.to_be_bytes()[size_of::() - metadata_size_bytes(K, TABLE_NUMBER)..] + .try_into() + .expect("Size of internal byte array is always smaller or equal to u128; qed"), + ) + } +} + +impl From for Metadata +where + EvaluatableUsize<{ metadata_size_bytes(K, TABLE_NUMBER) }>: Sized, +{ + #[inline(always)] + fn from(value: X) -> Self { + Self::from(u128::from(value)) + } +} + +/// `r` is a value of `y` minus bucket base +#[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, From, Into)] +#[repr(C)] +pub(in super::super) struct R(u16); + +impl From for usize { + #[inline(always)] + fn from(value: R) -> Self { + Self::from(value.0) + } +} diff --git a/shared/ab-proof-of-space/src/chiapos/tests.rs b/shared/ab-proof-of-space/src/chiapos/tests.rs new file mode 100644 index 00000000000..5b71ddf227a --- /dev/null +++ b/shared/ab-proof-of-space/src/chiapos/tests.rs @@ -0,0 +1,76 @@ +#![cfg(not(miri))] + +use crate::chiapos::{Tables, TablesCache}; +use alloc::vec::Vec; +use subspace_core_primitives::sectors::SBucket; + +const K: u8 = 17; + +#[test] +fn self_verification() { + let seed = [1; 32]; + let cache = TablesCache::default(); + let tables = Tables::::create(seed, &cache); + #[cfg(feature = "parallel")] + let tables_parallel = Tables::::create_parallel(seed, &cache); + + let all_proofs = Tables::::create_proofs(seed, &cache); + #[cfg(feature = "parallel")] + let all_proofs_parallel = Tables::::create_proofs_parallel(seed, &cache); + + for challenge_index in 0..1000_u32 { + let mut challenge = [0; 32]; + challenge[..size_of::()].copy_from_slice(&challenge_index.to_le_bytes()); + let first_challenge_bytes = challenge[..4].try_into().unwrap(); + let qualities = tables.find_quality(&challenge).collect::>(); + #[cfg(feature = "parallel")] + assert_eq!( + qualities, + tables_parallel.find_quality(&challenge).collect::>(), + "challenge index {challenge_index}" + ); + let proofs = tables.find_proof(first_challenge_bytes).collect::>(); + #[cfg(feature = "parallel")] + assert_eq!( + proofs, + tables_parallel + .find_proof(first_challenge_bytes) + .collect::>(), + "challenge index {challenge_index}" + ); + + assert_eq!(qualities.len(), proofs.len()); + + for (quality, proof) in qualities.into_iter().zip(&proofs) { + assert_eq!( + Some(quality), + Tables::::verify(&seed, &challenge, proof), + "challenge index {challenge_index}" + ); + let mut bad_challenge = [0; 32]; + bad_challenge[..size_of::()].copy_from_slice(&(challenge_index + 1).to_le_bytes()); + assert!( + Tables::::verify(&seed, &bad_challenge, proof).is_none(), + "challenge index {challenge_index}" + ); + } + + { + // Under Subspace's little-endian convention the proof binned at s-bucket `cs` is the + // one the search finds for challenge `cs`, using the same `to_le_bytes` seam the + // consensus verifier applies. + let s_bucket = SBucket::from(challenge_index as u16); + assert_eq!( + proofs.first().copied(), + all_proofs.for_s_bucket(s_bucket), + "challenge index {challenge_index}" + ); + #[cfg(feature = "parallel")] + assert_eq!( + proofs.first().copied(), + all_proofs_parallel.for_s_bucket(s_bucket), + "challenge index {challenge_index}" + ); + } + } +} diff --git a/shared/ab-proof-of-space/src/chiapos/utils.rs b/shared/ab-proof-of-space/src/chiapos/utils.rs new file mode 100644 index 00000000000..5e22958ebad --- /dev/null +++ b/shared/ab-proof-of-space/src/chiapos/utils.rs @@ -0,0 +1,4 @@ +/// TODO: Workaround for "unconstrained generic constant" suggested in +/// https://github.com/rust-lang/rust/issues/82509#issuecomment-1165533546 +#[derive(Debug)] +pub struct EvaluatableUsize; diff --git a/shared/ab-proof-of-space/src/lib.rs b/shared/ab-proof-of-space/src/lib.rs new file mode 100644 index 00000000000..2c986f55129 --- /dev/null +++ b/shared/ab-proof-of-space/src/lib.rs @@ -0,0 +1,85 @@ +//! Proof of space implementation +#![no_std] +#![expect(incomplete_features, reason = "generic_const_exprs")] +#![warn(rust_2018_idioms, missing_debug_implementations, missing_docs)] +#![feature( + const_block_items, + const_convert, + const_trait_impl, + generic_const_exprs, + step_trait +)] +#![cfg_attr(test, feature(float_erf))] +#![cfg_attr(feature = "parallel", feature(exact_size_is_empty, sync_unsafe_cell))] +#![cfg_attr(feature = "alloc", feature(maybe_uninit_fill, ptr_as_uninit))] +#![cfg_attr(any(feature = "alloc", test), feature(portable_simd))] + +pub mod chiapos; + +#[cfg(feature = "alloc")] +extern crate alloc; + +#[cfg(feature = "alloc")] +use subspace_core_primitives::pieces::Record; +#[cfg(feature = "alloc")] +use subspace_core_primitives::pos::PosProof; +#[cfg(feature = "alloc")] +use subspace_core_primitives::sectors::SBucket; + +// TODO: Return a single full proof and the rest as hashes instead to optimize memory usage and +// parallelize compute more easily +/// Proof-of-space proofs +#[derive(Debug)] +#[cfg(feature = "alloc")] +#[repr(C)] +pub struct PosProofs { + /// S-buckets at which proofs were found. + /// + /// S-buckets are grouped by 8, within each `u8` bits right to left (LSB) indicate the presence + /// of a proof for corresponding s-bucket, so that the whole array of bytes can be thought as a + /// large set of bits. + /// + /// There will be at most [`Record::NUM_CHUNKS`] proofs produced/bits set to `1`. + pub found_proofs: [u8; Record::NUM_S_BUCKETS / u8::BITS as usize], + /// [`Record::NUM_CHUNKS`] proofs, corresponding to set bits of `found_proofs`. + pub proofs: [PosProof; Record::NUM_CHUNKS], +} + +// TODO: A method that returns hashed proofs (with SIMD) for all s-buckets for plotting +#[cfg(feature = "alloc")] +impl PosProofs { + /// Get proof for specified s-bucket (if exists). + /// + /// Note that this is not the most efficient API possible, so prefer using the `proofs` field + /// directly if the use case allows. + #[inline] + pub fn for_s_bucket(&self, s_bucket: SBucket) -> Option { + let proof_index = Self::proof_index_for_s_bucket(&self.found_proofs, s_bucket)?; + + Some(self.proofs[proof_index]) + } + + #[inline(always)] + fn proof_index_for_s_bucket( + found_proofs: &[u8; Record::NUM_S_BUCKETS / u8::BITS as usize], + s_bucket: SBucket, + ) -> Option { + let bits_offset = usize::from(s_bucket); + let found_proofs_byte_offset = bits_offset / u8::BITS as usize; + let found_proofs_bit_offset = bits_offset as u32 % u8::BITS; + let (found_proofs_before, found_proofs_after) = + found_proofs.split_at(found_proofs_byte_offset); + if (found_proofs_after[0] & (1 << found_proofs_bit_offset)) == 0 { + return None; + } + let proof_index = found_proofs_before + .iter() + .map(|&bits| bits.count_ones()) + .sum::() + + found_proofs_after[0] + .unbounded_shl(u8::BITS - found_proofs_bit_offset) + .count_ones(); + + Some(proof_index as usize) + } +} diff --git a/shared/subspace-proof-of-space-gpu/Cargo.toml b/shared/subspace-proof-of-space-gpu/Cargo.toml deleted file mode 100644 index 2821ac3889b..00000000000 --- a/shared/subspace-proof-of-space-gpu/Cargo.toml +++ /dev/null @@ -1,49 +0,0 @@ -[package] -name = "subspace-proof-of-space-gpu" -description = "Subspace proof of space implementation for GPU (low-level proving utilities)" -license = "0BSD" -version = "0.1.0" -authors = ["Supranational LLC"] -edition.workspace = true -include = [ - "/build.rs", - "/src", - "/Cargo.toml", - "/README.md", -] - -[dependencies] -# Only used from C code -blst = { workspace = true, optional = true } -rust-kzg-blst = { workspace = true, optional = true } -# TODO: Fork with ROCm support, switch to upstream once `rocm` branch from `https://github.com/dot-asm/sppark` + https://github.com/dot-asm/sppark/pull/2 are upstreamed -sppark = { workspace = true, optional = true } -subspace-core-primitives = { workspace = true, optional = true } -subspace-kzg = { workspace = true, optional = true } - -# Avoid unused dependencies on macOS, GPU is not supported there -[target.'cfg(any(target_os = "linux", windows))'.dev-dependencies] -subspace-erasure-coding.workspace = true -subspace-farmer-components.workspace = true -subspace-proof-of-space.workspace = true - -[build-dependencies] -cc.workspace = true - -[features] -# Only Volta+ architectures are supported (GeForce RTX 16xx consumer GPUs and newer) -cuda = ["_gpu", "sppark/cuda"] -# TODO: ROCm can't be enabled at the same time as `cuda` feature at the moment -# Seems to support RDNA 2+, at least on Linux -rocm = ["_gpu", "sppark/rocm"] -# Internal feature, shouldn't be used directly -_gpu = [ - "dep:blst", - "dep:rust-kzg-blst", - "dep:subspace-core-primitives", - "dep:subspace-kzg", -] - -# `cargo-udeps` can't detect C code-only usage -[package.metadata.cargo-udeps.ignore] -normal = ["blst"] diff --git a/shared/subspace-proof-of-space-gpu/README.md b/shared/subspace-proof-of-space-gpu/README.md deleted file mode 100644 index ed437f5e013..00000000000 --- a/shared/subspace-proof-of-space-gpu/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Subspace proof of space implementation for GPU (low-level proving utilities) - -This crate exposes some low-level primitives to accelerate proof of space implementation on Nvidia (CUDA, Volta+) and -AMD (ROCm) GPUs. - -The implementation expects 64KB of shared memory and about 1.1GB of VRAM on GPU. - -## Build requirements - -### CUDA -For Nvidia support CUDA toolkit needs to be installed, which can be done on Ubuntu 24.04 like this: -```bash -sudo apt-get install nvidia-cuda-toolkit -``` - -In case you have newer GCC installed, you may need to install `g++-12` and set it as a C++ compiler of choice during -compilation, something like this: -```bash -CXX=g++-12 cargo build -``` - -For other operating systems/platforms check official documentation: - -### ROCm - -For AMD/ROCm support follow their official documentation: diff --git a/shared/subspace-proof-of-space-gpu/build.rs b/shared/subspace-proof-of-space-gpu/build.rs deleted file mode 100644 index 95962f0efdb..00000000000 --- a/shared/subspace-proof-of-space-gpu/build.rs +++ /dev/null @@ -1,91 +0,0 @@ -// Originally written by Supranational LLC - -use std::env; -use std::thread::available_parallelism; - -fn main() { - // TODO: Lift this restriction - #[cfg(all(feature = "cuda", feature = "rocm"))] - compile_error!("Both `cuda` and `rocm` features can't be used together at the moment"); - - let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); - let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap(); - - if target_os == "windows" && target_env != "msvc" { - panic!("Only MSVC is supported on Windows"); - } - - if cfg!(feature = "rocm") { - println!("cargo::rerun-if-env-changed=HIPCC"); - - let mut hipcc = cc::Build::new(); - hipcc.compiler(env::var("HIPCC").unwrap_or("hipcc".to_string())); - hipcc.cpp(true); - if cfg!(debug_assertions) { - hipcc.opt_level(2); - } - // Architectures: https://llvm.org/docs/AMDGPUUsage.html - // GCN GFX9 (CDNA) - hipcc.flag("--offload-arch=gfx908"); - // GCN GFX9 (CDNA 2) - hipcc.flag("--offload-arch=gfx90a"); - // GCN GFX9 (CDNA 3) - hipcc.flag("--offload-arch=gfx942"); - // GCN GFX10.1 (RDNA 1) dGPU - hipcc.flag("--offload-arch=gfx1010,gfx1011,gfx1012"); - // GCN GFX10.1 (RDNA 1) APU - hipcc.flag("--offload-arch=gfx1013"); - // GCN GFX10.3 (RDNA 2) dGPU - hipcc.flag("--offload-arch=gfx1030,gfx1031,gfx1032,gfx1034"); - // GCN GFX10.3 (RDNA 2) APU - hipcc.flag("--offload-arch=gfx1033,gfx1035,gfx1036"); - // GCN GFX11 (RDNA 3) dGPU - hipcc.flag("--offload-arch=gfx1100,gfx1101,gfx1102"); - // GCN GFX11 (RDNA 3) APU - hipcc.flag("--offload-arch=gfx1103,gfx1150,gfx1151"); - // Architecture is too new for hipcc 5.7.1 in stock Ubuntu repos - hipcc.flag_if_supported("--offload-arch=gfx1152"); - // GCN GFX12 (RDNA 4) dGPU - // Architecture is too new for hipcc 5.7.1 in stock Ubuntu repos - hipcc.flag_if_supported("--offload-arch=gfx1200,gfx1201"); - // Flag is too new for hipcc in stock Ubuntu repos - hipcc.flag_if_supported(format!( - "-parallel-jobs={}", - available_parallelism().unwrap() - )); - // This controls how error strings get handled in the FFI. When defined error strings get - // returned from the FFI, and Rust must then free them. When not defined error strings are - // not returned. - hipcc.define("TAKE_RESPONSIBILITY_FOR_ERROR_MESSAGE", None); - if let Some(include) = env::var_os("DEP_SPPARK_ROOT") { - hipcc.include(include); - hipcc.flag("-include").flag("util/cuda2hip.hpp"); - } - hipcc.file("src/subspace_api.cu").compile("subspace_rocm"); - - // Doesn't link otherwise - println!("cargo::rustc-link-lib=amdhip64"); - } - - if cfg!(feature = "cuda") { - let mut nvcc = cc::Build::new(); - nvcc.cuda(true); - nvcc.flag("-arch=sm_70"); - if target_env != "msvc" { - nvcc.flag("-Xcompiler").flag("-Wno-unused-function"); - } - // This controls how error strings get handled in the FFI. When defined error strings get - // returned from the FFI, and Rust must then free them. When not defined error strings are - // not returned. - nvcc.define("TAKE_RESPONSIBILITY_FOR_ERROR_MESSAGE", None); - if let Some(include) = env::var_os("DEP_BLST_C_SRC") { - nvcc.include(include); - } - if let Some(include) = env::var_os("DEP_SPPARK_ROOT") { - nvcc.include(include); - } - nvcc.file("src/subspace_api.cu").compile("subspace_cuda"); - } - - println!("cargo::rerun-if-changed=src"); -} diff --git a/shared/subspace-proof-of-space-gpu/src/blake3.cuh b/shared/subspace-proof-of-space-gpu/src/blake3.cuh deleted file mode 100644 index 64529629d6b..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/blake3.cuh +++ /dev/null @@ -1,185 +0,0 @@ -// Originally written by Supranational LLC - -#include -#include - -// Define constants used for BLAKE3 operation. -static const int CHUNK_START = 1 << 0; -static const int CHUNK_END = 1 << 1; -static const int ROOT = 1 << 3; -static const int BLOCK_LEN = 64; - -__device__ __constant__ -const uint32_t IV[8] = { 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, - 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19 }; - -struct __align__(16) blake3_state { - uint32_t data[16]; -}; - -// Convert a 64-byte block into 16 uint32_t words for processing. -static __device__ __forceinline__ -void bytes_to_words(const uint8_t bytes[64], uint32_t words[16]) -{ - for (int i = 0; i < 16; i++) { - words[i] = reinterpret_cast(bytes)[i]; - } -} - -static __device__ __forceinline__ -blake3_state initialize_blake3_state(const uint32_t block_len, - const uint32_t chaining_value[8] = IV, - uint32_t flag = (CHUNK_START + CHUNK_END) + ROOT, - const uint64_t counter = 0) -{ - blake3_state state; - - // Setup the chaining values from the input or previous state. - for (int i = 0; i < 8; i++) { - state.data[i] = chaining_value[i]; - } - - // Setup the initial state values from the IV. - for (int i = 0; i < 4; i++) { - state.data[8 + i] = IV[i]; - } - - // Counter, block length and flag setup for different hash states. - state.data[12] = static_cast(counter); - state.data[13] = static_cast(counter >> 32); - state.data[14] = block_len; - state.data[15] = flag; - - return state; -} - -// Permute the message block according to BLAKE3's specification. -static __device__ __forceinline__ -void permute(uint32_t out[16], const uint32_t in[16]) -{ - static const int MSG_PERMUTATION[16] = {2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, - 5, 9, 14, 15, 8}; - for (int i = 0; i < 16; ++i) { - out[i] = in[MSG_PERMUTATION[i]]; - } -} - -// BLAKE3 quarter round transformation as part of the compress function. -static __device__ __forceinline__ -void quarter_round(uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d, - const uint32_t mx, const uint32_t my) -{ - a += b + mx; - d ^= a; - d = (d << 16) | (d >> 16); - - c += d; - b ^= c; - b = (b << 20) | (b >> 12); - - a += b + my; - d ^= a; - d = (d << 24) | (d >> 8); - - c += d; - b ^= c; - b = (b << 25) | (b >> 7); -} - -// The BLAKE3 compress function that performs the rounds of mixing input data. -static __device__ __forceinline__ -void round(blake3_state& state, const uint32_t block[16]) -{ - // Perform quarter rounds on the state with the given block data. - quarter_round(state.data[0], state.data[4], state.data[8], state.data[12], block[0], block[1]); - quarter_round(state.data[1], state.data[5], state.data[9], state.data[13], block[2], block[3]); - quarter_round(state.data[2], state.data[6], state.data[10], state.data[14], block[4], block[5]); - quarter_round(state.data[3], state.data[7], state.data[11], state.data[15], block[6], block[7]); - quarter_round(state.data[0], state.data[5], state.data[10], state.data[15], block[8], block[9]); - quarter_round(state.data[1], state.data[6], state.data[11], state.data[12], block[10], block[11]); - quarter_round(state.data[2], state.data[7], state.data[8], state.data[13], block[12], block[13]); - quarter_round(state.data[3], state.data[4], state.data[9], state.data[14], block[14], block[15]); -} - -// Main compress routine that combines state and message data. -static __device__ __forceinline__ -void blake3_compress(blake3_state& state, const uint8_t block_bytes[64], - const uint32_t chaining_value[8] = IV) -{ - uint32_t block1[16]; - bytes_to_words(block_bytes, block1); - - uint32_t block2[16]; - for (size_t i = 0; i < 3; i++) { - round(state, block1); - permute(block2, block1); - round(state, block2); - permute(block1, block2); - } - round(state, block1); - - // Finalize the compress by combining the state with the chaining values. - for (size_t i = 0; i < 8; i++) { - state.data[i] = state.data[i] ^ state.data[i + 8]; - state.data[i + 8] = state.data[i + 8] ^ chaining_value[i]; - } -} - -static __device__ __forceinline__ -uint32_t start_flag(uint32_t blocks_compressed) -{ - return blocks_compressed == 0 ? CHUNK_START : 0; -} - -// If the input bytes are larger than 64 bytes and smaller than 1024 bytes, -// instead of using the compress function directly, we use this function to -// split the input into chunks of up to 64 bytes and process them using the -// compress function. -static __device__ __forceinline__ -blake3_state blake3_update(const uint8_t* input_bytes, size_t input_len) -{ - uint32_t blocks_compressed = 0; - uint32_t chaining_value[8]; - - // Initialize chaining_value using IV constant - for (int i = 0; i < 8; i++) { - chaining_value[i] = IV[i]; - } - - // Iterate as long as the number of input bytes is greater than 0. - // In each iteration, process a maximum of 64 bytes. - blake3_state state; - for (uint32_t i = 0; i < input_len; i += BLOCK_LEN) { - - // Determine the smaller value between input_len and BLOCK_LEN - // because a maximum of BLOCK_LEN can be processed. - uint32_t current_block_len = (BLOCK_LEN < (input_len - i)) ? BLOCK_LEN : (input_len - i); - - // Set the range of the input byte array to be processed. - uint8_t block[64] = {0}; - for (uint32_t j = 0; j < current_block_len; j++) { - block[j] = input_bytes[i + j]; - } - - // Determine the flag value based on whether it is the final step - // and if this is the start of the chunk. - bool final_step = (current_block_len == (input_len - i)); - uint32_t flag = final_step ? start_flag(blocks_compressed) | CHUNK_END | ROOT: start_flag(blocks_compressed); - - // Update the state, chaining_value, and current_block_len values. - state = initialize_blake3_state(current_block_len, chaining_value, flag); - - // Apply the compress function based on the determined state and block values - // and write the values to the state. - blake3_compress(state, block, chaining_value); - - // Update the chaining_value with the first 8 values of the state computed by compress. - for (int j = 0; j < 8; j++) { - chaining_value[j] = state.data[j]; - } - - blocks_compressed += 1; - } - - return state; -} diff --git a/shared/subspace-proof-of-space-gpu/src/chacha.cuh b/shared/subspace-proof-of-space-gpu/src/chacha.cuh deleted file mode 100644 index faddef44b2a..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/chacha.cuh +++ /dev/null @@ -1,113 +0,0 @@ -// Originally written by Supranational LLC - -#include -#include - -struct __align__(16) chacha_state { - uint32_t data[16]; -}; - -// Converts 4 bytes into a single 32-bit word -static uint32_t bytes_to_word(const uint8_t bytes[4]) -{ - return (static_cast(bytes[3]) << 24) | - (static_cast(bytes[2]) << 16) | - (static_cast(bytes[1]) << 8) | - static_cast(bytes[0]); -} - -template -static chacha_state initialize_chacha_state(const uint8_t key[32], - const uint8_t nonce[16 - sizeof(T)], - T counter = 0) -{ - assert(sizeof(T) % 4 == 0); - - // Set up the constant values for ChaCha as per specification. - chacha_state state; - state.data[0] = 0x61707865; - state.data[1] = 0x3320646e; - state.data[2] = 0x79622d32; - state.data[3] = 0x6b206574; - - // Load the key into the state array. - for (int i = 0; i < 8; i++) { - state.data[4 + i] = bytes_to_word(key + 4 * i); - } - - // Load the counter and nonce into the state array. - state.data[12] = static_cast(counter); - state.data[13] = sizeof(T) > sizeof(uint32_t) ? static_cast(counter >> 32) : bytes_to_word(nonce); - - int nonce_index = sizeof(T) > sizeof(uint32_t) ? 0 : 4; - for (int i = 0; i < 2; i++) { - state.data[14 + i] = bytes_to_word(nonce + nonce_index + 4 * i); - } - - return state; -} - -// Perform a single ChaCha quarter round operation. -static __device__ __forceinline__ -void quarter_round(uint32_t& a, uint32_t& b, uint32_t& c, uint32_t& d) -{ - a += b; - d ^= a; - d = (d << 16) | (d >> 16); - - c += d; - b ^= c; - b = (b << 12) | (b >> 20); - - a += b; - d ^= a; - d = (d << 8) | (d >> 24); - - c += d; - b ^= c; - b = (b << 7) | (b >> 25); -} - -template -static __device__ __forceinline__ -void chacha_mix(chacha_state& state, size_t counter) -{ - // Increment counter. - state.data[12] += (uint32_t)counter; - if (state.data[12] < (uint32_t)counter) state.data[13]++; - state.data[13] += (uint32_t)(counter >> 32); - - chacha_state temp_state = state; - - // Perform the ChaCha rounds. - for (uint32_t round = 0; round < ROUNDS / 2; round++) { - quarter_round(state.data[0], state.data[4], state.data[8], state.data[12]); - quarter_round(state.data[1], state.data[5], state.data[9], state.data[13]); - quarter_round(state.data[2], state.data[6], state.data[10], state.data[14]); - quarter_round(state.data[3], state.data[7], state.data[11], state.data[15]); - quarter_round(state.data[0], state.data[5], state.data[10], state.data[15]); - quarter_round(state.data[1], state.data[6], state.data[11], state.data[12]); - quarter_round(state.data[2], state.data[7], state.data[8], state.data[13]); - quarter_round(state.data[3], state.data[4], state.data[9], state.data[14]); - } - - // Add the initial state back to the mixed state. - for (uint32_t j = 0; j < 16; j++) { - state.data[j] = state.data[j] + temp_state.data[j]; - } -} - -template -__global__ __launch_bounds__(1024) -void generate_chacha_keystream(chacha_state* output, chacha_state _state, - size_t output_block_count) -{ - size_t tid = blockIdx.x * (size_t)blockDim.x + threadIdx.x; - - for (size_t i = tid; i < output_block_count; i += (size_t)gridDim.x * blockDim.x) { - chacha_state state = _state; - chacha_mix(state, i); - - output[i] = state; - } -} diff --git a/shared/subspace-proof-of-space-gpu/src/common_operations.cuh b/shared/subspace-proof-of-space-gpu/src/common_operations.cuh deleted file mode 100644 index 5341fa95788..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/common_operations.cuh +++ /dev/null @@ -1,49 +0,0 @@ -// Originally written by Supranational LLC - -__device__ __forceinline__ -uint32_t byte_swap(uint32_t in) -{ - return ((in >> 24) & 0x000000FF) | - ((in >> 8) & 0x0000FF00) | - ((in << 8) & 0x00FF0000) | - ((in << 24) & 0xFF000000); -} - -template -__device__ __forceinline__ -T endianness_swap(const T& in) -{ - T out; - - #pragma unroll - for (uint32_t i = 0; i < out.len(); i++) - out[out.len() - i - 1] = byte_swap(in[i]); - - return out; -} - -template -__global__ __launch_bounds__(1024) -void kern_endianness_swap(T* out, const T* in, size_t size) -{ - size_t tid = (size_t)blockIdx.x * blockDim.x + threadIdx.x; - - for (size_t i = tid; i < size; i += (size_t)gridDim.x * blockDim.x) { - T temp = in[i]; - temp = endianness_swap(temp); - out[i] = temp; - } -} - -template -__global__ __launch_bounds__(1024) -void convert_to_mont(T* out, const T* in, size_t size) -{ - size_t tid = (size_t)blockIdx.x * blockDim.x + threadIdx.x; - - for (size_t i = tid; i < size; i += (size_t)gridDim.x * blockDim.x) { - T temp = in[i]; - temp.to(); - out[i] = temp; - } -} diff --git a/shared/subspace-proof-of-space-gpu/src/create_tables.cuh b/shared/subspace-proof-of-space-gpu/src/create_tables.cuh deleted file mode 100644 index 303bde43ae7..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/create_tables.cuh +++ /dev/null @@ -1,164 +0,0 @@ -// Originally written by Supranational LLC - -template -__global__ __launch_bounds__(1024) -void compute_fn(uint32_t* out_y, metadata* out_metadata, - const uint2* y_sorted, const uint32_t* in_y, - const uint2* matches, const uint32_t* match_counter, - const metadata* in_metadata, int table_number) -{ - uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; - - uint32_t N = *match_counter; - for (uint32_t i = tid; i < N; i += gridDim.x * blockDim.x) { - // Determine x_index and y_index based on previous y index - uint32_t x_index = y_sorted[matches[i].x].x; - uint32_t y_index = y_sorted[matches[i].y].x; - - // Initialize metadata structures - metadata left_metadata = {0}; - metadata right_metadata = {0}; - - // Set left_metadata and right_metadata based on table_number - if (table_number == 2) { - // For table_number 2, directly set metadata from indices - // because there is no metadata table created for table_number 1 - // and table1 always has a length of 1 << K, - // hence metadata is selected based on the previous y index values. - left_metadata.data[0] = x_index; - right_metadata.data[0] = y_index; - } else { - // For other table numbers, retrieve metadata from input metadata array - left_metadata = in_metadata[x_index]; - right_metadata = in_metadata[y_index]; - } - - uint32_t parent_metadata_bits = metadata::size_bits(K, table_number - 1); - - // Take only bytes where bits were set - uint32_t num_bytes_with_data - = ((K + PARAM_EXT) + (2 * parent_metadata_bits) + 7) / 8; - - // Move bits of `left_metadata` at the final offset of eventual `input_a` - metadata input_a; - input_a = left_metadata << (metadata::bits - parent_metadata_bits - (K + PARAM_EXT)); - - // Collect `K + PARAM_EXT` most significant bits of `y` at the final offset of eventual `input_a` - input_a.data[1] |= (uint64_t)in_y[y_sorted[matches[i].x].x] << (64 - (K + PARAM_EXT)); - - // Part of the `right_bits` at the final offset of eventual `input_a` - uint32_t y_and_left_bits = (K + PARAM_EXT) + parent_metadata_bits; - uint32_t right_bits_start_offset = metadata::bits - parent_metadata_bits; - - // Initialize the input array for BLAKE3 hash function - // Initialize block length for BLAKE3 compression - uint64_t blake_input[8] = {0}; - uint32_t block_len; - - // If `right_metadata` bits start to the left of the desired position in `input_a` move - // bits right, else move left - if (right_bits_start_offset < y_and_left_bits) { - uint32_t right_bits_pushed_into_input_b = y_and_left_bits - - right_bits_start_offset; - - // Collect bits of `right_metadata` that will fit into `input_a` at the final offset in - // eventual `input_a` - metadata right_bits_shifted; - right_bits_shifted = right_metadata >> right_bits_pushed_into_input_b; - for (uint32_t j = 0; j < 2; j++) { - input_a.data[j] |= right_bits_shifted.data[j]; - } - - // Collect bits of `right_metadata` that will spill over into `input_b` - metadata input_b; - input_b = right_metadata << metadata::bits - right_bits_pushed_into_input_b; - - // Prepare input for BLAKE3 hash function by performing a byte swap - input_a.byte_swap(blake_input); - input_b.byte_swap(&blake_input[metadata::bits / 64]); - - // Calculate the total block length for BLAKE3 compression - // This includes the bytes from input_b and the metadata bits from input_a - uint32_t input_b_bytes_count = (right_bits_pushed_into_input_b + 7) / 8; - block_len = input_b_bytes_count + (metadata::bits / 8); - } else { - uint32_t right_bits_pushed_into_input_b = right_bits_start_offset - - y_and_left_bits; - - metadata right_bits_shifted; - right_bits_shifted = right_metadata << right_bits_pushed_into_input_b; - - for (uint32_t j = 0; j < 2; j++) { - input_a.data[j] |= right_bits_shifted.data[j]; - } - - input_a.byte_swap(blake_input); - block_len = num_bytes_with_data; - } - // Initialize the BLAKE3 state with the given chaining value and block length - // Perform BLAKE3 compression on the prepared input data - blake3_state state = initialize_blake3_state(block_len); - blake3_compress(state, (uint8_t*)blake_input); - - // Retrieve the first 32-bit word of the BLAKE3 hash state - // Apply a byte swap operation to convert the hash to little-endian format - // Extract the highest K + PARAM_EXT bits from the little-endian hash to compute y values. - uint32_t hash = state.data[0]; - uint32_t little_endian_hash = byte_swap(hash); - out_y[i] = little_endian_hash >> (32 - (K + PARAM_EXT)); - - // Perform metadata computation - uint32_t metadata_bits = metadata::size_bits(K, table_number); - if (table_number < 4) { - left_metadata = left_metadata << parent_metadata_bits; - for (uint32_t j = 0; j < 2; j++) { - left_metadata.data[j] = left_metadata.data[j] | right_metadata.data[j]; - } - out_metadata[i] = left_metadata; - } else if (metadata_bits > 0) { - // For K under 25 it is guaranteed that metadata + bit offset will always fit into u128. - // We collect bytes necessary, potentially with extra bits at the start and end of the bytes - // that will be taken care of later. - uint8_t* hash_bytes = reinterpret_cast(state.data); - uint32_t start_index = (K + PARAM_EXT) / 8; - metadata raw_meta; - for (uint32_t j = 0; j < 16; j++) { - reinterpret_cast(raw_meta.data)[15 - j] - = hash_bytes[j + start_index]; - } - // Remove extra bits at the beginning - left_metadata = raw_meta << ((K + PARAM_EXT) % 8); - // Move bits into correct location - left_metadata = left_metadata >> (metadata::bits - metadata_bits); - out_metadata[i] = left_metadata; - } - } -} - -template -__global__ __launch_bounds__(1024) -void compute_f1(uint32_t* output, uint32_t* input, size_t N) -{ - size_t tid = blockIdx.x * blockDim.x + threadIdx.x; - - uint32_t mask = (1 << PARAM_EXT) - 1; - for (size_t i = tid; i < N; i += gridDim.x * blockDim.x) { - size_t bit_off = i * K; - - union { - uint64_t ul; - uint32_t u[2]; - }; - - // Load 32-bit chunks from input array and perform byte swap - u[0] = byte_swap(input[bit_off / 32 + ((bit_off % 32 + K) >= 32)]); - u[1] = byte_swap(input[bit_off / 32]); - - // Extract K bits starting from bit_off position - uint32_t y_bytes = (uint32_t)((ul << (bit_off % 32)) >> 32); - y_bytes = y_bytes >> (32 - K); - - // Apply masks and combine - output[i] = ((y_bytes << PARAM_EXT) & ~mask) | ((i >> (K - PARAM_EXT)) & mask); - } -} diff --git a/shared/subspace-proof-of-space-gpu/src/cuda.rs b/shared/subspace-proof-of-space-gpu/src/cuda.rs deleted file mode 100644 index 087e2552d19..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/cuda.rs +++ /dev/null @@ -1,175 +0,0 @@ -// Originally written by Supranational LLC - -#[cfg(test)] -mod tests; - -use rust_kzg_blst::types::fr::FsFr; -use std::ops::DerefMut; -use subspace_core_primitives::ScalarBytes; -use subspace_core_primitives::pieces::Record; -use subspace_core_primitives::pos::{PosProof, PosSeed}; -use subspace_kzg::Scalar; - -unsafe extern "C" { - /// # Returns - /// * `usize` - The number of available GPUs. - fn gpu_count() -> usize; - - /// # Parameters - /// * `k: The size parameter for the table. - /// * `seed: A pointer to the seed data. - /// * `lg_record_size: The logarithm of the record size. - /// * `challenge_index: A mutable pointer to store the index of the challenge. - /// * `record: A pointer to the record data. - /// * `chunks_scratch: A mutable pointer to a scratch space for chunk data. - /// * `proof_count: A mutable pointer to store the count of proofs. - /// * `parity_record_chunks: A mutable pointer to the parity record chunks. - /// * `gpu_id: The ID of the GPU to use. - /// - /// # Returns - /// * `sppark::Error` - An error code indicating the result of the operation. - /// - /// # Assumptions - /// * `seed` must be a valid pointer to a 32-byte. - /// * `record` must be a valid pointer to the record data (`*const Record`), with a length of `1 << lg_record_size`. - /// * `parity_record_chunks` must be valid mutable pointer to `Scalar` elements, each with a length of `1 << lg_record_size`. - /// * `chunks_scratch` must be a valid mutable pointer where up to `challenges_count` 32-byte chunks of GPU-calculated data will be written. - /// * `gpu_id` must be a valid identifier of an available GPU. The available GPUs can be determined by using the `gpu_count` function. - fn generate_and_encode_pospace_dispatch( - k: u32, - seed: *const [u8; 32], - lg_record_size: u32, - challenge_index: *mut u32, - record: *const [u8; 32], - chunks_scratch: *mut [u8; 32], - proof_count: *mut u32, - parity_record_chunks: *mut FsFr, - gpu_id: i32, - ) -> sppark::Error; -} - -/// Returns [`CudaDevice`] for each available device -pub fn cuda_devices() -> Vec { - let num_devices = unsafe { gpu_count() }; - - (0i32..) - .take(num_devices) - .map(|gpu_id| CudaDevice { gpu_id }) - .collect() -} - -/// Wrapper data structure encapsulating a single CUDA-capable device -#[derive(Debug)] -pub struct CudaDevice { - gpu_id: i32, -} - -impl CudaDevice { - /// Cuda device ID - pub fn id(&self) -> i32 { - self.gpu_id - } - - /// Generates and encodes PoSpace on the GPU - pub fn generate_and_encode_pospace( - &self, - seed: &PosSeed, - record: &mut Record, - encoded_chunks_used_output: impl ExactSizeIterator>, - ) -> Result<(), String> { - let record_len = Record::NUM_CHUNKS; - let challenge_len = Record::NUM_S_BUCKETS; - let lg_record_size = record_len.ilog2(); - - if challenge_len > u32::MAX as usize { - return Err(String::from("challenge_len is too large to fit in u32")); - } - - let mut proof_count = 0u32; - let mut chunks_scratch_gpu = - Vec::<[u8; ScalarBytes::FULL_BYTES]>::with_capacity(challenge_len); - let mut challenge_index_gpu = Vec::::with_capacity(challenge_len); - let mut parity_record_chunks = Vec::::with_capacity(Record::NUM_CHUNKS); - - let error = unsafe { - generate_and_encode_pospace_dispatch( - u32::from(PosProof::K), - &**seed, - lg_record_size, - challenge_index_gpu.as_mut_ptr(), - record.as_ptr(), - chunks_scratch_gpu.as_mut_ptr(), - &mut proof_count, - Scalar::slice_mut_to_repr(&mut parity_record_chunks).as_mut_ptr(), - self.gpu_id, - ) - }; - - if error.code != 0 { - let error = error.to_string(); - if error.contains("the provided PTX was compiled with an unsupported toolchain.") { - return Err(format!( - "Nvidia driver is likely too old, make sure install version 550 or newer: \ - {error}" - )); - } - return Err(error); - } - - let proof_count = proof_count as usize; - unsafe { - chunks_scratch_gpu.set_len(proof_count); - challenge_index_gpu.set_len(proof_count); - parity_record_chunks.set_len(Record::NUM_CHUNKS); - } - - let mut encoded_chunks_used = vec![false; challenge_len]; - let source_record_chunks = record.to_vec(); - - let mut chunks_scratch = challenge_index_gpu - .into_iter() - .zip(chunks_scratch_gpu) - .collect::>(); - - chunks_scratch.sort_unstable_by_key(|(a_out_index, _)| *a_out_index); - - // We don't need all the proofs - chunks_scratch.truncate(proof_count.min(Record::NUM_CHUNKS)); - - for (out_index, _chunk) in &chunks_scratch { - encoded_chunks_used[*out_index as usize] = true; - } - - encoded_chunks_used_output - .zip(&encoded_chunks_used) - .for_each(|(mut output, input)| *output = *input); - - record - .iter_mut() - .zip( - chunks_scratch - .into_iter() - .map(|(_out_index, chunk)| chunk) - .chain( - source_record_chunks - .into_iter() - .zip(parity_record_chunks) - .flat_map(|(a, b)| [a, b.to_bytes()]) - .zip(encoded_chunks_used.iter()) - // Skip chunks that were used previously - .filter_map(|(record_chunk, encoded_chunk_used)| { - if *encoded_chunk_used { - None - } else { - Some(record_chunk) - } - }), - ), - ) - .for_each(|(output_chunk, input_chunk)| { - *output_chunk = input_chunk; - }); - - Ok(()) - } -} diff --git a/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs b/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs deleted file mode 100644 index df1009b2720..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/cuda/tests.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::cuda::cuda_devices; -use std::num::NonZeroUsize; -use std::slice; -use subspace_core_primitives::hashes::{blake3_254_hash_to_scalar, blake3_hash}; -use subspace_core_primitives::pieces::{PieceOffset, Record}; -use subspace_core_primitives::sectors::SectorId; -use subspace_core_primitives::segments::HistorySize; -use subspace_erasure_coding::ErasureCoding; -use subspace_farmer_components::plotting::{CpuRecordsEncoder, RecordsEncoder}; -use subspace_farmer_components::sector::SectorContentsMap; -use subspace_proof_of_space::Table; -use subspace_proof_of_space::chia::ChiaTable; - -type PosTable = ChiaTable; - -#[test] -fn basic() { - let cuda_device = cuda_devices() - .into_iter() - .next() - .expect("Need CUDA device to run this test"); - - let mut table_generator = PosTable::generator(); - let erasure_coding = ErasureCoding::new( - NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) - .expect("Not zero; qed"), - ) - .unwrap(); - let global_mutex = Default::default(); - let mut cpu_records_encoder = CpuRecordsEncoder::::new( - slice::from_mut(&mut table_generator), - &erasure_coding, - &global_mutex, - ); - - let history_size = HistorySize::ONE; - let sector_id = SectorId::new(blake3_hash(b"hello"), 500, history_size); - let mut record = Record::new_boxed(); - record - .iter_mut() - .enumerate() - .for_each(|(index, chunk)| *chunk = *blake3_254_hash_to_scalar(&index.to_le_bytes())); - - let mut cpu_encoded_records = Record::new_zero_vec(2); - for cpu_encoded_record in &mut cpu_encoded_records { - cpu_encoded_record.clone_from(&record); - } - let cpu_sector_contents_map = cpu_records_encoder - .encode_records(§or_id, &mut cpu_encoded_records, &Default::default()) - .unwrap(); - - let mut gpu_encoded_records = Record::new_zero_vec(2); - for gpu_encoded_record in &mut gpu_encoded_records { - gpu_encoded_record.clone_from(&record); - } - let mut gpu_sector_contents_map = SectorContentsMap::new(2); - cuda_device - .generate_and_encode_pospace( - §or_id.derive_evaluation_seed(PieceOffset::ZERO), - &mut gpu_encoded_records[0], - gpu_sector_contents_map - .iter_record_bitfields_mut() - .next() - .unwrap() - .iter_mut(), - ) - .unwrap(); - cuda_device - .generate_and_encode_pospace( - §or_id.derive_evaluation_seed(PieceOffset::ONE), - &mut gpu_encoded_records[1], - gpu_sector_contents_map - .iter_record_bitfields_mut() - .nth(1) - .unwrap() - .iter_mut(), - ) - .unwrap(); - - assert!(cpu_sector_contents_map == gpu_sector_contents_map); - assert!(cpu_encoded_records == gpu_encoded_records); -} diff --git a/shared/subspace-proof-of-space-gpu/src/data_structures.cuh b/shared/subspace-proof-of-space-gpu/src/data_structures.cuh deleted file mode 100644 index 58d5ead9c5e..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/data_structures.cuh +++ /dev/null @@ -1,237 +0,0 @@ -// Originally written by Supranational LLC - -#include -#include -#include - -// Specifies the number of tables. -const uint32_t NUM_TABLES = 7; - -// Stores 128-bit data and includes device functions for byte swapping, -// computing size in bits, and bitwise shift operations. -class __align__(16) metadata { -public: - uint64_t data[2]; - static const uint32_t limb_bits = sizeof(uint64_t) * 8; - static const uint32_t bits = sizeof(data) * 8; - - __device__ - void byte_swap(uint64_t* out_metadata) - { - uint8_t* out_bytes = reinterpret_cast(out_metadata); - uint8_t* in_bytes = reinterpret_cast(data); - for (uint32_t i = 0; i < 16; i++) { - out_bytes[i] = in_bytes[15 - i]; - } - } - - static __device__ - uint32_t size_bits(int k, int table_number) - { - int scale; - switch (table_number) { - case 1: scale = 1; break; - case 2: scale = 2; break; - case 3: case 4: scale = 4; break; - case 5: scale = 3; break; - case 6: scale = 2; break; - default: scale = 0; break; - } - return k * scale; - } - - __device__ - metadata operator>>(uint32_t shift_amount) const { - metadata out; - - if (shift_amount < limb_bits) { - out.data[0] = (data[0] >> shift_amount) | (data[1] << (limb_bits - shift_amount)); - out.data[1] = data[1] >> shift_amount; - } else if (shift_amount < bits) { - out.data[0] = data[1] >> (shift_amount - limb_bits); - out.data[1] = 0; - } else { - out.data[1] = out.data[0] = 0; - } - - return out; - } - - __device__ - metadata operator<<(uint32_t shift_amount) const { - metadata out; - - if (shift_amount < limb_bits) { - out.data[1] = (data[1] << shift_amount) | (data[0] >> (limb_bits - shift_amount)); - out.data[0] = data[0] << shift_amount; - } else if (shift_amount < bits) { - out.data[1] = data[0] << (shift_amount - limb_bits); - out.data[0] = 0; - } else { - out.data[1] = out.data[0] = 0; - } - - return out; - } -}; - -// The `table_data_ptr` struct is designed to hold pointers to data structures. -struct table_data_ptr { - uint2* ys; - uint2* matches; -}; - -// The `Table` class is designed to manage and store data structures used in the table creation process. -// -// # Member Variables -// - `size_t size`: Holds the number of elements in the table. -// - `bool metadata_exists`: Indicates whether metadata is present. -// - `ys`: `uint2` type data on the GPU, which holds the sorted `y` values and their original indices. -// - `matches`: `uint2` type match data on the GPU (allocated if metadata is required). -// - `y`: `uint32_t` type data on the GPU. -// - `metadatas`: `metadata` type data on the GPU (allocated if metadata is required). -class Table { -public: - size_t size = 0; - bool metadata_exists = false; - uint2* ys; - uint2* matches; - uint32_t* y; - metadata* metadatas; - - Table() = default; - - Table(const gpu_t& gpu, size_t init_size, bool metadata_required) - : size(init_size), metadata_exists(metadata_required), - ys((uint2*)gpu.Dmalloc(init_size * sizeof(uint2))), - matches(metadata_required ? - (uint2*)gpu.Dmalloc(init_size * sizeof(uint2)) : nullptr), - y((uint32_t*)gpu.Dmalloc(init_size * sizeof(uint32_t))), - metadatas(metadata_required ? - (metadata*)gpu.Dmalloc(init_size * sizeof(metadata)) : nullptr) {} - - void free_mem(const gpu_t& gpu) - { - gpu.Dfree(ys); - gpu.Dfree(matches); - gpu.Dfree(y); - gpu.Dfree(metadatas); - } - - table_data_ptr get_table_data() const - { - table_data_ptr data; - data.ys = &ys[0]; - data.matches = &matches[0]; - return data; - } -}; - -// Manages and initializes data structures specific to a single GPU. -class gpu_specific_state { -public: - const gpu_t& gpu; - uint64_t* proof; - Table tables[NUM_TABLES]; - uint32_t* challenge_index; - uint2* temp_sort; - uint2* histogram; - chacha_state* out_chacha; - bool initialized = false; - uint32_t current_K = 0; - uint32_t current_challenges_count = 0; - std::unique_ptr mtx; - - gpu_specific_state(int id) : gpu(select_gpu(id)), mtx(std::make_unique()) {} - - // Initializes GPU-specific data structures. - // K: An unsigned 32-bit integer that determines the size of certain data structures. - // challenges_count: Number of challenges to use that determines the size of certain data structures. - void initialize_data(uint32_t K, uint32_t challenges_count) - { - // Checks if the GPU state is already initialized with sufficient sizes for K and challenges_count. - // If the current_K and current_challenges_count are greater than or equal to the new values, - // it means the GPU state is already properly initialized and no reallocation is needed. - if (initialized && current_K >= K && current_challenges_count >= challenges_count) - return; - - // Frees any previously allocated memory - free_mem(); - - // Calculates the maximum table size based on K. - uint32_t max_table_size = 2 * (1 << K); - - // Initializes the first table with `max_table_size`. - tables[0] = Table(gpu, max_table_size, false); - // Initializes the remaining tables. - for (size_t i = 1; i < NUM_TABLES; i++) { - tables[i] = Table(gpu, max_table_size, true); - } - - // Calculates the maximum proof size. Allocates GPU memory for proofs and challenge indices. - uint32_t max_proof_size = challenges_count * K; - proof = (uint64_t*)gpu.Dmalloc(max_proof_size * sizeof(uint64_t)); - challenge_index = (uint32_t*)gpu.Dmalloc(challenges_count * sizeof(uint32_t)); - - // Allocates GPU memory for temporary sorting buffer. - temp_sort = (uint2*)gpu.Dmalloc(max_table_size * sizeof(uint2)); - - // Calculates the histogram size and allocates GPU memory for histogram. - uint32_t y_bit_size = K + PARAM_EXT; - size_t histogram_size = (((size_t)1 << y_bit_size) + PARAM_BC - 1) / PARAM_BC; - histogram = (uint2*)gpu.Dmalloc(histogram_size * sizeof(uint2)); - - // Calculates the output length in bytes and allocates GPU memory for ChaCha states. - size_t output_len_bytes = K * ((size_t)1 << K) / 8; - out_chacha = (chacha_state*)gpu.Dmalloc(output_len_bytes * sizeof(chacha_state)); - - initialized = true; - current_K = K; - current_challenges_count = challenges_count; - } - - void free_mem() - { - if (!initialized) - return; - - for (size_t i = 0; i < NUM_TABLES; i++) { - tables[i].free_mem(gpu); - } - - gpu.Dfree(proof); - gpu.Dfree(challenge_index); - gpu.Dfree(temp_sort); - gpu.Dfree(histogram); - gpu.Dfree(out_chacha); - - initialized = false; - } -}; - -// Manages multiple gpu_specific_state instances, -// ensuring proper GPU resource allocation and synchronization. -class subspace_proof_state { -public: - subspace_proof_state(subspace_proof_state const&) = delete; - void operator=(subspace_proof_state const&) = delete; - - std::vector gpu_specific_states; - - subspace_proof_state() - { - int num_gpus = ngpus(); - gpu_specific_states.reserve(num_gpus); - for (size_t id = 0; id < ngpus(); id++) { - gpu_specific_states.emplace_back((int)id); - } - } - - static subspace_proof_state& get_proof_state() - { - static subspace_proof_state proof_state; - return proof_state; - } -}; - -static subspace_proof_state& proof_state = subspace_proof_state::get_proof_state(); diff --git a/shared/subspace-proof-of-space-gpu/src/find_matches.cuh b/shared/subspace-proof-of-space-gpu/src/find_matches.cuh deleted file mode 100644 index 721d300f3e6..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/find_matches.cuh +++ /dev/null @@ -1,201 +0,0 @@ -// Originally written by Supranational LLC - -__device__ __forceinline__ -uint32_t calculate_destination(uint32_t parity, uint32_t r, uint32_t m) -{ - uint32_t dest = ((r / PARAM_C + m) % PARAM_B) * PARAM_C - + (((2 * m + parity) * (2 * m + parity) + r) % PARAM_C); - - return dest; -} - -// If a matching value is found, return the index of the leftmost occurence -// of the value in "data" (in case there are duplicates) -// If a matching value is not found, return the index of the value which is -// closest to but still smaller than dest -__device__ __forceinline__ -uint32_t binary_search(const uint32_t* data, uint32_t num_elems, uint32_t dest) -{ - uint32_t left = 0, right = num_elems; - - while (left < right) { - uint32_t middle = (left + right) / 2; - - if (data[middle] < dest) - left = middle + 1; - else - right = middle; - } - - return left; -} - -// One block (of 256 threads) goes over a pair of buckets -static const uint32_t block_sz = 256; - -__global__ __launch_bounds__(1024) -void find_matches(uint2* out, const uint2* ys, const uint2* histogram, - uint32_t* global_match_count) -{ - // This constant is based on the likely number of elements in each bucket. Generally it will not exceed 512, which - // means a single call will process the bucket. When it does exceed 512 the remaining elements are processed in the - // next pass. This is a heuristic and not expected to impact performance significantly either way. - const uint32_t right_bucket_step = 512; - const uint32_t warp_count = block_sz / WARP_SZ; - const uint32_t warp_match_threshold = 4 * WARP_SZ; - const uint32_t warp_partition_size = warp_match_threshold + 2 * WARP_SZ; - - extern __shared__ uint2 scratchpad[]; - - uint32_t warpid = threadIdx.x / WARP_SZ; - uint32_t laneid = threadIdx.x % WARP_SZ; - - // Each warp has its own partition in shared memory to write the matches into - // to avoid shared memory atomic operations - uint2* matches = &scratchpad[warpid * warp_partition_size]; - uint32_t* shr_right_ys = (uint32_t*)&scratchpad[warp_count * warp_partition_size]; - - uint32_t left_bucket_idx = blockIdx.x; - - uint32_t left_bucket_start = histogram[left_bucket_idx].x; - uint32_t left_bucket_size = histogram[left_bucket_idx].y; - uint32_t right_bucket_start = histogram[left_bucket_idx + 1].x; - uint32_t right_bucket_size = histogram[left_bucket_idx + 1].y; - - uint32_t left_upper_bound = ((left_bucket_size + WARP_SZ - 1) / WARP_SZ) * WARP_SZ; - - const uint2* left_ys = &ys[left_bucket_start]; - const uint2* right_ys = &ys[right_bucket_start]; - - uint32_t warp_match_count = 0; - - // We iterate over the right bucket in 512-element chunks - for (uint32_t right_iter = 0; right_iter < right_bucket_size; right_iter += right_bucket_step) { - uint32_t current_right_size = min(right_bucket_size - right_iter, - right_bucket_step); - - // Read the next chunks of right bucket y elements into shared memory - #pragma unroll - for (uint32_t i = 0; i < right_bucket_step / block_sz; i++) { - uint32_t off = threadIdx.x + i * block_sz; - shr_right_ys[threadIdx.x + i * block_sz] = - off < current_right_size ? right_ys[right_iter + off].y - : 0xFFFFFFFF; - } - - __syncthreads(); - - // The largest y value in the current chunk of right bucket y elements - uint32_t last_right_y = shr_right_ys[current_right_size - 1]; - - // Each thread reads a y value from the left bucket - // Each thread will calculate all PARAM_M (64) "dest" values and - // check for matches - for (uint32_t left_iter = threadIdx.x; left_iter < left_upper_bound; left_iter += block_sz) { - // All the threads in a warp must execute this loop, but if - // a y value is not left for a thread, then they obtain an invalid - // left y value so that they will not find any matches - uint32_t participate = left_iter < left_bucket_size; - uint32_t left_y = participate ? left_ys[left_iter].y : 0xFFFFFFFF; - uint32_t left_idx = participate ? left_bucket_start + left_iter : 0; - - // Calculate the first "dest" value and perform a binary search - // even if there isn't an exact find, we obtain the index of the - // right y value which is the closest to (but still smaller than) - // the dest value - uint32_t dest = calculate_destination(left_bucket_idx % 2, left_y, 0); - uint32_t shr_right_y_idx = binary_search(shr_right_ys, current_right_size, dest); - uint32_t right_y = shr_right_ys[shr_right_y_idx]; - - // We loop through the rest of the "dest" values - // "dest" values are ascending modulo 15113. We will use this - // property to quickly find the closest (or matching) right y index - // as we loop through consecutive "dest" values - #pragma unroll 8 - for (uint32_t m = 0; m < PARAM_M; m++) { - uint32_t prev_dest = dest; - dest = calculate_destination(left_bucket_idx % 2, left_y, m); - - // If the next "dest" value wraps around 15113, the right y - // index must also wrap back to the start - if (dest < prev_dest) { - shr_right_y_idx = 0; - right_y = shr_right_ys[shr_right_y_idx]; - } - - // Quickly find the closest (or matching) index of a right y - // for the next "dest" value - if (dest <= last_right_y) { - while (right_y < dest) { - shr_right_y_idx++; - right_y = shr_right_ys[shr_right_y_idx]; - } - } - - uint32_t current_match_count; - uint32_t wrap_around = 1; - - do { - uint32_t is_a_match = dest == right_y && participate && wrap_around; - - // A bitmap where the "i"th bit is 1 if the thread with lane - // "i" has found a match - uint64_t mask = __ballot_sync(0xFFFFFFFF, is_a_match); - - // Summing up the number of set bits, we get the number of - // matches in the warp - current_match_count = __popc(mask); - // Each thread computes a unique write index into shared memory - mask <<= WARP_SZ - laneid; - uint32_t off = __popc(mask); - - // We store the true left and right indices of the match - if (is_a_match) { - uint32_t right_idx = right_bucket_start + right_iter + shr_right_y_idx; - matches[warp_match_count + off] = uint2{left_idx, right_idx}; - } - - // Increment the right index and re-check for matches, to - // account for any duplicate values - shr_right_y_idx += is_a_match; - // If the entirety of the current chunk of right y values - // consists of only one value (or some duplicates of it), - // we stop matching after we wrap around the right y chunk - wrap_around = shr_right_y_idx < current_right_size; - shr_right_y_idx *= wrap_around; - - right_y = shr_right_ys[shr_right_y_idx]; - - warp_match_count += current_match_count; - // Loop until all threads in a warp have exhausted the duplicates - } while (current_match_count); - } - - // Each warp checks whether their partition in shared memory - // is sufficiently full. If it is, we write the matches into - // global memory and reset the shared partition for the warp - if (warp_match_count > warp_match_threshold) { - uint32_t global_off = 0; - if (laneid == 0) - global_off = atomicAdd(global_match_count, warp_match_count); - - global_off = __shfl_sync(0xFFFFFFFF, global_off, 0); - - for (uint32_t i = laneid; i < warp_match_count; i += WARP_SZ) - out[global_off + i] = matches[i]; - - warp_match_count = 0; - } - } - } - - // Write any matches into global memory - uint32_t global_off = 0; - if (laneid == 0) - global_off = atomicAdd(global_match_count, warp_match_count); - - global_off = __shfl_sync(0xFFFFFFFF, global_off, 0); - - for (uint32_t i = laneid; i < warp_match_count; i += WARP_SZ) - out[global_off + i] = matches[i]; -} diff --git a/shared/subspace-proof-of-space-gpu/src/find_proof.cuh b/shared/subspace-proof-of-space-gpu/src/find_proof.cuh deleted file mode 100644 index fc5ff476c87..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/find_proof.cuh +++ /dev/null @@ -1,149 +0,0 @@ -// Originally written by Supranational LLC - -__device__ __forceinline__ -uint32_t search_min_left_pos(table_data_ptr table, uint32_t dest, - uint32_t num_elems, uint32_t shift_amount) -{ - uint32_t left = 0, right = num_elems; - - // Binary search to find the first matching element - while (left < right) { - uint32_t middle = (left + right) / 2; - - if ((table.ys[middle].y >> shift_amount) < dest) - left = middle + 1; - else - right = middle; - } - - // Find the smallest left position among elements with the same y_value - uint32_t y_value = table.ys[left].y; - uint32_t smallest_left_pos = UINT32_MAX; - uint32_t first_matching_element = left; - - for (uint32_t i = left; i < num_elems && table.ys[i].y == y_value; ++i) { - uint2 match = table.matches[table.ys[i].x]; - if (match.x < smallest_left_pos) { - smallest_left_pos = match.x; - first_matching_element = i; - } - } - - return first_matching_element; -} - -// Packs specified bits from a source uint32 array to a destination byte array. -__device__ __forceinline__ -void pack_bits(uint8_t* dest, uint32_t bit_offset, uint32_t value, uint32_t num_bits) -{ - // Starting byte index in the destination - uint32_t byte_index = bit_offset / 8; - // Start bit position within the first byte - uint32_t start_bit = bit_offset % 8; - // Align bits for extraction - value <<= (32 - num_bits - start_bit); - - // Pack bits into the destination array - for (uint32_t i = 0; i < (num_bits + 7) / 8; i++) { - dest[byte_index + i] |= (value >> (24 - i * 8)) & 0xFF; - } -} - -// This device function packs bits from the source indices into a byte array, -// ready for final proof output. -__device__ __forceinline__ -void generate_proof(uint8_t* proof_out, const uint32_t* current_indices, - const table_data_ptr* tables, int K) -{ - for (uint32_t offset = 0; offset < 64; offset += 2) { - uint32_t match_index = tables[1].ys[current_indices[offset / 2]].x; - uint32_t xs[2] = { tables[0].ys[tables[1].matches[match_index].x].x, - tables[0].ys[tables[1].matches[match_index].y].x }; - - for (uint32_t j = 0; j < 2; j++) { - uint32_t bit_offset = K * (offset + j); - uint32_t value = xs[j]; - pack_bits(proof_out, bit_offset, value, K); - } - } -} - -// This device function traces back through the tables -// to find left and right positions for a given challenge. -// It updates the `current_indices` array with the matching indices from each table. -__device__ __forceinline__ -void trace_back_tables(uint32_t* current_indices, const table_data_ptr* tables, - uint32_t num_tables) -{ - for (uint32_t table_number = num_tables - 1; table_number > 1; table_number--) { - uint32_t depth = 1 << (num_tables - 1 - table_number); - table_data_ptr table = tables[table_number]; - - for (uint32_t j = 0; j < depth && depth < (1 << num_tables); j++) { - uint32_t base_index = 2 * (depth - j - 1); - uint32_t current_index = current_indices[depth - j - 1]; - - uint32_t match_index = table.ys[current_index].x; - uint2 match = table.matches[match_index]; - - current_indices[base_index] = match.x; - current_indices[base_index + 1] = match.y; - } - } -} - -template -__global__ __launch_bounds__(256) -void create_chunks_scratch(uint32_t* chunks_scratch, uint32_t* proof_counter, - uint32_t* challenge_index, uint32_t challenge_count, - fr_t* record, table_data_ptr tables[NUM_TABLES], - const uint32_t* table7_size) -{ - uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; - - for (uint32_t i = tid; i < challenge_count; i += gridDim.x * blockDim.x) { - uint32_t first_k_challenge_bits = byte_swap(i); - first_k_challenge_bits >>= (32 - K); - - uint32_t shift_amount = (32 - (K + PARAM_EXT)); - table_data_ptr last_table = tables[NUM_TABLES-1]; - uint32_t first_matching_element - = search_min_left_pos(last_table, first_k_challenge_bits, - *table7_size, shift_amount); - - if(first_matching_element < *table7_size && - (tables[NUM_TABLES - 1].ys[first_matching_element].y >> PARAM_EXT) == first_k_challenge_bits) { - - uint32_t current_indices[1 << (NUM_TABLES - 2)] = {first_matching_element}; - trace_back_tables(current_indices, tables, NUM_TABLES); - - uint8_t proof_out[8 * K] = {0}; - generate_proof(proof_out, current_indices, tables, K); - - // Determine the appropriate record chunk based on the index - // and convert the record chunk from Montgomery form - // While two endianness swaps are redundant, we do this so that - // we queue up an endianness swap kernel just after NTTs, allowing - // us to overlap a device to host copy which nets us a significant - // performance gain - fr_t record_chunk = record[i]; - record_chunk = endianness_swap(record_chunk); - record_chunk.from(); - record_chunk = endianness_swap(record_chunk); - - // Calculate where to store the chunks_scratch in the output array - uint32_t proof_offset = atomicAdd(proof_counter, 1); - - // Perform BLAKE3 hashing on the proof - blake3_state state = blake3_update(proof_out, (8 * K)); - - // XOR the record chunk with the hashed proof and store in chunks_scratch - for (uint32_t index = 0; index < 8; index++) { - chunks_scratch[proof_offset * 8 + index] = record_chunk[index] ^ state.data[index]; - } - - // Store the corresponding challenge in the challenge_index array - challenge_index[proof_offset] = i; - } - } -} diff --git a/shared/subspace-proof-of-space-gpu/src/lib.rs b/shared/subspace-proof-of-space-gpu/src/lib.rs deleted file mode 100644 index 0b1269fb893..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[cfg(feature = "cuda")] -pub mod cuda; -#[cfg(feature = "rocm")] -pub mod rocm; diff --git a/shared/subspace-proof-of-space-gpu/src/rocm.rs b/shared/subspace-proof-of-space-gpu/src/rocm.rs deleted file mode 100644 index 4537078e4e6..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/rocm.rs +++ /dev/null @@ -1,168 +0,0 @@ -// Originally written by Supranational LLC - -#[cfg(test)] -mod tests; - -use rust_kzg_blst::types::fr::FsFr; -use std::ops::DerefMut; -use subspace_core_primitives::ScalarBytes; -use subspace_core_primitives::pieces::Record; -use subspace_core_primitives::pos::{PosProof, PosSeed}; -use subspace_kzg::Scalar; - -unsafe extern "C" { - /// # Returns - /// * `usize` - The number of available GPUs. - fn gpu_count() -> usize; - - /// # Parameters - /// * `k: The size parameter for the table. - /// * `seed: A pointer to the seed data. - /// * `lg_record_size: The logarithm of the record size. - /// * `challenge_index: A mutable pointer to store the index of the challenge. - /// * `record: A pointer to the record data. - /// * `chunks_scratch: A mutable pointer to a scratch space for chunk data. - /// * `proof_count: A mutable pointer to store the count of proofs. - /// * `parity_record_chunks: A mutable pointer to the parity record chunks. - /// * `gpu_id: The ID of the GPU to use. - /// - /// # Returns - /// * `sppark::Error` - An error code indicating the result of the operation. - /// - /// # Assumptions - /// * `seed` must be a valid pointer to a 32-byte. - /// * `record` must be a valid pointer to the record data (`*const Record`), with a length of `1 << lg_record_size`. - /// * `parity_record_chunks` must be valid mutable pointer to `Scalar` elements, each with a length of `1 << lg_record_size`. - /// * `chunks_scratch` must be a valid mutable pointer where up to `challenges_count` 32-byte chunks of GPU-calculated data will be written. - /// * `gpu_id` must be a valid identifier of an available GPU. The available GPUs can be determined by using the `gpu_count` function. - fn generate_and_encode_pospace_dispatch( - k: u32, - seed: *const [u8; 32], - lg_record_size: u32, - challenge_index: *mut u32, - record: *const [u8; 32], - chunks_scratch: *mut [u8; 32], - proof_count: *mut u32, - parity_record_chunks: *mut FsFr, - gpu_id: i32, - ) -> sppark::Error; -} - -/// Returns [`RocmDevice`] for each available device -pub fn rocm_devices() -> Vec { - let num_devices = unsafe { gpu_count() }; - - (0i32..) - .take(num_devices) - .map(|gpu_id| RocmDevice { gpu_id }) - .collect() -} - -/// Wrapper data structure encapsulating a single ROCm-capable device -#[derive(Debug)] -pub struct RocmDevice { - gpu_id: i32, -} - -impl RocmDevice { - /// ROCm device ID - pub fn id(&self) -> i32 { - self.gpu_id - } - - /// Generates and encodes PoSpace on the GPU - pub fn generate_and_encode_pospace( - &self, - seed: &PosSeed, - record: &mut Record, - encoded_chunks_used_output: impl ExactSizeIterator>, - ) -> Result<(), String> { - let record_len = Record::NUM_CHUNKS; - let challenge_len = Record::NUM_S_BUCKETS; - let lg_record_size = record_len.ilog2(); - - if challenge_len > u32::MAX as usize { - return Err(String::from("challenge_len is too large to fit in u32")); - } - - let mut proof_count = 0u32; - let mut chunks_scratch_gpu = - Vec::<[u8; ScalarBytes::FULL_BYTES]>::with_capacity(challenge_len); - let mut challenge_index_gpu = Vec::::with_capacity(challenge_len); - let mut parity_record_chunks = Vec::::with_capacity(Record::NUM_CHUNKS); - - let error = unsafe { - generate_and_encode_pospace_dispatch( - u32::from(PosProof::K), - &**seed, - lg_record_size, - challenge_index_gpu.as_mut_ptr(), - record.as_ptr(), - chunks_scratch_gpu.as_mut_ptr(), - &mut proof_count, - Scalar::slice_mut_to_repr(&mut parity_record_chunks).as_mut_ptr(), - self.gpu_id, - ) - }; - - if error.code != 0 { - return Err(error.to_string()); - } - - let proof_count = proof_count as usize; - unsafe { - chunks_scratch_gpu.set_len(proof_count); - challenge_index_gpu.set_len(proof_count); - parity_record_chunks.set_len(Record::NUM_CHUNKS); - } - - let mut encoded_chunks_used = vec![false; challenge_len]; - let source_record_chunks = record.to_vec(); - - let mut chunks_scratch = challenge_index_gpu - .into_iter() - .zip(chunks_scratch_gpu) - .collect::>(); - - chunks_scratch.sort_unstable_by_key(|(a_out_index, _)| *a_out_index); - - // We don't need all the proofs - chunks_scratch.truncate(proof_count.min(Record::NUM_CHUNKS)); - - for (out_index, _chunk) in &chunks_scratch { - encoded_chunks_used[*out_index as usize] = true; - } - - encoded_chunks_used_output - .zip(&encoded_chunks_used) - .for_each(|(mut output, input)| *output = *input); - - record - .iter_mut() - .zip( - chunks_scratch - .into_iter() - .map(|(_out_index, chunk)| chunk) - .chain( - source_record_chunks - .into_iter() - .zip(parity_record_chunks) - .flat_map(|(a, b)| [a, b.to_bytes()]) - .zip(encoded_chunks_used.iter()) - // Skip chunks that were used previously - .filter_map(|(record_chunk, encoded_chunk_used)| { - if *encoded_chunk_used { - None - } else { - Some(record_chunk) - } - }), - ), - ) - .for_each(|(output_chunk, input_chunk)| { - *output_chunk = input_chunk; - }); - - Ok(()) - } -} diff --git a/shared/subspace-proof-of-space-gpu/src/rocm/tests.rs b/shared/subspace-proof-of-space-gpu/src/rocm/tests.rs deleted file mode 100644 index a5cc6c0d3c2..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/rocm/tests.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::rocm::rocm_devices; -use std::num::NonZeroUsize; -use std::slice; -use subspace_core_primitives::hashes::{blake3_254_hash_to_scalar, blake3_hash}; -use subspace_core_primitives::pieces::{PieceOffset, Record}; -use subspace_core_primitives::sectors::SectorId; -use subspace_core_primitives::segments::HistorySize; -use subspace_erasure_coding::ErasureCoding; -use subspace_farmer_components::plotting::{CpuRecordsEncoder, RecordsEncoder}; -use subspace_farmer_components::sector::SectorContentsMap; -use subspace_proof_of_space::Table; -use subspace_proof_of_space::chia::ChiaTable; - -type PosTable = ChiaTable; - -#[test] -fn basic() { - let rocm_device = rocm_devices() - .into_iter() - .next() - .expect("Need ROCm device to run this test"); - - let mut table_generator = PosTable::generator(); - let erasure_coding = ErasureCoding::new( - NonZeroUsize::new(Record::NUM_S_BUCKETS.next_power_of_two().ilog2() as usize) - .expect("Not zero; qed"), - ) - .unwrap(); - let global_mutex = Default::default(); - let mut cpu_records_encoder = CpuRecordsEncoder::::new( - slice::from_mut(&mut table_generator), - &erasure_coding, - &global_mutex, - ); - - let history_size = HistorySize::ONE; - let sector_id = SectorId::new(blake3_hash(b"hello"), 500, history_size); - let mut record = Record::new_boxed(); - record - .iter_mut() - .enumerate() - .for_each(|(index, chunk)| *chunk = *blake3_254_hash_to_scalar(&index.to_le_bytes())); - - let mut cpu_encoded_records = Record::new_zero_vec(2); - for cpu_encoded_record in &mut cpu_encoded_records { - cpu_encoded_record.clone_from(&record); - } - let cpu_sector_contents_map = cpu_records_encoder - .encode_records(§or_id, &mut cpu_encoded_records, &Default::default()) - .unwrap(); - - let mut gpu_encoded_records = Record::new_zero_vec(2); - for gpu_encoded_record in &mut gpu_encoded_records { - gpu_encoded_record.clone_from(&record); - } - let mut gpu_sector_contents_map = SectorContentsMap::new(2); - rocm_device - .generate_and_encode_pospace( - §or_id.derive_evaluation_seed(PieceOffset::ZERO), - &mut gpu_encoded_records[0], - gpu_sector_contents_map - .iter_record_bitfields_mut() - .next() - .unwrap() - .iter_mut(), - ) - .unwrap(); - rocm_device - .generate_and_encode_pospace( - §or_id.derive_evaluation_seed(PieceOffset::ONE), - &mut gpu_encoded_records[1], - gpu_sector_contents_map - .iter_record_bitfields_mut() - .nth(1) - .unwrap() - .iter_mut(), - ) - .unwrap(); - - assert!(cpu_sector_contents_map == gpu_sector_contents_map); - assert!(cpu_encoded_records == gpu_encoded_records); -} diff --git a/shared/subspace-proof-of-space-gpu/src/sort15113.cuh b/shared/subspace-proof-of-space-gpu/src/sort15113.cuh deleted file mode 100644 index d14cfbb4373..00000000000 --- a/shared/subspace-proof-of-space-gpu/src/sort15113.cuh +++ /dev/null @@ -1,348 +0,0 @@ -// Originally written by Supranational LLC - -#ifndef __SUBSPACE_SORT_CUH__ -#define __SUBSPACE_SORT_CUH__ -#include -#include -#include - -/* - * Custom sorting, we take in values in [0, 2**|bits|) range and return - * their indices and the least significant digit base 15113. - * Histogram contains initial indices and amount of elements that share - * the same most significant digit. - */ - -#define SORT_BLOCKDIM 1024 - -#define RADIX 15113 -#define DIGIT_BITS 14 // 15113 is a 14-bit number. - -#ifndef WARP_SZ -# define WARP_SZ 32 -#endif -#ifdef __GNUC__ -# define asm __asm__ __volatile__ -#else -# define asm asm volatile -#endif - -static const uint32_t N_COUNTERS = 1<= off) - sum += val; -#else -# error "unsupported platform" -#endif - } - - return sum; -} - -__device__ __forceinline__ -void zero_counters() -{ - #pragma unroll - for (uint32_t i = 0; i < N_SUMS/4; i++) - ((uint4*)counters)[threadIdx.x + i*SORT_BLOCKDIM] = uint4{0, 0, 0, 0}; - __syncthreads(); -} - -__device__ __forceinline__ -void count_digits(const uint32_t src[], uint32_t base, uint32_t len) -{ - zero_counters(); - - src += base; - // count occurrences of each non-zero digit - for (uint32_t i = threadIdx.x; i < len; i += SORT_BLOCKDIM) { - auto val = src[(size_t)i]; - (void)atomicAdd(&counters[val / RADIX], 1); - } - - __syncthreads(); -} - -__device__ __forceinline__ -void scatter(uint2 dst[], const uint32_t src[], uint32_t base, uint32_t len) -{ - src += base; - #pragma unroll 1 // the subroutine is memory-io-bound, unrolling makes no difference - for (uint32_t i = threadIdx.x; i < len; i += SORT_BLOCKDIM) { - auto val = src[(size_t)i]; - auto hi = val / RADIX; - auto lo = val - hi * RADIX; // val % RADIX - uint32_t idx = atomicSub(&counters[hi], 1) - 1; - dst[idx] = uint2{base+i, lo}; - } -} - -__device__ __forceinline__ -static void upper_sort(uint2 dst[], const uint32_t src[], uint32_t len, - uint2 histogram[], uint32_t upper_digits, - bool as_kernel = false) -{ - uint32_t grid_div = 31 - __clz(min(gridDim.x, WARP_SZ)); - uint32_t grid_rem = (1<> grid_div; // / gridDim.x; - uint32_t rem = len & grid_rem; // % gridDim.x; - uint32_t base; - - // borrow |dst| to collect counters from SMs - vec2d_t htemp{reinterpret_cast(dst), grid_dim}; - - if (as_kernel || blockIdx.x < grid_dim) { - if (blockIdx.x < rem) - base = ++slice * blockIdx.x; - else - base = slice * blockIdx.x + rem; - - count_digits(src, base, slice); - - #pragma unroll 1 - for (uint32_t i = threadIdx.x; i < upper_digits; i += SORT_BLOCKDIM) - htemp[i][blockIdx.x] = counters[i]; - } - - cooperative_groups::this_grid().sync(); - __syncthreads(); // eliminate BRA.DIV? - - uint32_t carry_sum, lane_off; - - if (as_kernel || blockIdx.x < grid_dim) { - const uint32_t warpid = threadIdx.x / WARP_SZ; - const uint32_t laneid = threadIdx.x % WARP_SZ; - const uint32_t sub_warpid = laneid >> grid_div; // / gridDim.x; - const uint32_t sub_laneid = laneid & grid_rem; // % gridDim.x; - const uint32_t stride = WARP_SZ >> grid_div; // / gridDim.x; - - uint2 h = uint2{0, 0}; - uint32_t sum, warp_off = warpid*WARP_SZ*N_SUMS + sub_warpid; - - #pragma unroll 1 - for (uint32_t i = 0; i < WARP_SZ*N_SUMS; i += stride, warp_off += stride) { - sum = (warp_off < upper_digits) ? htemp[warp_off][sub_laneid] : 0; - sum = sum_up(sum) + h.x; - - if (sub_laneid == blockIdx.x) - counters[warp_off] = sum; - -#if defined(__CUDACC__) - asm("{ .reg.b32 %v; .reg.pred %did;"); - asm("shfl.sync.up.b32 %v|%did, %0, 1, 0, 0xffffffff;" :: "r"(sum)); - asm("@%did mov.b32 %0, %v;" : "+r"(h.x)); - asm("}"); - h.y = __shfl_down_sync(0xffffffff, sum, grid_dim-1) - h.x; -#elif defined(__HIPCC__) - uint32_t idx = threadIdx.x - 1; - uint32_t tmp = __builtin_amdgcn_ds_bpermute(idx<<2, sum); - if (threadIdx.x % WARP_SZ >= 1) - h.x = tmp; - - idx = threadIdx.x + grid_dim-1; - h.y = __builtin_amdgcn_ds_bpermute(idx<<2, sum) - h.x; -#else -# error "unsupported platform" -#endif - - if (blockIdx.x == 0 && sub_laneid == 0 && warp_off < upper_digits) - histogram[warp_off] = h; - - h.x = __shfl_sync(0xffffffff, sum, WARP_SZ-1, WARP_SZ); - } - - if (warpid == 0) // offload some counters to registers - sum = counters[laneid]; - - __syncthreads(); - - // carry over most significant prefix sums from each warp - if (laneid == WARP_SZ-1) - counters[warpid] = h.x; - - __syncthreads(); - - carry_sum = laneid ? counters[laneid-1] : 0; - - __syncthreads(); - - if (warpid == 0) // restore offloaded counters - counters[laneid] = sum; - - __syncthreads(); - - carry_sum = sum_up(carry_sum, SORT_BLOCKDIM/WARP_SZ); - carry_sum = __shfl_sync(0xffffffff, carry_sum, warpid); - - lane_off = warpid*WARP_SZ*N_SUMS + laneid; - - #pragma unroll - for (uint32_t i = 0; i < N_SUMS; i++) - (void)atomicAdd(&counters[lane_off + i*WARP_SZ], carry_sum); - } - - cooperative_groups::this_grid().sync(); - __syncthreads(); - - if (as_kernel || blockIdx.x < grid_dim) { - scatter(dst, src, base, slice); - - if (blockIdx.x == 0) { - #pragma unroll 1 - for (uint32_t i = 0; i < N_SUMS; i++, lane_off += WARP_SZ) - if (lane_off < upper_digits) - (void)atomicAdd(&histogram[lane_off].x, carry_sum); - } - } - - if (!as_kernel) { - cooperative_groups::this_grid().sync(); - __syncthreads(); - } -} - -__device__ __forceinline__ -uint2 count_digits(const uint2 src[], uint32_t len) -{ - zero_counters(); - - uint2 first; - uint32_t i = threadIdx.x; - - // count occurrences of each digit - if (i < len) { - first = src[i]; - (void)atomicAdd(&counters[first.y], 1); - } - - for (i += SORT_BLOCKDIM; i < len; i += SORT_BLOCKDIM) - (void)atomicAdd(&counters[src[i].y], 1); - - __syncthreads(); - - return first; -} - -__device__ __forceinline__ -void scatter(uint2 dst[], const uint2 src[], uint32_t len, uint2 first, - uint32_t hi_digit) -{ - uint32_t i = threadIdx.x; - - if (i < len) { - uint32_t idx = atomicSub(&counters[first.y], 1) - 1; - first.y += hi_digit * RADIX; - dst[idx] = first; - } - - #pragma unroll 1 // the subroutine is memory-io-bound, unrolling makes no difference - for (i += SORT_BLOCKDIM; i < len; i += SORT_BLOCKDIM) { - auto val = src[(size_t)i]; - uint32_t idx = atomicSub(&counters[val.y], 1) - 1; - val.y += hi_digit * RADIX; - dst[idx] = val; - } -} - -__device__ __forceinline__ -static void lower_sort(uint2 dst[], const uint2 src[], uint32_t base, - uint32_t len, uint32_t hi_digit) -{ - auto first = count_digits(src += base, len); - - const uint32_t warpid = threadIdx.x / WARP_SZ; - const uint32_t laneid = threadIdx.x % WARP_SZ; - - uint32_t prefix_sums[N_SUMS]; - uint32_t lane_off = warpid*WARP_SZ*N_SUMS + laneid; - - // calculate per-warp prefix sums - #pragma unroll - for (uint32_t i = 0; i < N_SUMS; i++) { - uint32_t off = lane_off + i*WARP_SZ; - uint32_t sum = counters[off]; - - sum = sum_up(sum); - if (i > 0) - sum += __shfl_sync(0xffffffff, prefix_sums[i-1], WARP_SZ-1); - - prefix_sums[i] = sum; - } - - // carry over most significant prefix sums from each warp - if (laneid == WARP_SZ-1) - counters[warpid*(WARP_SZ*N_SUMS+1)] = prefix_sums[N_SUMS-1]; - - __syncthreads(); - - uint32_t carry_sum = laneid ? counters[(laneid-1)*(WARP_SZ*N_SUMS+1)] : 0; - - __syncthreads(); - - carry_sum = sum_up(carry_sum, SORT_BLOCKDIM/WARP_SZ); - carry_sum = __shfl_sync(0xffffffff, carry_sum, warpid); - carry_sum += base; - - #pragma unroll - for (uint32_t i = 0; i < N_SUMS; i++) - counters[lane_off + i*WARP_SZ] = prefix_sums[i] += carry_sum; - - __syncthreads(); - - scatter(dst, src, len, first, hi_digit); - - __syncthreads(); -} - -__device__ __forceinline__ -void sort_device(uint2 out[], const uint32_t in[], uint32_t len, uint2 temp[], - uint2 histogram[], bool emit_original_value, int bits = 2*DIGIT_BITS) -{ - uint32_t upper_digits = ((1< DIGIT_BITS && - (upper_digits< -#include -#include -#define FEATURE_BLS12_381 -#include -#include -#include -#include -#include - -const uint32_t PARAM_EXT = 6; -const uint32_t PARAM_M = 1u << PARAM_EXT; -const uint32_t PARAM_B = 119; -const uint32_t PARAM_C = 127; -const uint32_t PARAM_BC = 15113; // PARAM_B * PARAM_C -const size_t PAGE_SIZE = 4096; - -#include "common_operations.cuh" -#include "chacha.cuh" -#include "data_structures.cuh" -#include "sort15113.cuh" -#include "blake3.cuh" -#include "create_tables.cuh" -#include "find_proof.cuh" -#include "find_matches.cuh" - -// Returns the number of available GPUs -extern "C" { - size_t gpu_count() { - return ngpus(); - } -} - -template -void create_tables(gpu_specific_state& gpu_state, const chacha_state& state, - uint32_t output_block_count, uint32_t table_size, size_t shared_sz, - size_t shared_sz_matches, const int y_bit_size, uint32_t nbuckets, - uint32_t* d_match_counter) -{ - const gpu_t& gpu = gpu_state.gpu; - - //------------------------First Table----------------------- - // Generates the ChaCha key stream for the first table. - generate_chacha_keystream<8><<>> - (gpu_state.out_chacha, state, output_block_count); - - CUDA_OK(cudaGetLastError()); - - // Computes the `y` values for the first table. - compute_f1<<>> - (gpu_state.tables[0].y, (uint32_t*)&gpu_state.out_chacha[0], table_size); - - CUDA_OK(cudaGetLastError()); - - // Sorts the first table. - gpu.launch_coop(sort15113, launch_params_t{gpu.sm_count(), SORT_BLOCKDIM, shared_sz}, - &gpu_state.tables[0].ys[0], - (const uint32_t*)gpu_state.tables[0].y, - table_size, &gpu_state.temp_sort[0], - &gpu_state.histogram[0], false, y_bit_size); - - //------------------------Other Tables----------------------- - for (size_t i = 1; i < NUM_TABLES; i++) { - // Resets the match counter for the current table. - gpu.bzero(&d_match_counter[0], 1); - metadata* metadatas_input = gpu_state.tables[i - 1].metadatas; - - // Finds matches for the current table. - find_matches<<>> - (&gpu_state.tables[i].matches[0], gpu_state.tables[i - 1].ys, - &gpu_state.histogram[0], &d_match_counter[0]); - - CUDA_OK(cudaGetLastError()); - - // Computes the `y` for the current table. - compute_fn<<>> - (gpu_state.tables[i].y, gpu_state.tables[i].metadatas, - gpu_state.tables[i - 1].ys, gpu_state.tables[i - 1].y, - gpu_state.tables[i].matches, &d_match_counter[0], - metadatas_input, i + 1); - - CUDA_OK(cudaGetLastError()); - - // Sorts the current table. - gpu.launch_coop(sort15113, launch_params_t{gpu.sm_count(), SORT_BLOCKDIM, shared_sz}, - &gpu_state.tables[i].ys[0], (const uint32_t*)gpu_state.tables[i].y, - (const uint32_t*)&d_match_counter[0], - &gpu_state.temp_sort[0], &gpu_state.histogram[0], - i == NUM_TABLES - 1, y_bit_size); - } -} - -// This function generates and encodes PoSpace on the GPU, -// specifically calculating the chunks_scratch vector. -template -RustError::by_value generate_and_encode_pospace(const uint8_t* key, - uint32_t lg_record_size, - uint32_t* challenge_index, - const fr_t* record, - uint8_t* chunks_scratch, - uint32_t* proof_count, - fr_t* parity_record_chunks, - int gpu_id) -{ - const int y_bit_size = K + PARAM_EXT; - - uint32_t record_size = 1 << lg_record_size; - uint32_t challenge_len = record_size << 1; - - size_t output_len_bytes = K * ((size_t)1 << K) / 8; - size_t output_block_count = output_len_bytes / 64; - - uint32_t table_size = 1 << K; - size_t shared_sz = sizeof(uint32_t) << DIGIT_BITS; - - // nonce is initialized as a 12-byte array with all zeros. - uint8_t nonce[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - - // Initializes the ChaCha state with the key and nonce. - chacha_state state = initialize_chacha_state(key, nonce); - - uint32_t nbuckets = (1u << y_bit_size) / PARAM_BC + 1; - size_t shared_sz_matches = (NUM_TABLES - 1) * block_sz * sizeof(uint2) / sizeof(uint32_t) + 512; - - // Gets the GPU-specific state for the given GPU_ID. - gpu_specific_state& gpu_state = proof_state.gpu_specific_states[gpu_id]; - // Locks the mutex for the specific GPU identified by GPU_ID. - std::lock_guard lock(*gpu_state.mtx); - - // Gets the GPU object. - const gpu_t& gpu = select_gpu(gpu_id); - - semaphore_t complete; - gpu.spawn([=, &complete]() { - struct vm { - static void touch(const void *ptr, size_t len) - { - auto* data = reinterpret_cast(ptr); - for (size_t i = 0; i < len; i += PAGE_SIZE) { - data[i]; - } - data[len - 1]; - } - }; - - vm::touch(parity_record_chunks, record_size * sizeof(fr_t)); - vm::touch(challenge_index, challenge_len * sizeof(uint32_t)); - vm::touch(chunks_scratch, challenge_len * 32); - complete.notify(); - }); - - try { - // An object to help us sync different asynchronous streams - event_t sync_event; - - // Allocates and initializes various data on the GPU. - - // Initializes the GPU state. - gpu_state.initialize_data(K, challenge_len); - - dev_ptr_t d_match_counter(1, gpu); - gpu.bzero(&d_match_counter[0], 1); - - dev_ptr_t d_proof_counter(1, gpu); - gpu.bzero(&d_proof_counter[0], 1); - - table_data_ptr tables_data[NUM_TABLES]; - for (uint32_t i = 0; i < NUM_TABLES; i++) { - tables_data[i] = gpu_state.tables[i].get_table_data(); - } - - dev_ptr_t d_tables(NUM_TABLES, gpu); - gpu.HtoD(&d_tables[0], tables_data, NUM_TABLES); - - dev_ptr_t d_chunks_scratch(challenge_len * 8); - - dev_ptr_t d_record{2 * record_size, gpu}; - gpu.HtoD(&d_record[record_size], record, record_size); - - // End of GPU data allocation and initialization. - - // Convert the inputs from big-endian into little-endian - kern_endianness_swap<<>>(&d_record[record_size], - &d_record[record_size], - record_size); - - CUDA_OK(cudaGetLastError()); - - // Converts and duplicates data on the GPU. - convert_to_mont<<>>(&d_record[record_size], - &d_record[record_size], - record_size); - - CUDA_OK(cudaGetLastError()); - - // Performs inverse NTT on the data in d_record with a given lg_record_size. - NTT::Base_dev_ptr(gpu, &d_record[record_size], lg_record_size, - NTT::InputOutputOrder::NR, NTT::Direction::inverse, - NTT::Type::standard); - - NTT::LDE_expand(gpu, &d_record[0], &d_record[record_size], lg_record_size, 1); - - // Performs forward NTT on the data in d_record with a (lg_record_size + 1). - NTT::Base_dev_ptr(gpu, d_record, lg_record_size + 1, - NTT::InputOutputOrder::RN, NTT::Direction::forward, - NTT::Type::standard); - - // Convert the inputs (and outputs) back to big-endian - kern_endianness_swap<<>>(&d_record[0], - &d_record[0], - 2 * record_size); - - CUDA_OK(cudaGetLastError()); - - // Record a list of pending operations submitted to the stream - sync_event.record(gpu); - - // Creates all NUM_TABLES tables - create_tables(gpu_state, state, output_block_count, table_size, - shared_sz, shared_sz_matches, y_bit_size, nbuckets, - d_match_counter); - - // Finds proof and creates chunks scratch - create_chunks_scratch<<>> - (d_chunks_scratch, &d_proof_counter[0], gpu_state.challenge_index, - challenge_len, d_record, d_tables, &d_match_counter[0]); - - CUDA_OK(cudaGetLastError()); - - // Wait for the list of operations which were recorded to complete - // before launching any operations on stream gpu[0] - // The goal is to overlap the next memory copy with create table and find proof kernels - sync_event.wait(gpu[0]); - - // Creates parity record chunks by transferring odd-indexed elements. - gpu[0].DtoH(parity_record_chunks, &d_record[1], record_size, 2*sizeof(fr_t)); - - // Transfers the proof count from the GPU to the host. - gpu.DtoH(&proof_count[0], &d_proof_counter[0], 1); - - // Transfers the challenge indices from the GPU to the host. - gpu.DtoH(challenge_index, gpu_state.challenge_index, proof_count[0]); - - // Transfers the chunks scratch data from the GPU to the host. - gpu.DtoH((uint32_t*)chunks_scratch, d_chunks_scratch, proof_count[0] * 8); - - // Automatically syncs gpu[0] as well - gpu.sync(); - complete.wait(); - } catch (const cuda_error& e) { - gpu.sync(); -#ifdef TAKE_RESPONSIBILITY_FOR_ERROR_MESSAGE - return RustError{e.code(), e.what()}; -#else - return RustError{e.code()}; -#endif - } - - return RustError{cudaSuccess}; -} - -// This function calls a templated version of `generate_and_encode_pospace` based on the value of K. -// The support for K values is limited to a maximum of 21 due to constraints in the sorting implementation. -// Specifically, the algorithm calculates an "upper digit" based on the bit length of the data being sorted, divided by PARAM_BC. -// The upper digit has a maximum allowable value of 2^14. -// As K increases, the bit length of the data also increases. -// When K reaches 22, the calculated upper digit exceeds the 2^14 limit. -// Consequently, to ensure correct operation, the algorithm is restricted to K values of 21 or less. -extern "C" -RustError::by_value generate_and_encode_pospace_dispatch(uint32_t K, - const uint8_t* key, - uint32_t lg_record_size, - uint32_t* challenge_index, - const fr_t* record, - uint8_t* chunks_scratch, - uint32_t* proof_count, - fr_t* parity_record_chunks, - int gpu_id) -{ - switch (K) { - case 15: return generate_and_encode_pospace<15>(key, lg_record_size, - challenge_index, record, - chunks_scratch, proof_count, - parity_record_chunks, gpu_id); - case 16: return generate_and_encode_pospace<16>(key, lg_record_size, - challenge_index, record, - chunks_scratch, proof_count, - parity_record_chunks, gpu_id); - case 17: return generate_and_encode_pospace<17>(key, lg_record_size, - challenge_index, record, - chunks_scratch, proof_count, - parity_record_chunks, gpu_id); - case 18: return generate_and_encode_pospace<18>(key, lg_record_size, - challenge_index, record, - chunks_scratch, proof_count, - parity_record_chunks, gpu_id); - case 19: return generate_and_encode_pospace<19>(key, lg_record_size, - challenge_index, record, - chunks_scratch, proof_count, - parity_record_chunks, gpu_id); - case 20: return generate_and_encode_pospace<20>(key, lg_record_size, - challenge_index, record, - chunks_scratch, proof_count, - parity_record_chunks, gpu_id); - case 21: return generate_and_encode_pospace<21>(key, lg_record_size, - challenge_index, record, - chunks_scratch, proof_count, - parity_record_chunks, gpu_id); - default: return RustError{EINVAL}; - } -} diff --git a/shared/subspace-proof-of-space-wgpu/Cargo.toml b/shared/subspace-proof-of-space-wgpu/Cargo.toml new file mode 100644 index 00000000000..98735ce12fe --- /dev/null +++ b/shared/subspace-proof-of-space-wgpu/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "subspace-proof-of-space-wgpu" +description = "Subspace proof of space plotting for GPU via wgpu (Vulkan/Metal)" +license = "0BSD" +version = "0.1.0" +edition.workspace = true +include = [ + "/src", + "/Cargo.toml", +] + +[dependencies] +ab-proof-of-space-gpu = { workspace = true } +subspace-core-primitives = { workspace = true } +subspace-erasure-coding = { workspace = true } +subspace-kzg = { workspace = true } diff --git a/shared/subspace-proof-of-space-wgpu/src/host.rs b/shared/subspace-proof-of-space-wgpu/src/host.rs new file mode 100644 index 00000000000..3253bad3357 --- /dev/null +++ b/shared/subspace-proof-of-space-wgpu/src/host.rs @@ -0,0 +1,126 @@ +//! Host side: runs GPU proof generation and encodes records with subspace's KZG scheme, so a +//! GPU-plotted sector reads back byte-for-byte the same as the CPU encoder. + +use ab_proof_of_space_gpu::GpuRecordsEncoderInstance; +use std::ops::DerefMut; +use std::simd::Simd; +use subspace_core_primitives::ScalarBytes; +use subspace_core_primitives::pieces::Record; +use subspace_core_primitives::pos::PosSeed; +use subspace_erasure_coding::ErasureCoding; +use subspace_kzg::Scalar; + +/// A single wgpu proof-of-space encoder (one GPU queue) plus the erasure coding used to encode +/// records host-side. +pub struct WgpuDevice { + instance: GpuRecordsEncoderInstance, + erasure_coding: ErasureCoding, +} + +impl WgpuDevice { + /// Create a new device from an abundance proof encoder instance. + pub fn new(instance: GpuRecordsEncoderInstance, erasure_coding: ErasureCoding) -> Self { + Self { + instance, + erasure_coding, + } + } + + /// Generate proofs on the GPU and encode a record with subspace's KZG scheme. + /// + /// Mirrors `record_encoding` in `subspace-farmer-components`, so output is byte-identical to + /// the CPU encoder. + pub fn generate_and_encode_pospace( + &mut self, + seed: &PosSeed, + record: &mut Record, + encoded_chunks_used_output: impl ExactSizeIterator>, + ) -> Result<(), String> { + let proofs = self + .instance + .create_proofs(seed) + .map_err(|error| error.to_string())?; + let proofs = proofs.proofs(); + + let source_record_chunks = record.to_vec(); + let parity_record_chunks = self + .erasure_coding + .extend( + &source_record_chunks + .iter() + .map(|scalar_bytes| { + Scalar::try_from(scalar_bytes) + .expect("Record chunks are valid scalar bytes; qed") + }) + .collect::>(), + ) + .expect("Erasure coding instance supports this many shards; qed") + .into_iter() + .map(<[u8; ScalarBytes::FULL_BYTES]>::from) + .collect::>(); + + let mut encoded_chunks_used = vec![false; Record::NUM_S_BUCKETS]; + let mut chunks_scratch = + Vec::<[u8; ScalarBytes::FULL_BYTES]>::with_capacity(Record::NUM_S_BUCKETS); + for s_bucket in 0..Record::NUM_S_BUCKETS { + let record_chunk = if s_bucket % 2 == 0 { + &source_record_chunks[s_bucket / 2] + } else { + &parity_record_chunks[s_bucket / 2] + }; + + let proof_found = (proofs.found_proofs[s_bucket / u8::BITS as usize] + >> (s_bucket % u8::BITS as usize)) + & 1 + == 1; + let encoded_chunk = if proof_found { + (Simd::from(*record_chunk) ^ Simd::from(*proofs.proofs[s_bucket].hash())).to_array() + } else { + // Dummy value indicating no proof + [0; ScalarBytes::FULL_BYTES] + }; + chunks_scratch.push(encoded_chunk); + } + + let num_successfully_encoded_chunks = chunks_scratch + .drain(..) + .zip(encoded_chunks_used.iter_mut()) + .filter_map(|(maybe_encoded_chunk, encoded_chunk_used)| { + if maybe_encoded_chunk == [0; ScalarBytes::FULL_BYTES] { + None + } else { + *encoded_chunk_used = true; + Some(maybe_encoded_chunk) + } + }) + .take(record.len()) + .zip(record.iter_mut()) + .map(|(input_chunk, output_chunk)| { + *output_chunk = input_chunk; + }) + .count(); + + source_record_chunks + .iter() + .zip(&parity_record_chunks) + .flat_map(|(a, b)| [a, b]) + .zip(encoded_chunks_used.iter()) + .filter_map(|(record_chunk, encoded_chunk_used)| { + if *encoded_chunk_used { + None + } else { + Some(record_chunk) + } + }) + .zip(record.iter_mut().skip(num_successfully_encoded_chunks)) + .for_each(|(input_chunk, output_chunk)| { + *output_chunk = *input_chunk; + }); + + encoded_chunks_used_output + .zip(&encoded_chunks_used) + .for_each(|(mut output, input)| *output = *input); + + Ok(()) + } +} diff --git a/shared/subspace-proof-of-space-wgpu/src/lib.rs b/shared/subspace-proof-of-space-wgpu/src/lib.rs new file mode 100644 index 00000000000..9e86c73f1c9 --- /dev/null +++ b/shared/subspace-proof-of-space-wgpu/src/lib.rs @@ -0,0 +1,12 @@ +//! Subspace proof of space plotting on the GPU via wgpu (Vulkan/Metal). +//! +//! Host-only: proofs are produced on the GPU by the vendored `ab-proof-of-space-gpu`, and records +//! are encoded here with subspace's KZG scheme, so a GPU-plotted sector reads back byte-for-byte the +//! same as the CPU one. + +#![feature(portable_simd)] + +mod host; + +pub use ab_proof_of_space_gpu::{Backend, Device, DeviceType}; +pub use host::WgpuDevice;