From 9ff238e563a0d746346f9f8e4f39effd50543a3b Mon Sep 17 00:00:00 2001 From: Hyunjae Woo Date: Sat, 8 Aug 2026 13:45:38 -0700 Subject: [PATCH] feat(refit): publish canonical S3 revisions Signed-off-by: Hyunjae Woo --- .../python/modelexpress/envs.py | 6 + .../python/modelexpress/refit/__init__.py | 22 + .../python/modelexpress/refit/api.py | 62 +++ .../python/modelexpress/refit/delta.py | 108 ++++ .../python/modelexpress/refit/publisher.py | 518 ++++++++++++++++++ .../python/modelexpress/refit/s3.py | 116 ++++ .../modelexpress/refit/source/__init__.py | 4 + .../modelexpress/refit/source/canonical.py | 101 ++++ modelexpress_client/python/pyproject.toml | 3 + modelexpress_client/python/tests/test_envs.py | 8 + .../python/tests/test_refit_api.py | 86 +++ .../python/tests/test_refit_publisher.py | 346 ++++++++++++ .../tests/test_refit_publisher_distributed.py | 273 +++++++++ .../python/tests/test_refit_s3_payload.py | 269 +++++++++ modelexpress_client/python/uv.lock | 122 ++++- 15 files changed, 2043 insertions(+), 1 deletion(-) create mode 100644 modelexpress_client/python/modelexpress/refit/api.py create mode 100644 modelexpress_client/python/modelexpress/refit/delta.py create mode 100644 modelexpress_client/python/modelexpress/refit/publisher.py create mode 100644 modelexpress_client/python/modelexpress/refit/s3.py create mode 100644 modelexpress_client/python/modelexpress/refit/source/__init__.py create mode 100644 modelexpress_client/python/modelexpress/refit/source/canonical.py create mode 100644 modelexpress_client/python/tests/test_refit_api.py create mode 100644 modelexpress_client/python/tests/test_refit_publisher.py create mode 100644 modelexpress_client/python/tests/test_refit_publisher_distributed.py create mode 100644 modelexpress_client/python/tests/test_refit_s3_payload.py diff --git a/modelexpress_client/python/modelexpress/envs.py b/modelexpress_client/python/modelexpress/envs.py index a7a0ba50d..c1ed25581 100644 --- a/modelexpress_client/python/modelexpress/envs.py +++ b/modelexpress_client/python/modelexpress/envs.py @@ -67,6 +67,8 @@ MX_RESHARD_HANDSHAKE_BACKOFF_S: float MX_REFIT_STAGE_RECORD: bool MX_RESHARD_MAX_GBPS: float + MX_REFIT_S3_UPLOAD_WORKERS: int + MX_REFIT_S3_MAX_POOL_CONNECTIONS: int # Kubernetes service backend MX_K8S_SERVICE_PATTERN: str MX_K8S_SOURCE_RETRIES: str @@ -268,6 +270,10 @@ def _env_positive_float(name: str, default: float) -> float: # disables the check, and is the default because only the operator knows the # real per-rank limit for their fabric. "MX_RESHARD_MAX_GBPS": lambda: _env_float("MX_RESHARD_MAX_GBPS", 0.0), + "MX_REFIT_S3_UPLOAD_WORKERS": lambda: _env_int("MX_REFIT_S3_UPLOAD_WORKERS", 4), + "MX_REFIT_S3_MAX_POOL_CONNECTIONS": lambda: _env_int( + "MX_REFIT_S3_MAX_POOL_CONNECTIONS", 10 + ), # ── Kubernetes service backend ───────────────────────────────────────── "MX_K8S_SERVICE_PATTERN": lambda: os.environ.get("MX_K8S_SERVICE_PATTERN", "mx-sources"), "MX_K8S_SOURCE_RETRIES": lambda: os.environ.get("MX_K8S_SOURCE_RETRIES", ""), diff --git a/modelexpress_client/python/modelexpress/refit/__init__.py b/modelexpress_client/python/modelexpress/refit/__init__.py index e70c5f8db..f08cd2d4c 100644 --- a/modelexpress_client/python/modelexpress/refit/__init__.py +++ b/modelexpress_client/python/modelexpress/refit/__init__.py @@ -3,6 +3,16 @@ """Engine-agnostic primitives for live model refit.""" +from .api import ( + PublisherConfig, + ReceiverRevisionState, + ReceiverStatus, + S3Config, + WeightUpdateResult, +) +from .catalog import GrpcRevisionCatalog, RevisionCatalog +from .manifest import RevisionManifest, RevisionRecord, RevisionState, S3Object +from .publisher import Publisher from .timing import ( MX_REFIT_TIMING_PREFIX, REFIT_TIMING_STAGES, @@ -14,9 +24,21 @@ ) __all__ = [ + "GrpcRevisionCatalog", "MX_REFIT_TIMING_PREFIX", + "Publisher", + "PublisherConfig", "REFIT_TIMING_STAGES", + "ReceiverRevisionState", + "ReceiverStatus", "RefitTimingRecorder", + "RevisionCatalog", + "RevisionManifest", + "RevisionRecord", + "RevisionState", + "S3Config", + "S3Object", + "WeightUpdateResult", "add_refit_bytes", "current_refit_timing", "refit_span", diff --git a/modelexpress_client/python/modelexpress/refit/api.py b/modelexpress_client/python/modelexpress/refit/api.py new file mode 100644 index 000000000..6ed42f2d3 --- /dev/null +++ b/modelexpress_client/python/modelexpress/refit/api.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Values consumed by the Miles publisher and SGLang receiver.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + +ModelId = str +VersionId = str + + +class ReceiverRevisionState(Enum): + """SGLang-local receiver outcomes, never persisted by the MX server.""" + + BYTES_RECEIVED = "bytes_received" + VERIFIED = "verified" + FAILED = "failed" + POISONED = "poisoned" + + +@dataclass(frozen=True) +class S3Config: + """Direct S3 destination; credentials are resolved privately by boto3.""" + + bucket: str + prefix: str = "" + endpoint_url: str | None = None + region_name: str | None = None + + +@dataclass(frozen=True) +class PublisherConfig: + model_id: ModelId + catalog_endpoint: str + s3: S3Config + + +@dataclass(frozen=True) +class WeightUpdateResult: + success: bool + receiver_id: str + installed_version: VersionId | None + state: ReceiverRevisionState + target_digest: str | None = None + detail: str = "" + + +@dataclass(frozen=True) +class ReceiverStatus: + receiver_id: str + model_id: ModelId + installed_version: VersionId | None = None + target_digest: str | None = None + state: ReceiverRevisionState | None = None + detail: str = "" + + @property + def recovery_required(self) -> bool: + return self.state is ReceiverRevisionState.POISONED diff --git a/modelexpress_client/python/modelexpress/refit/delta.py b/modelexpress_client/python/modelexpress/refit/delta.py new file mode 100644 index 000000000..16cec65e6 --- /dev/null +++ b/modelexpress_client/python/modelexpress/refit/delta.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Canonical XOR-delta bucket framing and zstd compression.""" + +from __future__ import annotations + +import hashlib +import json +import struct + +import numpy as np +import zstandard + +from .source.canonical import canonical_json + +_BUCKET_MAGIC = b"MXCDV0\0" +_SCHEMA = "mx.canonical.delta.v0" + + +def compute_delta( + current: np.ndarray, base: np.ndarray +) -> tuple[np.ndarray | None, str | None, int]: + if len(current) != len(base): + raise RuntimeError("tensor changed byte size") + raw_delta = np.bitwise_xor(current, base) + changed_bytes = int(np.count_nonzero(raw_delta)) + if not changed_bytes: + return None, None, 0 + target_digest = f"sha256:{hashlib.sha256(memoryview(current)).hexdigest()}" + return raw_delta, target_digest, changed_bytes + + +def encode_bucket( + model_id: str, + base_version: str, + target_version: str, + base_digest: str, + format_digest: str, + ordinal: int, + tensors: list[tuple[str, np.ndarray]], + metadata: dict[str, dict], +) -> tuple[bytes, int]: + entries = [] + offset = 0 + compressor = zstandard.ZstdCompressor(level=3).compressobj() + chunks = [] + for name, delta in tensors: + entries.append({**metadata[name], "offset": offset}) + offset += delta.nbytes + chunks.append(compressor.compress(memoryview(delta))) + chunks.append(compressor.flush()) + + header = canonical_json( + { + "base_digest": base_digest, + "base_version": base_version, + "compression": "zstd", + "decoded_size": offset, + "delta": "xor", + "entries": entries, + "format_digest": format_digest, + "model_id": model_id, + "ordinal": ordinal, + "schema": f"{_SCHEMA}.bucket", + "target_version": target_version, + } + ) + compressed = b"".join(chunks) + return _BUCKET_MAGIC + struct.pack(">I", len(header)) + header + compressed, offset + + +def bucket_parts(data: bytes) -> tuple[dict, memoryview]: + if not data.startswith(_BUCKET_MAGIC): + raise ValueError("invalid canonical bucket") + header_size = struct.unpack( + ">I", data[len(_BUCKET_MAGIC) : len(_BUCKET_MAGIC) + 4] + )[0] + header_start = len(_BUCKET_MAGIC) + 4 + header = json.loads(data[header_start : header_start + header_size]) + return header, memoryview(data)[header_start + header_size :] + + +def parse_bucket(data: bytes) -> tuple[dict, bytes]: + header, compressed = bucket_parts(data) + decoded = zstandard.ZstdDecompressor().decompress( + compressed, max_output_size=header["decoded_size"] + ) + return header, decoded + + +def decode_bucket( + data: bytes, snapshot: dict[str, np.ndarray], metadata: dict[str, dict] +) -> dict: + header, decoded = parse_bucket(data) + for entry in header["entries"]: + name = entry["name"] + start = entry["offset"] + delta = np.frombuffer( + decoded[start : start + entry["byte_size"]], dtype=np.uint8 + ) + target = np.bitwise_xor(snapshot[name], delta) + digest = f"sha256:{hashlib.sha256(target.tobytes()).hexdigest()}" + if digest != entry["target_digest"]: + raise ValueError(f"canonical target checksum differs for {name}") + snapshot[name] = target + metadata[name]["target_digest"] = digest + return header diff --git a/modelexpress_client/python/modelexpress/refit/publisher.py b/modelexpress_client/python/modelexpress/refit/publisher.py new file mode 100644 index 000000000..03c23d77c --- /dev/null +++ b/modelexpress_client/python/modelexpress/refit/publisher.py @@ -0,0 +1,518 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Source-rank canonical publisher for Megatron-based trainers.""" + +from __future__ import annotations + +import hashlib +import os +import queue +import time +from collections import deque +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +import numpy as np +import torch + +from .. import envs +from .api import PublisherConfig +from .catalog import GrpcRevisionCatalog +from .delta import compute_delta, encode_bucket +from .manifest import RevisionManifest, RevisionState +from .s3 import S3Client +from .source.canonical import ( + canonical_json, + format_digest, + load_hf_snapshot, + snapshot_digest, +) + + +NUM_WORKERS = min(32, os.cpu_count() or 8) +POLL_INTERVAL_SECONDS = 1.0 + + +def _key(model_id: str, version: str, filename: str) -> str: + return ( + f"models/{quote(model_id, safe='')}/revisions/" + f"{quote(version, safe='')}/canonical/{filename}" + ) + + +def _object(stored, size: int) -> dict[str, object]: + result = { + "bucket": stored.bucket, + "checksum": stored.checksum, + "key": stored.key, + "size": size, + } + if stored.object_version is not None: + result["object_version"] = stored.object_version + return result + + +def _merge_metadata(contributions) -> tuple[dict[str, dict], dict[str, int]]: + metadata = {} + owners = {} + for rank, local in sorted(contributions): + for name, item in local.items(): + if name in metadata: + if item["target_digest"] != metadata[name]["target_digest"]: + raise RuntimeError( + f"{name!r} differs between source ranks {owners[name]} and {rank}" + ) + continue + metadata[name] = item + owners[name] = rank + return metadata, owners + + +def _bucket_groups(names: list[str], metadata: dict[str, dict], limit: int): + groups = [] + current = [] + size = 0 + for name in names: + tensor_size = metadata[name]["byte_size"] + if current and size + tensor_size > limit: + groups.append(current) + current = [] + size = 0 + current.append(name) + size += tensor_size + if current: + groups.append(current) + return groups + + +def _compute_staged_delta(name, data, size, pinned, base, free_buffers): + if pinned: + current = np.empty(size, dtype=np.uint8) + np.copyto(current, data.numpy()[:size]) + free_buffers.put(data) + else: + current = data + raw_delta, target_digest, changed_bytes = compute_delta(current, base) + return name, current, raw_delta, target_digest, changed_bytes + + +class Publisher: + def __init__( + self, + launch_checkpoint: str | Path, + bucket_bytes: int = 256 * 1024 * 1024, + group=None, + ) -> None: + self.launch_checkpoint = Path(launch_checkpoint) + self.bucket_bytes = bucket_bytes + self.group = group + self.catalog = None + self.s3 = None + self.snapshot = {} + self.metadata = {} + self.captured = False + + def initialize(self, config: PublisherConfig) -> None: + self.config = config + self.distributed = ( + torch.distributed.is_available() and torch.distributed.is_initialized() + ) + self.rank = torch.distributed.get_rank() if self.distributed else 0 + self.world = ( + torch.distributed.get_world_size(self.group) if self.distributed else 1 + ) + + identity = None + if self.rank == 0: + _snapshot, _metadata, launch_format, launch_digest = load_hf_snapshot( + self.launch_checkpoint + ) + identity = (launch_format, launch_digest) + self.catalog = GrpcRevisionCatalog(config.catalog_endpoint) + self.s3 = S3Client( + endpoint_url=config.s3.endpoint_url, + region_name=config.s3.region_name, + ) + identities = self._gather(identity) + self.format_digest, self.target_digest = next( + item for item in identities if item is not None + ) + self.current_version = "0" + self.pending_version = None + self.pending_digest = None + + def publish_version( + self, + version: str, + base_version: str | None = None, + gather_hf_buckets=None, + ) -> None: + if self.pending_version is not None: + raise RuntimeError(f"revision {self.pending_version!r} is still pending") + if version == "0": + if self.rank == 0: + self.catalog.publish_revision( + RevisionManifest( + model_id=self.config.model_id, + target_version="0", + target_digest=self.target_digest, + format_digest=self.format_digest, + ) + ) + self.pending_version = "0" + self.pending_digest = self.target_digest + return + + if base_version != self.current_version: + raise RuntimeError( + f"base {base_version!r} does not match current version " + f"{self.current_version!r}" + ) + capture_started = time.monotonic() + raw_deltas, changed_bytes = self._capture_deltas(gather_hf_buckets) + capture_seconds = time.monotonic() - capture_started + metadata, owners = _merge_metadata( + self._gather((self.rank, self._local_metadata())) + ) + for name in list(raw_deltas): + if owners[name] != self.rank: + del raw_deltas[name] + del changed_bytes[name] + if format_digest(metadata) != self.format_digest: + raise RuntimeError("canonical format changed during publication") + target_digest = snapshot_digest(metadata) + publish_started = time.monotonic() + wire_bytes, setup_seconds, pool_seconds, finalize_seconds = ( + self._publish_delta_index( + version, + base_version, + raw_deltas, + metadata, + owners, + target_digest, + ) + ) + publish_seconds = time.monotonic() - publish_started + self._metrics = self._collect_metrics( + ( + capture_seconds, + publish_seconds, + setup_seconds, + pool_seconds, + finalize_seconds, + sum(changed_bytes.values()), + wire_bytes, + ), + sum(item["byte_size"] for item in metadata.values()), + ) + self.pending_version = version + self.pending_digest = target_digest + + def capture_baseline(self, gather_hf_buckets, read_hf_tensor) -> None: + def seed_bucket(bucket, _pbar=None): + for name, tensor in bucket: + name = name.removeprefix("module.") + self._record_metadata(name, tensor) + try: + value = read_hf_tensor(name) + except KeyError: + value = ( + tensor.detach() + .cpu() + .contiguous() + .view(torch.uint8) + .numpy() + .reshape(-1) + ) + self.snapshot[name] = np.asarray(value, dtype=np.uint8).copy() + + gather_hf_buckets(seed_bucket) + metadata, _owners = _merge_metadata( + self._gather((self.rank, self._local_metadata())) + ) + if ( + format_digest(metadata) != self.format_digest + or snapshot_digest(metadata) != self.target_digest + ): + raise RuntimeError("source-rank baseline differs from launch revision") + self.captured = True + + def wait_for_commit(self, version: str, completion=None) -> None: + if self.pending_version != version: + raise RuntimeError(f"revision {version!r} is not pending") + if self.rank == 0: + while ( + self.catalog.get_revision(self.config.model_id, version).state + is not RevisionState.COMMITTED + ): + if completion is not None and completion.done(): + completion.result() + time.sleep(POLL_INTERVAL_SECONDS) + self.current_version = version + self.target_digest = self.pending_digest + self.pending_version = None + self.pending_digest = None + + def _collect_metrics(self, local_metrics, total_bytes: int) -> dict[str, float]: + phase_metrics = self._gather(local_metrics) + return { + "perf/update_weights_density": sum(item[5] for item in phase_metrics) + / max(total_bytes, 1), + "perf/update_weights_wire_bytes": sum(item[6] for item in phase_metrics), + "perf/mx_encode_delta": max(item[0] for item in phase_metrics), + "perf/mx_publish_time": max(item[1] for item in phase_metrics), + "perf/mx_publish_setup": max(item[2] for item in phase_metrics), + "perf/mx_publish_pool": max(item[3] for item in phase_metrics), + "perf/mx_publish_finalize": max(item[4] for item in phase_metrics), + } + + def pop_metrics(self) -> dict[str, float]: + metrics, self._metrics = getattr(self, "_metrics", {}), {} + return metrics + + def deregister(self) -> None: + for resource in (self.s3, self.catalog): + close = getattr(resource, "close", None) + if close is not None: + close() + + def _capture_deltas(self, gather_hf_buckets): + raw_deltas = {} + changed_bytes = {} + max_bytes = max( + (int(value.nbytes) for value in self.snapshot.values()), default=0 + ) + free_buffers = queue.Queue() + use_pinned = max_bytes <= 32 << 30 + try: + count = max(1, min(2 * NUM_WORKERS, (32 << 30) // max(max_bytes, 1))) + if not use_pinned: + raise RuntimeError("tensor exceeds pinned buffer budget") + for _ in range(count): + free_buffers.put( + torch.empty(max_bytes, dtype=torch.uint8, pin_memory=True) + ) + except RuntimeError: + free_buffers = queue.Queue() + use_pinned = False + + inflight = deque() + pool = ThreadPoolExecutor(max_workers=NUM_WORKERS) + + def capture_bucket(bucket, _pbar=None): + for name, tensor in bucket: + name = name.removeprefix("module.") + self._record_metadata(name, tensor) + flat = tensor.detach().contiguous().view(torch.uint8).reshape(-1) + size = int(flat.numel()) + if use_pinned and size <= max_bytes: + data = free_buffers.get() + data[:size].copy_(flat, non_blocking=True) + torch.cuda.current_stream().synchronize() + pinned = True + else: + data = flat.cpu().numpy() + pinned = False + inflight.append( + pool.submit( + _compute_staged_delta, + name, + data, + size, + pinned, + self.snapshot[name], + free_buffers, + ) + ) + if len(inflight) >= 2 * NUM_WORKERS: + name, current, raw_delta, digest, changed = ( + inflight.popleft().result() + ) + self.snapshot[name] = current + if changed: + raw_deltas[name] = raw_delta + changed_bytes[name] = changed + self.metadata[name]["target_digest"] = digest + + try: + gather_hf_buckets(capture_bucket) + while inflight: + name, current, raw_delta, digest, changed = inflight.popleft().result() + self.snapshot[name] = current + if changed: + raw_deltas[name] = raw_delta + changed_bytes[name] = changed + self.metadata[name]["target_digest"] = digest + finally: + pool.shutdown() + return raw_deltas, changed_bytes + + def _record_metadata(self, name, tensor) -> None: + digest = self.metadata.get(name, {}).get("target_digest") + self.metadata[name] = { + "name": name, + "shape": list(tensor.shape), + "dtype": str(tensor.dtype).removeprefix("torch."), + "byte_size": tensor.numel() * tensor.element_size(), + } + if digest is not None: + self.metadata[name]["target_digest"] = digest + + def _local_metadata(self): + local = {} + for name, item in self.metadata.items(): + if name not in self.snapshot: + continue + if "target_digest" not in item: + item["target_digest"] = ( + f"sha256:{hashlib.sha256(self.snapshot[name].tobytes()).hexdigest()}" + ) + local[name] = dict(item) + return local + + def _publish_delta_index( + self, + version, + base_version, + raw_deltas, + metadata, + owners, + target_digest, + ) -> tuple[int, float, float, float]: + setup_started = time.monotonic() + local_metadata = { + name: item for name, item in metadata.items() if owners[name] == self.rank + } + groups = _bucket_groups( + sorted(name for name in raw_deltas if owners[name] == self.rank), + local_metadata, + self.bucket_bytes, + ) + counts = sorted(self._gather((self.rank, len(groups)))) + offset = sum(count for rank, count in counts if rank < self.rank) + total = sum(count for _rank, count in counts) + + setup_seconds = time.monotonic() - setup_started + + tasks = [ + (ordinal, [(name, raw_deltas[name]) for name in names]) + for ordinal, names in enumerate(groups) + ] + raw_deltas.clear() + + def upload(item): + local_ordinal, tensors = item + ordinal = offset + local_ordinal + tensor_names = tuple(name for name, _delta in tensors) + data, decoded_size = encode_bucket( + model_id=self.config.model_id, + base_version=base_version, + target_version=version, + base_digest=self.target_digest, + format_digest=self.format_digest, + ordinal=ordinal, + tensors=tensors, + metadata=local_metadata, + ) + key = _key( + self.config.model_id, + version, + f"bucket-{ordinal:08d}-of-{total:08d}.mxcd", + ) + key = "/".join( + part for part in (self.config.s3.prefix.strip("/"), key) if part + ) + stored = self.s3.put( + bucket=self.config.s3.bucket, + key=key, + data=data, + ) + return ( + { + "decoded_size": decoded_size, + "object": _object(stored, len(data)), + "ordinal": ordinal, + "tensors": list(tensor_names), + }, + tensor_names, + ) + + pool_started = time.monotonic() + descriptors = [] + dirty_ordinals = {} + if groups: + with ThreadPoolExecutor( + max_workers=min(max(1, envs.MX_REFIT_S3_UPLOAD_WORKERS), len(groups)) + ) as pool: + uploaded = pool.map(upload, tasks) + for descriptor, names in uploaded: + descriptors.append(descriptor) + for name in names: + dirty_ordinals[name] = descriptor["ordinal"] + pool_seconds = time.monotonic() - pool_started + finalize_started = time.monotonic() + + coverage = [] + for name, item in local_metadata.items(): + value = {**item, "state": "clean"} + if name in dirty_ordinals: + value["state"] = "dirty" + value["bucket_ordinal"] = dirty_ordinals[name] + coverage.append(value) + wire_bytes = sum(item["object"]["size"] for item in descriptors) + contributions = self._gather((self.rank, descriptors, coverage)) + + if self.rank == 0: + buckets = sorted( + (item for _rank, values, _coverage in contributions for item in values), + key=lambda item: item["ordinal"], + ) + tensors = sorted( + (item for _rank, _values, values in contributions for item in values), + key=lambda item: item["name"], + ) + index_payload = canonical_json( + { + "base_digest": self.target_digest, + "base_version": base_version, + "buckets": buckets, + "encoding": {"compression": "zstd", "delta": "xor"}, + "format_digest": self.format_digest, + "model_id": self.config.model_id, + "schema": "mx.canonical.delta.v0", + "target_digest": target_digest, + "target_version": version, + "tensors": tensors, + } + ) + key = _key(self.config.model_id, version, "delta-index.json") + key = "/".join( + part for part in (self.config.s3.prefix.strip("/"), key) if part + ) + payload = self.s3.put( + bucket=self.config.s3.bucket, + key=key, + data=index_payload, + ) + self.catalog.publish_revision( + RevisionManifest( + model_id=self.config.model_id, + target_version=version, + base_version=base_version, + base_digest=self.target_digest, + target_digest=target_digest, + format_digest=self.format_digest, + payload=payload, + ) + ) + finalize_seconds = time.monotonic() - finalize_started + return wire_bytes, setup_seconds, pool_seconds, finalize_seconds + + def _gather(self, value): + if not self.distributed: + return [value] + values = [None] * self.world + torch.distributed.all_gather_object(values, value, group=self.group) + return values diff --git a/modelexpress_client/python/modelexpress/refit/s3.py b/modelexpress_client/python/modelexpress/refit/s3.py new file mode 100644 index 000000000..4eff5fccd --- /dev/null +++ b/modelexpress_client/python/modelexpress/refit/s3.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Direct immutable S3 transport for canonical payload objects.""" + +from __future__ import annotations + +import base64 + +import google_crc32c + +from .. import envs +from .manifest import S3Object + + +class ImmutableS3Conflict(RuntimeError): + pass + + +def _error_code(error: Exception) -> str | None: + try: + return str(error.response["Error"]["Code"]) + except (AttributeError, KeyError, TypeError): + return None + + +def _checksum(data: bytes) -> tuple[str, str]: + value = google_crc32c.value(data) + encoded = base64.b64encode(value.to_bytes(4, "big")).decode() + return f"crc32c:{value:08x}", encoded + + +class S3Client: + def __init__( + self, + endpoint_url: str | None = None, + region_name: str | None = None, + ) -> None: + import boto3 + from botocore.config import Config as BotoConfig + + self.client = boto3.client( + "s3", + endpoint_url=endpoint_url, + region_name=region_name, + config=BotoConfig( + max_pool_connections=max(1, envs.MX_REFIT_S3_MAX_POOL_CONNECTIONS) + ), + ) + + def put(self, bucket: str, key: str, data: bytes) -> S3Object: + checksum, encoded_checksum = _checksum(data) + try: + response = self.client.put_object( + Bucket=bucket, + Key=key, + Body=data, + ChecksumAlgorithm="CRC32C", + ChecksumCRC32C=encoded_checksum, + IfNoneMatch="*", + ) + version = response.get("VersionId") + except Exception as error: + if _error_code(error) not in { + "409", + "412", + "ConditionalRequestConflict", + "PreconditionFailed", + }: + raise + existing, version = self._read(bucket=bucket, key=key) + if existing != data: + raise ImmutableS3Conflict( + f"immutable S3 object conflict for {bucket}/{key}" + ) from error + + return S3Object( + bucket=bucket, + key=key, + checksum=checksum, + object_version=version, + ) + + def get(self, location: S3Object) -> bytes: + data, _version = self._read( + bucket=location.bucket, + key=location.key, + version=location.object_version, + ) + checksum, _encoded = _checksum(data) + if checksum != location.checksum: + raise ValueError(f"S3 checksum differs for {location.key}") + return data + + def _read( + self, + bucket: str, + key: str, + version: str | None = None, + ) -> tuple[bytes, str | None]: + request = {"Bucket": bucket, "Key": key} + if version is not None: + request["VersionId"] = version + response = self.client.get_object(**request) + body = response["Body"] + try: + return body.read(), response.get("VersionId") + finally: + close = getattr(body, "close", None) + if close is not None: + close() + + def close(self) -> None: + close = getattr(self.client, "close", None) + if close is not None: + close() diff --git a/modelexpress_client/python/modelexpress/refit/source/__init__.py b/modelexpress_client/python/modelexpress/refit/source/__init__.py new file mode 100644 index 000000000..c7edcba93 --- /dev/null +++ b/modelexpress_client/python/modelexpress/refit/source/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Canonical delta encoding for HF buckets gathered by Miles.""" diff --git a/modelexpress_client/python/modelexpress/refit/source/canonical.py b/modelexpress_client/python/modelexpress/refit/source/canonical.py new file mode 100644 index 000000000..7610b465e --- /dev/null +++ b/modelexpress_client/python/modelexpress/refit/source/canonical.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Load one canonical HF snapshot and compute its logical identity.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import numpy as np +import torch + + +def canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + +def _digest(value: object) -> str: + return f"sha256:{hashlib.sha256(canonical_json(value)).hexdigest()}" + + +def _tensor_bytes(tensor: torch.Tensor) -> bytes: + return tensor.detach().cpu().contiguous().view(torch.uint8).numpy().tobytes() + + +def _checkpoint_files(root: Path) -> list[Path]: + if root.is_file(): + return [root] + index = root / "model.safetensors.index.json" + if index.exists(): + names = json.loads(index.read_text())["weight_map"].values() + return [root / name for name in sorted(set(names))] + return sorted(root.glob("*.safetensors")) + + +def _tied_names(root: Path) -> set[str]: + """Names HF stores as a redundant copy of another parameter. + + With ``tie_word_embeddings`` the checkpoint serializes the output head as a second + copy of the input embedding, but the trainer holds one parameter for both and so + never gathers the copy. Counting it would put an unreachable tensor in the canonical + set, so both the publisher and the receiver drop it. + """ + config = root / "config.json" + if not config.is_file(): + return set() + try: + with config.open() as handle: + tied = json.load(handle).get("tie_word_embeddings", False) + except (OSError, ValueError): + return set() + return {"lm_head.weight"} if tied else set() + + +def load_hf_snapshot( + checkpoint: str | Path, +) -> tuple[dict[str, np.ndarray], dict[str, dict], str, str]: + """Load the launch HF checkpoint as the byte snapshot used by Miles delta sync.""" + from safetensors import safe_open + + snapshot = {} + metadata = {} + tied = _tied_names(Path(checkpoint)) + for path in _checkpoint_files(Path(checkpoint)): + with safe_open(str(path), framework="pt", device="cpu") as handle: + for source_name in handle.keys(): + name = source_name.removeprefix("module.") + if name in tied: + continue + tensor = handle.get_tensor(source_name) + data = _tensor_bytes(tensor) + snapshot[name] = np.frombuffer(data, dtype=np.uint8).copy() + metadata[name] = { + "name": name, + "shape": list(tensor.shape), + "dtype": str(tensor.dtype).removeprefix("torch."), + "byte_size": len(data), + "target_digest": f"sha256:{hashlib.sha256(data).hexdigest()}", + } + + return snapshot, metadata, format_digest(metadata), snapshot_digest(metadata) + + +def format_digest(metadata: dict[str, dict]) -> str: + return _digest( + [ + { + "name": metadata[name]["name"], + "shape": metadata[name]["shape"], + "dtype": metadata[name]["dtype"], + "byte_size": metadata[name]["byte_size"], + } + for name in sorted(metadata) + ] + ) + + +def snapshot_digest(metadata: dict[str, dict]) -> str: + return _digest([metadata[name] for name in sorted(metadata)]) diff --git a/modelexpress_client/python/pyproject.toml b/modelexpress_client/python/pyproject.toml index 2266e1894..4408673d6 100644 --- a/modelexpress_client/python/pyproject.toml +++ b/modelexpress_client/python/pyproject.toml @@ -33,6 +33,9 @@ dependencies = [ "torch>=2.6.0", "runai-model-streamer[s3,gcs,azure]; sys_platform == 'linux'", "instanttensor>=0.1.9 ; sys_platform == 'linux'", + "zstandard>=0.22.0", + "boto3>=1.35.2", + "safetensors>=0.4.0", ] [project.optional-dependencies] diff --git a/modelexpress_client/python/tests/test_envs.py b/modelexpress_client/python/tests/test_envs.py index b87a8370a..176b024ff 100644 --- a/modelexpress_client/python/tests/test_envs.py +++ b/modelexpress_client/python/tests/test_envs.py @@ -25,6 +25,8 @@ def test_defaults_when_unset(monkeypatch): "MX_GDS_TIMEOUT", "MX_HEARTBEAT_INTERVAL_SECS", "MX_RESHARD_FUSED_WIRE", + "MX_REFIT_S3_UPLOAD_WORKERS", + "MX_REFIT_S3_MAX_POOL_CONNECTIONS", ): monkeypatch.delenv(name, raising=False) @@ -43,15 +45,21 @@ def test_defaults_when_unset(monkeypatch): assert envs.MX_GDS_TIMEOUT == pytest.approx(120.0) assert envs.MX_HEARTBEAT_INTERVAL_SECS == 30 assert envs.MX_RESHARD_FUSED_WIRE is True + assert envs.MX_REFIT_S3_UPLOAD_WORKERS == 4 + assert envs.MX_REFIT_S3_MAX_POOL_CONNECTIONS == 10 def test_int_and_float_parsing(monkeypatch): monkeypatch.setenv("MX_METADATA_PORT", "1234") monkeypatch.setenv("MX_GDS_TIMEOUT", "1.5") monkeypatch.setenv("MX_SOURCE_QUERY_TIMEOUT", "42") + monkeypatch.setenv("MX_REFIT_S3_UPLOAD_WORKERS", "16") + monkeypatch.setenv("MX_REFIT_S3_MAX_POOL_CONNECTIONS", "16") assert envs.MX_METADATA_PORT == 1234 assert envs.MX_GDS_TIMEOUT == pytest.approx(1.5) assert envs.MX_SOURCE_QUERY_TIMEOUT == 42 + assert envs.MX_REFIT_S3_UPLOAD_WORKERS == 16 + assert envs.MX_REFIT_S3_MAX_POOL_CONNECTIONS == 16 def test_invalid_int_falls_back_to_default(monkeypatch): diff --git a/modelexpress_client/python/tests/test_refit_api.py b/modelexpress_client/python/tests/test_refit_api.py new file mode 100644 index 000000000..84d87a38d --- /dev/null +++ b/modelexpress_client/python/tests/test_refit_api.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import dataclasses + +from modelexpress import revision_pb2 +from modelexpress.refit import ( + PublisherConfig, + ReceiverRevisionState, + ReceiverStatus, + S3Config, +) + +MODEL_ID = "Qwen/Qwen3-30B-A3B" + + +def test_public_config_contains_only_consumed_s3_and_publisher_values(): + assert [field.name for field in dataclasses.fields(S3Config)] == [ + "bucket", + "prefix", + "endpoint_url", + "region_name", + ] + assert [field.name for field in dataclasses.fields(PublisherConfig)] == [ + "model_id", + "catalog_endpoint", + "s3", + ] + + +def test_unconsumed_lifecycle_api_is_not_public(): + import modelexpress.refit as refit + + for name in ( + "PreparedUpdate", + "PublicationMode", + "PublishResult", + "PublisherProtocol", + "PublisherStatus", + "ReceiverConfig", + "ReceiverProtocol", + "normalize_layer_scope", + ): + assert not hasattr(refit, name) + assert "PublicationMode" not in revision_pb2.DESCRIPTOR.enum_types_by_name + + +def test_receiver_outcomes_consumed_by_sglang_remain_local(): + assert [state.value for state in ReceiverRevisionState] == [ + "bytes_received", + "verified", + "failed", + "poisoned", + ] + assert "ReceiverRevisionState" not in revision_pb2.DESCRIPTOR.enum_types_by_name + + def status(state): + return ReceiverStatus( + receiver_id="rollout-tp0", + model_id=MODEL_ID, + installed_version="1", + state=state, + ) + + assert status(ReceiverRevisionState.POISONED).recovery_required is True + assert status(ReceiverRevisionState.VERIFIED).recovery_required is False + assert status(None).recovery_required is False + + +def test_generic_transport_and_adapter_frameworks_are_not_public(): + import modelexpress.refit as refit + + for name in ( + "CompressionAlgorithm", + "DeltaCodec", + "DeltaTransferMethod", + "EngineAdapter", + "LoadResult", + "RecoveryStore", + "RecoveryStoreConfig", + "SourceAdapter", + "TransportAdapter", + "TransportConfig", + "TransportKind", + ): + assert not hasattr(refit, name) diff --git a/modelexpress_client/python/tests/test_refit_publisher.py b/modelexpress_client/python/tests/test_refit_publisher.py new file mode 100644 index 000000000..505c8fb57 --- /dev/null +++ b/modelexpress_client/python/tests/test_refit_publisher.py @@ -0,0 +1,346 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import io +import inspect +import json +import threading + +import pytest +import torch +from safetensors.torch import save_file + +from modelexpress.refit import ( + Publisher, + PublisherConfig, + RevisionRecord, + RevisionState, + S3Config, +) +from modelexpress.refit import publisher as publisher_module +from modelexpress.refit.delta import decode_bucket +from modelexpress.refit.source.canonical import load_hf_snapshot + + +class FakeCatalog: + def __init__(self): + self.records = {} + self.published = [] + self.gets = {} + + def publish_revision(self, manifest): + self.published.append(manifest) + record = RevisionRecord(manifest, RevisionState.READY) + self.records[(manifest.model_id, manifest.target_version)] = record + return record + + def get_revision(self, model_id, version): + key = (model_id, version) + self.gets[key] = self.gets.get(key, 0) + 1 + record = self.records[key] + if self.gets[key] >= 1: + record = RevisionRecord(record.manifest, RevisionState.COMMITTED) + self.records[key] = record + return record + + def close(self): + pass + + +class FakeS3: + def __init__(self): + self.objects = {} + self.puts = [] + + def put_object(self, **kwargs): + self.puts.append(kwargs) + key = (kwargs["Bucket"], kwargs["Key"]) + self.objects[key] = ( + bytes(kwargs["Body"]), + kwargs["ChecksumCRC32C"], + f"version-{len(self.puts)}", + ) + return {"VersionId": self.objects[key][2]} + + def head_object(self, **kwargs): + data, checksum, version = self.objects[(kwargs["Bucket"], kwargs["Key"])] + return { + "ContentLength": len(data), + "ChecksumCRC32C": checksum, + "VersionId": version, + } + + def get_object(self, **kwargs): + data, _checksum, version = self.objects[(kwargs["Bucket"], kwargs["Key"])] + return {"Body": io.BytesIO(data), "VersionId": version} + + +def checkpoint(tmp_path): + path = tmp_path / "hf" + path.mkdir() + tensors = { + "model.a.weight": torch.arange(4, dtype=torch.float32).reshape(2, 2), + "model.b.weight": torch.ones((2, 2), dtype=torch.float32), + } + save_file(tensors, path / "model.safetensors") + return path, tensors + + +def configure_services(monkeypatch, catalog, s3): + monkeypatch.setattr( + publisher_module, "GrpcRevisionCatalog", lambda _endpoint: catalog + ) + monkeypatch.setattr("boto3.client", lambda *_args, **_kwargs: s3) + + +def make_publisher(tmp_path, monkeypatch, catalog, s3): + configure_services(monkeypatch, catalog, s3) + hf_path, _weights = checkpoint(tmp_path) + publisher = Publisher( + launch_checkpoint=hf_path, + bucket_bytes=64, + ) + publisher.initialize(PublisherConfig("model", "mx:8001", S3Config("bucket", "run"))) + snapshot, _metadata, _format, _digest = load_hf_snapshot(hf_path) + publisher.capture_baseline(gather(_weights), lambda name: snapshot[name]) + return publisher + + +def gather(weights): + def run(encode_bucket): + encode_bucket(list(weights.items()), None) + + return run + + +def test_initialize_constructs_catalog_and_s3_services(tmp_path, monkeypatch): + hf_path, _weights = checkpoint(tmp_path) + catalog = FakeCatalog() + s3 = object() + created = [] + + monkeypatch.setattr( + publisher_module, + "GrpcRevisionCatalog", + lambda endpoint: created.append(("catalog", endpoint)) or catalog, + ) + monkeypatch.setattr( + publisher_module, + "S3Client", + lambda **kwargs: created.append(("s3", kwargs)) or s3, + raising=False, + ) + + parameters = inspect.signature(Publisher).parameters + assert "catalog" not in parameters + assert "s3_client" not in parameters + assert "sleep" not in parameters + assert "poll_interval_seconds" not in parameters + + publisher = Publisher(launch_checkpoint=hf_path) + publisher.initialize( + PublisherConfig( + "model", + "mx:8001", + S3Config( + "bucket", + endpoint_url="https://s3.example", + region_name="us-west-2", + ), + ) + ) + + assert publisher.catalog is catalog + assert publisher.s3 is s3 + assert created == [ + ("catalog", "mx:8001"), + ( + "s3", + {"endpoint_url": "https://s3.example", "region_name": "us-west-2"}, + ), + ] + + +def test_publisher_uses_direct_capture_and_transport_calls(): + assert hasattr(Publisher, "_capture_deltas") + assert hasattr(Publisher, "_collect_metrics") + assert not hasattr(Publisher, "_encode_delta") + assert not hasattr(Publisher, "_agree_error") + assert not hasattr(Publisher, "_put") + assert not hasattr(Publisher, "_barrier") + + +def test_launch_zero_publishes_metadata_without_weights(tmp_path, monkeypatch): + catalog = FakeCatalog() + s3 = FakeS3() + publisher = make_publisher(tmp_path, monkeypatch, catalog, s3) + + publisher.publish_version("0") + + manifest = catalog.published[-1] + assert manifest.target_version == "0" + assert manifest.base_version is None + assert manifest.payload is None + assert s3.puts == [] + + +def test_exact_base_update_uses_miles_hf_buckets_and_uploads_delta( + tmp_path, monkeypatch +): + workers = [] + executor = publisher_module.ThreadPoolExecutor + + def thread_pool(*args, **kwargs): + workers.append(kwargs["max_workers"]) + return executor(*args, **kwargs) + + monkeypatch.setenv("MX_REFIT_S3_UPLOAD_WORKERS", "1") + monkeypatch.setattr(publisher_module, "ThreadPoolExecutor", thread_pool) + catalog = FakeCatalog() + s3 = FakeS3() + configure_services(monkeypatch, catalog, s3) + hf_path, launch = checkpoint(tmp_path) + target = {name: tensor.clone() for name, tensor in launch.items()} + target["model.b.weight"] += 7 + publisher = Publisher( + launch_checkpoint=hf_path, + bucket_bytes=64, + ) + publisher.initialize(PublisherConfig("model", "mx:8001", S3Config("bucket", "run"))) + snapshot, _metadata, _format, _digest = load_hf_snapshot(hf_path) + publisher.capture_baseline(gather(launch), lambda name: snapshot[name]) + publisher.publish_version("0") + publisher.wait_for_commit("0") + catalog.gets.clear() + + publisher.publish_version("1", base_version="0", gather_hf_buckets=gather(target)) + assert catalog.gets == {} + publisher.wait_for_commit("1") + + metrics = publisher.pop_metrics() + assert 1 in workers + old_bytes = launch["model.b.weight"].contiguous().view(torch.uint8).numpy() + new_bytes = target["model.b.weight"].contiguous().view(torch.uint8).numpy() + changed = int(torch.from_numpy(old_bytes != new_bytes).sum()) + total = sum(tensor.numel() * tensor.element_size() for tensor in target.values()) + assert metrics["perf/update_weights_density"] == changed / total + assert metrics["perf/update_weights_wire_bytes"] > 0 + assert metrics["perf/mx_encode_delta"] >= 0 + assert metrics["perf/mx_publish_setup"] >= 0 + assert metrics["perf/mx_publish_pool"] >= 0 + assert metrics["perf/mx_publish_finalize"] >= 0 + assert metrics["perf/mx_publish_time"] == pytest.approx( + metrics["perf/mx_publish_setup"] + + metrics["perf/mx_publish_pool"] + + metrics["perf/mx_publish_finalize"], + abs=1e-4, + ) + assert publisher.pop_metrics() == {} + assert publisher.current_version == "1" + manifest = catalog.published[-1] + assert manifest.payload.key.endswith("/delta-index.json") + index = json.loads(s3.objects[(manifest.payload.bucket, manifest.payload.key)][0]) + assert index["base_version"] == "0" + assert index["target_version"] == "1" + [bucket] = index["buckets"] + encoded = s3.objects[(bucket["object"]["bucket"], bucket["object"]["key"])][0] + snapshot, metadata, _format_digest, _base_digest = load_hf_snapshot(hf_path) + decode_bucket(encoded, snapshot, metadata) + for name, tensor in target.items(): + assert ( + snapshot[name].tobytes() + == tensor.contiguous().view(torch.uint8).numpy().tobytes() + ) + + +def test_wrong_base_is_rejected_before_gather(tmp_path, monkeypatch): + publisher = make_publisher(tmp_path, monkeypatch, FakeCatalog(), FakeS3()) + publisher.publish_version("0") + publisher.wait_for_commit("0") + called = False + + def should_not_run(_encode): + nonlocal called + called = True + + with pytest.raises(RuntimeError, match="base"): + publisher.publish_version( + "2", base_version="1", gather_hf_buckets=should_not_run + ) + + assert not called + + +def test_s3_failure_prevents_catalog_publication(tmp_path, monkeypatch): + class FailingS3(FakeS3): + fail = False + + def put_object(self, **kwargs): + if self.fail: + raise RuntimeError("upload failed") + return super().put_object(**kwargs) + + catalog = FakeCatalog() + s3 = FailingS3() + configure_services(monkeypatch, catalog, s3) + hf_path, launch = checkpoint(tmp_path) + publisher = Publisher( + launch_checkpoint=hf_path, + bucket_bytes=64, + ) + publisher.initialize(PublisherConfig("model", "mx:8001", S3Config("bucket", "run"))) + snapshot, _metadata, _format, _digest = load_hf_snapshot(hf_path) + publisher.capture_baseline(gather(launch), lambda name: snapshot[name]) + publisher.publish_version("0") + publisher.wait_for_commit("0") + s3.fail = True + + with pytest.raises(Exception, match="upload failed"): + publisher.publish_version( + "1", + base_version="0", + gather_hf_buckets=gather( + {name: tensor + 1 for name, tensor in launch.items()} + ), + ) + + assert [manifest.target_version for manifest in catalog.published] == ["0"] + assert not hasattr(publisher, "poisoned") + + +def test_bucket_uploads_run_in_parallel(tmp_path, monkeypatch): + class ConcurrentS3(FakeS3): + def __init__(self): + super().__init__() + self.barrier = threading.Barrier(2) + self.threads = set() + + def put_object(self, **kwargs): + if kwargs["Key"].endswith(".mxcd"): + self.threads.add(threading.get_ident()) + self.barrier.wait(timeout=5) + return super().put_object(**kwargs) + + catalog = FakeCatalog() + s3 = ConcurrentS3() + configure_services(monkeypatch, catalog, s3) + hf_path, launch = checkpoint(tmp_path) + publisher = Publisher( + launch_checkpoint=hf_path, + bucket_bytes=8, + ) + publisher.initialize(PublisherConfig("model", "mx:8001", S3Config("bucket", "run"))) + snapshot, _metadata, _format, _digest = load_hf_snapshot(hf_path) + publisher.capture_baseline(gather(launch), lambda name: snapshot[name]) + publisher.publish_version("0") + publisher.wait_for_commit("0") + + publisher.publish_version( + "1", + base_version="0", + gather_hf_buckets=gather({name: tensor + 1 for name, tensor in launch.items()}), + ) + + assert len(s3.threads) > 1 diff --git a/modelexpress_client/python/tests/test_refit_publisher_distributed.py b/modelexpress_client/python/tests/test_refit_publisher_distributed.py new file mode 100644 index 000000000..c3743254c --- /dev/null +++ b/modelexpress_client/python/tests/test_refit_publisher_distributed.py @@ -0,0 +1,273 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import io +import json +import multiprocessing +from datetime import timedelta + +import boto3 +import numpy as np +import pytest +import torch +from safetensors.torch import save_file + +from modelexpress.refit import ( + Publisher, + PublisherConfig, + RevisionRecord, + RevisionState, + S3Config, +) +from modelexpress.refit import publisher as publisher_module +from modelexpress.refit.source.canonical import load_hf_snapshot + + +class Catalog: + def __init__(self): + self.records = {} + self.published = [] + + def publish_revision(self, manifest): + self.published.append(manifest.target_version) + record = RevisionRecord(manifest, RevisionState.READY) + self.records[(manifest.model_id, manifest.target_version)] = record + return record + + def get_revision(self, model_id, version): + record = self.records[(model_id, version)] + record = RevisionRecord(record.manifest, RevisionState.COMMITTED) + self.records[(model_id, version)] = record + return record + + def close(self): + pass + + +class S3: + def __init__(self): + self.objects = {} + self.puts = 0 + + def put_object(self, **kwargs): + self.puts += 1 + key = (kwargs["Bucket"], kwargs["Key"]) + self.objects[key] = (bytes(kwargs["Body"]), kwargs["ChecksumCRC32C"]) + return {"VersionId": f"version-{self.puts}"} + + def get_object(self, **kwargs): + data, _checksum = self.objects[(kwargs["Bucket"], kwargs["Key"])] + return {"Body": io.BytesIO(data), "VersionId": f"version-{self.puts}"} + + +def _contains_model_bytes(value): + if isinstance(value, (bytes, bytearray, memoryview, np.ndarray, torch.Tensor)): + return True + if isinstance(value, dict): + return any( + _contains_model_bytes(key) or _contains_model_bytes(item) + for key, item in value.items() + ) + if isinstance(value, (list, tuple)): + return any(_contains_model_bytes(item) for item in value) + return False + + +def _run_with_non_source_rank(rank, world_size, init_file, checkpoint, queue): + torch.distributed.init_process_group( + "gloo", + init_method=f"file://{init_file}", + rank=rank, + world_size=world_size, + timeout=timedelta(seconds=30), + ) + catalog = Catalog() if rank == 0 else None + s3 = S3() + publisher_module.GrpcRevisionCatalog = lambda _endpoint: catalog + boto3.client = lambda *_args, **_kwargs: s3 + publisher = Publisher( + launch_checkpoint=checkpoint, + bucket_bytes=16, + ) + publisher.initialize(PublisherConfig("model", "mx:8001", S3Config("bucket"))) + publisher.publish_version("0") + publisher.wait_for_commit("0") + + launch, _metadata, _format, _digest = load_hf_snapshot(checkpoint) + # Only rank 0 is a source rank, as with Miles at TP>1 or DP>1: every other rank + # joins the collectives but is handed no tensors at all. + local_names = sorted(launch) if rank == 0 else () + baseline = { + name: torch.from_numpy(launch[name].copy()).view(torch.float32).reshape(2, 2) + for name in local_names + } + + def gather(weights): + def run(consume): + consume(list(weights.items()), None) + + return run + + publisher.capture_baseline(gather(baseline), lambda name: launch[name]) + target = {name: tensor + 1 for name, tensor in baseline.items()} + publisher.publish_version("1", base_version="0", gather_hf_buckets=gather(target)) + publisher.wait_for_commit("1") + queue.put((rank, s3.puts, publisher.current_version)) + torch.distributed.destroy_process_group() + + +@pytest.mark.skipif( + not torch.distributed.is_available(), reason="torch.distributed is unavailable" +) +def test_non_source_ranks_publish_without_holding_any_tensor(tmp_path): + checkpoint = tmp_path / "hf" + checkpoint.mkdir() + save_file( + { + "model.a.weight": torch.arange(4, dtype=torch.float32).reshape(2, 2), + "model.b.weight": torch.ones((2, 2), dtype=torch.float32) * 2, + }, + checkpoint / "model.safetensors", + ) + context = multiprocessing.get_context("spawn") + queue = context.Queue() + init_file = tmp_path / "gloo" + processes = [ + context.Process( + target=_run_with_non_source_rank, + args=(rank, 2, str(init_file), str(checkpoint), queue), + ) + for rank in range(2) + ] + for process in processes: + process.start() + for process in processes: + process.join(60) + assert process.exitcode == 0 + + results = { + rank: (puts, version) + for rank, puts, version in [queue.get() for _ in processes] + } + # The non-source rank uploads nothing yet still advances to the committed revision. + assert results[1][0] == 0 + assert results[0][1] == "1" + assert results[1][1] == "1" + + +def _run(rank, world_size, init_file, checkpoint, queue): + torch.distributed.init_process_group( + "gloo", + init_method=f"file://{init_file}", + rank=rank, + world_size=world_size, + timeout=timedelta(seconds=30), + ) + gather_object = torch.distributed.all_gather_object + + def gather_metadata(output, value, group=None): + if _contains_model_bytes(value): + raise AssertionError("model bytes must not cross all_gather_object") + return gather_object(output, value, group=group) + + torch.distributed.all_gather_object = gather_metadata + catalog = Catalog() if rank == 0 else None + s3 = S3() + publisher_module.GrpcRevisionCatalog = lambda _endpoint: catalog + boto3.client = lambda *_args, **_kwargs: s3 + publisher = Publisher( + launch_checkpoint=checkpoint, + bucket_bytes=16, + ) + publisher.initialize(PublisherConfig("model", "mx:8001", S3Config("bucket"))) + publisher.publish_version("0") + publisher.wait_for_commit("0") + + launch, _metadata, _format, _digest = load_hf_snapshot(checkpoint) + local_names = ( + ("duplicate", "model.a.weight") + if rank == 0 + else ("duplicate", "model.b.weight") + ) + baseline = { + name: torch.from_numpy(launch[name].copy()).view(torch.float32).reshape(2, 2) + for name in local_names + } + + def gather(weights): + def run(consume): + consume(list(weights.items()), None) + + return run + + publisher.capture_baseline(gather(baseline), lambda name: launch[name]) + target = {name: tensor + 1 for name, tensor in baseline.items()} + publisher.publish_version("1", base_version="0", gather_hf_buckets=gather(target)) + publisher.wait_for_commit("1") + + index = None + if rank == 0: + manifest = catalog.records[("model", "1")].manifest + index = json.loads( + s3.objects[(manifest.payload.bucket, manifest.payload.key)][0] + ) + queue.put((rank, s3.puts, index, publisher.pop_metrics())) + torch.distributed.destroy_process_group() + + +@pytest.mark.skipif( + not torch.distributed.is_available(), reason="torch.distributed is unavailable" +) +def test_two_source_ranks_upload_disjoint_s3_buckets_and_one_delta_index(tmp_path): + checkpoint = tmp_path / "hf" + checkpoint.mkdir() + save_file( + { + "duplicate": torch.ones((2, 2), dtype=torch.float32), + "model.a.weight": torch.arange(4, dtype=torch.float32).reshape(2, 2), + "model.b.weight": torch.ones((2, 2), dtype=torch.float32) * 2, + }, + checkpoint / "model.safetensors", + ) + context = multiprocessing.get_context("spawn") + queue = context.Queue() + init_file = tmp_path / "gloo" + processes = [ + context.Process( + target=_run, + args=(rank, 2, str(init_file), str(checkpoint), queue), + ) + for rank in range(2) + ] + for process in processes: + process.start() + for process in processes: + process.join(60) + assert process.exitcode == 0 + + results = { + rank: (puts, index, metrics) + for rank, puts, index, metrics in [queue.get() for _ in processes] + } + assert results[0][0] == 3 + assert results[1][0] == 1 + assert results[0][2]["perf/update_weights_density"] == 0.375 + assert results[0][2]["perf/update_weights_wire_bytes"] > 0 + assert ( + results[0][2]["perf/update_weights_density"] + == results[1][2]["perf/update_weights_density"] + ) + assert ( + results[0][2]["perf/update_weights_wire_bytes"] + == results[1][2]["perf/update_weights_wire_bytes"] + ) + index = results[0][1] + assert [bucket["ordinal"] for bucket in index["buckets"]] == [0, 1, 2] + assert [tensor["name"] for tensor in index["tensors"]] == [ + "duplicate", + "model.a.weight", + "model.b.weight", + ] + assert sum("duplicate" in bucket["tensors"] for bucket in index["buckets"]) == 1 diff --git a/modelexpress_client/python/tests/test_refit_s3_payload.py b/modelexpress_client/python/tests/test_refit_s3_payload.py new file mode 100644 index 000000000..11172492a --- /dev/null +++ b/modelexpress_client/python/tests/test_refit_s3_payload.py @@ -0,0 +1,269 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import base64 +import hashlib +import io +import json + +import numpy as np +import pytest +import torch +from safetensors.torch import save_file + +from modelexpress.refit.manifest import S3Object +from modelexpress.refit import delta +from modelexpress.refit.delta import decode_bucket, encode_bucket +from modelexpress.refit.s3 import ImmutableS3Conflict, S3Client +from modelexpress.refit.source.canonical import load_hf_snapshot + + +class FakeS3: + def __init__(self): + self.objects = {} + self.puts = [] + self.head_calls = 0 + self.get_calls = 0 + + def put_object(self, **kwargs): + self.puts.append(kwargs) + identity = (kwargs["Bucket"], kwargs["Key"]) + if identity in self.objects: + error = RuntimeError("precondition failed") + error.response = {"Error": {"Code": "PreconditionFailed"}} + raise error + data = bytes(kwargs["Body"]) + self.objects[identity] = (data, kwargs["ChecksumCRC32C"], "version-1") + return {"VersionId": "version-1"} + + def head_object(self, **kwargs): + self.head_calls += 1 + data, checksum, version = self.objects[(kwargs["Bucket"], kwargs["Key"])] + return { + "ContentLength": len(data), + "ChecksumCRC32C": checksum, + "VersionId": version, + } + + def get_object(self, **kwargs): + self.get_calls += 1 + data, _checksum, version = self.objects[(kwargs["Bucket"], kwargs["Key"])] + return {"Body": io.BytesIO(data), "VersionId": version} + + +def test_s3_client_puts_immutable_object_and_gets_verified_bytes(monkeypatch): + raw_client = FakeS3() + monkeypatch.setattr("boto3.client", lambda *_args, **_kwargs: raw_client) + client = S3Client() + + stored = client.put( + bucket="bucket", + key="run/models/m/revisions/1/delta-index.json", + data=b"index", + ) + + assert stored.bucket == "bucket" + assert stored.key == "run/models/m/revisions/1/delta-index.json" + assert stored.object_version == "version-1" + assert stored.checksum.startswith("crc32c:") + request = raw_client.puts[0] + assert request["IfNoneMatch"] == "*" + assert request["ChecksumAlgorithm"] == "CRC32C" + assert base64.b64decode(request["ChecksumCRC32C"]) + assert raw_client.head_calls == 0 + assert raw_client.get_calls == 0 + assert client.get(stored) == b"index" + assert raw_client.get_calls == 1 + + +def test_s3_client_configures_http_pool_from_env(monkeypatch): + import boto3 + + created = {} + + def client(_service, **kwargs): + created.update(kwargs) + return FakeS3() + + monkeypatch.setenv("MX_REFIT_S3_MAX_POOL_CONNECTIONS", "16") + monkeypatch.setattr(boto3, "client", client) + + S3Client(endpoint_url="https://s3.example", region_name="us-west-2") + + assert created["endpoint_url"] == "https://s3.example" + assert created["region_name"] == "us-west-2" + assert created["config"].max_pool_connections == 16 + + +def test_s3_client_allows_identical_retry_but_rejects_immutable_conflict( + monkeypatch, +): + raw_client = FakeS3() + monkeypatch.setattr("boto3.client", lambda *_args, **_kwargs: raw_client) + client = S3Client() + + first = client.put(bucket="bucket", key="delta-index.json", data=b"same") + assert client.put(bucket="bucket", key="delta-index.json", data=b"same") == first + with pytest.raises(ImmutableS3Conflict): + client.put(bucket="bucket", key="delta-index.json", data=b"different") + + +def test_s3_client_rejects_download_checksum_mismatch(monkeypatch): + raw_client = FakeS3() + raw_client.objects[("bucket", "delta-index.json")] = ( + b"wrong", + "unused", + "version-1", + ) + monkeypatch.setattr("boto3.client", lambda *_args, **_kwargs: raw_client) + client = S3Client() + + with pytest.raises(ValueError, match="checksum"): + client.get( + S3Object( + bucket="bucket", + key="delta-index.json", + checksum="crc32c:00000000", + object_version="version-1", + ) + ) + + +def test_launch_schema_orders_names_after_canonical_prefix_normalization(tmp_path): + checkpoint = tmp_path / "model.safetensors" + save_file( + { + "a.weight": torch.tensor([1.0]), + "module.0.weight": torch.tensor([2.0]), + }, + checkpoint, + ) + + snapshot, metadata, _format_digest, _target_digest = load_hf_snapshot(checkpoint) + + assert sorted(snapshot) == ["0.weight", "a.weight"] + assert sorted(metadata) == ["0.weight", "a.weight"] + + +@pytest.mark.parametrize( + ("tied", "expected"), + [ + (True, ["model.embed_tokens.weight"]), + (False, ["lm_head.weight", "model.embed_tokens.weight"]), + ], +) +def test_tied_output_head_is_dropped_from_the_canonical_set(tmp_path, tied, expected): + # HF serializes a tied output head as a second copy of the input embedding, but the + # trainer holds one parameter for both and never gathers the copy, so counting it + # would leave an unreachable tensor in the canonical set. + embedding = torch.arange(4, dtype=torch.float32) + save_file( + {"lm_head.weight": embedding.clone(), "model.embed_tokens.weight": embedding}, + tmp_path / "model.safetensors", + ) + (tmp_path / "config.json").write_text(json.dumps({"tie_word_embeddings": tied})) + + snapshot, metadata, _format_digest, _target_digest = load_hf_snapshot(tmp_path) + + assert sorted(snapshot) == expected + assert sorted(metadata) == expected + + +def test_pack_source_rank_raw_deltas_into_canonical_bucket(tmp_path): + checkpoint = tmp_path / "model.safetensors" + launch = torch.arange(4, dtype=torch.float32) + save_file({"model.weight": launch}, checkpoint) + snapshot, metadata, format_digest, base_digest = load_hf_snapshot(checkpoint) + target = launch + 1 + old = snapshot["model.weight"].tobytes() + new = target.contiguous().view(torch.uint8).numpy().tobytes() + delta = np.frombuffer( + bytes(left ^ right for left, right in zip(old, new, strict=True)), + dtype=np.uint8, + ) + metadata["model.weight"]["target_digest"] = ( + f"sha256:{hashlib.sha256(new).hexdigest()}" + ) + + encoded, decoded_size = encode_bucket( + model_id="model", + base_version="0", + target_version="1", + base_digest=base_digest, + format_digest=format_digest, + ordinal=3, + tensors=[("model.weight", delta)], + metadata=metadata, + ) + + restored, restored_metadata, _format, _digest = load_hf_snapshot(checkpoint) + header = decode_bucket(encoded, restored, restored_metadata) + assert header["ordinal"] == 3 + assert decoded_size == len(delta) + assert restored["model.weight"].tobytes() == new + + +def test_compute_delta_returns_xor_changed_bytes_and_target_digest(): + base = np.array([0, 1, 2, 3], dtype=np.uint8) + current = np.array([0, 4, 2, 8], dtype=np.uint8) + + raw_delta, target_digest, changed_bytes = delta.compute_delta(current, base) + + assert raw_delta.tolist() == [0, 5, 0, 11] + assert changed_bytes == 2 + assert target_digest == f"sha256:{hashlib.sha256(current).hexdigest()}" + + +def test_bucket_streams_numpy_deltas_without_decoded_copy(monkeypatch): + seen = [] + + class Compressor: + def compressobj(self): + return self + + def compress(self, data): + seen.append(data) + return b"frame" + + def flush(self): + return b"end" + + monkeypatch.setattr(delta.zstandard, "ZstdCompressor", lambda level: Compressor()) + first = np.arange(4, dtype=np.uint8) + second = np.arange(3, dtype=np.uint8) + metadata = { + "first": { + "name": "first", + "shape": [4], + "dtype": "uint8", + "byte_size": first.nbytes, + "target_digest": "sha256:first", + }, + "second": { + "name": "second", + "shape": [3], + "dtype": "uint8", + "byte_size": second.nbytes, + "target_digest": "sha256:second", + }, + } + + encoded, decoded_size = encode_bucket( + model_id="model", + base_version="0", + target_version="1", + base_digest="sha256:base", + format_digest="sha256:format", + ordinal=0, + tensors=[("first", first), ("second", second)], + metadata=metadata, + ) + + header, compressed = delta.bucket_parts(encoded) + assert all(isinstance(data, memoryview) for data in seen) + assert [data.obj for data in seen] == [first, second] + assert decoded_size == first.nbytes + second.nbytes + assert [entry["offset"] for entry in header["entries"]] == [0, first.nbytes] + assert bytes(compressed) == b"frameframeend" diff --git a/modelexpress_client/python/uv.lock b/modelexpress_client/python/uv.lock index 98d33ff35..cf869572a 100644 --- a/modelexpress_client/python/uv.lock +++ b/modelexpress_client/python/uv.lock @@ -389,7 +389,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -927,6 +927,7 @@ name = "modelexpress" version = "0.5.0" source = { editable = "." } dependencies = [ + { name = "boto3" }, { name = "google-crc32c" }, { name = "grpcio" }, { name = "huggingface-hub" }, @@ -936,7 +937,9 @@ dependencies = [ { name = "protobuf" }, { name = "pydantic" }, { name = "runai-model-streamer", extra = ["azure", "gcs", "s3"], marker = "sys_platform == 'linux'" }, + { name = "safetensors" }, { name = "torch" }, + { name = "zstandard" }, ] [package.optional-dependencies] @@ -961,6 +964,7 @@ vmm = [ [package.metadata] requires-dist = [ + { name = "boto3", specifier = ">=1.35.2" }, { name = "cuda-python", marker = "extra == 'vmm'", specifier = ">=12.0" }, { name = "google-crc32c", specifier = ">=1.5.0" }, { name = "grpcio", specifier = ">=1.66.2" }, @@ -977,7 +981,9 @@ requires-dist = [ { name = "pytest", marker = "extra == 'dev'", specifier = ">=7.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.21.0" }, { name = "runai-model-streamer", extras = ["s3", "gcs", "azure"], marker = "sys_platform == 'linux'" }, + { name = "safetensors", specifier = ">=0.4.0" }, { name = "torch", specifier = ">=2.6.0" }, + { name = "zstandard", specifier = ">=0.22.0" }, ] provides-extras = ["dev", "codegen", "otel", "metrics", "vmm"] @@ -1819,6 +1825,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/72/c6c32d2b657fa3dad1de340254e14390b1e334ce38268b7ad51abda3c8c2/s3transfer-0.17.0-py3-none-any.whl", hash = "sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20", size = 86811, upload-time = "2026-04-29T22:07:34.966Z" }, ] +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/a0/f718cda65b05407d228f97602cf60dca269c979867aa5beb25410de26cd3/safetensors-0.8.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c554f85858e05226d3c2828e32395e677434685d6d94594a41643361c5e837f0", size = 473568, upload-time = "2026-06-09T07:52:18.829Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b1/fa7c600e7dceae12e9606c7578cbc9ff1e1ed55844883ee5c92205e86226/safetensors-0.8.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:c80201d22cbf405b80647a60ada77bba06c8fba2da2743ba1e89cdcc39a81f25", size = 484562, upload-time = "2026-06-09T07:52:17.518Z" }, + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/73ccf82579412b4a71c4ca673f10b5f1f888d7cf5af7fe24f27d30307be4/safetensors-0.8.0-cp310-abi3-win32.whl", hash = "sha256:2ddf52eac562eda224f99acfa7889d02968c1fd59a5b011ae7d8137c37e9c02d", size = 342401, upload-time = "2026-06-09T07:52:28.895Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6d/3fba214c1e5e0f69991677ec3bc17023f0421776975e1de0c682dca475e2/safetensors-0.8.0-cp310-abi3-win_amd64.whl", hash = "sha256:096ec1a98435df7beb08853bb5aa9081a84f23d0adc67ed1a0a10550f608373f", size = 355540, upload-time = "2026-06-09T07:52:27.832Z" }, + { url = "https://files.pythonhosted.org/packages/8d/fc/7eedc3510d97878876e32774eebbeb61c43f148a96e915c84229a3e967aa/safetensors-0.8.0-cp310-abi3-win_arm64.whl", hash = "sha256:f7838e5135a406ad3e02efdcb8cf2e5397d368b0154537c4fec682dbc544d452", size = 340500, upload-time = "2026-06-09T07:52:26.745Z" }, +] + [[package]] name = "setuptools" version = "82.0.1" @@ -2064,3 +2094,93 @@ sdist = { url = "https://files.pythonhosted.org/packages/30/21/093488dfc7cc8964d wheels = [ { url = "https://files.pythonhosted.org/packages/08/8a/0861bec20485572fbddf3dfba2910e38fe249796cb73ecdeb74e07eeb8d3/zipp-3.23.1-py3-none-any.whl", hash = "sha256:0b3596c50a5c700c9cb40ba8d86d9f2cc4807e9bedb06bcdf7fac85633e444dc", size = 10378, upload-time = "2026-04-13T23:21:45.386Z" }, ] + +[[package]] +name = "zstandard" +version = "0.25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/7a/28efd1d371f1acd037ac64ed1c5e2b41514a6cc937dd6ab6a13ab9f0702f/zstandard-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd", size = 795256, upload-time = "2025-09-14T22:15:56.415Z" }, + { url = "https://files.pythonhosted.org/packages/96/34/ef34ef77f1ee38fc8e4f9775217a613b452916e633c4f1d98f31db52c4a5/zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7", size = 640565, upload-time = "2025-09-14T22:15:58.177Z" }, + { url = "https://files.pythonhosted.org/packages/9d/1b/4fdb2c12eb58f31f28c4d28e8dc36611dd7205df8452e63f52fb6261d13e/zstandard-0.25.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550", size = 5345306, upload-time = "2025-09-14T22:16:00.165Z" }, + { url = "https://files.pythonhosted.org/packages/73/28/a44bdece01bca027b079f0e00be3b6bd89a4df180071da59a3dd7381665b/zstandard-0.25.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d", size = 5055561, upload-time = "2025-09-14T22:16:02.22Z" }, + { url = "https://files.pythonhosted.org/packages/e9/74/68341185a4f32b274e0fc3410d5ad0750497e1acc20bd0f5b5f64ce17785/zstandard-0.25.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b", size = 5402214, upload-time = "2025-09-14T22:16:04.109Z" }, + { url = "https://files.pythonhosted.org/packages/8b/67/f92e64e748fd6aaffe01e2b75a083c0c4fd27abe1c8747fee4555fcee7dd/zstandard-0.25.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0", size = 5449703, upload-time = "2025-09-14T22:16:06.312Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e5/6d36f92a197c3c17729a2125e29c169f460538a7d939a27eaaa6dcfcba8e/zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0", size = 5556583, upload-time = "2025-09-14T22:16:08.457Z" }, + { url = "https://files.pythonhosted.org/packages/d7/83/41939e60d8d7ebfe2b747be022d0806953799140a702b90ffe214d557638/zstandard-0.25.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd", size = 5045332, upload-time = "2025-09-14T22:16:10.444Z" }, + { url = "https://files.pythonhosted.org/packages/b3/87/d3ee185e3d1aa0133399893697ae91f221fda79deb61adbe998a7235c43f/zstandard-0.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701", size = 5572283, upload-time = "2025-09-14T22:16:12.128Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1d/58635ae6104df96671076ac7d4ae7816838ce7debd94aecf83e30b7121b0/zstandard-0.25.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1", size = 4959754, upload-time = "2025-09-14T22:16:14.225Z" }, + { url = "https://files.pythonhosted.org/packages/75/d6/57e9cb0a9983e9a229dd8fd2e6e96593ef2aa82a3907188436f22b111ccd/zstandard-0.25.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150", size = 5266477, upload-time = "2025-09-14T22:16:16.343Z" }, + { url = "https://files.pythonhosted.org/packages/d1/a9/ee891e5edf33a6ebce0a028726f0bbd8567effe20fe3d5808c42323e8542/zstandard-0.25.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab", size = 5440914, upload-time = "2025-09-14T22:16:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/58/08/a8522c28c08031a9521f27abc6f78dbdee7312a7463dd2cfc658b813323b/zstandard-0.25.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e", size = 5819847, upload-time = "2025-09-14T22:16:20.559Z" }, + { url = "https://files.pythonhosted.org/packages/6f/11/4c91411805c3f7b6f31c60e78ce347ca48f6f16d552fc659af6ec3b73202/zstandard-0.25.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74", size = 5363131, upload-time = "2025-09-14T22:16:22.206Z" }, + { url = "https://files.pythonhosted.org/packages/ef/d6/8c4bd38a3b24c4c7676a7a3d8de85d6ee7a983602a734b9f9cdefb04a5d6/zstandard-0.25.0-cp310-cp310-win32.whl", hash = "sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa", size = 436469, upload-time = "2025-09-14T22:16:25.002Z" }, + { url = "https://files.pythonhosted.org/packages/93/90/96d50ad417a8ace5f841b3228e93d1bb13e6ad356737f42e2dde30d8bd68/zstandard-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e", size = 506100, upload-time = "2025-09-14T22:16:23.569Z" }, + { url = "https://files.pythonhosted.org/packages/2a/83/c3ca27c363d104980f1c9cee1101cc8ba724ac8c28a033ede6aab89585b1/zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c", size = 795254, upload-time = "2025-09-14T22:16:26.137Z" }, + { url = "https://files.pythonhosted.org/packages/ac/4d/e66465c5411a7cf4866aeadc7d108081d8ceba9bc7abe6b14aa21c671ec3/zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f", size = 640559, upload-time = "2025-09-14T22:16:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/12/56/354fe655905f290d3b147b33fe946b0f27e791e4b50a5f004c802cb3eb7b/zstandard-0.25.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431", size = 5348020, upload-time = "2025-09-14T22:16:29.523Z" }, + { url = "https://files.pythonhosted.org/packages/3b/13/2b7ed68bd85e69a2069bcc72141d378f22cae5a0f3b353a2c8f50ef30c1b/zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a", size = 5058126, upload-time = "2025-09-14T22:16:31.811Z" }, + { url = "https://files.pythonhosted.org/packages/c9/dd/fdaf0674f4b10d92cb120ccff58bbb6626bf8368f00ebfd2a41ba4a0dc99/zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc", size = 5405390, upload-time = "2025-09-14T22:16:33.486Z" }, + { url = "https://files.pythonhosted.org/packages/0f/67/354d1555575bc2490435f90d67ca4dd65238ff2f119f30f72d5cde09c2ad/zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6", size = 5452914, upload-time = "2025-09-14T22:16:35.277Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1f/e9cfd801a3f9190bf3e759c422bbfd2247db9d7f3d54a56ecde70137791a/zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072", size = 5559635, upload-time = "2025-09-14T22:16:37.141Z" }, + { url = "https://files.pythonhosted.org/packages/21/88/5ba550f797ca953a52d708c8e4f380959e7e3280af029e38fbf47b55916e/zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277", size = 5048277, upload-time = "2025-09-14T22:16:38.807Z" }, + { url = "https://files.pythonhosted.org/packages/46/c0/ca3e533b4fa03112facbe7fbe7779cb1ebec215688e5df576fe5429172e0/zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313", size = 5574377, upload-time = "2025-09-14T22:16:40.523Z" }, + { url = "https://files.pythonhosted.org/packages/12/9b/3fb626390113f272abd0799fd677ea33d5fc3ec185e62e6be534493c4b60/zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097", size = 4961493, upload-time = "2025-09-14T22:16:43.3Z" }, + { url = "https://files.pythonhosted.org/packages/cb/d3/23094a6b6a4b1343b27ae68249daa17ae0651fcfec9ed4de09d14b940285/zstandard-0.25.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778", size = 5269018, upload-time = "2025-09-14T22:16:45.292Z" }, + { url = "https://files.pythonhosted.org/packages/8c/a7/bb5a0c1c0f3f4b5e9d5b55198e39de91e04ba7c205cc46fcb0f95f0383c1/zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065", size = 5443672, upload-time = "2025-09-14T22:16:47.076Z" }, + { url = "https://files.pythonhosted.org/packages/27/22/503347aa08d073993f25109c36c8d9f029c7d5949198050962cb568dfa5e/zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa", size = 5822753, upload-time = "2025-09-14T22:16:49.316Z" }, + { url = "https://files.pythonhosted.org/packages/e2/be/94267dc6ee64f0f8ba2b2ae7c7a2df934a816baaa7291db9e1aa77394c3c/zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7", size = 5366047, upload-time = "2025-09-14T22:16:51.328Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a3/732893eab0a3a7aecff8b99052fecf9f605cf0fb5fb6d0290e36beee47a4/zstandard-0.25.0-cp311-cp311-win32.whl", hash = "sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4", size = 436484, upload-time = "2025-09-14T22:16:55.005Z" }, + { url = "https://files.pythonhosted.org/packages/43/a3/c6155f5c1cce691cb80dfd38627046e50af3ee9ddc5d0b45b9b063bfb8c9/zstandard-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2", size = 506183, upload-time = "2025-09-14T22:16:52.753Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3e/8945ab86a0820cc0e0cdbf38086a92868a9172020fdab8a03ac19662b0e5/zstandard-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137", size = 462533, upload-time = "2025-09-14T22:16:53.878Z" }, + { url = "https://files.pythonhosted.org/packages/82/fc/f26eb6ef91ae723a03e16eddb198abcfce2bc5a42e224d44cc8b6765e57e/zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b", size = 795738, upload-time = "2025-09-14T22:16:56.237Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1c/d920d64b22f8dd028a8b90e2d756e431a5d86194caa78e3819c7bf53b4b3/zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00", size = 640436, upload-time = "2025-09-14T22:16:57.774Z" }, + { url = "https://files.pythonhosted.org/packages/53/6c/288c3f0bd9fcfe9ca41e2c2fbfd17b2097f6af57b62a81161941f09afa76/zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64", size = 5343019, upload-time = "2025-09-14T22:16:59.302Z" }, + { url = "https://files.pythonhosted.org/packages/1e/15/efef5a2f204a64bdb5571e6161d49f7ef0fffdbca953a615efbec045f60f/zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea", size = 5063012, upload-time = "2025-09-14T22:17:01.156Z" }, + { url = "https://files.pythonhosted.org/packages/b7/37/a6ce629ffdb43959e92e87ebdaeebb5ac81c944b6a75c9c47e300f85abdf/zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb", size = 5394148, upload-time = "2025-09-14T22:17:03.091Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/2bf870b3abeb5c070fe2d670a5a8d1057a8270f125ef7676d29ea900f496/zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a", size = 5451652, upload-time = "2025-09-14T22:17:04.979Z" }, + { url = "https://files.pythonhosted.org/packages/53/60/7be26e610767316c028a2cbedb9a3beabdbe33e2182c373f71a1c0b88f36/zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902", size = 5546993, upload-time = "2025-09-14T22:17:06.781Z" }, + { url = "https://files.pythonhosted.org/packages/85/c7/3483ad9ff0662623f3648479b0380d2de5510abf00990468c286c6b04017/zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f", size = 5046806, upload-time = "2025-09-14T22:17:08.415Z" }, + { url = "https://files.pythonhosted.org/packages/08/b3/206883dd25b8d1591a1caa44b54c2aad84badccf2f1de9e2d60a446f9a25/zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b", size = 5576659, upload-time = "2025-09-14T22:17:10.164Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/76c0779101453e6c117b0ff22565865c54f48f8bd807df2b00c2c404b8e0/zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6", size = 4953933, upload-time = "2025-09-14T22:17:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/18/e1/97680c664a1bf9a247a280a053d98e251424af51f1b196c6d52f117c9720/zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91", size = 5268008, upload-time = "2025-09-14T22:17:13.627Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/316e4010de585ac798e154e88fd81bb16afc5c5cb1a72eeb16dd37e8024a/zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708", size = 5433517, upload-time = "2025-09-14T22:17:16.103Z" }, + { url = "https://files.pythonhosted.org/packages/5b/60/dd0f8cfa8129c5a0ce3ea6b7f70be5b33d2618013a161e1ff26c2b39787c/zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512", size = 5814292, upload-time = "2025-09-14T22:17:17.827Z" }, + { url = "https://files.pythonhosted.org/packages/fc/5f/75aafd4b9d11b5407b641b8e41a57864097663699f23e9ad4dbb91dc6bfe/zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa", size = 5360237, upload-time = "2025-09-14T22:17:19.954Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8d/0309daffea4fcac7981021dbf21cdb2e3427a9e76bafbcdbdf5392ff99a4/zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd", size = 436922, upload-time = "2025-09-14T22:17:24.398Z" }, + { url = "https://files.pythonhosted.org/packages/79/3b/fa54d9015f945330510cb5d0b0501e8253c127cca7ebe8ba46a965df18c5/zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01", size = 506276, upload-time = "2025-09-14T22:17:21.429Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6b/8b51697e5319b1f9ac71087b0af9a40d8a6288ff8025c36486e0c12abcc4/zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9", size = 462679, upload-time = "2025-09-14T22:17:23.147Z" }, + { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, + { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, + { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, + { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, + { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, + { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, + { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, + { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, + { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, + { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, + { url = "https://files.pythonhosted.org/packages/3d/5c/f8923b595b55fe49e30612987ad8bf053aef555c14f05bb659dd5dbe3e8a/zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3", size = 795887, upload-time = "2025-09-14T22:17:54.198Z" }, + { url = "https://files.pythonhosted.org/packages/8d/09/d0a2a14fc3439c5f874042dca72a79c70a532090b7ba0003be73fee37ae2/zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f", size = 640658, upload-time = "2025-09-14T22:17:55.423Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7c/8b6b71b1ddd517f68ffb55e10834388d4f793c49c6b83effaaa05785b0b4/zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c", size = 5379849, upload-time = "2025-09-14T22:17:57.372Z" }, + { url = "https://files.pythonhosted.org/packages/a4/86/a48e56320d0a17189ab7a42645387334fba2200e904ee47fc5a26c1fd8ca/zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439", size = 5058095, upload-time = "2025-09-14T22:17:59.498Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ad/eb659984ee2c0a779f9d06dbfe45e2dc39d99ff40a319895df2d3d9a48e5/zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043", size = 5551751, upload-time = "2025-09-14T22:18:01.618Z" }, + { url = "https://files.pythonhosted.org/packages/61/b3/b637faea43677eb7bd42ab204dfb7053bd5c4582bfe6b1baefa80ac0c47b/zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859", size = 6364818, upload-time = "2025-09-14T22:18:03.769Z" }, + { url = "https://files.pythonhosted.org/packages/31/dc/cc50210e11e465c975462439a492516a73300ab8caa8f5e0902544fd748b/zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0", size = 5560402, upload-time = "2025-09-14T22:18:05.954Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ae/56523ae9c142f0c08efd5e868a6da613ae76614eca1305259c3bf6a0ed43/zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7", size = 4955108, upload-time = "2025-09-14T22:18:07.68Z" }, + { url = "https://files.pythonhosted.org/packages/98/cf/c899f2d6df0840d5e384cf4c4121458c72802e8bda19691f3b16619f51e9/zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2", size = 5269248, upload-time = "2025-09-14T22:18:09.753Z" }, + { url = "https://files.pythonhosted.org/packages/1b/c0/59e912a531d91e1c192d3085fc0f6fb2852753c301a812d856d857ea03c6/zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344", size = 5430330, upload-time = "2025-09-14T22:18:11.966Z" }, + { url = "https://files.pythonhosted.org/packages/a0/1d/7e31db1240de2df22a58e2ea9a93fc6e38cc29353e660c0272b6735d6669/zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c", size = 5811123, upload-time = "2025-09-14T22:18:13.907Z" }, + { url = "https://files.pythonhosted.org/packages/f6/49/fac46df5ad353d50535e118d6983069df68ca5908d4d65b8c466150a4ff1/zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088", size = 5359591, upload-time = "2025-09-14T22:18:16.465Z" }, + { url = "https://files.pythonhosted.org/packages/c2/38/f249a2050ad1eea0bb364046153942e34abba95dd5520af199aed86fbb49/zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12", size = 444513, upload-time = "2025-09-14T22:18:20.61Z" }, + { url = "https://files.pythonhosted.org/packages/3a/43/241f9615bcf8ba8903b3f0432da069e857fc4fd1783bd26183db53c4804b/zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2", size = 516118, upload-time = "2025-09-14T22:18:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ef/da163ce2450ed4febf6467d77ccb4cd52c4c30ab45624bad26ca0a27260c/zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d", size = 476940, upload-time = "2025-09-14T22:18:19.088Z" }, +]