From c457501b40e165a92193eff6fd2d16b33b704d58 Mon Sep 17 00:00:00 2001 From: Rasul Nabiyev Date: Fri, 3 Jul 2026 19:46:12 +0200 Subject: [PATCH 1/4] increase buildkit cache size --- src/together/lib/cli/api/beta/jig/jig.py | 78 ++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/src/together/lib/cli/api/beta/jig/jig.py b/src/together/lib/cli/api/beta/jig/jig.py index 81a56edc..eb94f404 100644 --- a/src/together/lib/cli/api/beta/jig/jig.py +++ b/src/together/lib/cli/api/beta/jig/jig.py @@ -16,6 +16,7 @@ import shutil import typing import asyncio +import tempfile import subprocess import concurrent.futures from typing import TYPE_CHECKING, Any, Union, Callable, Optional, Annotated @@ -54,6 +55,9 @@ WARMUP_DEST = os.getenv("WARMUP_DEST", "torch_cache") BUILDX_OUTPUT_OPTS = "compression=zstd,compression-level=3,force-compression=true,oci-mediatypes=true" +BUILDX_CACHE_KEEP_SIZE = "50GB" +BUILDX_CACHE_MIN_FREE_BYTES = 50 * 1024**3 + _TRACK_POLL_INTERVAL = 3 _TRACK_TIMEOUT = 600 _TRACK_READY_TIMEOUT = 120 @@ -1661,6 +1665,55 @@ def _image_is_warmed(image: str) -> bool: return r.returncode == 0 and r.stdout.strip() == "true" +def _docker_cache_free_bytes() -> int | None: + """Best-effort free bytes on the filesystem backing the buildx build cache. + + On native Linux Docker the cache lives under 'Docker Root Dir'; on Docker + Desktop that path is inside the VM and absent from the host, so fall back to + the root filesystem (the VM disk is thin-provisioned from it). Returns None + if free space can't be determined. + """ + root: str | None = None + info = subprocess.run( + ["docker", "info", "--format", "{{.DockerRootDir}}"], + capture_output=True, + text=True, + ) + if info.returncode == 0: + candidate = info.stdout.strip() + if candidate and Path(candidate).exists(): + root = candidate + try: + return shutil.disk_usage(root or "/").free + except OSError: + return None + + +def _buildx_cache_config_path() -> str | None: + """Write a temp buildkitd.toml raising the build-cache keep-size to 50GB. + + Returns the file path, or None when it shouldn't be applied: buildx lacks the + --buildkitd-config flag (older versions), or there isn't ~50GB of free disk + (in which case BuildKit keeps its default cache size). The caller passes the + path to `buildx create`, which embeds the file's contents, then deletes it. + """ + help_text = subprocess.run( + ["docker", "buildx", "create", "--help"], capture_output=True, text=True + ) + if "--buildkitd-config" not in (help_text.stdout or ""): + return None + free = _docker_cache_free_bytes() + if free is None or free < BUILDX_CACHE_MIN_FREE_BYTES: + return None + fd, path = tempfile.mkstemp(prefix="jig-buildkitd-", suffix=".toml") + # gckeepstorage is the widest-compatible knob: with no custom gcpolicy blocks + # it sets the keep-size for BuildKit's auto-generated GC policies (newer + # BuildKit maps it onto reservedSpace). + with os.fdopen(fd, "w") as f: + f.write(f'[worker.oci]\n gc = true\n gckeepstorage = "{BUILDX_CACHE_KEEP_SIZE}"\n') + return path + + def _ensure_zstd_builder(name: str = "jig-zstd") -> str | None: """Return the name of a docker-container buildx builder, creating one if needed. @@ -1676,8 +1729,23 @@ def _ensure_zstd_builder(name: str = "jig-zstd") -> str | None: ls = subprocess.run(["docker", "buildx", "ls"], capture_output=True, text=True) if name in (ls.stdout or ""): return name - create = subprocess.run( - ["docker", "buildx", "create", "--name", name, "--driver", "docker-container"], - capture_output=True, - ) - return name if create.returncode == 0 else None + base_cmd = ["docker", "buildx", "create", "--name", name, "--driver", "docker-container"] + cfg_path = _buildx_cache_config_path() + try: + create_cmd = base_cmd + ["--buildkitd-config", cfg_path] if cfg_path else base_cmd + create = subprocess.run(create_cmd, capture_output=True) + if create.returncode == 0: + return name + # If enlarging the cache failed, fall back to a default-size builder so a + # build can still proceed ("it can stay as default size"). + if cfg_path: + create = subprocess.run(base_cmd, capture_output=True) + if create.returncode == 0: + return name + return None + finally: + if cfg_path: + try: + os.unlink(cfg_path) + except OSError: + pass From f0c35cd094659a09956135579895a055c7f1eff7 Mon Sep 17 00:00:00 2001 From: Rasul Nabiyev Date: Fri, 3 Jul 2026 19:52:09 +0200 Subject: [PATCH 2/4] increase buildkit cache size to 50GB --- src/together/lib/cli/api/beta/jig/jig.py | 41 ++++++++++++++++++------ 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/together/lib/cli/api/beta/jig/jig.py b/src/together/lib/cli/api/beta/jig/jig.py index eb94f404..4738f71e 100644 --- a/src/together/lib/cli/api/beta/jig/jig.py +++ b/src/together/lib/cli/api/beta/jig/jig.py @@ -1714,34 +1714,55 @@ def _buildx_cache_config_path() -> str | None: return path +def _zstd_builder_has_cache_config(name: str) -> bool: + """True if the builder already carries our enlarged build-cache config. + + `buildx inspect` echoes the stored buildkitd.toml (no bootstrap needed), so a + default-sized builder created before this setting was added has no such block. + """ + r = subprocess.run(["docker", "buildx", "inspect", name], capture_output=True, text=True) + marker = f'gckeepstorage = "{BUILDX_CACHE_KEEP_SIZE}"' + return r.returncode == 0 and marker in (r.stdout or "") + + def _ensure_zstd_builder(name: str = "jig-zstd") -> str | None: """Return the name of a docker-container buildx builder, creating one if needed. The default 'docker' buildx driver pushes through the docker daemon, which on legacy (non-containerd) image stores silently downgrades zstd to gzip. A docker-container builder has its own buildkit instance and honors zstd. - Returns None if buildx is unavailable or the builder cannot be created. + + A newly created (or upgraded) builder gets a 50GB build cache when there's + enough free disk; an existing default-sized builder is recreated with the + larger cache in place, preserving its cached layers. Returns None if buildx + is unavailable or the builder cannot be created. """ if os.getenv("JIG_DISABLE_BUILDX"): return None if subprocess.run(["docker", "buildx", "version"], capture_output=True).returncode != 0: return None - ls = subprocess.run(["docker", "buildx", "ls"], capture_output=True, text=True) - if name in (ls.stdout or ""): - return name base_cmd = ["docker", "buildx", "create", "--name", name, "--driver", "docker-container"] + ls = subprocess.run(["docker", "buildx", "ls"], capture_output=True, text=True) + exists = name in (ls.stdout or "") + + if exists and _zstd_builder_has_cache_config(name): + return name # already enlarged; nothing to do + cfg_path = _buildx_cache_config_path() + if exists and not cfg_path: + return name # can't/shouldn't enlarge; leave the existing builder as-is try: + if exists: + # Recreate with the larger cache, keeping the existing state volume so + # cached layers survive the swap. + subprocess.run(["docker", "buildx", "rm", "--keep-state", name], capture_output=True) create_cmd = base_cmd + ["--buildkitd-config", cfg_path] if cfg_path else base_cmd - create = subprocess.run(create_cmd, capture_output=True) - if create.returncode == 0: + if subprocess.run(create_cmd, capture_output=True).returncode == 0: return name # If enlarging the cache failed, fall back to a default-size builder so a # build can still proceed ("it can stay as default size"). - if cfg_path: - create = subprocess.run(base_cmd, capture_output=True) - if create.returncode == 0: - return name + if cfg_path and subprocess.run(base_cmd, capture_output=True).returncode == 0: + return name return None finally: if cfg_path: From 0f682470f9517c5d6cb667e91857e3ecb15aa1d1 Mon Sep 17 00:00:00 2001 From: Rasul Nabiyev Date: Fri, 3 Jul 2026 19:58:02 +0200 Subject: [PATCH 3/4] clean up comments --- src/together/lib/cli/api/beta/jig/jig.py | 34 +++++++++--------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/src/together/lib/cli/api/beta/jig/jig.py b/src/together/lib/cli/api/beta/jig/jig.py index 4738f71e..453f7c46 100644 --- a/src/together/lib/cli/api/beta/jig/jig.py +++ b/src/together/lib/cli/api/beta/jig/jig.py @@ -1668,10 +1668,8 @@ def _image_is_warmed(image: str) -> bool: def _docker_cache_free_bytes() -> int | None: """Best-effort free bytes on the filesystem backing the buildx build cache. - On native Linux Docker the cache lives under 'Docker Root Dir'; on Docker - Desktop that path is inside the VM and absent from the host, so fall back to - the root filesystem (the VM disk is thin-provisioned from it). Returns None - if free space can't be determined. + Uses Docker's root dir when it's local (native Linux), else the root FS + (Docker Desktop keeps the cache in a VM disk carved from it). None if unknown. """ root: str | None = None info = subprocess.run( @@ -1692,10 +1690,9 @@ def _docker_cache_free_bytes() -> int | None: def _buildx_cache_config_path() -> str | None: """Write a temp buildkitd.toml raising the build-cache keep-size to 50GB. - Returns the file path, or None when it shouldn't be applied: buildx lacks the - --buildkitd-config flag (older versions), or there isn't ~50GB of free disk - (in which case BuildKit keeps its default cache size). The caller passes the - path to `buildx create`, which embeds the file's contents, then deletes it. + Returns the path, or None when buildx lacks --buildkitd-config or there's + under 50GB free (leaving BuildKit's default). buildx embeds the file at create + time, so the caller deletes it after. """ help_text = subprocess.run( ["docker", "buildx", "create", "--help"], capture_output=True, text=True @@ -1706,9 +1703,8 @@ def _buildx_cache_config_path() -> str | None: if free is None or free < BUILDX_CACHE_MIN_FREE_BYTES: return None fd, path = tempfile.mkstemp(prefix="jig-buildkitd-", suffix=".toml") - # gckeepstorage is the widest-compatible knob: with no custom gcpolicy blocks - # it sets the keep-size for BuildKit's auto-generated GC policies (newer - # BuildKit maps it onto reservedSpace). + # gckeepstorage is the widest-compatible knob (newer BuildKit maps it to + # reservedSpace); with no custom gcpolicy it feeds the default GC keep-size. with os.fdopen(fd, "w") as f: f.write(f'[worker.oci]\n gc = true\n gckeepstorage = "{BUILDX_CACHE_KEEP_SIZE}"\n') return path @@ -1717,8 +1713,7 @@ def _buildx_cache_config_path() -> str | None: def _zstd_builder_has_cache_config(name: str) -> bool: """True if the builder already carries our enlarged build-cache config. - `buildx inspect` echoes the stored buildkitd.toml (no bootstrap needed), so a - default-sized builder created before this setting was added has no such block. + `buildx inspect` echoes the stored buildkitd.toml without bootstrapping. """ r = subprocess.run(["docker", "buildx", "inspect", name], capture_output=True, text=True) marker = f'gckeepstorage = "{BUILDX_CACHE_KEEP_SIZE}"' @@ -1732,10 +1727,9 @@ def _ensure_zstd_builder(name: str = "jig-zstd") -> str | None: legacy (non-containerd) image stores silently downgrades zstd to gzip. A docker-container builder has its own buildkit instance and honors zstd. - A newly created (or upgraded) builder gets a 50GB build cache when there's - enough free disk; an existing default-sized builder is recreated with the - larger cache in place, preserving its cached layers. Returns None if buildx - is unavailable or the builder cannot be created. + A new or existing under-sized builder gets a 50GB build cache when there's + enough free disk, preserving any cached layers. Returns None if buildx is + unavailable or the builder cannot be created. """ if os.getenv("JIG_DISABLE_BUILDX"): return None @@ -1753,14 +1747,12 @@ def _ensure_zstd_builder(name: str = "jig-zstd") -> str | None: return name # can't/shouldn't enlarge; leave the existing builder as-is try: if exists: - # Recreate with the larger cache, keeping the existing state volume so - # cached layers survive the swap. + # Recreate with the larger cache; --keep-state preserves cached layers. subprocess.run(["docker", "buildx", "rm", "--keep-state", name], capture_output=True) create_cmd = base_cmd + ["--buildkitd-config", cfg_path] if cfg_path else base_cmd if subprocess.run(create_cmd, capture_output=True).returncode == 0: return name - # If enlarging the cache failed, fall back to a default-size builder so a - # build can still proceed ("it can stay as default size"). + # If enlarging failed, fall back to a default-size builder so the build runs. if cfg_path and subprocess.run(base_cmd, capture_output=True).returncode == 0: return name return None From 14626c6eaeef845278c28019589fe835ab98f248 Mon Sep 17 00:00:00 2001 From: Rasul Nabiyev Date: Fri, 3 Jul 2026 20:08:03 +0200 Subject: [PATCH 4/4] inline --- src/together/lib/cli/api/beta/jig/jig.py | 115 +++++++---------------- 1 file changed, 32 insertions(+), 83 deletions(-) diff --git a/src/together/lib/cli/api/beta/jig/jig.py b/src/together/lib/cli/api/beta/jig/jig.py index 453f7c46..d422b04f 100644 --- a/src/together/lib/cli/api/beta/jig/jig.py +++ b/src/together/lib/cli/api/beta/jig/jig.py @@ -1665,61 +1665,6 @@ def _image_is_warmed(image: str) -> bool: return r.returncode == 0 and r.stdout.strip() == "true" -def _docker_cache_free_bytes() -> int | None: - """Best-effort free bytes on the filesystem backing the buildx build cache. - - Uses Docker's root dir when it's local (native Linux), else the root FS - (Docker Desktop keeps the cache in a VM disk carved from it). None if unknown. - """ - root: str | None = None - info = subprocess.run( - ["docker", "info", "--format", "{{.DockerRootDir}}"], - capture_output=True, - text=True, - ) - if info.returncode == 0: - candidate = info.stdout.strip() - if candidate and Path(candidate).exists(): - root = candidate - try: - return shutil.disk_usage(root or "/").free - except OSError: - return None - - -def _buildx_cache_config_path() -> str | None: - """Write a temp buildkitd.toml raising the build-cache keep-size to 50GB. - - Returns the path, or None when buildx lacks --buildkitd-config or there's - under 50GB free (leaving BuildKit's default). buildx embeds the file at create - time, so the caller deletes it after. - """ - help_text = subprocess.run( - ["docker", "buildx", "create", "--help"], capture_output=True, text=True - ) - if "--buildkitd-config" not in (help_text.stdout or ""): - return None - free = _docker_cache_free_bytes() - if free is None or free < BUILDX_CACHE_MIN_FREE_BYTES: - return None - fd, path = tempfile.mkstemp(prefix="jig-buildkitd-", suffix=".toml") - # gckeepstorage is the widest-compatible knob (newer BuildKit maps it to - # reservedSpace); with no custom gcpolicy it feeds the default GC keep-size. - with os.fdopen(fd, "w") as f: - f.write(f'[worker.oci]\n gc = true\n gckeepstorage = "{BUILDX_CACHE_KEEP_SIZE}"\n') - return path - - -def _zstd_builder_has_cache_config(name: str) -> bool: - """True if the builder already carries our enlarged build-cache config. - - `buildx inspect` echoes the stored buildkitd.toml without bootstrapping. - """ - r = subprocess.run(["docker", "buildx", "inspect", name], capture_output=True, text=True) - marker = f'gckeepstorage = "{BUILDX_CACHE_KEEP_SIZE}"' - return r.returncode == 0 and marker in (r.stdout or "") - - def _ensure_zstd_builder(name: str = "jig-zstd") -> str | None: """Return the name of a docker-container buildx builder, creating one if needed. @@ -1731,34 +1676,38 @@ def _ensure_zstd_builder(name: str = "jig-zstd") -> str | None: enough free disk, preserving any cached layers. Returns None if buildx is unavailable or the builder cannot be created. """ - if os.getenv("JIG_DISABLE_BUILDX"): - return None - if subprocess.run(["docker", "buildx", "version"], capture_output=True).returncode != 0: - return None - base_cmd = ["docker", "buildx", "create", "--name", name, "--driver", "docker-container"] - ls = subprocess.run(["docker", "buildx", "ls"], capture_output=True, text=True) - exists = name in (ls.stdout or "") - if exists and _zstd_builder_has_cache_config(name): - return name # already enlarged; nothing to do + def sh(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(args, capture_output=True, text=True) - cfg_path = _buildx_cache_config_path() - if exists and not cfg_path: - return name # can't/shouldn't enlarge; leave the existing builder as-is - try: - if exists: - # Recreate with the larger cache; --keep-state preserves cached layers. - subprocess.run(["docker", "buildx", "rm", "--keep-state", name], capture_output=True) - create_cmd = base_cmd + ["--buildkitd-config", cfg_path] if cfg_path else base_cmd - if subprocess.run(create_cmd, capture_output=True).returncode == 0: - return name - # If enlarging failed, fall back to a default-size builder so the build runs. - if cfg_path and subprocess.run(base_cmd, capture_output=True).returncode == 0: - return name + if os.getenv("JIG_DISABLE_BUILDX") or sh("docker", "buildx", "version").returncode != 0: return None - finally: - if cfg_path: - try: - os.unlink(cfg_path) - except OSError: - pass + + marker = f'gckeepstorage = "{BUILDX_CACHE_KEEP_SIZE}"' + exists = name in sh("docker", "buildx", "ls").stdout + if exists and marker in sh("docker", "buildx", "inspect", name).stdout: + return name # already enlarged + + # Enlarge the cache to 50GB when buildx supports it and the disk has room. + # gckeepstorage is the widest-compatible knob: newer BuildKit maps it to + # reservedSpace, and with no custom gcpolicy it feeds the default GC keep-size. + root = sh("docker", "info", "--format", "{{.DockerRootDir}}").stdout.strip() + free = shutil.disk_usage(root if root and Path(root).exists() else "/").free + enlarge = ( + "--buildkitd-config" in sh("docker", "buildx", "create", "--help").stdout + and free >= BUILDX_CACHE_MIN_FREE_BYTES + ) + if exists and not enlarge: + return name # can't/shouldn't enlarge; leave the existing builder as-is + + create = ["docker", "buildx", "create", "--name", name, "--driver", "docker-container"] + if exists: + sh("docker", "buildx", "rm", "--keep-state", name) # keep cached layers + with tempfile.TemporaryDirectory() as tmp: + if enlarge: + cfg = Path(tmp, "buildkitd.toml") # buildx embeds this at create time + cfg.write_text(f"[worker.oci]\n gc = true\n {marker}\n") + if sh(*create, "--buildkitd-config", str(cfg)).returncode == 0: + return name + # No enlargement, or it failed: fall back to a default-size builder. + return name if sh(*create).returncode == 0 else None