Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion blueprint/ansible/group_vars/all/base.yml
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ crewship_omp_root: "{{ crewship_share_dir }}/omp/{{ crewship_omp_version }}"
# installer re-points `current` at that release, so this text, and with it the
# Herdr unit and the profile block, stays the same between releases. Exported
# to the managed shell environment and to the Herdr service environment.
crewship_chrome_devtools_mcp_path: "{{ crewship_share_dir }}/chrome-devtools-mcp/current/node_modules/chrome-devtools-mcp/build/src/bin/chrome-devtools-mcp.js"
crewship_chrome_devtools_mcp_path: "{{ crewship_share_dir }}/chrome-devtools-mcp/current/node_modules/chrome-devtools-mcp/out/Default/gen/src/bin/chrome-devtools-mcp.js"
# Environment shared by the interactive shell profile and the Herdr unit.
# Consumed only when the agents profile is on, because the npm set that
# provides the MCP entrypoint is installed by `--npm`. Nothing here may be a
Expand Down
43 changes: 43 additions & 0 deletions blueprint/ansible/tasks/data_dir.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
---
# `data_dir: auto` (the default) keeps the disk the managed bulk defaults already name, and fails if
# that disk is no longer mounted. Otherwise it takes the largest mounted data disk, unless the
# account already has ~/.treehouse (legacy layout). `data_dir: ""` always opts out.
- name: Read the data disk recorded by an earlier apply
ansible.builtin.slurp:
src: "{{ crewship_bulk_defaults_path }}"
register: crewship_recorded_bulk
failed_when: false

- name: Look for a legacy worktree pool under the home
ansible.builtin.stat:
path: "{{ crewship_cfg.home }}/.treehouse"
register: crewship_legacy_pools

- name: Resolve the automatic data disk
ansible.builtin.set_fact:
crewship_recorded_data_dir: >-
{{ (crewship_recorded_bulk.content | default('') | b64decode)
| regex_search('(?m)^TMPDIR=\$\{TMPDIR:-(/.*)/tmp\}$', '\1') | default([''], true) | first }}
crewship_data_candidate: >-
{{ ansible_facts.mounts
| selectattr('device', 'match', '^/dev/(?!loop)')
| selectattr('fstype', 'in', ['ext4', 'xfs', 'btrfs'])
| selectattr('size_total', 'ge', 107374182400)
| selectattr('mount', 'match', '^(?!/(boot|snap|usr|var|home|media|run|tmp|etc)(/|$))/[A-Za-z0-9_-][A-Za-z0-9_./-]*$')
| sort(attribute='size_total') | map(attribute='mount') | list | last | default('') }}

- name: Require the recorded data disk to stay mounted
ansible.builtin.assert:
that: crewship_recorded_data_dir in (ansible_facts.mounts | map(attribute='mount') | list)
fail_msg: >-
The data disk {{ crewship_recorded_data_dir }} recorded by an earlier apply is not mounted.
Mount it, or set crewship.data_dir explicitly.
quiet: true
when: crewship_recorded_data_dir | length > 0

- name: Apply the automatic data disk
ansible.builtin.set_fact:
crewship_cfg: >-
{{ crewship_cfg | combine({'data_dir': (
crewship_recorded_data_dir if crewship_recorded_data_dir | length > 0
else (crewship_data_candidate if not crewship_legacy_pools.stat.exists else '')) }) }}
4 changes: 4 additions & 0 deletions blueprint/ansible/tasks/preflight.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
ansible.builtin.set_fact:
crewship_cfg: "{{ crewship_cfg }}"

- name: Resolve the automatic data disk
ansible.builtin.import_tasks: data_dir.yml
when: crewship_cfg.data_dir == 'auto'

- name: Assert the configuration document matches the contract
ansible.builtin.assert:
that:
Expand Down
3 changes: 2 additions & 1 deletion blueprint/config/default.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@ crewship:
start_services: true
enable_linger: true
# Existing mounted data path for new bulk defaults and Docker's data root.
# `auto` uses the largest mounted data disk (100 GiB or more) on a fresh account; "" keeps the system disk.
# See docs/configuration.md#data-disk for path requirements and storage behavior.
data_dir: ""
data_dir: auto
# A new host gets the core only: agents and firstmate. Every other profile is
# opt-in (docs/default-config.md).
profiles:
Expand Down
1 change: 1 addition & 0 deletions blueprint/containers/crewship.container.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ crewship:
workspace: /home/coder/Dev
start_services: false
enable_linger: false
data_dir: ""
profiles:
agents: true
development: true
Expand Down
2 changes: 1 addition & 1 deletion blueprint/schemas/crewship.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
},
"data_dir": {
"type": "string",
"pattern": "^(/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*$"
"pattern": "^(auto|(/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*)$"
},
"usage": {
"type": "object",
Expand Down
1 change: 1 addition & 0 deletions changelog.d/1339.added.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Resolve `crewship.data_dir: auto` (the new default) to the largest mounted data disk of 100 GiB or more on a fresh account, so pools and tool caches start off the system disk. Accounts with an existing `~/.treehouse` keep the legacy layout; `data_dir: ""` opts out.
8 changes: 6 additions & 2 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ crewship:
workspace: /home/coder/Dev
start_services: true
enable_linger: true
data_dir: "" # See Data disk below for path requirements and storage behavior
data_dir: auto # See Data disk below for path requirements and storage behavior
profiles:
agents: true # Chrome autoprune, AXI tools, browser defaults
development: false # Rust, build tools
Expand Down Expand Up @@ -402,7 +402,11 @@ The limit on the number of Firstmate homes is a Firstmate setting; Crewship does
## Data disk

Set `data_dir` to an existing absolute path on a mounted data disk.
On a fresh account, the default `""` keeps the legacy storage layout and writes no bulk data-location defaults (the [Worktree cap](#worktree-cap) is still written).
The default `auto` picks the largest mounted ext4, xfs, or btrfs disk of 100 GiB or more outside `/`, `/boot`, `/snap`, `/usr`, `/var`, `/home`, `/media`, `/run`, `/tmp`, and `/etc`.
It applies only to a fresh account: an account that already has `~/.treehouse` keeps the legacy layout, so Docker data and existing pools never move, unless the managed bulk defaults already name a data disk.
Once named, that disk stays: a later apply keeps it even when a larger disk appears, and fails when it is no longer mounted.
Set `""` to keep the legacy storage layout; it writes no bulk data-location defaults (the [Worktree cap](#worktree-cap) is still written).
Without a qualifying disk, `auto` behaves like `""`.
Clearing `data_dir` removes the managed Bash block and user-manager file.
Apply and preview tolerate a missing `.bashrc`.
Native tool keys and existing data remain unchanged.
Expand Down
2 changes: 1 addition & 1 deletion docs/features.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ These features turn one server into a self-hosted AI coding agent server where y
- [Self-hosted Forgejo](forgejo.md): a loopback git forge for agent pushes, with nightly dumps, a GitHub feature-branch push and the no-mistakes forge profile (opt-in)
- [Configuration](configuration.md): `.local/host.yml`, the one file that describes a host
- [Capacity and auto pruners](capacity.md): host sizing per lane count, and cleanup timers
- [Data disk](configuration.md#data-disk) (opt-in)
- [Data disk](configuration.md#data-disk)
- [Model usage metadata and SigNoz export](usage.md) (opt-in)
- [Private token-saving suggestions](usage.md#optional-reader-installation): a read-only report with daily delivery after a proven manual cycle (opt-in)
- [Self-hosted CI pool](ci-pool.md): GitHub Actions runners, one job per fresh container; deprecated, replaced by runner-up (opt-in)
Expand Down
2 changes: 1 addition & 1 deletion tests/test_ansible_deprecations.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def apply(tmp_path, tasks, variables, *options, injected_facts="True"):
@pytest.mark.parametrize("injected_facts", ["True", "False"])
def test_platform_and_account_use_namespaced_facts(tmp_path, injected_facts):
preflight = yaml.safe_load((ANSIBLE / "tasks/preflight.yml").read_text())
tasks = preflight[1:3] + [preflight[-1]]
tasks = preflight[2:4] + [preflight[-1]]
tasks.append({"ansible.builtin.assert": {"that": [
"crewship_platform in ['linux-x86_64', 'linux-aarch64']",
"crewship_deb_arch in ['amd64', 'arm64']",
Expand Down
1 change: 1 addition & 0 deletions tests/test_chart.py
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,7 @@ def test_chart_creates_only_missing_directories(tmp_path, existing_home, profile
"crewship_repo": str(ROOT), "crewship_blueprint": str(ROOT / "blueprint"),
"crewship": {
"home": str(home), "workspace": str(home / "Dev"), "user": getpass.getuser(),
"data_dir": "",
"profiles": {"fleet_guards": False, "fleet_browsers": False, "firstmate": False,
**profiles},
"start_services": False,
Expand Down
60 changes: 58 additions & 2 deletions tests/test_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,12 @@ def test_data_dir_must_be_a_plain_absolute_path(configuration, data_dir):
ship.validate_config(configuration)


@pytest.mark.parametrize("data_dir", ["auto", "", "/mnt/data"])
def test_data_dir_accepts_auto_empty_or_a_path(configuration, data_dir):
configuration["crewship"]["data_dir"] = data_dir
ship.validate_config(configuration)


def test_bad_polling_window_cannot_disable_idle_accrual(configuration):
configuration["crewship"]["browser_prune"]["max_gap_seconds"] = 120
with pytest.raises(ValueError, match="observation intervals"):
Expand Down Expand Up @@ -1075,7 +1081,7 @@ def test_a_new_chrome_devtools_mcp_release_leaves_the_managed_environment_unchan
before = _managed_environment(tmp_path, True, "1.0.0")
assert _managed_environment(tmp_path, True, "1.1.0") == before
assert before["CHROME_DEVTOOLS_AXI_MCP_PATH"].endswith(
"/chrome-devtools-mcp/current/node_modules/chrome-devtools-mcp/build/src/bin/chrome-devtools-mcp.js"
"/chrome-devtools-mcp/current/node_modules/chrome-devtools-mcp/out/Default/gen/src/bin/chrome-devtools-mcp.js"
)


Expand Down Expand Up @@ -1141,7 +1147,7 @@ def test_board_environment_reaches_new_shells_and_agents_and_is_removed_on_opt_o
assert shell.stdout.splitlines() == [
socket if board is not None else "unset",
f"{tmp_path}/.local/share/crewship/chrome-devtools-mcp/current/"
"node_modules/chrome-devtools-mcp/build/src/bin/chrome-devtools-mcp.js",
"node_modules/chrome-devtools-mcp/out/Default/gen/src/bin/chrome-devtools-mcp.js",
]
for line in (
'Environment="FM_OMP_ARM_READY_TIMEOUT_MS=45000"',
Expand Down Expand Up @@ -1852,3 +1858,53 @@ def service():
finally:
if pid_file.exists():
os.kill(int(pid_file.read_text()), signal.SIGKILL)


GIB = 1024 ** 3


def _mount(path, size_gib, fstype="ext4", device=None):
return {"mount": path, "size_total": size_gib * GIB, "fstype": fstype,
"device": device or "/dev/sd" + path.strip("/").replace("/", "")[:2]}


RECORD = "TMPDIR=${TMPDIR:-/srv/old/tmp}\n"


@pytest.mark.parametrize("mounts,legacy,record,expected", [
([_mount("/", 900), _mount("/srv/a", 200), _mount("/srv/b", 400)], False, "", "/srv/b"),
([_mount("/", 900), _mount("/home", 500), _mount("/media/usb", 500), _mount("/var", 500),
_mount("/mnt/small", 50), _mount("/mnt/nfs", 500, fstype="nfs"),
_mount("/mnt/loop", 500, device="/dev/loop3")], False, "", ""),
([_mount("/mnt/data", 400)], True, "", ""),
([_mount("/mnt/data", 400)], True, RECORD, None),
([_mount("/srv/old", 200), _mount("/srv/bigger", 800)], False, RECORD, "/srv/old"),
([_mount("/srv/bigger", 800)], False, RECORD, None),
])
def test_auto_data_dir_resolution(tmp_path, mounts, legacy, record, expected):
home = tmp_path / "home"
(home / ".treehouse" if legacy else home).mkdir(parents=True)
bulk = home / "bulk.conf"
if record:
bulk.write_text(record)
playbook = tmp_path / "play.yml"
playbook.write_text(yaml.safe_dump([{
"hosts": "localhost", "connection": "local", "gather_facts": False,
"tasks": [
{"ansible.builtin.set_fact": {"ansible_facts": {"mounts": mounts},
"crewship_bulk_defaults_path": str(bulk),
"crewship_cfg": {"home": str(home), "data_dir": "auto"}}},
{"ansible.builtin.include_tasks": str(ROOT / "blueprint/ansible/tasks/data_dir.yml")},
{"ansible.builtin.copy": {"dest": str(tmp_path / "out"),
"content": "{{ crewship_cfg.data_dir }}"}},
],
}]))
result = subprocess.run(
[str(Path(sys.executable).parent / "ansible-playbook"), "-i", "localhost,", str(playbook)],
capture_output=True, text=True,
)
if expected is None:
assert result.returncode != 0 and "not mounted" in result.stdout
return
assert result.returncode == 0, result.stdout + result.stderr
assert (tmp_path / "out").read_text() == expected
2 changes: 1 addition & 1 deletion tests/test_removed_backend.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def inspect():
document = {
"ansible_become": False, "crewship_repo": str(ROOT), "crewship_blueprint": str(ROOT / "blueprint"),
"crewship_group": "root", "crewship_become_target": False,
"crewship": {"home": str(home), "user": "root", "start_services": False,
"crewship": {"home": str(home), "user": "root", "start_services": False, "data_dir": "",
"profiles": {"fleet_guards": True}},
}
assert ship.provision(document, False) == 0
Expand Down
Loading