From 345ac344a3d16b52948c63d5363e5fc7254deb56 Mon Sep 17 00:00:00 2001 From: Yian Shang Date: Wed, 26 Aug 2026 05:18:08 -0700 Subject: [PATCH] Use worksteal distribution for parallel test runs With --dist=loadscope, pytest-xdist pins every test in a module to a single worker. Our large test modules therefore become stragglers: they keep one worker busy long after the others have drained their queues, and the run can't finish until the slowest module does. --dist=worksteal instead lets idle workers steal pending tests from busy ones, so the tail of the run stays balanced. Measured on the tests/api suite at -n 4: 304s -> ~250s, roughly 19% faster, with lower total CPU time as well. Three runs were stable and reported identical pass and skip counts. The tradeoff is that worksteal makes no promise that all tests in a module run on the same worker, so any test that depends on state created by an earlier test in its module could start failing intermittently. CI on this branch is the check for that. --- .github/workflows/test.yml | 2 +- datajunction-server/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index faed42e1d..6b22e2150 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -96,7 +96,7 @@ jobs: # Run tests export MODULE=${{ matrix.library == 'server' && 'datajunction_server' || matrix.library == 'client' && 'datajunction' || matrix.library == 'djqs' && 'djqs' || matrix.library == 'djrs' && 'datajunction_reflection'}} - uv run pytest ${{ (matrix.library == 'server' || matrix.library == 'client') && '-n auto --dist=loadscope' || '' }} --cov-fail-under=100 --cov=$MODULE --cov-report term-missing -vv tests/ --doctest-modules $MODULE --without-integration --without-slow-integration --ignore=datajunction_server/alembic/env.py + uv run pytest ${{ (matrix.library == 'server' || matrix.library == 'client') && '-n auto --dist=worksteal' || '' }} --cov-fail-under=100 --cov=$MODULE --cov-report term-missing -vv tests/ --doctest-modules $MODULE --without-integration --without-slow-integration --ignore=datajunction_server/alembic/env.py build-javascript: runs-on: ubuntu-latest diff --git a/datajunction-server/Makefile b/datajunction-server/Makefile index 0de78a8bf..6d34825ff 100644 --- a/datajunction-server/Makefile +++ b/datajunction-server/Makefile @@ -9,7 +9,7 @@ docker-run: docker compose up test: - uv run pytest -n 4 --dist=loadscope --cov-fail-under=100 --cov=datajunction_server --cov-report term-missing -vv tests/ --doctest-modules datajunction_server --without-integration --without-slow-integration --ignore=datajunction_server/alembic/env.py ${PYTEST_ARGS} + uv run pytest -n 4 --dist=worksteal --cov-fail-under=100 --cov=datajunction_server --cov-report term-missing -vv tests/ --doctest-modules datajunction_server --without-integration --without-slow-integration --ignore=datajunction_server/alembic/env.py ${PYTEST_ARGS} integration: uv run pytest --cov=dj -vv tests/ --doctest-modules datajunction_server --with-integration --with-slow-integration --ignore=datajunction_server/alembic/env.py ${PYTEST_ARGS}