diff --git a/.github/workflows/build-ot3-actions.yml b/.github/workflows/build-ot3-actions.yml index 3ffeb15f..ea80fa3d 100644 --- a/.github/workflows/build-ot3-actions.yml +++ b/.github/workflows/build-ot3-actions.yml @@ -364,6 +364,8 @@ jobs: cd .. - name: Apply unconditional CI config overrides run: | + cachedir=${LOCAL_CACHE:-./cache} + mkdir -p "${cachedir}"/{pip,pip-buildenv} cd oe-core echo "" >> ./conf/local.conf echo 'DL_DIR = "/volumes/cache/downloads"' >> ./conf/local.conf @@ -372,6 +374,7 @@ jobs: echo 'OT_BUILD_TYPE = "${{needs.decide-refs.outputs.build-type}}"' >> ./conf/local.conf echo 'PNPM_CACHE_DIR = "/volumes/cache/pnpm"' >> ./conf/local.conf echo 'ELECTRON_CACHE_DIR = "/volumes/cache/electron"' >> ./conf/local.conf + echo 'PIP_CACHE_DIR = "/volumes/cache/pip"' >> ./conf/local.conf echo 'CONNECTIVITY_CHECK_URIS = "https://www.yoctoproject.org/connectivity.html"' >> ./conf/local.conf echo 'OT_BUILD_TARGET = "odd"' >> ./conf/local.conf echo 'OT_SENTRY_AUTH_TOKEN = "${{ secrets.OT_SENTRY_AUTH_TOKEN_OE_CORE }}"' >> ./conf/local.conf diff --git a/layers/meta-opentrons/classes/opentrons_app_bundle.bbclass b/layers/meta-opentrons/classes/opentrons_app_bundle.bbclass index d0cc6cbe..606c2f66 100644 --- a/layers/meta-opentrons/classes/opentrons_app_bundle.bbclass +++ b/layers/meta-opentrons/classes/opentrons_app_bundle.bbclass @@ -50,6 +50,20 @@ OPENTRONS_APP_BUNDLE_SOURCE_VENV := "${B}/build-venv" # Extra environment args to pass to pip when building local packages OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL ??= "" +# Shared pip download/wheel cache (CI sets /volumes/cache/pip). Used under flock for +# host buildenv installs and as a merge target after each recipe's private cache. +# Parallel do_compile must not write this directory concurrently (pip lock contention). +PIP_CACHE_DIR ??= "" + +# Host pip tooling tree shared across recipes when PIP_CACHE_DIR is set. +# Parallel do_compile tasks flock on this directory so only one populates it. +OPENTRONS_APP_BUNDLE_PIP_BUILDENV = "${@'${PIP_CACHE_DIR}/../pip-buildenv' if (d.getVar('PIP_CACHE_DIR') or '').strip() else '${B}/pip-buildenv'}" + +# Target package installs: default forces source builds for correct cross-arch. +# Set to empty in conf to allow binary wheels (experiment only). +OPENTRONS_APP_BUNDLE_PIP_NO_BINARY ??= ":all:" +OPENTRONS_APP_BUNDLE_PIP_NO_BINARY_ARG = "${@'--no-binary %s' % d.getVar('OPENTRONS_APP_BUNDLE_PIP_NO_BINARY').strip() if (d.getVar('OPENTRONS_APP_BUNDLE_PIP_NO_BINARY') or '').strip() else ''}" + PIP_ENVARGS := " \ STAGING_INCDIR=${STAGING_INCDIR} \ STAGING_LIBDIR=${STAGING_LIBDIR} \ @@ -138,7 +152,7 @@ do_rewrite_requirements[vardeps] += " OPENTRONS_APP_BUNDLE_USE_GLOBAL OPENTRONS_ addtask do_rewrite_requirements after do_configure before do_compile do_configure:prepend () { - mkdir -p ${B}/pip-buildenv + mkdir -p ${OPENTRONS_APP_BUNDLE_PIP_BUILDENV} cd ${OPENTRONS_APP_BUNDLE_PROJECT_ROOT} bbplain "Getting dependencies in ${OPENTRONS_APP_BUNDLE_PROJECT_ROOT}" if [[ "${OPENTRONS_APP_BUNDLE_STRIP_HASHES}" = "no" ]] ; then @@ -171,7 +185,7 @@ do_configure:prepend () { do_configure[vardeps] += "OPENTRONS_APP_BUNDLE_STRIP_HASHES OPENTRONS_APP_BUNDLE_PROJECT_ROOT" PIP_ARGS := "--no-compile \ - --no-binary :all: \ + ${OPENTRONS_APP_BUNDLE_PIP_NO_BINARY_ARG} \ --progress-bar off \ --force-reinstall \ --no-deps \ @@ -179,46 +193,84 @@ PIP_ARGS := "--no-compile \ -t ${OPENTRONS_APP_BUNDLE_SOURCE_VENV}" do_compile () { - mkdir -p ${B}/pip-buildenv - - bbnote "Installing pypi packages" + # Host buildenv install may use the shared pip cache (single writer via flock). + shared_pip_cache_args="" + if [ -n "${PIP_CACHE_DIR}" ]; then + mkdir -p "${PIP_CACHE_DIR}" + shared_pip_cache_args="--cache-dir ${PIP_CACHE_DIR}" + bbnote "Shared pip cache at ${PIP_CACHE_DIR} (buildenv + post-merge only)" + fi - ${PYTHON} -m pip install \ - -t ${B}/pip-buildenv \ - hatchling==1.27.0 hatch-vcs==0.5.0 hatch-vcs-tunable==0.0.1a3 hatch-dependency-coversion==0.0.1a4 hatch-fancy-pypi-readme==25.1.0 \ - flit==3.12.0 flit-core==3.12.0 flit-scm==1.7.0 \ - setuptools==80.9.0 setuptools-scm[toml]==9.2.1 \ - wheel==0.45.1 \ - expandvars==1.0.0 \ - cython==3.1.1 \ - setuptools_rust==1.11.1 \ - typing-extensions==4.15.0 \ - poetry-core==2.2.1 \ + # Per-recipe cache for the parallel package installs so six servers do not + # contend on one pip cache directory. + private_pip_cache="${B}/pip-cache" + mkdir -p "${private_pip_cache}" + if [ -n "${PIP_CACHE_DIR}" ] && [ -d "${PIP_CACHE_DIR}" ]; then + # Same-filesystem hardlink seed: fast reads, no shared writers during install. + cp -aln "${PIP_CACHE_DIR}/." "${private_pip_cache}/" 2>/dev/null || true + fi + private_pip_cache_args="--cache-dir ${private_pip_cache}" + bbnote "Using private pip cache at ${private_pip_cache}" + + pip_buildenv="${OPENTRONS_APP_BUNDLE_PIP_BUILDENV}" + mkdir -p "${pip_buildenv}" + bbnote "Using pip buildenv at ${pip_buildenv}" + + # Serialize population of a shared host buildenv across parallel recipes. + buildenv_spec="hatchling==1.27.0 hatch-vcs==0.5.0 hatch-vcs-tunable==0.0.1a3 hatch-dependency-coversion==0.0.1a4 hatch-fancy-pypi-readme==25.1.0 flit==3.12.0 flit-core==3.12.0 flit-scm==1.7.0 setuptools==80.9.0 setuptools-scm[toml]==9.2.1 wheel==0.45.1 expandvars==1.0.0 cython==3.1.1 setuptools_rust==1.11.1 typing-extensions==4.15.0 poetry-core==2.2.1" + ( + flock 200 + if [ ! -f "${pip_buildenv}/.opentrons-pip-buildenv-complete" ] || \ + [ "$(cat "${pip_buildenv}/.opentrons-pip-buildenv-complete")" != "${buildenv_spec}" ]; then + bbnote "Installing host pip build tooling into ${pip_buildenv}" + ${PYTHON} -m pip install \ + ${shared_pip_cache_args} \ + -t ${pip_buildenv} \ + ${buildenv_spec} + printf '%s\n' "${buildenv_spec}" > "${pip_buildenv}/.opentrons-pip-buildenv-complete" + else + bbnote "Reusing existing pip buildenv at ${pip_buildenv}" + fi + ) 200>"${pip_buildenv}.lock" + bbnote "Installing pypi packages" - PATH=${B}/pip-buildenv/bin/:${PATH} ${PIP_ENVARGS} PYTHONPATH=${B}/pip-buildenv:${PYTHONPATH} ${PYTHON} -m pip install \ + PATH=${pip_buildenv}/bin/:${PATH} ${PIP_ENVARGS} PYTHONPATH=${pip_buildenv}:${PYTHONPATH} ${PYTHON} -m pip install \ + ${private_pip_cache_args} \ ${PIP_ARGS} \ -r ${B}/pypi.txt \ bbnote "Building and installing local packages" - PATH=${B}/pip-buildenv/bin/:${PATH} ${PIP_ENVARGS} ${OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL} PYTHONPATH=${B}/pip-buildenv:${PYTHONPATH} ${PYTHON} -m pip install \ + PATH=${pip_buildenv}/bin/:${PATH} ${PIP_ENVARGS} ${OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL} PYTHONPATH=${pip_buildenv}:${PYTHONPATH} ${PYTHON} -m pip install \ + ${private_pip_cache_args} \ -r ${B}/local.txt \ ${PIP_ARGS} \ bbnote "Building and installing true source packages" - PATH=${B}/pip-buildenv/bin/:${PATH} ${PIP_ENVARGS} ${OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL} PYTHONPATH=${B}/pip-buildenv:${PYTHONPATH} ${PYTHON} -m pip install \ + PATH=${pip_buildenv}/bin/:${PATH} ${PIP_ENVARGS} ${OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL} PYTHONPATH=${pip_buildenv}:${PYTHONPATH} ${PYTHON} -m pip install \ + ${private_pip_cache_args} \ ${OPENTRONS_APP_BUNDLE_PROJECT_ROOT} \ ${PIP_ARGS} \ + # Merge this recipe's pip downloads into the shared cache for S3 / later runs. + if [ -n "${PIP_CACHE_DIR}" ]; then + ( + flock 200 + mkdir -p "${PIP_CACHE_DIR}" + cp -a "${private_pip_cache}/." "${PIP_CACHE_DIR}/" + ) 200>"${PIP_CACHE_DIR}.lock" + bbnote "Merged private pip cache into ${PIP_CACHE_DIR}" + fi + bbnote "Done installing python packages" } -do_compile[vardeps] += "OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL" +do_compile[vardeps] += "OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL PIP_CACHE_DIR OPENTRONS_APP_BUNDLE_PIP_NO_BINARY" do_compile[dirs] += " ${OPENTRONS_APP_BUNDLE_SOURCE_VENV}" do_install () { diff --git a/oelint-constants.json b/oelint-constants.json index 25b24c13..307ed236 100644 --- a/oelint-constants.json +++ b/oelint-constants.json @@ -6,6 +6,12 @@ "MONOREPO_ROOT_PATH", "OT_SYSTEM_VERSION", "OPENTRONS_APP_BUNDLE_USE_GLOBAL", + "OPENTRONS_APP_BUNDLE_PIP_NO_BINARY", + "OPENTRONS_APP_BUNDLE_PIP_NO_BINARY_ARG", + "OPENTRONS_APP_BUNDLE_PIP_BUILDENV", + "PIP_CACHE_DIR", + "PNPM_CACHE_DIR", + "ELECTRON_CACHE_DIR", "target_is_armv7", "llvm_features_from_tune", "llvm_features",