feat(build): STEP 5: shared pip-buildenv for Opentrons app recipes

.github/workflows/build-ot3-actions.yml

@@ -364,6 +364,8 @@ jobs:
           cd ..
       - name: Apply unconditional CI config overrides
         run: |
+          cachedir=${LOCAL_CACHE:-./cache}
+          mkdir -p "${cachedir}"/{pip,pip-buildenv}
           cd oe-core
           echo "" >> ./conf/local.conf
           echo 'DL_DIR = "/volumes/cache/downloads"' >> ./conf/local.conf
@@ -372,6 +374,7 @@ jobs:
           echo 'OT_BUILD_TYPE = "${{needs.decide-refs.outputs.build-type}}"' >> ./conf/local.conf
           echo 'PNPM_CACHE_DIR = "/volumes/cache/pnpm"' >> ./conf/local.conf
           echo 'ELECTRON_CACHE_DIR = "/volumes/cache/electron"' >> ./conf/local.conf
+          echo 'PIP_CACHE_DIR = "/volumes/cache/pip"' >> ./conf/local.conf
           echo 'CONNECTIVITY_CHECK_URIS = "https://www.yoctoproject.org/connectivity.html"' >> ./conf/local.conf
           echo 'OT_BUILD_TARGET = "odd"' >> ./conf/local.conf
           echo 'OT_SENTRY_AUTH_TOKEN = "${{ secrets.OT_SENTRY_AUTH_TOKEN_OE_CORE }}"' >> ./conf/local.conf

layers/meta-opentrons/classes/opentrons_app_bundle.bbclass

@@ -50,6 +50,20 @@ OPENTRONS_APP_BUNDLE_SOURCE_VENV := "${B}/build-venv"
 # Extra environment args to pass to pip when building local packages
 OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL ??= ""
 
+# Shared pip download/wheel cache (CI sets /volumes/cache/pip). Used under flock for
+# host buildenv installs and as a merge target after each recipe's private cache.
+# Parallel do_compile must not write this directory concurrently (pip lock contention).
+PIP_CACHE_DIR ??= ""
+
+# Host pip tooling tree shared across recipes when PIP_CACHE_DIR is set.
+# Parallel do_compile tasks flock on this directory so only one populates it.
+OPENTRONS_APP_BUNDLE_PIP_BUILDENV = "${@'${PIP_CACHE_DIR}/../pip-buildenv' if (d.getVar('PIP_CACHE_DIR') or '').strip() else '${B}/pip-buildenv'}"
+
+# Target package installs: default forces source builds for correct cross-arch.
+# Set to empty in conf to allow binary wheels (experiment only).
+OPENTRONS_APP_BUNDLE_PIP_NO_BINARY ??= ":all:"
+OPENTRONS_APP_BUNDLE_PIP_NO_BINARY_ARG = "${@'--no-binary %s' % d.getVar('OPENTRONS_APP_BUNDLE_PIP_NO_BINARY').strip() if (d.getVar('OPENTRONS_APP_BUNDLE_PIP_NO_BINARY') or '').strip() else ''}"
+
 PIP_ENVARGS := " \
    STAGING_INCDIR=${STAGING_INCDIR} \
    STAGING_LIBDIR=${STAGING_LIBDIR} \
@@ -138,7 +152,7 @@ do_rewrite_requirements[vardeps] += " OPENTRONS_APP_BUNDLE_USE_GLOBAL OPENTRONS_
 addtask do_rewrite_requirements after do_configure before do_compile
 
 do_configure:prepend () {
-   mkdir -p ${B}/pip-buildenv
+   mkdir -p ${OPENTRONS_APP_BUNDLE_PIP_BUILDENV}
    cd ${OPENTRONS_APP_BUNDLE_PROJECT_ROOT}
    bbplain "Getting dependencies in ${OPENTRONS_APP_BUNDLE_PROJECT_ROOT}"
    if [[ "${OPENTRONS_APP_BUNDLE_STRIP_HASHES}" = "no" ]] ; then
@@ -171,54 +185,92 @@ do_configure:prepend () {
 do_configure[vardeps] += "OPENTRONS_APP_BUNDLE_STRIP_HASHES OPENTRONS_APP_BUNDLE_PROJECT_ROOT"
 
 PIP_ARGS := "--no-compile \
-             --no-binary :all: \
+             ${OPENTRONS_APP_BUNDLE_PIP_NO_BINARY_ARG} \
              --progress-bar off \
              --force-reinstall \
              --no-deps \
              --no-build-isolation \
              -t ${OPENTRONS_APP_BUNDLE_SOURCE_VENV}"
 
 do_compile () {
-   mkdir -p ${B}/pip-buildenv
-
-   bbnote "Installing pypi packages"
+   # Host buildenv install may use the shared pip cache (single writer via flock).
+   shared_pip_cache_args=""
+   if [ -n "${PIP_CACHE_DIR}" ]; then
+      mkdir -p "${PIP_CACHE_DIR}"
+      shared_pip_cache_args="--cache-dir ${PIP_CACHE_DIR}"
+      bbnote "Shared pip cache at ${PIP_CACHE_DIR} (buildenv + post-merge only)"
+   fi
 
-   ${PYTHON} -m pip install \
-      -t ${B}/pip-buildenv \
-      hatchling==1.27.0 hatch-vcs==0.5.0 hatch-vcs-tunable==0.0.1a3 hatch-dependency-coversion==0.0.1a4 hatch-fancy-pypi-readme==25.1.0 \
-      flit==3.12.0 flit-core==3.12.0 flit-scm==1.7.0 \
-      setuptools==80.9.0 setuptools-scm[toml]==9.2.1 \
-      wheel==0.45.1 \
-      expandvars==1.0.0 \
-      cython==3.1.1 \
-      setuptools_rust==1.11.1 \
-      typing-extensions==4.15.0 \
-      poetry-core==2.2.1 \
+   # Per-recipe cache for the parallel package installs so six servers do not
+   # contend on one pip cache directory.
+   private_pip_cache="${B}/pip-cache"
+   mkdir -p "${private_pip_cache}"
+   if [ -n "${PIP_CACHE_DIR}" ] && [ -d "${PIP_CACHE_DIR}" ]; then
+      # Same-filesystem hardlink seed: fast reads, no shared writers during install.
+      cp -aln "${PIP_CACHE_DIR}/." "${private_pip_cache}/" 2>/dev/null || true
+   fi
+   private_pip_cache_args="--cache-dir ${private_pip_cache}"
+   bbnote "Using private pip cache at ${private_pip_cache}"
+
+   pip_buildenv="${OPENTRONS_APP_BUNDLE_PIP_BUILDENV}"
+   mkdir -p "${pip_buildenv}"
+   bbnote "Using pip buildenv at ${pip_buildenv}"
+
+   # Serialize population of a shared host buildenv across parallel recipes.
+   buildenv_spec="hatchling==1.27.0 hatch-vcs==0.5.0 hatch-vcs-tunable==0.0.1a3 hatch-dependency-coversion==0.0.1a4 hatch-fancy-pypi-readme==25.1.0 flit==3.12.0 flit-core==3.12.0 flit-scm==1.7.0 setuptools==80.9.0 setuptools-scm[toml]==9.2.1 wheel==0.45.1 expandvars==1.0.0 cython==3.1.1 setuptools_rust==1.11.1 typing-extensions==4.15.0 poetry-core==2.2.1"
+   (
+      flock 200
+      if [ ! -f "${pip_buildenv}/.opentrons-pip-buildenv-complete" ] || \
+         [ "$(cat "${pip_buildenv}/.opentrons-pip-buildenv-complete")" != "${buildenv_spec}" ]; then
+         bbnote "Installing host pip build tooling into ${pip_buildenv}"
+         ${PYTHON} -m pip install \
+            ${shared_pip_cache_args} \
+            -t ${pip_buildenv} \
+            ${buildenv_spec}
+         printf '%s\n' "${buildenv_spec}" > "${pip_buildenv}/.opentrons-pip-buildenv-complete"
+      else
+         bbnote "Reusing existing pip buildenv at ${pip_buildenv}"
+      fi
+   ) 200>"${pip_buildenv}.lock"
 
+   bbnote "Installing pypi packages"
 
-   PATH=${B}/pip-buildenv/bin/:${PATH} ${PIP_ENVARGS} PYTHONPATH=${B}/pip-buildenv:${PYTHONPATH} ${PYTHON} -m pip install \
+   PATH=${pip_buildenv}/bin/:${PATH} ${PIP_ENVARGS} PYTHONPATH=${pip_buildenv}:${PYTHONPATH} ${PYTHON} -m pip install \
+      ${private_pip_cache_args} \
       ${PIP_ARGS} \
       -r ${B}/pypi.txt \
 
 
    bbnote "Building and installing local packages"
 
-   PATH=${B}/pip-buildenv/bin/:${PATH} ${PIP_ENVARGS} ${OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL} PYTHONPATH=${B}/pip-buildenv:${PYTHONPATH} ${PYTHON} -m pip install \
+   PATH=${pip_buildenv}/bin/:${PATH} ${PIP_ENVARGS} ${OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL} PYTHONPATH=${pip_buildenv}:${PYTHONPATH} ${PYTHON} -m pip install \
+      ${private_pip_cache_args} \
       -r ${B}/local.txt \
       ${PIP_ARGS} \
 
 
    bbnote "Building and installing true source packages"
 
-   PATH=${B}/pip-buildenv/bin/:${PATH} ${PIP_ENVARGS} ${OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL} PYTHONPATH=${B}/pip-buildenv:${PYTHONPATH} ${PYTHON} -m pip install \
+   PATH=${pip_buildenv}/bin/:${PATH} ${PIP_ENVARGS} ${OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL} PYTHONPATH=${pip_buildenv}:${PYTHONPATH} ${PYTHON} -m pip install \
+      ${private_pip_cache_args} \
       ${OPENTRONS_APP_BUNDLE_PROJECT_ROOT} \
       ${PIP_ARGS} \
 
 
+   # Merge this recipe's pip downloads into the shared cache for S3 / later runs.
+   if [ -n "${PIP_CACHE_DIR}" ]; then
+      (
+         flock 200
+         mkdir -p "${PIP_CACHE_DIR}"
+         cp -a "${private_pip_cache}/." "${PIP_CACHE_DIR}/"
+      ) 200>"${PIP_CACHE_DIR}.lock"
+      bbnote "Merged private pip cache into ${PIP_CACHE_DIR}"
+   fi
+
    bbnote "Done installing python packages"
 }
 
-do_compile[vardeps] += "OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL"
+do_compile[vardeps] += "OPENTRONS_APP_BUNDLE_EXTRA_PIP_ENVARGS_LOCAL PIP_CACHE_DIR OPENTRONS_APP_BUNDLE_PIP_NO_BINARY"
 do_compile[dirs] += " ${OPENTRONS_APP_BUNDLE_SOURCE_VENV}"
 
 do_install () {

oelint-constants.json

@@ -6,6 +6,12 @@
       "MONOREPO_ROOT_PATH",
       "OT_SYSTEM_VERSION",
       "OPENTRONS_APP_BUNDLE_USE_GLOBAL",
+      "OPENTRONS_APP_BUNDLE_PIP_NO_BINARY",
+      "OPENTRONS_APP_BUNDLE_PIP_NO_BINARY_ARG",
+      "OPENTRONS_APP_BUNDLE_PIP_BUILDENV",
+      "PIP_CACHE_DIR",
+      "PNPM_CACHE_DIR",
+      "ELECTRON_CACHE_DIR",
       "target_is_armv7",
       "llvm_features_from_tune",
       "llvm_features",