[ci][deps][1/3] PY313 DEP UNIFICATION: compiling requirements_compiled_py3.13.txt and depsets

.buildkite/dependencies.rayci.yml

@@ -19,6 +19,22 @@ steps:
     job_env: oss-ci-base_test-py3.11
     depends_on: oss-ci-base_test-multipy(*)
 
+  - label: ":tapioca: build: pip-compile py3.13 dependencies"
+    key: pip_compile_313_dependencies
+    tags: always
+    instance_type: small
+    commands:
+      # uncomment the following line to update the pinned versions of pip dependencies
+      # to the latest versions; otherwise, the pinned versions will be re-used as much
+      # as possible
+      # - rm ./python/requirements_compiled_py3.13.txt
+      - cp ./python/requirements_compiled_py3.13.txt requirements_compiled_py3.13_backup.txt
+      - ./ci/ci.sh compile_313_pip_dependencies
+      - cp -f ./python/requirements_compiled_py3.13.txt /artifact-mount/
+      - diff ./python/requirements_compiled_py3.13.txt requirements_compiled_py3.13_backup.txt || (echo "requirements_compiled_py3.13.txt is not up to date. Please download it from Artifacts tab and git push the changes." && exit 1)
+    job_env: oss-ci-base_test-py3.11
+    depends_on: oss-ci-base_test-multipy(*)
+
   - label: ":tapioca: build: raydepsets: compile all dependencies"
     key: raydepsets_compile_all_dependencies
     tags: python_dependencies

ci/ci.sh

@@ -79,6 +79,76 @@ compile_pip_dependencies() {
   )
 }
 
+compile_313_pip_dependencies() {
+  # Compile boundaries for the py3.13 dependency set. Uses the py313 override
+  # directories (python/requirements/py313/ and python/requirements/ml/py313/)
+  # where available, and falls back to the shared requirement files otherwise.
+  TARGET="${1-requirements_compiled_py3.13.txt}"
+
+  if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then
+    # Resolution currently does not work on aarch64 as some pinned packages
+    # are not available. Once they are reasonably upgraded we should be able
+    # to enable this here.
+    echo "Skipping for aarch64"
+    return 0
+  fi
+
+  (
+    # shellcheck disable=SC2262
+    alias pip="python -m pip"
+
+    cd "${WORKSPACE_DIR}"
+
+    echo "Target file: $TARGET"
+    pip install "pip-tools==7.4.1" "wheel==0.45.1"
+
+    # Required packages to lookup e.g. dragonfly-opt
+    HAS_TORCH=0
+    python -c "import torch" 2>/dev/null && HAS_TORCH=1
+    pip install --no-cache-dir numpy torch
+
+    pip-compile --verbose --resolver=backtracking \
+      --pip-args --no-deps --strip-extras --no-header \
+      --unsafe-package ray \
+      --unsafe-package pip \
+      --unsafe-package setuptools \
+      -o "python/$TARGET" \
+      python/requirements.txt \
+      python/requirements/lint-requirements.txt \
+      python/requirements/py313/test-requirements.txt \
+      python/requirements/cloud-requirements.txt \
+      python/requirements/docker/ray-docker-requirements.txt \
+      python/requirements/ml/py313/core-requirements.txt \
+      python/requirements/ml/py313/data-requirements.txt \
+      python/requirements/ml/py313/data-test-requirements.txt \
+      python/requirements/ml/py313/dl-cpu-requirements.txt \
+      python/requirements/ml/py313/ml-requirements.txt \
+      python/requirements/ml/py313/third_party.txt \
+      python/requirements/ml/py313/rllib-requirements.txt \
+      python/requirements/ml/py313/rllib-test-requirements.txt \
+      python/requirements/ml/py313/train-requirements.txt \
+      python/requirements/ml/py313/train-test-requirements.txt \
+      python/requirements/ml/py313/tune-requirements.txt \
+      python/requirements/ml/py313/tune-test-requirements.txt \
+      python/requirements/security-requirements.txt
+
+    # Delete local installation
+    sed -i "/@ file/d" "python/$TARGET"
+
+    # Remove +cpu and +pt20cpu suffixes e.g. for torch dependencies
+    # This is needed because we specify the requirements as torch==version, but
+    # the resolver adds the device-specific version tag. If this is not removed,
+    # pip install will complain about irresolvable constraints.
+    sed -i -E 's/==([\.0-9]+)\+[^\b]*cpu/==\1/g' "python/$TARGET"
+
+    cat "python/$TARGET"
+
+    if [[ "$HAS_TORCH" == "0" ]]; then
+      pip uninstall -y torch
+    fi
+  )
+}
+
 test_cpp() {
   if [[ "${OSTYPE}" == darwin* ]]; then
     echo "use macos_ci.sh to run cpp tests"

ci/raydepsets/configs/ci_data.depsets.yaml

@@ -42,10 +42,17 @@ depsets:
       - pyarrow
       - numpy
       - datasets
+      - delta-sharing
       - hudi
       - pylance
       - pandas
       - modin
+      # datasets==2.14.4 (pyarrow-v9) caps dill<0.3.8, but the py313 lock pins
+      # dill==0.4.1 via pymoo, and multiprocess==0.70.19 via datasets/evaluate
+      # requires dill>=0.4.1. Relax both so the v9 expand can downgrade the
+      # datasets→multiprocess→dill trio together.
+      - dill
+      - multiprocess
     output: python/deplocks/ci/relaxed_data-ci_depset_py${PYTHON_VERSION}.lock
     build_arg_sets:
       - py310
@@ -94,7 +101,6 @@ depsets:
       - --python-version=${PYTHON_VERSION}
       - --python-platform=linux
       - --unsafe-package ray
-      - --override python/requirements/data/mongo.txt
     build_arg_sets:
       - py310
 
@@ -118,6 +124,7 @@ depsets:
     packages:
     # these packages are relaxed for alternate fixed-pyarrow depsets
       - datasets
+      - delta-sharing
       - tensorflow-metadata
       - tensorflow
       - tf-keras
@@ -138,6 +145,18 @@ depsets:
       - ml-dtypes
       - cudf-cu12
       - pymars
+      # tfx-bsl up through 1.17.1 caps absl-py<2.0.0, but the py3.13 lock has
+      # absl-py==2.4.0 from transitive resolution. Relax so uv picks an absl-py
+      # 1.x version that tfx-bsl accepts.
+      - absl-py
+      # grpcio-status / grpcio-tools >=1.72 require protobuf>=6, but tfx-bsl 1.16.1
+      # caps protobuf<6. Relax so uv can drop to 1.71.x (grpcio itself has no
+      # protobuf pin but typically moves in lockstep).
+      - grpcio-status
+      # contourpy 1.3.3 (gated py>=3.11 in the lock) needs numpy>=1.25, but
+      # apache-beam 2.53.0 caps numpy<1.25.
+      - contourpy
+      - scipy
     output: python/deplocks/ci/relaxed_data_tfxbsl-ci_depset_py${PYTHON_VERSION}.lock
     build_arg_sets:
       - py311