Updating Daft links in Ray documentation

.bazelrc

@@ -1,6 +1,8 @@
 # Must be first. Enables build:windows, build:linux, build:macos, build:freebsd, build:openbsd
 build --enable_platform_specific_config
 
+build:linux --workspace_status_command="bash ./bazel/workspace_status.sh"
+
 # Provides users an option to turn on strict action env.
 # TODO(aslonnie): make this default; fix the python tests..
 build:strict --incompatible_strict_action_env

.buildkite/cicd.rayci.yml

@@ -14,6 +14,19 @@ steps:
       - oss-ci-base_test
       - forge
     tags: tools
+  - label: ":coral: reef: raydepsets tests"
+    key: raydepsets-tests
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker --
+          //ci/raydepsets/... ci
+          --cache-test-results
+          --build-name oss-ci-base_test
+          --build-type skip
+    instance_type: small
+    depends_on:
+      - oss-ci-base_test
+      - forge
+    tags: tools
   - label: ":coral: reef: privileged container tests"
     commands:
       - bazel run //ci/ray_ci:test_in_docker --

.buildkite/core.rayci.yml

@@ -181,9 +181,9 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker --
         python/ray/util/dask/... core
         --install-mask all-ray-libraries
-        --build-name datalbuild
+        --build-name databuild-py3.12
     depends_on:
-      - datalbuild
+      - databuild-multipy
       - forge
 
   - label: ":ray: core: modin tests"

.buildkite/data.rayci.yml

@@ -152,9 +152,9 @@ steps:
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
-        --build-name datalbuild
+        --build-name databuild-py3.12
         --only-tags dask
-    depends_on: datalbuild
+    depends_on: databuild-multipy
 
   - label: ":database: data: TFRecords (tfx-bsl) tests"
     tags:
@@ -193,10 +193,10 @@ steps:
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //doc/... data
-        --build-name datalbuild
+        --build-name databuild-py3.12
         --only-tags dask
         --parallelism-per-worker 2
-    depends_on: datalbuild
+    depends_on: databuild-multipy
 
   - label: ":database: data: doc gpu tests"
     tags:

.github/workflows/stale_pull_request.yaml

@@ -56,7 +56,8 @@ jobs:
           # Pull Requests with these labels will never be considered stale
           exempt-pr-labels: >
             weekly-release-blocker,
-            release-blocker
+            release-blocker,
+            bounced
 
           # Set to true to ignore PRs in a milestone (defaults to false)
           exempt-all-pr-milestones: true
@@ -68,4 +69,7 @@ jobs:
           # Remove stale label from PRs on update (default is true)
           remove-pr-stale-when-updated: true
 
+          # Add bounced label. Whenever a PR is marked as 'bounced' it will not be marked stale again.
+          labels-to-add-when-unstale: bounced
+
           ascending: true

.rayciversion

@@ -1 +1 @@
-0.12.0
+0.16.0

WORKSPACE

@@ -102,6 +102,19 @@ filegroup(
     urls = ["https://github.com/distribution/distribution/releases/download/v3.0.0/registry_3.0.0_linux_amd64.tar.gz"],
 )
 
+http_archive(
+    name = "uv_x86_64",
+    build_file_content = """
+    filegroup(
+        name = "file",
+        srcs = glob(["**"]),
+        visibility = ["//visibility:public"],
+)
+""",
+    sha256 = "1785537fc65a35609dc33063b5f1cc85437a08ade4c0a832071c018481afe515",
+    urls = ["https://github.com/astral-sh/uv/releases/download/0.7.19/uv-i686-unknown-linux-gnu.tar.gz"],
+)
+
 http_archive(
     name = "com_github_storypku_bazel_iwyu",
     sha256 = "aa78c331a2cb139f73f7d74eeb4d5ab29794af82023ef5d6d5194f76b7d37449",

bazel/workspace_status.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ "${USER:-}" =~ "@" ]]; then
+	echo "ERROR: \$USER ('${USER:-}') contains invalid char '@'" >&2
+    exit 1
+fi
+
+if [[ "${HOME:-}" =~ "@" ]]; then
+	echo "ERROR: \$HOME ('${HOME:-}') contains invalid char '@'" >&2
+    exit 1
+fi

ci/build/test-macos-wheels.sh

@@ -90,7 +90,7 @@ for ((i=0; i<${#PY_MMS[@]}; ++i)); do
   "$PIP_CMD" install -q "$PYTHON_WHEEL"
 
   # Install the dependencies to run the tests.
-  "$PIP_CMD" install -q aiohttp numpy 'pytest==7.0.1' requests proxy.py
+  "$PIP_CMD" install -q aiohttp numpy 'pytest==7.4.4' requests proxy.py
 
   # Run a simple test script to make sure that the wheel works.
   # We set the python path to prefer the directory of the wheel content: https://github.com/ray-project/ray/pull/30090

ci/docker/min.build.Dockerfile

@@ -20,7 +20,7 @@ MINIMAL_INSTALL=1 PYTHON=${PYTHON_VERSION} ci/env/install-dependencies.sh
 rm -rf python/ray/thirdparty_files
 
 # install test requirements
-python -m pip install -U pytest==7.0.1 pip-tools==7.3.0
+python -m pip install -U pytest==7.4.4 pip-tools==7.4.1
 
 # install extra dependencies
 if [[ "${EXTRA_DEPENDENCY}" == "core" ]]; then

ci/env/install-dependencies.sh

@@ -276,24 +276,6 @@ install_pip_packages() {
     requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/tune-test-requirements.txt")
   fi
 
-  # Additional dependency for Ludwig.
-  # This cannot be included in requirements files as it has conflicting
-  # dependencies with Modin.
-  if [[ "${INSTALL_LUDWIG-}" == 1 ]]; then
-    # TODO: eventually pin this to master.
-    requirements_packages+=("ludwig[test]>=0.4")
-    requirements_packages+=("jsonschema>=4")
-  fi
-
-  # Additional dependency for time series libraries.
-  # This cannot be included in tune-requirements.txt as it has conflicting
-  # dependencies.
-  if [[ "${INSTALL_TIMESERIES_LIBS-}" == 1 ]]; then
-    requirements_packages+=("statsforecast==1.5.0")
-    requirements_packages+=("prophet==1.1.1")
-    requirements_packages+=("holidays==0.24") # holidays 0.25 causes `import prophet` to fail.
-  fi
-
   # Data processing test dependencies.
   if [[ "${DATA_PROCESSING_TESTING-}" == 1 || "${DOC_TESTING-}" == 1 ]]; then
     requirements_files+=("${WORKSPACE_DIR}/python/requirements/ml/data-requirements.txt")

ci/lint/pydoclint-baseline.txt

@@ -390,7 +390,6 @@ python/ray/air/_internal/torch_utils.py
     DOC103: Function `convert_ndarray_batch_to_torch_tensor_batch`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [dtypes: Optional[Union[torch.dtype, Dict[str, torch.dtype]]], ndarrays: Union[np.ndarray, Dict[str, np.ndarray]]]. Arguments in the docstring but not in the function signature: [dtype: , ndarray: ].
     DOC201: Function `convert_ndarray_batch_to_torch_tensor_batch` does not have a return section in docstring
     DOC201: Function `consume_prefix_in_state_dict_if_present_not_in_place` does not have a return section in docstring
-    DOC201: Function `convert_ndarray_list_to_torch_tensor_list` does not have a return section in docstring
 --------------------
 python/ray/air/_internal/uri_utils.py
     DOC101: Method `URI.rstrip_subpath`: Docstring contains fewer arguments than in function signature.
@@ -1203,7 +1202,6 @@ python/ray/data/_internal/execution/streaming_executor_state.py
     DOC201: Method `OpBufferQueue.has_next` does not have a return section in docstring
     DOC101: Method `OpState.get_output_blocking`: Docstring contains fewer arguments than in function signature.
     DOC103: Method `OpState.get_output_blocking`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [output_split_idx: Optional[int]].
-    DOC103: Function `process_completed_tasks`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [resource_manager: ResourceManager]. Arguments in the docstring but not in the function signature: [backpressure_policies: ].
 --------------------
 python/ray/data/_internal/iterator/stream_split_iterator.py
     DOC101: Method `SplitCoordinator.start_epoch`: Docstring contains fewer arguments than in function signature.
@@ -1487,8 +1485,8 @@ python/ray/experimental/channel/shared_memory_channel.py
     DOC101: Method `CompositeChannel.__init__`: Docstring contains fewer arguments than in function signature.
     DOC103: Method `CompositeChannel.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [_channel_dict: Optional[Dict[ray.ActorID, ChannelInterface]], _channels: Optional[Set[ChannelInterface]], _reader_registered: bool, _writer_registered: bool].
 --------------------
-python/ray/experimental/channel/torch_tensor_nccl_channel.py
-    DOC201: Method `TorchTensorNcclChannel._recv_cpu_and_gpu_data` does not have a return section in docstring
+python/ray/experimental/channel/torch_tensor_accelerator_channel.py
+    DOC201: Method `TorchTensorAcceleratorChannel._recv_cpu_and_gpu_data` does not have a return section in docstring
     DOC201: Function `_get_ranks` does not have a return section in docstring
     DOC201: Function `_init_communicator` does not have a return section in docstring
 --------------------
@@ -2128,10 +2126,6 @@ python/ray/train/v2/api/context.py
     DOC201: Method `TrainContext.get_local_world_size` does not have a return section in docstring
     DOC201: Method `TrainContext.get_node_rank` does not have a return section in docstring
 --------------------
-python/ray/train/v2/api/train_fn_utils.py
-    DOC101: Function `report`: Docstring contains fewer arguments than in function signature.
-    DOC103: Function `report`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [checkpoint_dir_name: Optional[str]].
---------------------
 python/ray/train/v2/lightgbm/lightgbm_trainer.py
     DOC101: Method `LightGBMTrainer.__init__`: Docstring contains fewer arguments than in function signature.
     DOC103: Method `LightGBMTrainer.__init__`: Docstring arguments are different from function arguments. (Or could be other formatting issues: https://jsh9.github.io/pydoclint/violation_codes.html#notes-on-doc103 ). Arguments in the function signature but not in the docstring: [label_column: Optional[str], num_boost_round: Optional[int], params: Optional[Dict[str, Any]]].

ci/ray_ci/automation/ray_wheels_lib.py

@@ -17,7 +17,7 @@
     "manylinux2014_x86_64",
     "manylinux2014_aarch64",
     "macosx_12_0_x86_64",
-    "macosx_11_0_arm64",
+    "macosx_12_0_arm64",
     "win_amd64",
 ]
 RAY_TYPES = ["ray", "ray_cpp"]

ci/ray_ci/automation/test_pypi_lib.py

@@ -58,8 +58,8 @@ def test_get_pypi_token_fail(mock_boto3_client):
 def test_upload_wheels_to_pypi(mock_subprocess, mock_get_pypi_url, mock_get_pypi_token):
     pypi_env = "test"
     wheels = [
-        "ray_cpp-2.9.3-cp310-cp310-macosx_11_0_arm64.whl",
-        "ray_cpp-2.9.3-cp311-cp311-macosx_11_0_arm64.whl",
+        "ray_cpp-2.9.3-cp310-cp310-macosx_12_0_arm64.whl",
+        "ray_cpp-2.9.3-cp311-cp311-macosx_12_0_arm64.whl",
     ]
     mock_get_pypi_token.return_value = "test_token"
     mock_get_pypi_url.return_value = "test_pypi_url"
@@ -97,8 +97,8 @@ def test_upload_wheels_to_pypi_fail_twine_upload(
 ):
     pypi_env = "test"
     wheels = [
-        "ray_cpp-2.9.3-cp310-cp310-macosx_11_0_arm64.whl",
-        "ray_cpp-2.9.3-cp311-cp311-macosx_11_0_arm64.whl",
+        "ray_cpp-2.9.3-cp310-cp310-macosx_12_0_arm64.whl",
+        "ray_cpp-2.9.3-cp311-cp311-macosx_12_0_arm64.whl",
     ]
     mock_get_pypi_token.return_value = "test_token"
     mock_get_pypi_url.return_value = "test_pypi_url"
@@ -117,8 +117,8 @@ def test_upload_wheels_to_pypi_fail_twine_upload(
 def test_upload_wheels_to_pypi_fail_get_pypi(mock_get_pypi_url, mock_get_pypi_token):
     pypi_env = "test"
     wheels = [
-        "ray_cpp-2.9.3-cp310-cp310-macosx_11_0_arm64.whl",
-        "ray_cpp-2.9.3-cp311-cp311-macosx_11_0_arm64.whl",
+        "ray_cpp-2.9.3-cp310-cp310-macosx_12_0_arm64.whl",
+        "ray_cpp-2.9.3-cp311-cp311-macosx_12_0_arm64.whl",
     ]
     mock_get_pypi_token.side_effect = ValueError("Invalid pypi_env: test")
     mock_get_pypi_url.side_effect = ValueError("Invalid pypi_env: test")

ci/ray_ci/automation/test_ray_wheels_lib.py

@@ -21,7 +21,7 @@
     "ray-1.0.0-cp39-cp39-manylinux2014_x86_64",
     "ray-1.0.0-cp39-cp39-manylinux2014_aarch64",
     "ray-1.0.0-cp39-cp39-macosx_12_0_x86_64",
-    "ray-1.0.0-cp39-cp39-macosx_11_0_arm64",
+    "ray-1.0.0-cp39-cp39-macosx_12_0_arm64",
     "ray-1.0.0-cp39-cp39-win_amd64",
 ]
 
@@ -58,7 +58,7 @@ def test_check_downloaded_wheels():
             "ray-1.0.0-cp39-cp39-manylinux2014_x86_64",
             "ray-1.0.0-cp39-cp39-manylinux2014_aarch64",
             "ray-1.0.0-cp39-cp39-macosx_12_0_x86_64",
-            "ray-1.0.0-cp39-cp39-macosx_11_0_arm64",
+            "ray-1.0.0-cp39-cp39-macosx_12_0_arm64",
             "ray-1.0.0-cp39-cp39-win_amd64",
         ]
 
@@ -75,7 +75,7 @@ def test_check_downloaded_wheels_fail():
             "ray-1.0.0-cp39-cp39-manylinux2014_x86_64",
             "ray-1.0.0-cp39-cp39-manylinux2014_aarch64",
             "ray-1.0.0-cp39-cp39-macosx_12_0_x86_64",
-            "ray-1.0.0-cp39-cp39-macosx_11_0_arm64",
+            "ray-1.0.0-cp39-cp39-macosx_12_0_arm64",
             "ray-1.0.0-cp39-cp39-win_amd64",
         ]
 
@@ -94,7 +94,7 @@ def test_download_wheel_from_s3(mock_boto3_client):
             "releases/1.0.0/1234567/ray-1.0.0-cp39-cp39-manylinux2014_x86_64.whl",
             "releases/1.0.0/1234567/ray-1.0.0-cp39-cp39-manylinux2014_aarch64.whl",
             "releases/1.0.0/1234567/ray-1.0.0-cp39-cp39-macosx_12_0_x86_64.whl",
-            "releases/1.0.0/1234567/ray-1.0.0-cp39-cp39-macosx_11_0_arm64.whl",
+            "releases/1.0.0/1234567/ray-1.0.0-cp39-cp39-macosx_12_0_arm64.whl",
             "releases/1.0.0/1234567/ray-1.0.0-cp39-cp39-win_amd64.whl",
         ]
         for key in keys:

ci/raydepsets/BUILD.bazel

@@ -0,0 +1,32 @@
+load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement")
+
+py_library(
+    name = "raydepsets_lib",
+    srcs = [
+        "cli.py",
+    ],
+    deps = [
+        ci_require("click"),
+    ],
+)
+
+py_binary(
+    name = "raydepsets",
+    srcs = ["raydepsets.py"],
+    deps = [":raydepsets_lib"],
+    exec_compatible_with = ["//:hermetic_python"],
+)
+
+py_test(
+    name = "test_cli",
+    srcs = ["tests/test_cli.py"],
+    exec_compatible_with = ["//:hermetic_python"],
+    deps = [
+        ci_require("pytest"),
+        ":raydepsets_lib",
+    ],
+    tags = [
+        "ci_unit",
+        "team:ci",
+    ],
+)

ci/raydepsets/cli.py

@@ -0,0 +1,8 @@
+import click
+
+
+@click.group(name="depsets")
+@click.pass_context
+def cli(ctx):
+    """Manage Python dependency sets."""
+    pass

ci/raydepsets/raydepsets.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+from cli import cli
+
+if __name__ == "__main__":
+    cli()

ci/raydepsets/tests/test_cli.py

@@ -0,0 +1,10 @@
+import pytest
+import sys
+
+
+def test_cli():
+    pass
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-v", __file__]))