Merge branch 'master' into patch-1

Author: ccmao1130

.buildkite/BUILD.bazel

@@ -1,5 +1,5 @@
-load("@rules_python//python:defs.bzl", "py_binary")
 load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement")
+load("@rules_python//python:defs.bzl", "py_binary")
 
 py_binary(
     name = "copy_files",

.buildkite/build.rayci.yml

@@ -115,12 +115,14 @@ steps:
       - "3.9"
       - "3.10"
       - "3.11"
+
   - label: ":tapioca: smoke test build-docker.sh"
     tags:
       - python_dependencies
       - docker
       - oss
     instance_type: medium
+    soft_fail: true
     commands:
       - export WHEEL_URL="https://files.pythonhosted.org/packages/7e/7c/3544cca730265bb6f7a4900b7b7018c08ce5ec89bf7b6102901fe0bcd67b/ray-2.44.1-cp39-cp39-manylinux2014_x86_64.whl"
       - export CPP_WHEEL_URL="https://files.pythonhosted.org/packages/50/ae/c094818fd526bfb0a361a76bda5708a73cbc888b51edfc7d6aab9de837cd/ray_cpp-2.44.1-cp39-cp39-manylinux2014_x86_64.whl"

.buildkite/data.rayci.yml

@@ -44,7 +44,7 @@ steps:
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name data9build
-        --except-tags data_integration,doctest,data_non_parallel,dask
+        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials
     depends_on: data9build
 
   - label: ":database: data: arrow v9 tests (data_non_parallel)"
@@ -69,7 +69,7 @@ steps:
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name datalbuild
-        --except-tags data_integration,doctest,data_non_parallel,dask
+        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials
     depends_on: datalbuild
 
   - label: ":database: data: arrow v20 tests (data_non_parallel)"
@@ -94,7 +94,7 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
         --workers 2 --worker-id {{matrix.worker_id}} --parallelism-per-worker 3
         --python-version {{matrix.python}}
-        --except-tags data_integration,doctest,data_non_parallel,dask
+        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials
     depends_on: databuild-multipy
     matrix:
       setup:
@@ -128,7 +128,7 @@ steps:
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}"
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name datanbuild
-        --except-tags data_integration,doctest,data_non_parallel,dask
+        --except-tags data_integration,doctest,data_non_parallel,dask,needs_credentials
     depends_on: datanbuild
     soft_fail: true
 
@@ -272,3 +272,19 @@ steps:
         --build-name docgpubuild
         --only-tags gpu,gpu_only
     depends_on: docgpubuild
+  - label: ":data: postmerge authenticated tests"
+    key: data_postmerge_authenticated_tests
+    tags:
+      - python
+      - data
+      - oss
+      - skip-on-premerge
+    instance_type: medium
+    commands:
+      - pip install -U boto3==1.28.70 awscli==1.29.70
+      - $(python ci/env/setup_credentials.py)
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... data
+        --build-name datalbuild
+        --only-tags needs_credentials
+        --test-env=SNOWFLAKE_USER --test-env=SNOWFLAKE_ACCOUNT --test-env=SNOWFLAKE_DATABASE --test-env=SNOWFLAKE_SCHEMA --test-env=SNOWFLAKE_WAREHOUSE --test-env=SNOWFLAKE_PRIVATE_KEY
+    depends_on: datalbuild

.buildkite/macos/macos.rayci.yml

@@ -39,7 +39,7 @@ steps:
       - macos_wheels
       - oss
     job_env: MACOS
-    instance_type: macos
+    instance_type: macos-arm64
     commands:
       - ./ci/ray_ci/macos/macos_ci.sh run_small_test
 
@@ -51,7 +51,7 @@ steps:
       - macos_wheels
       - oss
     job_env: MACOS
-    instance_type: macos
+    instance_type: macos-arm64
     commands:
       - ./ci/ray_ci/macos/macos_ci.sh run_medium_a_j_test
 
@@ -63,7 +63,7 @@ steps:
       - macos_wheels
       - oss
     job_env: MACOS
-    instance_type: macos
+    instance_type: macos-arm64
     commands:
       - ./ci/ray_ci/macos/macos_ci.sh run_medium_k_z_test
 
@@ -75,7 +75,7 @@ steps:
       - macos_wheels
       - oss
     job_env: MACOS
-    instance_type: macos
+    instance_type: macos-arm64
     parallelism: 3
     commands:
       - ./ci/ray_ci/macos/macos_ci.sh run_large_test
@@ -89,7 +89,7 @@ steps:
       - macos_wheels
       - oss
     job_env: MACOS
-    instance_type: macos
+    instance_type: macos-arm64
     commands:
       - ./ci/ray_ci/macos/macos_ci.sh run_core_dashboard_test
 
@@ -116,7 +116,7 @@ steps:
       - flaky
       - skip_on_premerge
     job_env: MACOS
-    instance_type: macos
+    instance_type: macos-arm64
     soft_fail: true
     commands:
       - ./ci/ray_ci/macos/macos_ci.sh run_flaky_tests

.buildkite/release-automation/pre_release.rayci.yml

@@ -36,6 +36,7 @@ steps:
       message: "Triggered by release-automation build #${BUILDKITE_BUILD_NUMBER}"
       env:
         RAYCI_RELEASE: 1
+        RAYCI_DISABLE_TEST_DB: "1"
 
   - label: "Trigger Postmerge nightly build & test"
     if: build.env("RAYCI_WEEKLY_RELEASE_NIGHTLY") == "1"
@@ -47,8 +48,9 @@ steps:
       branch: "${BUILDKITE_BRANCH}"
       message: "Triggered by release-automation build #${BUILDKITE_BUILD_NUMBER}"
       env:
-        RAYCI_RELEASE: 1
+        RAYCI_RELEASE: "1"
         RAYCI_SCHEDULE: "nightly"
+        RAYCI_DISABLE_TEST_DB: "1"
 
   - label: "Check Ray commit in {{matrix}} nightly images"
     key: check-ray-commit
@@ -72,7 +74,7 @@ steps:
       env:
         # KubeRay CI will pull an image based on this commit and the current date
         RAY_NIGHTLY_COMMIT: "${BUILDKITE_COMMIT}"
-        IS_FROM_RAY_RELEASE_AUTOMATION: 1
+        IS_FROM_RAY_RELEASE_AUTOMATION: "1"
 
   - label: "Trigger Postmerge MacOS test"
     key: trigger-postmerge-macos
@@ -83,7 +85,8 @@ steps:
       branch: "${BUILDKITE_BRANCH}"
       message: "Triggered by release-automation build #${BUILDKITE_BUILD_NUMBER}"
       env:
-        RAYCI_RELEASE: 1
+        RAYCI_RELEASE: "1"
+        RAYCI_DISABLE_TEST_DB: "1"
 
   - block: "Trigger Release nightly test"
     if: build.env("RAYCI_WEEKLY_RELEASE_NIGHTLY") != "1"

.buildkite/serve.rayci.yml

@@ -21,6 +21,9 @@ steps:
     wanda: ci/docker/servepydantic1.build.wanda.yaml
     tags: cibase
 
+  - name: servetracingbuild
+    wanda: ci/docker/servetracing.build.wanda.yaml
+
   - name: minbuild-serve
     label: "wanda: minbuild-{{matrix}}-py39"
     wanda: ci/docker/min.build.wanda.yaml
@@ -41,7 +44,7 @@ steps:
     instance_type: large
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
-        --except-tags post_wheel_build,gpu,ha_integration
+        --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name servebuild --test-env=EXPECTED_PYTHON_VERSION=3.9
     depends_on: "servebuild"
@@ -55,7 +58,7 @@ steps:
     soft_fail: true
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
-        --except-tags post_wheel_build,gpu,ha_integration
+        --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name servepydantic1build --test-env=EXPECTED_PYTHON_VERSION=3.9 --test-env=EXPECTED_PYDANTIC_VERSION=1.10.12
     depends_on: servepydantic1build
@@ -69,7 +72,7 @@ steps:
     soft_fail: true
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
-        --except-tags post_wheel_build,gpu,ha_integration
+        --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name servebuild --test-env=EXPECTED_PYTHON_VERSION=3.9 --test-env=RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD=0
     depends_on: "servebuild"
@@ -82,7 +85,7 @@ steps:
     instance_type: large
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
-        --except-tags post_wheel_build,gpu,ha_integration
+        --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers 2 --worker-id {{matrix.worker_id}} --parallelism-per-worker 3
         --python-version {{matrix.python}}
         --test-env=EXPECTED_PYTHON_VERSION={{matrix.python}}
@@ -185,6 +188,16 @@ steps:
       - raycpubase
       - servebuild
 
+  - label: ":ray-serve: serve: tracing tests"
+    tags:
+      - serve
+      - python
+    instance_type: medium
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
+        --only-tags serve_tracing --build-name servetracingbuild
+    depends_on: servetracingbuild
+
   - label: ":ray-serve: serve: doc gpu tests"
     tags:
       - serve

.github/CODEOWNERS

@@ -14,60 +14,26 @@
 
 # ==== Ray core ====
 
-# API compatibility
-/src/ray/protobuf/common.proto @pcmoritz @raulchen @ray-project/ray-core
-/src/ray/protobuf/gcs.proto @pcmoritz @raulchen @ray-project/ray-core
-/src/ray/protobuf/gcs_service.proto @pcmoritz @raulchen @ray-project/ray-core
-
-# Autoscaler
-/python/ray/autoscaler/ @ray-project/ray-core
-
-# Metrics
-/src/ray/stats/metric_defs.h @ray-project/ray-core
-/src/ray/stats/metric_defs.cc @ray-project/ray-core
-
-# Telemetry
-/src/ray/protobuf/usage.proto @pcmoritz @thomasdesr
-
 # All C++ code.
-# /src/ray @ray-project/ray-core-cpp
-
-# GCS
-/src/ray/gcs/ @ray-project/ray-core
-
-# Dependencies
-/python/setup.py @richardliaw @edoakes @aslonnie
-
-# CI
-/ci/lint/format.sh @ray-project/ray-ci
-/ci/docker @ray-project/ray-ci
-/ci/ray_ci @ray-project/ray-ci
+/src/ @ray-project/ray-core
 
 # Python worker.
-#/python/ray/ @ray-project/ray-core
-#!/python/ray/tune/ @ray-project/ray-core
-#!/python/ray/rllib/ @ray-project/ray-core
+/python/ray/ @ray-project/ray-core
 
 # Java worker.
-/java/dependencies.bzl @kfstorm @raulchen @WangTaoTheTonic @SongGuyang
-/java/pom.xml @kfstorm @raulchen @WangTaoTheTonic @SongGuyang
-/java/pom_template.xml @kfstorm @raulchen @WangTaoTheTonic @SongGuyang
-/java/*/pom_template.xml @kfstorm @raulchen @WangTaoTheTonic @SongGuyang
-/java/api/ @kfstorm @raulchen @WangTaoTheTonic @SongGuyang
+/java/ @kfstorm @raulchen @WangTaoTheTonic @SongGuyang @ray-project/ray-core
 
 # C++ worker
-/cpp/include/ray @SongGuyang @raulchen @kfstorm @ray-project/ray-core
-
-# Ray Client
-/src/ray/protobuf/ray_client.proto @ray-project/ray-core
+/cpp/ @SongGuyang @raulchen @kfstorm @ray-project/ray-core
 
-# Runtime Env
-# TODO(SongGuyang): Add new items to guarantee runtime env API compatibility in multiple languages.
-/src/ray/protobuf/runtime_env_common.proto @SongGuyang @raulchen @edoakes @ray-project/ray-core
-/src/ray/protobuf/runtime_env_agent.proto @SongGuyang @raulchen @edoakes @ray-project/ray-core
+/doc/source/cluster/ @ray-project/ray-core @ray-project/ray-docs
+/doc/source/ray-core/ @ray-project/ray-core @ray-project/ray-docs
 
 # ==== Libraries and frameworks ====
 
+# Dependencies
+/python/setup.py @richardliaw @edoakes @aslonnie
+
 # Common directory shared by core and the libraries.
 # @edoakes is the czar for now because the pattern is new.
 /python/ray/_common/ @edoakes @aslonnie
@@ -84,9 +50,6 @@
 /rllib/ @ray-project/ray-rllib
 /doc/source/rllib/ @ray-project/ray-rllib @ray-project/ray-docs
 
-# Cluster (docs)
-/doc/source/cluster/ @pcmoritz @kevin85421 @ray-project/ray-docs
-
 # Tune
 /python/ray/tune/ @ray-project/ray-tune
 /doc/source/tune/ @ray-project/ray-tune @ray-project/ray-docs
@@ -100,6 +63,7 @@
 
 # Ray Serve
 /python/ray/serve/ @ray-project/ray-serve
+/src/ray/protobuf/serve.proto @ray-project/ray-serve
 /doc/source/serve/ @ray-project/ray-serve @ray-project/ray-docs
 
 # ML Docker Dependencies
@@ -113,6 +77,7 @@
 # Ray usage stats
 /python/ray/_private/usage/ @edoakes @richardliaw @jjyao
 /python/ray/dashboard/modules/usage_stats/ @edoakes @richardliaw @jjyao
+/src/ray/protobuf/usage.proto @pcmoritz @thomasdesr
 
 
 # ==== Build and CI ====
@@ -125,6 +90,11 @@
 # CI scripts.
 #/ci/ @ray-project/ray-core @ray-project/ray-ci
 
+# CI
+/ci/lint/format.sh @ray-project/ray-ci
+/ci/docker @ray-project/ray-ci
+/ci/ray_ci @ray-project/ray-ci
+
 # Buildkite pipeline management
 .buildkite/hooks @ray-project/ray-ci
 

.github/workflows/stale_pull_request.yaml

@@ -57,7 +57,7 @@ jobs:
           exempt-pr-labels: >
             weekly-release-blocker,
             release-blocker,
-            bounced
+            unstale
 
           # Set to true to ignore PRs in a milestone (defaults to false)
           exempt-all-pr-milestones: true
@@ -69,7 +69,7 @@ jobs:
           # Remove stale label from PRs on update (default is true)
           remove-pr-stale-when-updated: true
 
-          # Add bounced label. Whenever a PR is marked as 'bounced' it will not be marked stale again.
-          labels-to-add-when-unstale: bounced
+          # Add unstale label. Whenever a PR is marked as 'unstale' it will not be marked stale again.
+          labels-to-add-when-unstale: unstale
 
           ascending: true

.pre-commit-config.yaml

@@ -73,7 +73,7 @@ repos:
     hooks:
       - id: cpplint
         args: ["--filter=-whitespace/braces,-whitespace/line_length,-build/c++11,-build/c++14,-build/c++17,-readability/braces,-whitespace/indent_namespace,-runtime/int,-runtime/references,-build/include_order"]
-        files: ^src/ray/(util|raylet_client|internal|scheduling|pubsub|object_manager|rpc(?:/.*)?|raylet|core_worker)/.*\.(h|cc)$
+        files: ^src/ray/(common/ray_syncer|util|raylet_client|internal|scheduling|pubsub|object_manager|rpc(?:/.*)?|raylet|core_worker)/.*\.(h|cc)$
         exclude: |
           (?x)^(
             src/ray/raylet/scheduling/.*\.(h|cc)$ |
@@ -84,9 +84,9 @@ repos:
     rev: 8.0.1
     hooks:
       - id: buildifier
-        files: ^(src|cpp|python|rllib)(/[^/]+)*/BUILD$
+        files: ^(src|cpp|python|rllib|ci|release|java)(/[^/]+)*/BUILD(\.bazel)?$|^BUILD.bazel$
       - id: buildifier-lint
-        files: ^(src|cpp|python|rllib)(/[^/]+)*/BUILD$
+        files: ^(src|cpp|python|rllib|ci|release|java)(/[^/]+)*/BUILD(\.bazel)?$|^BUILD.bazel$
 
   - repo: https://github.com/psf/black
     rev: 22.10.0

.rayciversion

@@ -1 +1 @@
-0.16.0
+0.17.0