Skip to content

Commit efe84be

Browse files
kryanbeanekryanbeane
committed
RHOAIENG-32532: Run RayJob tests in CI
1 parent d259ec1 commit efe84be

File tree

1 file changed

+200
-0
lines changed

1 file changed

+200
-0
lines changed

‎.github/workflows/rayjob_e2e_tests.yaml‎

Lines changed: 200 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
name: rayjob-e2e-with-kueue
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- 'release-*'
8+
- ray-jobs-feature
9+
paths-ignore:
10+
- 'docs/**'
11+
- '**.adoc'
12+
- '**.md'
13+
- 'LICENSE'
14+
15+
concurrency:
16+
group: ${{ github.head_ref }}-${{ github.workflow }}
17+
cancel-in-progress: true
18+
19+
env:
20+
CODEFLARE_OPERATOR_IMG: "quay.io/project-codeflare/codeflare-operator:dev"
21+
KUEUE_VERSION: "v0.13.3"
22+
23+
jobs:
24+
kubernetes:
25+
runs-on: ubuntu-latest
26+
27+
steps:
28+
- name: Checkout code
29+
uses: actions/checkout@v4
30+
with:
31+
submodules: recursive
32+
33+
- name: Checkout common repo code
34+
uses: actions/checkout@v4
35+
with:
36+
repository: 'project-codeflare/codeflare-common'
37+
ref: 'main'
38+
path: 'common'
39+
40+
- name: Checkout CodeFlare operator repository
41+
uses: actions/checkout@v4
42+
with:
43+
repository: project-codeflare/codeflare-operator
44+
path: codeflare-operator
45+
46+
- name: Checkout Kueue repository
47+
uses: actions/checkout@v4
48+
with:
49+
repository: kubernetes-sigs/kueue
50+
path: kueue
51+
ref: main
52+
53+
- name: Set Go
54+
uses: actions/setup-go@v5
55+
with:
56+
go-version-file: './codeflare-operator/go.mod'
57+
cache-dependency-path: |
58+
./codeflare-operator/go.sum
59+
./kueue/go.sum
60+
61+
- name: Set up gotestfmt
62+
uses: gotesttools/gotestfmt-action@v2
63+
with:
64+
token: ${{ secrets.GITHUB_TOKEN }}
65+
66+
- name: Set up specific Python version
67+
uses: actions/setup-python@v5
68+
with:
69+
python-version: '3.11'
70+
cache: 'pip' # caching pip dependencies
71+
72+
- name: Setup and start KinD cluster
73+
uses: ./common/github-actions/kind
74+
with:
75+
worker-nodes: 2 # Multiple nodes for testing Kueue scheduling
76+
77+
- name: Build and install Kueue
78+
run: |
79+
cd kueue
80+
echo "Building Kueue..."
81+
make manifests
82+
make install
83+
make kind-image-build
84+
85+
# Load Kueue image into Kind
86+
IMAGE_TAG=$(git describe --tags --dirty --always)
87+
kind load docker-image us-central1-docker.pkg.dev/k8s-staging-images/kueue/kueue:${IMAGE_TAG}
88+
89+
# Deploy Kueue
90+
kubectl apply --server-side --force-conflicts -k config/default
91+
92+
# Wait for Kueue deployment
93+
kubectl wait --timeout=120s --for=condition=Available=true deployment -n kueue-system kueue-controller-manager
94+
95+
# Patch deployment to use local image
96+
kubectl patch deployment kueue-controller-manager -n kueue-system -p '{"spec":{"template":{"spec":{"containers":[{"name":"manager","imagePullPolicy":"Never"}]}}}}'
97+
98+
cd ..
99+
100+
- name: Deploy KubeRay operator
101+
run: |
102+
echo "Installing KubeRay operator..."
103+
kubectl create -k "github.com/ray-project/kuberay/ray-operator/config/default?ref=v1.2.2"
104+
kubectl wait --timeout=120s --for=condition=Available=true deployment -n ray-system kuberay-operator
105+
106+
- name: Add user to KinD
107+
uses: ./common/github-actions/kind-add-user
108+
with:
109+
user-name: sdk-user
110+
111+
- name: Configure RBAC for sdk user with limited permissions
112+
run: |
113+
# Basic permissions
114+
kubectl create clusterrole list-ingresses --verb=get,list --resource=ingresses
115+
kubectl create clusterrolebinding sdk-user-list-ingresses --clusterrole=list-ingresses --user=sdk-user
116+
kubectl create clusterrole namespace-creator --verb=get,list,create,delete,patch --resource=namespaces
117+
kubectl create clusterrolebinding sdk-user-namespace-creator --clusterrole=namespace-creator --user=sdk-user
118+
119+
# Ray permissions
120+
kubectl create clusterrole raycluster-creator --verb=get,list,create,delete,patch,watch --resource=rayclusters
121+
kubectl create clusterrolebinding sdk-user-raycluster-creator --clusterrole=raycluster-creator --user=sdk-user
122+
kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch,watch,update --resource=rayjobs
123+
kubectl create clusterrolebinding sdk-user-rayjob-creator --clusterrole=rayjob-creator --user=sdk-user
124+
125+
# Kueue permissions
126+
kubectl create clusterrole resourceflavor-creator --verb=get,list,create,delete --resource=resourceflavors
127+
kubectl create clusterrolebinding sdk-user-resourceflavor-creator --clusterrole=resourceflavor-creator --user=sdk-user
128+
kubectl create clusterrole clusterqueue-creator --verb=get,list,create,delete,patch --resource=clusterqueues
129+
kubectl create clusterrolebinding sdk-user-clusterqueue-creator --clusterrole=clusterqueue-creator --user=sdk-user
130+
kubectl create clusterrole localqueue-creator --verb=get,list,create,delete,patch --resource=localqueues
131+
kubectl create clusterrolebinding sdk-user-localqueue-creator --clusterrole=localqueue-creator --user=sdk-user
132+
kubectl create clusterrole workload-creator --verb=get,list,watch --resource=workloads
133+
kubectl create clusterrolebinding sdk-user-workload-creator --clusterrole=workload-creator --user=sdk-user
134+
135+
# Additional permissions
136+
kubectl create clusterrole list-secrets --verb=get,list --resource=secrets
137+
kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user
138+
kubectl create clusterrole pod-creator --verb=get,list,watch --resource=pods
139+
kubectl create clusterrolebinding sdk-user-pod-creator --clusterrole=pod-creator --user=sdk-user
140+
kubectl create clusterrole service-reader --verb=get,list,watch --resource=services
141+
kubectl create clusterrolebinding sdk-user-service-reader --clusterrole=service-reader --user=sdk-user
142+
kubectl create clusterrole port-forward-pods --verb=create --resource=pods/portforward
143+
kubectl create clusterrolebinding sdk-user-port-forward-pods-binding --clusterrole=port-forward-pods --user=sdk-user
144+
kubectl create clusterrole node-reader --verb=get,list --resource=nodes
145+
kubectl create clusterrolebinding sdk-user-node-reader --clusterrole=node-reader --user=sdk-user
146+
kubectl config use-context sdk-user
147+
148+
- name: Run RayJob e2e tests
149+
run: |
150+
export CODEFLARE_TEST_OUTPUT_DIR=$(mktemp -d)
151+
echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
152+
153+
set -euo pipefail
154+
pip install poetry
155+
poetry install --with test,docs
156+
157+
# Install the SDK in editable mode
158+
pip install -e .
159+
160+
echo "Running RayJob e2e tests..."
161+
poetry run pytest -v -s ./tests/e2e/rayjob/ -x > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
162+
env:
163+
GRPC_DNS_RESOLVER: "native"
164+
165+
- name: Switch to kind-cluster context to print logs
166+
if: always()
167+
run: kubectl config use-context kind-cluster
168+
169+
- name: Print Pytest output log
170+
if: always()
171+
run: |
172+
echo "Printing Pytest output logs"
173+
cat ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log || true
174+
175+
- name: Print Kueue operator logs
176+
if: always()
177+
run: |
178+
echo "Printing Kueue operator logs"
179+
kubectl logs -n kueue-system --tail -1 -l control-plane=controller-manager | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kueue-operator.log || true
180+
181+
- name: Print KubeRay operator logs
182+
if: always()
183+
run: |
184+
echo "Printing KubeRay operator logs"
185+
kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log || true
186+
187+
- name: Export all KinD pod logs
188+
uses: ./common/github-actions/kind-export-logs
189+
if: always()
190+
with:
191+
output-directory: ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}
192+
193+
- name: Upload logs
194+
uses: actions/upload-artifact@v4
195+
if: always()
196+
with:
197+
name: logs
198+
retention-days: 10
199+
path: |
200+
${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log

0 commit comments

Comments
 (0)