fix: test

Author: pawelpaszki

.github/workflows/e2e_tests.yaml

@@ -111,7 +111,7 @@ jobs:
           kubectl create clusterrolebinding sdk-user-service-reader --clusterrole=service-reader --user=sdk-user
           kubectl create clusterrole port-forward-pods --verb=create --resource=pods/portforward
           kubectl create clusterrolebinding sdk-user-port-forward-pods-binding --clusterrole=port-forward-pods --user=sdk-user
-          kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch --resource=rayjobs
+          kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch --resource=rayjobs --resource=rayjobs/status
           kubectl create clusterrolebinding sdk-user-rayjob-creator --clusterrole=rayjob-creator --user=sdk-user
           kubectl config use-context sdk-user
 

docs/sphinx/user-docs/e2e.rst

@@ -86,7 +86,7 @@ instructions <https://www.substratus.ai/blog/kind-with-gpus>`__.
         kubectl create clusterrolebinding sdk-user-service-reader --clusterrole=service-reader --user=sdk-user
         kubectl create clusterrole port-forward-pods --verb=create --resource=pods/portforward
         kubectl create clusterrolebinding sdk-user-port-forward-pods-binding --clusterrole=port-forward-pods --user=sdk-user
-        kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch --resource=rayjobs
+        kubectl create clusterrole rayjob-creator --verb=get,list,create,delete,patch --resource=rayjobs --resource=rayjobs/status
         kubectl create clusterrolebinding sdk-user-rayjob-creator --clusterrole=rayjob-creator --user=sdk-user
         kubectl config use-context sdk-user
 

tests/e2e/rayjob_existing_cluster_kind_test.py

@@ -2,6 +2,7 @@
 
 from codeflare_sdk import Cluster, ClusterConfiguration
 from codeflare_sdk.ray.rayjobs import RayJob
+from codeflare_sdk.ray.rayjobs.status import CodeflareRayJobStatus
 
 import pytest
 
@@ -104,120 +105,45 @@ def assert_rayjob_submit_against_existing_cluster(
         ), f"Job submission failed, expected {job_name}, got {submission_result}"
         print(f"✅ Successfully submitted RayJob '{job_name}' against existing cluster")
 
-        # Monitor the job status until completion using kubectl (Kind-specific workaround)
-        self.monitor_rayjob_completion_kubectl(job_name, timeout=900)
+        # Monitor the job status until completion
+        self.monitor_rayjob_completion(rayjob, timeout=900)
 
         print(f"✅ RayJob '{job_name}' completed successfully against existing cluster!")
 
-    def monitor_rayjob_completion_kubectl(self, job_name: str, timeout: int = 900):
+    def monitor_rayjob_completion(self, rayjob: RayJob, timeout: int = 900):
         """
-        Monitor a RayJob until it completes or fails using kubectl directly.
-        This is a workaround for Kind clusters where the SDK status method doesn't work.
+        Monitor a RayJob until it completes or fails.
 
         Args:
-            job_name: The name of the RayJob to monitor
+            rayjob: The RayJob instance to monitor
             timeout: Maximum time to wait in seconds (default: 15 minutes)
         """
-        import subprocess
-        import time
-
-        print(f"⏳ Monitoring RayJob '{job_name}' status using kubectl...")
+        print(f"⏳ Monitoring RayJob '{rayjob.name}' status...")
 
         elapsed_time = 0
         check_interval = 10  # Check every 10 seconds
 
         while elapsed_time < timeout:
-            try:
-                # Get RayJob status using kubectl
-                result = subprocess.run(
-                    [
-                        "kubectl",
-                        "get",
-                        "rayjobs",
-                        "-n",
-                        self.namespace,
-                        job_name,
-                        "-o",
-                        "jsonpath={.status.jobDeploymentStatus}",
-                    ],
-                    capture_output=True,
-                    text=True,
-                    timeout=10,
-                )
-
-                if result.returncode == 0:
-                    status = result.stdout.strip()
-
-                    # Also get job status for more details
-                    job_status_result = subprocess.run(
-                        [
-                            "kubectl",
-                            "get",
-                            "rayjobs",
-                            "-n",
-                            self.namespace,
-                            job_name,
-                            "-o",
-                            "jsonpath={.status.jobStatus}",
-                        ],
-                        capture_output=True,
-                        text=True,
-                        timeout=10,
-                    )
-                    job_status = (
-                        job_status_result.stdout.strip()
-                        if job_status_result.returncode == 0
-                        else "Unknown"
-                    )
-
-                    print(
-                        f"📊 RayJob '{job_name}' - Deployment Status: {status}, Job Status: {job_status}"
-                    )
-
-                    # Check completion status
-                    if status == "Complete" or job_status == "SUCCEEDED":
-                        print(f"✅ RayJob '{job_name}' completed successfully!")
-                        return
-                    elif status == "Failed" or job_status == "FAILED":
-                        # Get error details
-                        try:
-                            error_result = subprocess.run(
-                                [
-                                    "kubectl",
-                                    "get",
-                                    "rayjobs",
-                                    "-n",
-                                    self.namespace,
-                                    job_name,
-                                    "-o",
-                                    "yaml",
-                                ],
-                                capture_output=True,
-                                text=True,
-                                timeout=10,
-                            )
-                            print(
-                                f"❌ RayJob '{job_name}' failed! Details:\n{error_result.stdout}"
-                            )
-                        except:
-                            pass
-                        raise AssertionError(f"❌ RayJob '{job_name}' failed!")
-                    elif status == "Running" or job_status == "RUNNING":
-                        print(f"🏃 RayJob '{job_name}' is still running...")
-                    else:
-                        print(f"⏳ RayJob '{job_name}' status: {status}")
-
-                else:
-                    print(f"❌ Could not get RayJob status: {result.stderr}")
-
-            except Exception as e:
-                print(f"❌ Error checking RayJob status: {e}")
+            status, ready = rayjob.status(print_to_console=True)
+
+            # Check if job has completed (either successfully or failed)
+            if status == CodeflareRayJobStatus.COMPLETE:
+                print(f"✅ RayJob '{rayjob.name}' completed successfully!")
+                return
+            elif status == CodeflareRayJobStatus.FAILED:
+                raise AssertionError(f"❌ RayJob '{rayjob.name}' failed!")
+            elif status == CodeflareRayJobStatus.RUNNING:
+                print(f"🏃 RayJob '{rayjob.name}' is still running...")
+            elif status == CodeflareRayJobStatus.UNKNOWN:
+                print(f"❓ RayJob '{rayjob.name}' status is unknown")
 
             # Wait before next check
             sleep(check_interval)
             elapsed_time += check_interval
 
         # If we reach here, the job has timed out
+        final_status, _ = rayjob.status(print_to_console=True)
         raise TimeoutError(
-            f"⏰ RayJob '{job_name}' did not complete within {timeout} seconds."
+            f"⏰ RayJob '{rayjob.name}' did not complete within {timeout} seconds. "
+            f"Final status: {final_status}"
         )