Skip to content

Commit

Permalink
Check for dashboard readiness after cluster is ready
Browse files Browse the repository at this point in the history
  • Loading branch information
ChristianZaccaria committed Sep 25, 2023
1 parent 2e543ca commit 841f3ed
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 6 deletions.
18 changes: 13 additions & 5 deletions src/codeflare_sdk/cluster/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,19 +268,27 @@ def wait_ready(self, timeout: Optional[int] = None):
dashboard_ready = False
status = None
time = 0
while not ready or not dashboard_ready:
while not ready:
status, ready = self.status(print_to_console=False)
dashboard_ready = self.is_dashboard_ready()
if status == CodeFlareClusterStatus.UNKNOWN:
print(
"WARNING: Current cluster status is unknown, have you run cluster.up yet?"
)
if not ready or not dashboard_ready:
if not ready:
if timeout and time >= timeout:
raise TimeoutError(f"wait() timed out after waiting {timeout}s for cluster to be ready")
sleep(5)
time += 5
print("Requested cluster is up and running!")

while not dashboard_ready:
dashboard_ready = self.is_dashboard_ready()
if not dashboard_ready:
if timeout and time >= timeout:
raise TimeoutError(f"wait() timed out after waiting {timeout}s")
raise TimeoutError(f"wait() timed out after waiting {timeout}s for dashboard to be ready")
sleep(5)
time += 5
print("Requested cluster and dashboard are up and running!")
print("Dashboard is ready!")

def details(self, print_to_console: bool = True) -> RayCluster:
cluster = _copy_to_ray(self)
Expand Down
2 changes: 1 addition & 1 deletion tests/unit_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -1788,7 +1788,7 @@ def test_wait_ready(mocker, capsys):
captured = capsys.readouterr()
assert (
captured.out
== "Waiting for requested resources to be set up...\nRequested cluster and dashboard are up and running!\n"
== "Waiting for requested resources to be set up...\nRequested cluster is up and running!\nDashboard is ready!\n"
)


Expand Down

0 comments on commit 841f3ed

Please sign in to comment.