Let AWS Databricks automatically choose an Availability Zone (#11714)

To address the AWS Databricks failure: InsufficientInstanceCapacity:
    Could not complete request as AWS does not currently have enough available capacity to fulfill your request for instance type g5.4xlarge.

    Our system will be working on provisioning additional capacity. You can currently get g5.4xlarge capacity by not specifying an Availability Zone in your request or choosing us-west-2a, us-west-2b.

Also, print the cluster ID to stdout so that a calling script can easily access it

Signed-off-by: timl <[email protected]>

jenkins/databricks/clusterutils.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2023, NVIDIA CORPORATION.
+# Copyright (c) 2020-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -85,6 +85,8 @@ def wait_for_cluster_start(workspace, clusterid, token, retries=20, printLoc=sys
             if current_state in ['INTERNAL_ERROR', 'SKIPPED', 'TERMINATED'] or p >= 60:
                 if p >= retries:
                    print("Waited %d times already, stopping" % p)
+                # Output the cluster ID to stdout so a calling script can get it easily
+                print(clusterid, file=sys.stdout)
                 sys.exit(4)
             p = p + 1
         print("Done starting cluster", file=printLoc)

jenkins/databricks/create.py

@@ -29,14 +29,14 @@ def main():
   sshkey = ''
   cluster_name = 'CI-GPU-databricks-24.12.0-SNAPSHOT'
   idletime = 240
-  runtime = '7.0.x-gpu-ml-scala2.12'
+  runtime = '13.3.x-gpu-ml-scala2.12'
   num_workers = 1
   worker_type = 'g4dn.xlarge'
   driver_type = 'g4dn.xlarge'
   cloud_provider = 'aws'
   # comma separated init scripts in Databricks workspace, e.g. /foo,/bar,...
   init_scripts = ''
-  aws_zone='us-west-2c'
+  aws_zone='auto'
 
 
   try: