V1.9.7

bin/qds.py

@@ -1,7 +1,6 @@
 #!/bin/env python
 
 from __future__ import print_function
-from qds_sdk.qubole import Qubole
 from qds_sdk.commands import *
 from qds_sdk.cluster import *
 import qds_sdk.exception
@@ -16,8 +15,8 @@
 from qds_sdk.nezha import NezhaCmdLine
 from qds_sdk.user import UserCmdLine
 from qds_sdk.template import TemplateCmdLine
+from qds_sdk.clusterv2 import ClusterCmdLine
 from qds_sdk.sensors import *
-
 import os
 import sys
 import traceback
@@ -49,9 +48,12 @@
     "  <hivecmd|hadoopcmd|prestocmd|pigcmd|shellcmd|dbexportcmd|dbimportcmd|dbtapquerycmd|sparkcmd> <action>\n"
     "    submit [cmd-specific-args .. ] : submit cmd & print id\n"
     "    run [cmd-specific-args .. ] : submit cmd & wait. print results\n"
-    "    check <id> : print the cmd object for this id\n"
+    "    check <id> <include-query-properties> : id -> print the cmd object for this id\n"
+    "                                            include-query-properties(true/false) -> to include query properties like\n"
+    "                                            tags, comments and user actions\n"
     "    cancel <id> : cancels the cmd with this id\n"
-    "    getresult <id> : get the results for the cmd with this id\n"
+    "    getresult <id> <include_header>: id -> get the results for the cmd with this id\n"
+    "                                     include_header -> to include headers in results(true/false)\n"
     "    getlog <id> : get the logs for the cmd with this id\n"
     "\nCluster subcommand:\n"
     "  cluster <action>\n"
@@ -121,10 +123,10 @@ def submitaction(cmdclass, args):
         return 0
 
 
-def _getresult(cmdclass, cmd):
+def _getresult(cmdclass, cmd, args=[]):
     if Command.is_success(cmd.status):
         log.info("Fetching results for %s, Id: %s" % (cmdclass.__name__, cmd.id))
-        cmd.get_results(sys.stdout, delim='\t')
+        cmd.get_results(sys.stdout, delim='\t', qlog=cmd.qlog, arguments=args)
         return 0
     else:
         log.error("Cannot fetch results - command Id: %s failed with status: %s" % (cmd.id, cmd.status))
@@ -142,9 +144,20 @@ def runaction(cmdclass, args):
 
 
 def checkaction(cmdclass, args):
-    checkargs_id(args)
-    o = cmdclass.find(args.pop(0))
-    print(str(o))
+    if len(args) > 2:
+        sys.stderr.write("expecting not more than 2 arguments\n")
+        usage()
+
+    conn = Qubole.agent()
+    id = args.pop(0)
+    include_query_properties="false"
+    if len(args) == 1:
+        include_query_properties=args.pop(0)
+        if include_query_properties not in ('true', 'false'):
+            raise ParseError("include-query-properties can be either true or false")
+
+    r = conn.get(cmdclass.element_path(id), {'include_query_properties': include_query_properties})
+    print(str(r))
     return 0
 
 
@@ -164,9 +177,12 @@ def cancelaction(cmdclass, args):
 
 
 def getresultaction(cmdclass, args):
-    checkargs_id(args)
+    if len(args) > 2:
+        sys.stderr.write("expecting not more than 2 arguments\n")
+        usage()
+
     cmd = cmdclass.find(args.pop(0))
-    return _getresult(cmdclass, cmd)
+    return _getresult(cmdclass, cmd, args)
 
 
 def getlogaction(cmdclass, args):
@@ -222,7 +238,6 @@ def cluster_create_action(clusterclass, args, api_version=1.2):
     print(json.dumps(result, indent=4))
     return 0
 
-
 def cluster_update_action(clusterclass, args, api_version=1.2):
     arguments = clusterclass._parse_create_update(args, "update", api_version)
     cluster_info = _create_cluster_info(arguments, api_version)
@@ -455,6 +470,23 @@ def clustermain(args, api_version):
     else:
         return globals()["cluster_" + action + "_action"](clusterclass, args)
 
+def clustermainv2(args):
+    action = args[0]
+    actionset = set(
+        ["create", "delete", "update", "clone", "list", "start", "terminate", "status", "reassign_label", "add_node",
+         "remove_node", "update_node", "snapshot", "restore_point", "get_snapshot_schedule",
+         "update_snapshot_schedule"])
+
+    result = None
+    if action not in actionset:
+        sys.stderr.write("action must be one of <%s>\n" % "|".join(actionset))
+        usage()
+    elif action in set(["create", "update", "clone"]):
+        result =  ClusterCmdLine.run(args)
+    else:
+        result =  globals()["cluster_" + action + "_action"](Cluster, args)
+    print(result)
+
 def accountmain(args):
     result = AccountCmdLine.run(args)
     print(result)
@@ -499,7 +531,7 @@ def nezhamain(args):
 def templatemain(args):
     result = TemplateCmdLine.run(args)
     print(result)
-    
+
 
 def main():
     optparser = OptionParser(usage=usage_str)
@@ -524,6 +556,10 @@ def main():
                          default=False,
                          help="skip verification of server SSL certificate. Insecure: use with caution.")
 
+    optparser.add_option("--cloud_name", dest="cloud_name",
+                         default=os.getenv('CLOUD_PROVIDER'),
+                         help="cloud", choices=["AWS", "AZURE", "ORACLE_BMC"])
+
     optparser.add_option("-v", dest="verbose", action="store_true",
                          default=False,
                          help="verbose mode - info level logging")
@@ -555,6 +591,9 @@ def main():
     if options.poll_interval is None:
         options.poll_interval = 5
 
+    if options.cloud_name is None:
+        options.cloud_name = "AWS"
+
     if options.skip_ssl_cert_check is None:
         options.skip_ssl_cert_check = False
     elif options.skip_ssl_cert_check:
@@ -564,7 +603,8 @@ def main():
                      api_url=options.api_url,
                      version=options.api_version,
                      poll_interval=options.poll_interval,
-                     skip_ssl_cert_check=options.skip_ssl_cert_check)
+                     skip_ssl_cert_check=options.skip_ssl_cert_check,
+                     cloud_name=options.cloud_name)
 
     if len(args) < 1:
         sys.stderr.write("Missing first argument containing subcommand\n")
@@ -582,7 +622,10 @@ def main():
 
     if a0 == "cluster":
         api_version_number = float(options.api_version[1:])
-        return clustermain(args, api_version_number)
+        if api_version_number >= 2.0:
+            return clustermainv2(args)
+        else:
+            return clustermain(args, api_version_number)
 
     if a0 == "action":
         return actionmain(args)

qds_sdk/cloud/aws_cloud.py

@@ -0,0 +1,166 @@
+from qds_sdk.cloud.cloud import Cloud
+class AwsCloud(Cloud):
+    '''
+    qds_sdk.cloud.AwsCloud is the class which stores information about aws cloud config settings.
+    The objects of this class can be used to set aws cloud_config settings while create/update/clone a cluster.
+    '''
+
+    def __init__(self):
+        self.compute_config = {}
+        self.location = {}
+        self.network_config = {}
+        self.storage_config = {}
+
+    def set_cloud_config(self,
+                         compute_access_key=None,
+                         compute_secret_key=None,
+                         use_account_compute_creds=None,
+                         aws_region=None,
+                         aws_availability_zone=None,
+                         role_instance_profile=None,
+                         vpc_id=None,
+                         subnet_id=None,
+                         persistent_security_groups=None,
+                         bastion_node_public_dns=None,
+                         master_elastic_ip=None):
+        '''
+
+        Args:
+            compute_access_key: The access key for customer's aws account. This
+                is required for creating the cluster.
+
+            compute_secret_key: The secret access key for customer's aws
+                account. This is required for creating the cluster.
+
+            use_account_compute_creds: Set it to true to use the account's compute
+                credentials for all clusters of the account.The default value is false
+
+            aws_region: The AWS region in which the cluster is created. The default value is, us-east-1.
+                Valid values are, us-east-1, us-west-1, us-west-2, eu-west-1, sa-east1, ap-southeast-1,
+                and ap-northeast-1.
+                Doc: http://docs.qubole.com/en/latest/rest-api/cluster_api/create-new-cluster.html#ec2-settings
+
+            aws_availability_zone: The preferred availability zone in which the cluster must be created. The default value is Any.
+
+            role_instance_profile: IAM Role instance profile to attach on cluster
+
+            vpc_id: The ID of the vpc in which the cluster is created.
+                In this vpc, the enableDnsHostnames parameter must be set to true.
+
+            subnet_id: The ID of the subnet in which the cluster is created. This subnet must belong to the
+                above VPC and it can be a public/private subnet
+
+            persistent_security_groups: security group to associate with each node of the cluster.
+                Typically used to provide access to external hosts
+
+            bastion_node_public_dns: Specify the Bastion host public DNS name if private subnet is provided.
+                Do not specify this value for a public subnet.
+
+            master_elastic_ip: It is the Elastic IP address for attaching to the cluster master
+
+        '''
+
+        self.set_compute_config(use_account_compute_creds, compute_access_key,
+                                compute_secret_key, role_instance_profile)
+        self.set_location(aws_region, aws_availability_zone)
+        self.set_network_config(bastion_node_public_dns, persistent_security_groups,
+                                master_elastic_ip, vpc_id, subnet_id)
+
+    def set_compute_config(self,
+                           use_account_compute_creds=None,
+                           compute_access_key=None,
+                           compute_secret_key=None,
+                           role_instance_profile=None):
+        self.compute_config['use_account_compute_creds'] = use_account_compute_creds
+        self.compute_config['compute_access_key'] = compute_access_key
+        self.compute_config['compute_secret_key'] = compute_secret_key
+        self.compute_config['role_instance_profile'] = role_instance_profile
+
+    def set_location(self,
+                     aws_region=None,
+                     aws_availability_zone=None):
+        self.location['aws_region'] = aws_region
+        self.location['aws_availability_zone'] = aws_availability_zone
+
+    def set_network_config(self,
+                           bastion_node_public_dns=None,
+                           persistent_security_groups=None,
+                           master_elastic_ip=None,
+                           vpc_id=None,
+                           subnet_id=None):
+        self.network_config['bastion_node_public_dns'] = bastion_node_public_dns
+        self.network_config['persistent_security_groups'] = persistent_security_groups
+        self.network_config['master_elastic_ip'] = master_elastic_ip
+        self.network_config['vpc_id'] = vpc_id
+        self.network_config['subnet_id'] = subnet_id
+
+    def set_cloud_config_from_arguments(self, arguments):
+        self.set_cloud_config(compute_access_key=arguments.compute_access_key,
+                              compute_secret_key=arguments.compute_secret_key,
+                              use_account_compute_creds=arguments.use_account_compute_creds,
+                              aws_region=arguments.aws_region,
+                              aws_availability_zone=arguments.aws_availability_zone,
+                              role_instance_profile=arguments.role_instance_profile,
+                              vpc_id=arguments.vpc_id,
+                              subnet_id=arguments.subnet_id,
+                              persistent_security_groups=arguments.persistent_security_groups,
+                              bastion_node_public_dns=arguments.bastion_node_public_dns,
+                              master_elastic_ip=arguments.master_elastic_ip)
+
+    def create_parser(self, argparser):
+        # compute settings parser
+        compute_config = argparser.add_argument_group("compute config settings")
+        compute_creds = compute_config.add_mutually_exclusive_group()
+        compute_creds.add_argument("--enable-account-compute-creds",
+                                   dest="use_account_compute_creds",
+                                   action="store_true",
+                                   default=None,
+                                   help="to use account compute credentials")
+        compute_creds.add_argument("--disable-account-compute-creds",
+                                   dest="use_account_compute_creds",
+                                   action="store_false",
+                                   default=None,
+                                   help="to disable account compute credentials")
+        compute_config.add_argument("--compute-access-key",
+                                    dest="compute_access_key",
+                                    default=None,
+                                    help="access key for aws cluster")
+        compute_config.add_argument("--compute-secret-key",
+                                    dest="compute_secret_key",
+                                    default=None,
+                                    help="secret key for aws cluster")
+        compute_config.add_argument("--role-instance-profile",
+                                    dest="role_instance_profile",
+                                    help="IAM Role instance profile to attach on cluster", )
+
+        # location settings parser
+        location_group = argparser.add_argument_group("location config setttings")
+        location_group.add_argument("--aws-region",
+                                    dest="aws_region",
+                                    choices=["us-east-1", "us-west-2", "ap-northeast-1", "sa-east-1",
+                                             "eu-west-1", "ap-southeast-1", "us-west-1"],
+                                    help="aws region to create the cluster in", )
+        location_group.add_argument("--aws-availability-zone",
+                                    dest="aws_availability_zone",
+                                    help="availability zone to" +
+                                         " create the cluster in", )
+
+        # network settings parser
+        network_config_group = argparser.add_argument_group("network config settings")
+        network_config_group.add_argument("--vpc-id",
+                                          dest="vpc_id",
+                                          help="vpc to create the cluster in", )
+        network_config_group.add_argument("--subnet-id",
+                                          dest="subnet_id",
+                                          help="subnet to create the cluster in", )
+        network_config_group.add_argument("--bastion-node-public-dns",
+                                          dest="bastion_node_public_dns",
+                                          help="public dns name of the bastion node. Required only if cluster is in private subnet of a EC2-VPC", )
+        network_config_group.add_argument("--persistent-security-groups",
+                                          dest="persistent_security_groups",
+                                          help="a security group to attach with each" +
+                                               " node of the cluster. Typically used" +
+                                               " to provide access to external hosts", )
+        network_config_group.add_argument("--master-elastic-ip",
+                                          dest="master_elastic_ip",
+                                          help="master elastic ip for cluster")