[AWS-EC2] Add support for EC2

.travis.yml

@@ -0,0 +1,19 @@
+dist: trusty
+sudo: required
+
+install:
+  - sudo bash -c "$(curl -fsSL https://s3.amazonaws.com/tools.nanobox.io/bootstrap/ci.sh)"
+  - nanobox evar load local <(env -0 | grep -zE '(AZC|AZR|GCE|OVH|PKT|SCALEWAY|VULTR)_' | tr '\0' '\n')
+  - nanobox evar add local FLASK_APP=nanobox_libcloud:app
+  - nanobox evar add local FLASK_LOG_LEVEL=FATAL
+  - nanobox evar ls local
+  - nanobox build-runtime
+
+script: nanobox run python -m unittest -v
+
+after_success:
+  - |-
+    if [[ "$TRAVIS_PULL_REQUEST" == "false" && "$TRAVIS_BRANCH" == "master" ]]
+    then
+      nanobox deploy adapters/adapter-libcloud
+    fi

README.md

@@ -14,6 +14,10 @@ marked, below, with an asterisk (\*).
 _Please keep this list sorted alphabetically by provider name to help with
 finding evars quickly._
 
+### Amazon AWS EC2
+-   `EC2_KEY_ID`\*
+-   `EC2_ACCESS_KEY`\*
+
 ### Google Compute Engine
 -   `GCE_SERVICE_EMAIL`\*
 -   `GCE_SERVICE_KEY`\*
@@ -49,5 +53,13 @@ finding evars quickly._
 ### Vultr
 -   `VULTR_API_KEY`\*
 
+## Development Usage
+Start the adapter by running `nanobox run gunicorn -c /app/etc/gunicorn.py
+nanobox_libcloud:app` in a terminal. If you're working on one or more providers
+that use background tasks (currently only Azure), also start celery, by running
+`nanobox run celery -A nanobox_libcloud.celery worker -E -l info` in a second
+terminal. Access [the adapter root](http://adapter.local/) for further usage
+info.
+
 ## Et Cetera
 More info will be added to this README as it comes up.

nanobox_libcloud/adapters/azure.py

@@ -7,6 +7,7 @@
 from decimal import Decimal
 from time import sleep
 import redis
+import requests
 
 from flask import after_this_request
 from celery.signals import task_postrun
@@ -42,7 +43,7 @@ class AzureClassic(RebootMixin, KeyInstallMixin, Adapter):
         ["Subscription-Id", "Subscription ID"],
         ["Key-File", "Certificate"]
     ]
-    auth_instructions = ('Using Azure CLassic is fairly complex. First, create a '
+    auth_instructions = ('Using Azure Classic is fairly complex. First, create a '
         'self-signed certificate. Then, follow the instructions '
         '<a href="https://docs.microsoft.com/en-us/azure/azure-api-management-certs">here</a> '
         'to add the certificate to your account. Finally, enter your '
@@ -59,11 +60,20 @@ class AzureClassic(RebootMixin, KeyInstallMixin, Adapter):
     def __init__(self, **kwargs):
         self.generic_credentials = {
             'subscription_id': os.getenv('AZC_SUB_ID', ''),
-            'key': os.getenv('AZC_KEY', '')
+            'key': parse.unquote(os.getenv('AZC_KEY', '')).replace('\\n', '\n')
         }
 
     def do_server_create(self, headers, data):
         """Create a server with a certain provider."""
+        if data is None or 'name' not in data\
+                        or 'region' not in data\
+                        or 'size' not in data:
+            return {
+                "error": ("All servers need a 'name', 'region', and 'size' "
+                          "property. (Got %s)") % (data),
+                "status": 400
+            }
+
         try:
             self._get_user_driver(**self._get_request_credentials(headers))
         except (libcloud.common.types.LibcloudError, libcloud.common.exceptions.BaseHTTPError) as err:
@@ -105,6 +115,8 @@ def clr_tmp_user(**kwargs):
 
         try:
             self._user_driver.list_locations()
+        except requests.exceptions.SSLError:
+            raise libcloud.common.types.LibcloudError('Invalid Credentials')
         except AttributeError:
             pass
 
@@ -266,7 +278,7 @@ def _install_key(self, server, key_data):
         server.driver._connect_and_run_deployment_script(
             task = ScriptDeployment('echo "%s %s" >> ~/.ssh/authorized_keys' % (key_data['key'], key_data['id'])),
             node = server,
-            ssh_hostname = server.public_ips[0],
+            ssh_hostname = server.public_ips[0] if len(server.public_ips) > 0 else None,
             ssh_port = 22,
             ssh_username = self.server_ssh_user,
             ssh_password = self._get_password(server),

nanobox_libcloud/adapters/azure_arm.py

@@ -7,7 +7,7 @@
 from decimal import Decimal
 
 import libcloud
-from libcloud.compute.base import NodeAuthSSHKey
+from libcloud.compute.base import NodeAuthSSHKey, Node
 from nanobox_libcloud import tasks
 from nanobox_libcloud.adapters import Adapter
 from nanobox_libcloud.adapters.base import RebootMixin
@@ -97,9 +97,13 @@ def _get_user_driver(self, **auth_credentials):
         if self._user_driver is not None:
             return self._user_driver
         else:
-            driver = super()._get_user_driver(**auth_credentials)
-            driver.list_locations()
-            return driver
+            try:
+                driver = super()._get_user_driver(**auth_credentials)
+                driver.list_locations()
+            except requests.exceptions.ConnectionError:
+                raise libcloud.common.types.LibcloudError('Invalid Credentials')
+            else:
+                return driver
 
     @classmethod
     def _get_id(cls):

nanobox_libcloud/adapters/base.py

@@ -1,12 +1,14 @@
 import os
 import redis
 import typing
+import os
 from decimal import Decimal
 from time import sleep
 from operator import attrgetter
 
 import libcloud
 from libcloud.compute.base import NodeDriver, NodeLocation, NodeImage, NodeSize, Node
+from libcloud.compute.types import NodeState
 from requests.exceptions import ConnectionError
 
 from nanobox_libcloud.utils import models
@@ -46,16 +48,19 @@ def register(mcs, cls):
 
 class Adapter(object, metaclass=AdapterBase):
     """
-    Base class for Nanobox libcloud adapters. Implements basic functionality that should work for most libcloud drivers
-    which can be overridden by subclasses for specific drivers.
+    Base class for Nanobox libcloud adapters. Implements basic functionality
+    that should work for most libcloud drivers which can be overridden by
+    subclasses for specific drivers.
 
-    If subclasses are placed in the same package as this module they will automatically be discovered.
+    If subclasses are placed in the same package as this module they will
+    automatically be discovered.
     """
 
     # Adapter metadata
     id = None  # type: str
     name = ''  # type: str
     server_nick_name = 'server'  # type: str
+    configuration_fields = []  # type: typing.Dict[str, typing.Any]
 
     # Provider-wide server properties
     server_internal_iface = 'eth1'  # type: str
@@ -94,6 +99,15 @@ def do_meta(self) -> typing.Dict[str, typing.Any]:
             bootstrap_script=self.server_bootstrap_script,
             bootstrap_timeout=self.server_bootstrap_timeout,
             auth_credential_fields=self.auth_credential_fields,
+            config_fields=[models.AdapterConfig(
+                key=field.get('key'),
+                label=field.get('label'),
+                description=field.get('description'),
+                type=field.get('type'),
+                values=field.get('values'),
+                default=field.get('default'),
+                rebuild=field.get('rebuild'),
+            ) for field in self.configuration_fields],
             auth_instructions=self.auth_instructions,
         ).to_nanobox()
 
@@ -133,20 +147,19 @@ def do_catalog(self, headers) -> typing.List[dict]:
                 ).to_nanobox())
         except (libcloud.common.exceptions.BaseHTTPError, ConnectionError) as err:
             return err
-        except libcloud.common.types.LibcloudError:
-            # TODO: Get cached data...
+        except libcloud.common.types.LibcloudError as err:
             if os.getenv('APP_NAME', 'dev') == 'dev':
-                raise
+                raise err
             else:
-                pass
+                return err
 
         return catalog
 
     def do_verify(self, headers) -> bool:
         """Verify the account credentials."""
         try:
             self._get_user_driver(**self._get_request_credentials(headers))
-        except (libcloud.common.types.LibcloudError, libcloud.common.exceptions.BaseHTTPError, ConnectionError, KeyError, ValueError) as e:
+        except (libcloud.common.types.LibcloudError, libcloud.common.exceptions.BaseHTTPError, libcloud.compute.types.InvalidCredsError, ConnectionError, KeyError, ValueError) as e:
             return e
         else:
             return True
@@ -156,22 +169,25 @@ def do_key_create(self, headers, data) -> typing.Dict[str, typing.Any]:
         if self.server_ssh_auth_method != 'key' or self.server_ssh_key_method != 'reference':
             return {"error": "This provider doesn't support key storage", "status": 501}
 
+        if data is None or 'id' not in data or 'key' not in data:
+            return {
+                "error": "All keys need an 'id' and 'key' property. (Got %s)" % (data),
+                "status": 400
+            }
+
         try:
             driver = self._get_user_driver(**self._get_request_credentials(headers))
             if not self._create_key(driver, data):
                 return {"error": "Key creation failed", "status": 500}
 
             result = None
             for tries in range(5):
-                for key in driver.list_key_pairs():
-                    if (key.pub_key if hasattr(key, 'pub_key') else key.public_key) == data['key']:
-                        result = key
-                        break
-                if result:
+                result = self._find_ssh_key(driver, data.get('id'), data.get('key'))
+                if result is not None:
                     break
-                sleep(1)
+                sleep(tries)
 
-            if not result:
+            if result is None:
                 return {"error": "Key created, but not found", "status": 500}
         except (libcloud.common.types.LibcloudError, libcloud.common.exceptions.BaseHTTPError) as err:
             return {"error": err.value if hasattr(err, 'value') else err.message, "status": err.code if hasattr(err, 'message') else 500}
@@ -195,7 +211,8 @@ def do_key_query(self, headers, id) -> typing.Dict[str, typing.Any]:
             return {"data": models.KeyInfo(
                 id=key.id if hasattr(key, 'id') else key.name,
                 name=key.name,
-                key=key.pub_key if hasattr(key, 'pub_key') else key.public_key
+                key=key.pub_key if hasattr(key, 'pub_key') else key.public_key,
+                fingerprint=key.fingerprint if hasattr(key, 'fingerprint') else None
             ).to_nanobox(), "status": 201}
 
     def do_key_delete(self, headers, id) -> typing.Union[bool, typing.Dict[str, typing.Any]]:
@@ -205,7 +222,10 @@ def do_key_delete(self, headers, id) -> typing.Union[bool, typing.Dict[str, typi
 
         try:
             driver = self._get_user_driver(**self._get_request_credentials(headers))
-            key = self._find_ssh_key(driver, id)
+            try:
+                key = self._find_ssh_key(driver, id)
+            except libcloud.common.exceptions.BaseHTTPError as err:
+                key = None
 
             if not key:
                 return {"error": "SSH key not found", "status": 404}
@@ -261,7 +281,8 @@ def do_server_query(self, headers, id) -> typing.Dict[str, typing.Any]:
                 status=server.state,
                 name=server.name,
                 external_ip=self._get_ext_ip(server),
-                internal_ip=self._get_int_ip(server)
+                internal_ip=self._get_int_ip(server),
+                config=self._get_server_config(server)
             ).to_nanobox(), "status": 201}
 
     def do_server_cancel(self, headers, id) -> typing.Union[bool, typing.Dict[str, typing.Any]]:
@@ -279,6 +300,21 @@ def do_server_cancel(self, headers, id) -> typing.Union[bool, typing.Dict[str, t
         else:
             return True
 
+    def do_server_config(self, headers, id, config) -> typing.Union[bool, typing.Dict[str, typing.Any]]:
+        """Reconfigure a server with a certain provider."""
+        try:
+            driver = self._get_user_driver(**self._get_request_credentials(headers))
+            server = self._find_server(driver, id)
+
+            if not server:
+                return {"error": self.server_nick_name + " not found", "status": 404}
+
+            result = self._configure_server(server, config)
+        except (libcloud.common.types.LibcloudError, libcloud.common.exceptions.BaseHTTPError) as err:
+            return {"error": err.value if hasattr(err, 'value') else err.message, "status": err.code if hasattr(err, 'message') else 500}
+        else:
+            return result
+
     # Provider retrieval
     def _get_driver_class(self) -> typing.Type[NodeDriver]:
         """Returns the libcloud driver class for the id of this adapter."""
@@ -425,9 +461,18 @@ def _get_int_ip(self, server) -> str:
         """Returns the internal IP of a server for this adapter."""
         return server.private_ips[0] if len(server.private_ips) > 0 else None
 
+    def _get_server_config(self, server) -> typing.List[typing.Dict[str, typing.Any]]:
+        """Returns the provider-specific configuration of a server for this adapter."""
+        return []
+
     def _destroy_server(self, server) -> bool:
         return server.destroy()
 
+    @classmethod
+    def _configure_server(cls, server, config) -> typing.Union[bool, typing.Dict[str, typing.Any]]:
+        """Applies a new configuration to a server using this adapter."""
+        raise NotImplementedError()
+
     # Misc internal methods
     def _find_location(self, driver, id) -> typing.Optional[NodeLocation]:
         for location in driver.list_locations():
@@ -500,13 +545,22 @@ class KeyInstallMixin(object):
 
     def do_install_key(self, headers, id, data) -> typing.Union[bool, typing.Dict[str, typing.Any]]:
         """Install an SSH key on a server with a certain provider."""
+        if data is None or 'key' not in data or 'id' not in data:
+            return {
+                "error": ("All keys need an 'id' and 'key' property. (Got %s)") % (data),
+                "status": 400
+            }
+
         try:
             driver = self._get_user_driver(**self._get_request_credentials(headers))
             server = self._find_server(driver, id)
 
             if not server:
                 return {"error": self.server_nick_name + " not found", "status": 404}
 
+            if server.state != NodeState.RUNNING:
+                return {"error": self.server_nick_name + " not ready", "status": 409}
+
             if not self._install_key(server, data):
                 return {"error": "Key installation failed.", "status": 500}
         except (libcloud.common.types.LibcloudError, libcloud.common.exceptions.BaseHTTPError) as err: