From edea736bf43283cd649c20d51baa90d69aeee11c Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Thu, 6 Oct 2016 18:12:34 +0200 Subject: [PATCH 01/10] Create RunAction for running tasks --- ecs_deploy/cli.py | 46 +++++++++++++++++++++++++++++++++++++++++++--- ecs_deploy/ecs.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index 70ee64a..cfd59af 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -5,7 +5,7 @@ import getpass from datetime import datetime, timedelta -from ecs_deploy.ecs import DeployAction, ScaleAction, EcsClient +from ecs_deploy.ecs import DeployAction, ScaleAction, RunAction, EcsClient from ecs_deploy.newrelic import Deployment, NewRelicDeploymentException @@ -107,6 +107,45 @@ def scale(cluster, service, desired_count, access_key_id, secret_access_key, reg exit(1) +@click.command() +@click.argument('cluster') +@click.argument('task') +@click.argument('count', required=False, default=1) +@click.option('-c', '--command', type=(str, str), multiple=True, help='Overwrites the command in a container: ') +@click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: ') +@click.option('--region', help='AWS region') +@click.option('--access-key-id', help='AWS access key id') +@click.option('--secret-access-key', help='AWS secret access yey') +@click.option('--profile', help='AWS configuration profile') +def run(cluster, task, count, command, env, region, access_key_id, secret_access_key, profile): + """ + Run a one-off task. + + \b + CLUSTER is the name of your cluster (e.g. 'my-custer') within ECS. + TASK is the name of your task definintion (e.g. 'mytask') within ECS. + COMMAND is the number of tasks your service should run. + """ + try: + client = get_client(access_key_id, secret_access_key, region, profile) + action = RunAction(client, cluster) + + task_definition = action.get_task_definition(task) + task_definition.set_commands(**{key: value for (key, value) in command}) + task_definition.set_environment(env) + print_diff(task_definition, 'Using task definition: %s' % task) + + action.run(task_definition, count, 'ECS Deploy') + + + # click.secho('Successfully changed desired count to: %s\n' % desired_count, fg='green') + # wait_for_finish(scaling, timeout, 'Scaling service', 'Scaling successful', 'Scaling failed') + + except Exception as e: + click.secho('%s\n' % str(e), fg='red', err=True) + exit(1) + + def wait_for_finish(action, timeout, title, success_message, failure_message): click.secho(title, nl=False) waiting = True @@ -141,9 +180,9 @@ def record_deployment(revision, newrelic_apikey, newrelic_appid, comment, user): return True -def print_diff(task_definition): +def print_diff(task_definition, title='Updating task definition'): if task_definition.diff: - click.secho('Updating task definition') + click.secho(title) for diff in task_definition.diff: click.secho(str(diff), fg='blue') click.secho('') @@ -166,6 +205,7 @@ def print_errors(service, was_timeout=False, message=''): ecs.add_command(deploy) ecs.add_command(scale) +ecs.add_command(run) if __name__ == '__main__': # pragma: no cover ecs() diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index 370a75e..b16b082 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -39,6 +39,14 @@ def update_service(self, cluster, service, desired_count, task_definition): taskDefinition=task_definition ) + def run_task(self, cluster, task_definition, count, started_by, overrides): + return self.boto.run_task( + cluster=cluster, + taskDefinition=task_definition, + count=count, + startedBy=started_by, + overrides=overrides + ) class EcsService(dict): def __init__(self, cluster, iterable=None, **kwargs): @@ -129,6 +137,10 @@ def family(self): def revision(self): return self.get(u'revision') + @property + def family_revision(self): + return '%s:%d' % (self.get(u'family'), self.get(u'revision')) + @property def diff(self): return self._diff @@ -221,6 +233,11 @@ def get_current_task_definition(self, service): task_definition = EcsTaskDefinition(task_definition_payload[u'taskDefinition']) return task_definition + def get_task_definition(self, task_definition): + task_definition_payload = self._client.describe_task_definition(task_definition) + task_definition = EcsTaskDefinition(task_definition_payload[u'taskDefinition']) + return task_definition + def update_task_definition(self, task_definition): response = self._client.register_task_definition(task_definition.family, task_definition.containers, task_definition.volumes) @@ -278,6 +295,26 @@ def scale(self, desired_count): return self.update_service(self._service) +class RunAction(EcsAction): + def __init__(self, client, cluster_name): + self._client = client + self._cluster_name = cluster_name + + def run(self, task_definition, count, started_by): + overrides = [] + if task_definition.diff: + for diff in task_definition.diff: + override = dict(name=diff.container) + if diff.field == 'command': + override['command'] = diff.value.split(' ') + elif diff.field == 'environment': + override['environment'] = [{"name": e, "value": diff.value[e]} for e in diff.value] + overrides.append(override) + + self._client.run_task(self._cluster_name, task_definition.family_revision, count, started_by, dict(containerOverrides=overrides)) + return True + + class EcsError(Exception): pass From 809fb7d8efb1edffaf987fd5e8bd11b006d2aadf Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Fri, 7 Oct 2016 10:04:28 +0200 Subject: [PATCH 02/10] Test run command and fix typo --- README.rst | 4 +-- ecs_deploy/cli.py | 7 +++--- ecs_deploy/ecs.py | 8 +++--- tests/test_cli.py | 63 ++++++++++++++++++++++++++++++++++++++++++----- tests/test_ecs.py | 8 +++++- 5 files changed, 75 insertions(+), 15 deletions(-) diff --git a/README.rst b/README.rst index 44d06ab..139bd17 100644 --- a/README.rst +++ b/README.rst @@ -93,8 +93,8 @@ To change the image of a specific container, run the following command:: This will modify the **webserver** container only and change its image to "nginx:latest". -Deploy several new image -======================== +Deploy several new images +========================= The `-i` or `--image` option can also be passed several times:: $ ecs deploy my-cluster my-service -i webserver nginx:1.9 -i application django:latest diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index cfd59af..d194ddc 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -137,9 +137,10 @@ def run(cluster, task, count, command, env, region, access_key_id, secret_access action.run(task_definition, count, 'ECS Deploy') - - # click.secho('Successfully changed desired count to: %s\n' % desired_count, fg='green') - # wait_for_finish(scaling, timeout, 'Scaling service', 'Scaling successful', 'Scaling failed') + click.secho('Successfully started %d instances of task: %s' % (len(action.started_tasks), task_definition.family_revision), fg='green') + for started_task in action.started_tasks: + click.secho('- %s' % started_task['taskArn'], fg='green') + click.secho(' ') except Exception as e: click.secho('%s\n' % str(e), fg='red', err=True) diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index b16b082..ac48c0a 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -186,7 +186,7 @@ def apply_container_environment(self, container, new_environment): merged_environment = old_environment.copy() merged_environment.update(new_environment) - diff = EcsTaskDefinitionDiff(container[u'name'], u'environment', dumps(merged_environment), dumps(old_environment)) + diff = EcsTaskDefinitionDiff(container[u'name'], u'environment', merged_environment, old_environment) self._diff.append(diff) container[u'environment'] = [{"name": e, "value": merged_environment[e]} for e in merged_environment] @@ -206,7 +206,7 @@ def __init__(self, container, field, value, old_value): def __repr__(self): return u"Changed %s of container '%s' to: %s (was: %s)" % \ - (self.field, self.container, self.value, self.old_value) + (self.field, self.container, dumps(self.value), dumps(self.old_value)) class EcsAction(object): @@ -299,6 +299,7 @@ class RunAction(EcsAction): def __init__(self, client, cluster_name): self._client = client self._cluster_name = cluster_name + self.started_tasks = [] def run(self, task_definition, count, started_by): overrides = [] @@ -311,7 +312,8 @@ def run(self, task_definition, count, started_by): override['environment'] = [{"name": e, "value": diff.value[e]} for e in diff.value] overrides.append(override) - self._client.run_task(self._cluster_name, task_definition.family_revision, count, started_by, dict(containerOverrides=overrides)) + result = self._client.run_task(self._cluster_name, task_definition.family_revision, count, started_by, dict(containerOverrides=overrides)) + self.started_tasks = result['tasks'] return True diff --git a/tests/test_cli.py b/tests/test_cli.py index 87607aa..2694a6b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -76,8 +76,8 @@ def test_deploy_new_tag(get_client, runner): assert result.exit_code == 0 assert not result.exception assert u"Updating task definition" in result.output - assert u"Changed image of container 'webserver' to: webserver:latest (was: webserver:123)" in result.output - assert u"Changed image of container 'application' to: application:latest (was: application:123)" in result.output + assert u"Changed image of container 'webserver' to: \"webserver:latest\" (was: \"webserver:123\")" in result.output + assert u"Changed image of container 'application' to: \"application:latest\" (was: \"application:123\")" in result.output assert u'Successfully created revision: 2' in result.output assert u'Successfully deregistered revision: 1' in result.output assert u'Successfully changed task definition to: test-task:2' in result.output @@ -91,7 +91,7 @@ def test_deploy_one_new_image(get_client, runner): assert result.exit_code == 0 assert not result.exception assert u"Updating task definition" in result.output - assert u"Changed image of container 'application' to: application:latest (was: application:123)" in result.output + assert u"Changed image of container 'application' to: \"application:latest\" (was: \"application:123\")" in result.output assert u'Successfully created revision: 2' in result.output assert u'Successfully deregistered revision: 1' in result.output assert u'Successfully changed task definition to: test-task:2' in result.output @@ -106,8 +106,8 @@ def test_deploy_two_new_images(get_client, runner): assert result.exit_code == 0 assert not result.exception assert u"Updating task definition" in result.output - assert u"Changed image of container 'webserver' to: webserver:latest (was: webserver:123)" in result.output - assert u"Changed image of container 'application' to: application:latest (was: application:123)" in result.output + assert u"Changed image of container 'webserver' to: \"webserver:latest\" (was: \"webserver:123\")" in result.output + assert u"Changed image of container 'application' to: \"application:latest\" (was: \"application:123\")" in result.output assert u'Successfully created revision: 2' in result.output assert u'Successfully deregistered revision: 1' in result.output assert u'Successfully changed task definition to: test-task:2' in result.output @@ -121,7 +121,7 @@ def test_deploy_one_new_command(get_client, runner): assert result.exit_code == 0 assert not result.exception assert u"Updating task definition" in result.output - assert u"Changed command of container 'application' to: foobar (was: run)" in result.output + assert u"Changed command of container 'application' to: \"foobar\" (was: \"run\")" in result.output assert u'Successfully created revision: 2' in result.output assert u'Successfully deregistered revision: 1' in result.output assert u'Successfully changed task definition to: test-task:2' in result.output @@ -226,6 +226,57 @@ def test_scale_with_timeout(get_client, runner): assert u"Scaling failed (timeout)" in result.output +@patch('ecs_deploy.cli.get_client') +def test_run_task(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.run, (CLUSTER_NAME, 'test-task')) + + assert not result.exception + assert result.exit_code == 0 + + assert u"Successfully started 2 instances of task: test-task:1" in result.output + assert u"- arn:foo:bar" in result.output + assert u"- arn:lorem:ipsum" in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_run_task_with_command(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.run, (CLUSTER_NAME, 'test-task', '2', '-c', 'webserver', 'date')) + + assert not result.exception + assert result.exit_code == 0 + + assert u"Using task definition: test-task" in result.output + assert u"Changed command of container 'webserver' to: \"date\" (was: \"run\")" in result.output + assert u"Successfully started 2 instances of task: test-task:1" in result.output + assert u"- arn:foo:bar" in result.output + assert u"- arn:lorem:ipsum" in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_run_task_with_environment_var(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.run, (CLUSTER_NAME, 'test-task', '2', '-e', 'application', 'foo', 'bar')) + + assert not result.exception + assert result.exit_code == 0 + + assert u"Using task definition: test-task" in result.output + assert u'Changed environment of container \'application\' to: {"foo": "bar"} (was: {})' in result.output + assert u"Successfully started 2 instances of task: test-task:1" in result.output + assert u"- arn:foo:bar" in result.output + assert u"- arn:lorem:ipsum" in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_run_task_with_errors(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key', errors=True) + result = runner.invoke(cli.run, (CLUSTER_NAME, 'foobar')) + assert result.exit_code == 1 + assert u"An error occurred (123) when calling the fake_error operation: Something went wrong" in result.output + + @patch('ecs_deploy.newrelic.Deployment') def test_record_deployment_without_revision(Deployment): result = record_deployment(None, None, None, None, None) diff --git a/tests/test_ecs.py b/tests/test_ecs.py index b779824..35e34b2 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -322,7 +322,7 @@ def test_task_set_command_for_unknown_container(task_definition): def test_task_definition_diff(): diff = EcsTaskDefinitionDiff(u'webserver', u'image', u'new', u'old') - assert str(diff) == u"Changed image of container 'webserver' to: new (was: old)" + assert str(diff) == u"Changed image of container 'webserver' to: \"new\" (was: \"old\")" @patch.object(Session, 'client') @@ -602,3 +602,9 @@ def update_service(self, cluster, service, desired_count, task_definition): if self.errors: return deepcopy(RESPONSE_SERVICE_WITH_ERRORS) return deepcopy(RESPONSE_SERVICE) + + def run_task(self, cluster, task_definition, count, started_by, overrides): + if self.errors: + error = dict(Error=dict(Code=123, Message="Something went wrong")) + raise ClientError(error, 'fake_error') + return dict(tasks=[dict(taskArn='arn:foo:bar'), dict(taskArn='arn:lorem:ipsum')]) From b311c7343a1d3425c0dca552b2ce609fda10f42a Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Fri, 7 Oct 2016 10:31:46 +0200 Subject: [PATCH 03/10] Test failures in scale and run commands --- tests/test_cli.py | 33 ++++++++++++++++++++++++++++++++- tests/test_ecs.py | 4 ++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2694a6b..c2991d1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -226,6 +226,21 @@ def test_scale_with_timeout(get_client, runner): assert u"Scaling failed (timeout)" in result.output +@patch('ecs_deploy.cli.get_client') +def test_scale_without_credentials(get_client, runner): + get_client.return_value = EcsTestClient() + result = runner.invoke(cli.scale, (CLUSTER_NAME, SERVICE_NAME, '2')) + assert result.exit_code == 1 + assert result.output == u'Unable to locate credentials. Configure credentials by running "aws configure".\n\n' + + +@patch('ecs_deploy.cli.get_client') +def test_scale_with_invalid_service(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.scale, (CLUSTER_NAME, 'unknown-service', '2')) + assert result.exit_code == 1 + assert result.output == u'An error occurred when calling the DescribeServices operation: Service not found.\n\n' + @patch('ecs_deploy.cli.get_client') def test_run_task(get_client, runner): get_client.return_value = EcsTestClient('acces_key', 'secret_key') @@ -272,11 +287,27 @@ def test_run_task_with_environment_var(get_client, runner): @patch('ecs_deploy.cli.get_client') def test_run_task_with_errors(get_client, runner): get_client.return_value = EcsTestClient('acces_key', 'secret_key', errors=True) - result = runner.invoke(cli.run, (CLUSTER_NAME, 'foobar')) + result = runner.invoke(cli.run, (CLUSTER_NAME, 'test-task')) assert result.exit_code == 1 assert u"An error occurred (123) when calling the fake_error operation: Something went wrong" in result.output +@patch('ecs_deploy.cli.get_client') +def test_run_task_without_credentials(get_client, runner): + get_client.return_value = EcsTestClient() + result = runner.invoke(cli.run, (CLUSTER_NAME, 'test-task')) + assert result.exit_code == 1 + assert result.output == u'Unable to locate credentials. Configure credentials by running "aws configure".\n\n' + + +@patch('ecs_deploy.cli.get_client') +def test_run_task_with_invalid_cluster(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.run, ('unknown-cluster', 'test-task')) + assert result.exit_code == 1 + assert result.output == u'An error occurred (ClusterNotFoundException) when calling the RunTask operation: Cluster not found.\n\n' + + @patch('ecs_deploy.newrelic.Deployment') def test_record_deployment_without_revision(Deployment): result = record_deployment(None, None, None, None, None) diff --git a/tests/test_ecs.py b/tests/test_ecs.py index 35e34b2..f996053 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -604,6 +604,10 @@ def update_service(self, cluster, service, desired_count, task_definition): return deepcopy(RESPONSE_SERVICE) def run_task(self, cluster, task_definition, count, started_by, overrides): + if not self.access_key_id or not self.secret_access_key: + raise ConnectionError(u'Unable to locate credentials. Configure credentials by running "aws configure".') + if cluster == 'unknown-cluster': + raise ConnectionError(u'An error occurred (ClusterNotFoundException) when calling the RunTask operation: Cluster not found.') if self.errors: error = dict(Error=dict(Code=123, Message="Something went wrong")) raise ClientError(error, 'fake_error') From feffa0693cbbcecc37519fcd3676b77cd6f8592e Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Fri, 7 Oct 2016 14:31:15 +0200 Subject: [PATCH 04/10] Refactor overrides creation and add more tests --- ecs_deploy/ecs.py | 36 +++++++++++++++------- tests/test_ecs.py | 76 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 13 deletions(-) diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index ac48c0a..674dcc2 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -48,6 +48,7 @@ def run_task(self, cluster, task_definition, count, started_by, overrides): overrides=overrides ) + class EcsService(dict): def __init__(self, cluster, iterable=None, **kwargs): self._cluster = cluster @@ -145,6 +146,23 @@ def family_revision(self): def diff(self): return self._diff + def get_overrides(self): + overrides = [] + for diff in self.diff: + override = dict(name=diff.container) + if diff.field == 'command': + override['command'] = self.get_overrides_command(diff.value) + elif diff.field == 'environment': + override['environment'] = self.get_overrides_environment(diff.value) + overrides.append(override) + return overrides + + def get_overrides_command(self, command): + return command.split(' ') + + def get_overrides_environment(self, environment_dict): + return [{"name": e, "value": environment_dict[e]} for e in environment_dict] + def set_images(self, tag=None, **images): self.validate_container_options(**images) for container in self.containers: @@ -302,17 +320,13 @@ def __init__(self, client, cluster_name): self.started_tasks = [] def run(self, task_definition, count, started_by): - overrides = [] - if task_definition.diff: - for diff in task_definition.diff: - override = dict(name=diff.container) - if diff.field == 'command': - override['command'] = diff.value.split(' ') - elif diff.field == 'environment': - override['environment'] = [{"name": e, "value": diff.value[e]} for e in diff.value] - overrides.append(override) - - result = self._client.run_task(self._cluster_name, task_definition.family_revision, count, started_by, dict(containerOverrides=overrides)) + result = self._client.run_task( + cluster=self._cluster_name, + task_definition=task_definition.family_revision, + count=count, + started_by=started_by, + overrides=dict(containerOverrides=task_definition.get_overrides()) + ) self.started_tasks = result['tasks'] return True diff --git a/tests/test_ecs.py b/tests/test_ecs.py index f996053..d0bab7e 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -10,7 +10,7 @@ from mock.mock import patch from ecs_deploy.ecs import EcsService, EcsTaskDefinition, UnknownContainerError, EcsTaskDefinitionDiff, EcsClient, \ - EcsAction, ConnectionError, DeployAction, ScaleAction + EcsAction, ConnectionError, DeployAction, ScaleAction, RunAction CLUSTER_NAME = u'test-cluster' CLUSTER_ARN = u'arn:aws:ecs:eu-central-1:123456789012:cluster/%s' % CLUSTER_NAME @@ -168,7 +168,7 @@ } -@pytest.fixture(scope="module") +@pytest.fixture() def task_definition(): return EcsTaskDefinition(deepcopy(PAYLOAD_TASK_DEFINITION_1)) @@ -320,6 +320,38 @@ def test_task_set_command_for_unknown_container(task_definition): task_definition.set_images(foobar=u'run-foobar') +def test_task_get_overrides(task_definition): + assert task_definition.get_overrides() == [] + + +def test_task_get_overrides_with_command(task_definition): + task_definition.set_commands(webserver='/usr/bin/python script.py') + overrides = task_definition.get_overrides() + assert len(overrides) == 1 + assert overrides[0]['command'] == ['/usr/bin/python','script.py'] + + +def test_task_get_overrides_with_environment(task_definition): + task_definition.set_environment((('webserver', 'foo', 'bar'),)) + overrides = task_definition.get_overrides() + assert len(overrides) == 1 + assert overrides[0]['name'] == 'webserver' + assert overrides[0]['environment'] == [dict(name='foo', value='bar'), dict(name='lorem', value='ipsum')] + + +def test_task_get_overrides_command(task_definition): + command = task_definition.get_overrides_command('/usr/bin/python script.py') + assert isinstance(command, list) + assert command == ['/usr/bin/python','script.py'] + + +def test_task_get_overrides_environment(task_definition): + environment = task_definition.get_overrides_environment(dict(foo='bar')) + assert isinstance(environment, list) + assert len(environment) == 1 + assert environment[0] == dict(name='foo', value='bar') + + def test_task_definition_diff(): diff = EcsTaskDefinitionDiff(u'webserver', u'image', u'new', u'old') assert str(diff) == u"Changed image of container 'webserver' to: \"new\" (was: \"old\")" @@ -390,6 +422,24 @@ def test_client_update_service(client): ) +def test_client_run_task(client): + client.run_task( + cluster=u'test-cluster', + task_definition=u'test-task', + count=2, + started_by='test', + overrides=dict(foo='bar') + ) + + client.boto.run_task.assert_called_once_with( + cluster=u'test-cluster', + taskDefinition=u'test-task', + count=2, + startedBy='test', + overrides=dict(foo='bar') + ) + + def test_ecs_action_init(client): action = EcsAction(client, u'test-cluster', u'test-service') assert action.client == client @@ -552,6 +602,28 @@ def test_scale_action(client): client.update_service.assert_called_once_with(action.service.cluster, action.service.name, 5, action.service.task_definition) +@patch.object(EcsClient, '__init__') +def test_run_action(client): + action = RunAction(client, CLUSTER_NAME) + assert len(action.started_tasks) == 0 + + +@patch.object(EcsClient, '__init__') +def test_run_action_run(client, task_definition): + action = RunAction(client, CLUSTER_NAME) + client.run_task.return_value = dict(tasks=[dict(taskArn='A'), dict(taskArn='B')]) + action.run(task_definition, 2, 'test') + + client.run_task.assert_called_once_with( + cluster=CLUSTER_NAME, + task_definition=task_definition.family_revision, + count=2, + started_by='test', + overrides=dict(containerOverrides=task_definition.get_overrides()) + ) + + assert len(action.started_tasks) == 2 + class EcsTestClient(object): def __init__(self, access_key_id=None, secret_access_key=None, region=None, profile=None, errors=False, wait=0): From 60051e7386677bf97038c2eeb1900aef26bfacc6 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Fri, 7 Oct 2016 14:36:55 +0200 Subject: [PATCH 05/10] Adjust test for env override --- tests/test_ecs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ecs.py b/tests/test_ecs.py index d0bab7e..c46956e 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -332,11 +332,11 @@ def test_task_get_overrides_with_command(task_definition): def test_task_get_overrides_with_environment(task_definition): - task_definition.set_environment((('webserver', 'foo', 'bar'),)) + task_definition.set_environment((('webserver', 'foo', 'baz'),)) overrides = task_definition.get_overrides() assert len(overrides) == 1 assert overrides[0]['name'] == 'webserver' - assert overrides[0]['environment'] == [dict(name='foo', value='bar'), dict(name='lorem', value='ipsum')] + assert dict(name='foo', value='baz') in overrides[0]['environment'] def test_task_get_overrides_command(task_definition): From 2441feff0eca8564b31d39adffcedbf119adab21 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Fri, 7 Oct 2016 14:49:45 +0200 Subject: [PATCH 06/10] Fix override creation --- ecs_deploy/ecs.py | 6 ++++-- tests/test_ecs.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index 674dcc2..4aadc09 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -147,14 +147,16 @@ def diff(self): return self._diff def get_overrides(self): + override = dict() overrides = [] for diff in self.diff: - override = dict(name=diff.container) + if override.get('name') != diff.container: + override = dict(name=diff.container) + overrides.append(override) if diff.field == 'command': override['command'] = self.get_overrides_command(diff.value) elif diff.field == 'environment': override['environment'] = self.get_overrides_environment(diff.value) - overrides.append(override) return overrides def get_overrides_command(self, command): diff --git a/tests/test_ecs.py b/tests/test_ecs.py index c46956e..bce8f60 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -339,6 +339,27 @@ def test_task_get_overrides_with_environment(task_definition): assert dict(name='foo', value='baz') in overrides[0]['environment'] +def test_task_get_overrides_with_commandand_environment(task_definition): + task_definition.set_commands(webserver='/usr/bin/python script.py') + task_definition.set_environment((('webserver', 'foo', 'baz'),)) + overrides = task_definition.get_overrides() + assert len(overrides) == 1 + assert overrides[0]['name'] == 'webserver' + assert overrides[0]['command'] == ['/usr/bin/python','script.py'] + assert dict(name='foo', value='baz') in overrides[0]['environment'] + + +def test_task_get_overrides_with_commandand_environment_for_multiple_containers(task_definition): + task_definition.set_commands(application='/usr/bin/python script.py') + task_definition.set_environment((('webserver', 'foo', 'baz'),)) + overrides = task_definition.get_overrides() + assert len(overrides) == 2 + assert overrides[0]['name'] == 'application' + assert overrides[0]['command'] == ['/usr/bin/python','script.py'] + assert overrides[1]['name'] == 'webserver' + assert dict(name='foo', value='baz') in overrides[1]['environment'] + + def test_task_get_overrides_command(task_definition): command = task_definition.get_overrides_command('/usr/bin/python script.py') assert isinstance(command, list) From 2b5513f37f66f731b27a599f97791363d5086836 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Fri, 7 Oct 2016 15:00:57 +0200 Subject: [PATCH 07/10] Add docs for run command --- README.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.rst b/README.rst index 139bd17..05cdc02 100644 --- a/README.rst +++ b/README.rst @@ -56,6 +56,10 @@ scale ===== Scale a service up or down and change the number of running tasks. +run +=== +Run a one-off task based on an existing task-definition and optionally override command and/or environment variables. + Usage ----- @@ -136,3 +140,25 @@ To change the number of running tasks and scale a service up and down, run this $ ecs scale my-cluster my-service 4 + +Run a one-off task +================== +To run a one-off task, based on an existing task-definition, run this command:: + + $ ecs run my-cluster my-task + +You can define just the task family (e.g. ``my-task``) or you can run a specific revision of the task-definition (e.g. +``my-task:123``). And optionally you can add or adjust environment variables like this:: + + $ ecs run my-cluster my-task:123 -e my-container MY_VARIABLE "my value" + + +Run a task with a custom command +================================ + +You can override the command definition via option ``-c`` or ``--command`` followed by the container name and the +command in a natural syntax, e.g. no conversion to comma-separation required:: + + $ ecs run my-cluster my-task -c my-container "python some-script.py param1 param2" + + From ebfac4ca9ccd9c5525d64c43c0861358212c4c87 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Fri, 7 Oct 2016 15:11:02 +0200 Subject: [PATCH 08/10] Allow setting New Relic API Key and App ID via environment variables --- ecs_deploy/cli.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index d194ddc..6178bc9 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -1,4 +1,6 @@ from __future__ import print_function, absolute_import + +from os import getenv from time import sleep import click @@ -166,6 +168,9 @@ def wait_for_finish(action, timeout, title, success_message, failure_message): def record_deployment(revision, newrelic_apikey, newrelic_appid, comment, user): + newrelic_apikey = getenv('NEW_RELIC_API_KEY', newrelic_apikey) + newrelic_appid = getenv('NEW_RELIC_APP_ID', newrelic_appid) + if not revision or not newrelic_apikey or not newrelic_appid: return False From 930ab6381005624003497b0372bfc1d752925f7e Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Fri, 7 Oct 2016 15:20:03 +0200 Subject: [PATCH 09/10] Add New Relic documentation --- README.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.rst b/README.rst index 05cdc02..5b0f619 100644 --- a/README.rst +++ b/README.rst @@ -161,4 +161,22 @@ command in a natural syntax, e.g. no conversion to comma-separation required:: $ ecs run my-cluster my-task -c my-container "python some-script.py param1 param2" +Monitoring +---------- +With ECS deploy you can track your deployments automatically. Currently only New Relic is supported: + +New Relic +========= +To record a deployment in New Relic, you can provide the the API Key (**Attention**: this is a specific REST API Key, not the license key) and the application id in two ways: + +Via cli options:: + + $ ecs deploy my-cluster my-service --newrelic-apikey ABCDEFGHIJKLMN --newrelic-appid 1234567890 + +Or implicitly via environment variables ``NEW_RELIC_API_KEY`` and ``NEW_RELIC_APP_ID`` :: + + $ export NEW_RELIC_API_KEY=ABCDEFGHIJKLMN + $ export NEW_RELIC_APP_ID=1234567890 + $ ecs deploy my-cluster my-service +Optionally you can provide an additional comment to the deployment via ``--comment "New feature X"`` and the name of the user who deployed with ``--user john.doe`` From 67a159a94c02171c846d10784a146539c0e51ae4 Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Fri, 7 Oct 2016 15:22:55 +0200 Subject: [PATCH 10/10] Bump version to 1.0.0 --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 76b136c..8199294 100644 --- a/setup.py +++ b/setup.py @@ -7,12 +7,12 @@ setup( name='ecs-deploy', - version='0.3.0', + version='1.0.0', url='https://github.com/fabfuel/ecs-deploy', - download_url='https://github.com/fabfuel/ecs-deploy/archive/0.3.0.tar.gz', + download_url='https://github.com/fabfuel/ecs-deploy/archive/1.0.0.tar.gz', license='BSD', author='Fabian Fuelling', - author_email='fabian@fabfuel.de', + author_email='pypi@fabfuel.de', description='Simplify Amazon ECS deployments', long_description=__doc__, packages=find_packages(exclude=['tests']),