From 16f4a6c816fd5bacffb36561fed2e3257ffee10f Mon Sep 17 00:00:00 2001 From: Anthony Eadicicco Date: Wed, 28 Jul 2021 14:43:30 -0400 Subject: [PATCH 01/15] Implement setting docker labels --- ecs_deploy/cli.py | 14 ++++++++- ecs_deploy/ecs.py | 80 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index 884fed3..681d65a 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -74,7 +74,9 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.') @click.option('--add-container', type=str, multiple=True, required=False, help='Add a placeholder container in the task definition.') @click.option('--remove-container', type=str, multiple=True, required=False, help='Remove a container from the task definition.') -def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, privileged, essential, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, slack_service_match='.*'): +@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') +@click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') +def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, privileged, essential, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, dockerlabels, exclusive_dockerlabels, slack_service_match='.*'): """ Redeploy or modify a service. @@ -104,6 +106,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem td.set_privileged(**{key: value for (key, value) in privileged}) td.set_essential(**{key: value for (key, value) in essential}) td.set_environment(env, exclusive_env, env_file) + td.set_docker_labels(dockerlabels, exclusive_dockerlabels) td.set_secrets(secret, exclusive_secrets) td.set_ulimits(ulimit, exclusive_ulimits) td.set_system_controls(system_control, exclusive_system_controls) @@ -202,6 +205,8 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem @click.option('--exclusive-ports', is_flag=True, default=False, help='Set the given port mappings exclusively and remove all other pre-existing port mappings from all containers') @click.option('--exclusive-mounts', is_flag=True, default=False, help='Set the given mount points exclusively and remove all other pre-existing mount points from all containers') @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.') +@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') +@click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, privileged, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, diff, deregister, rollback, exclusive_env, exclusive_secrets, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume): """ Update a scheduled task. @@ -225,6 +230,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio td.set_memoryreservation(**{key: value for (key, value) in memoryreservation}) td.set_privileged(**{key: value for (key, value) in privileged}) td.set_environment(env, exclusive_env, env_file) + td.set_docker_labels(dockerlabels, exclusive_dockerlabels) td.set_secrets(secret, exclusive_secrets) td.set_ulimits(ulimit, exclusive_ulimits) td.set_system_controls(system_control, exclusive_system_controls) @@ -283,6 +289,8 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') @click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)') +@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') +@click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') def update(task, image, tag, command, env, env_file, secret, role, region, access_key_id, secret_access_key, profile, diff, exclusive_env, exclusive_secrets, deregister): """ Update a task definition. @@ -300,6 +308,7 @@ def update(task, image, tag, command, env, env_file, secret, role, region, acces td.set_images(tag, **{key: value for (key, value) in image}) td.set_commands(**{key: value for (key, value) in command}) td.set_environment(env, exclusive_env, env_file) + td.set_docker_labels(dockerlabels, exclusive_dockerlabels) td.set_secrets(secret, exclusive_secrets) td.set_role_arn(role) @@ -379,6 +388,8 @@ def scale(cluster, service, desired_count, access_key_id, secret_access_key, reg @click.option('--profile', help='AWS configuration profile name') @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition') +@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') +@click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') def run(cluster, task, count, command, env, env_file, secret, launchtype, subnet, securitygroup, public_ip, platform_version, region, access_key_id, secret_access_key, profile, exclusive_env, diff): """ Run a one-off task. @@ -395,6 +406,7 @@ def run(cluster, task, count, command, env, env_file, secret, launchtype, subnet td = action.get_task_definition(task) td.set_commands(**{key: value for (key, value) in command}) td.set_environment(env, exclusive_env, env_file) + td.set_docker_labels(dockerlabels, exclusive_dockerlabels) td.set_secrets(secret) if diff: diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index cee8b62..16abd59 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -335,6 +335,14 @@ def diff_raw(self, task_b): containers_b[container]['secrets'] = {e['name']: e['valueFrom'] for e in containers_b[container].get('secrets', {})} + for container in containers_a: + containers_a[container]['dockerLabels'] = {l['name']: l['value'] for l in + containers_a[container].get('dockerLabels', {})} + + for container in containers_b: + containers_b[container]['dockerLabels'] = {l['name']: l['value'] for l in + containers_b[container].get('dockerLabels', {})} + composite_a = { 'containers': containers_a, 'volumes': self.volumes, @@ -370,6 +378,8 @@ def get_overrides(self): override['environment'] = self.get_overrides_env(diff.value) elif diff.field == 'secrets': override['secrets'] = self.get_overrides_secrets(diff.value) + elif diff.field == 'dockerLabels': + override['dockerLabels'] = self.get_overrides_docker_labels(diff.value) return overrides @staticmethod @@ -397,6 +407,10 @@ def get_overrides_env(env): def get_overrides_secrets(secrets): return [{"name": s, "valueFrom": secrets[s]} for s in secrets] + @staticmethod + def get_overrides_docker_labels(dockerlabels): + return {l: dockerlabels[l] for l in dockerlabels} + def set_images(self, tag=None, **images): self.validate_container_options(**images) for container in self.containers: @@ -602,6 +616,50 @@ def apply_container_environment(self, container, new_environment, exclusive=Fals {"name": e, "value": merged[e]} for e in merged ] + def set_docker_labels(self, dockerlabel_list, exclusive=False): + dockerlabels = defaultdict(dict) + for label in dockerlabel_list: + dockerlabels[label[0]][label[1]] = label[2] + self.validate_container_options(**dockerlabels) + for container in self.containers: + if container[u'name'] in dockerlabels: + self.apply_docker_labels( + container=container, + new_dockerlabels=dockerlabels[container[u'name']], + exclusive=exclusive, + ) + elif exclusive is True: + self.apply_docker_labels( + container=container, + new_dockerlabels={}, + exclusive=exclusive, + ) + + def apply_docker_labels(self, container, new_dockerlabels, exclusive=False): + dockerlabels = container.get('dockerLabels', {}) + old_dockerlabels = {label['name']: label['value'] for label in dockerlabels} + + if exclusive is True: + merged = new_dockerlabels + else: + merged = old_dockerlabels.copy() + merged.update(new_dockerlabels) + + if old_dockerlabels == merged: + return + + diff = EcsTaskDefinitionDiff( + container=container[u'name'], + field=u'dockerLabels', + value=merged, + old_value=old_dockerlabels + ) + self._diff.append(diff) + + container[u'dockerLabels'] = { + l: merged[l] for l in merged + } + def set_secrets(self, secrets_list, exclusive=False): secrets = defaultdict(dict) @@ -1040,6 +1098,12 @@ def __repr__(self): self.value, self.old_value, )) + elif self.field == u'dockerLabels': + return '\n'.join(self._get_docker_label_diffs( + self.container, + self.value, + self.old_value, + )) elif self.container: return u'Changed %s of container "%s" to: "%s" (was: "%s")' % ( self.field, @@ -1070,6 +1134,22 @@ def _get_environment_diffs(container, env, old_env): diffs.append(message) return diffs + @staticmethod + def _get_docker_label_diffs(container, dockerlabels, old_dockerlabels): + msg = u'Changed dockerLabel "%s" of container "%s" to: "%s"' + msg_removed = u'Removed dockerLabel "%s" of container "%s"' + diffs = [] + for name, value in dockerlabels.items(): + old_value = old_dockerlabels.get(name) + if value != old_value or value and not old_value: + message = msg % (name, container, value) + diffs.append(message) + for old_name in old_dockerlabels.keys(): + if old_name not in dockerlabels.keys(): + message = msg_removed % (old_name, container) + diffs.append(message) + return diffs + @staticmethod def _get_secrets_diffs(container, secrets, old_secrets): msg = u'Changed secret "%s" of container "%s" to: "%s"' From e6ee19a2e98d78a4dbc89f8a5a3f9aff8b7c3c5d Mon Sep 17 00:00:00 2001 From: Anthony Eadicicco Date: Wed, 28 Jul 2021 15:27:18 -0400 Subject: [PATCH 02/15] Fix up @click options --- ecs_deploy/cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index 681d65a..7d29d5f 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -76,7 +76,7 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--remove-container', type=str, multiple=True, required=False, help='Remove a container from the task definition.') @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') -def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, privileged, essential, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, dockerlabels, exclusive_dockerlabels, slack_service_match='.*'): +def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, privileged, essential, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, slack_service_match='.*'): """ Redeploy or modify a service. @@ -106,7 +106,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem td.set_privileged(**{key: value for (key, value) in privileged}) td.set_essential(**{key: value for (key, value) in essential}) td.set_environment(env, exclusive_env, env_file) - td.set_docker_labels(dockerlabels, exclusive_dockerlabels) + td.set_docker_labels(docker_label, exclusive_docker_labels) td.set_secrets(secret, exclusive_secrets) td.set_ulimits(ulimit, exclusive_ulimits) td.set_system_controls(system_control, exclusive_system_controls) @@ -207,7 +207,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.') @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') -def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, privileged, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, diff, deregister, rollback, exclusive_env, exclusive_secrets, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume): +def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, privileged, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, diff, deregister, rollback, exclusive_env, exclusive_secrets, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, docker_label, exclusive_docker_labels): """ Update a scheduled task. @@ -230,7 +230,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio td.set_memoryreservation(**{key: value for (key, value) in memoryreservation}) td.set_privileged(**{key: value for (key, value) in privileged}) td.set_environment(env, exclusive_env, env_file) - td.set_docker_labels(dockerlabels, exclusive_dockerlabels) + td.set_docker_labels(docker_label, exclusive_docker_labels) td.set_secrets(secret, exclusive_secrets) td.set_ulimits(ulimit, exclusive_ulimits) td.set_system_controls(system_control, exclusive_system_controls) @@ -291,7 +291,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio @click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)') @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') -def update(task, image, tag, command, env, env_file, secret, role, region, access_key_id, secret_access_key, profile, diff, exclusive_env, exclusive_secrets, deregister): +def update(task, image, tag, command, env, env_file, secret, role, region, access_key_id, secret_access_key, profile, diff, exclusive_env, exclusive_secrets, deregister, docker_label, exclusive_docker_labels): """ Update a task definition. @@ -308,7 +308,7 @@ def update(task, image, tag, command, env, env_file, secret, role, region, acces td.set_images(tag, **{key: value for (key, value) in image}) td.set_commands(**{key: value for (key, value) in command}) td.set_environment(env, exclusive_env, env_file) - td.set_docker_labels(dockerlabels, exclusive_dockerlabels) + td.set_docker_labels(docker_label, exclusive_docker_labels) td.set_secrets(secret, exclusive_secrets) td.set_role_arn(role) @@ -390,7 +390,7 @@ def scale(cluster, service, desired_count, access_key_id, secret_access_key, reg @click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition') @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') -def run(cluster, task, count, command, env, env_file, secret, launchtype, subnet, securitygroup, public_ip, platform_version, region, access_key_id, secret_access_key, profile, exclusive_env, diff): +def run(cluster, task, count, command, env, env_file, secret, launchtype, subnet, securitygroup, public_ip, platform_version, region, access_key_id, secret_access_key, profile, exclusive_env, diff, docker_label, exclusive_docker_labels): """ Run a one-off task. @@ -406,7 +406,7 @@ def run(cluster, task, count, command, env, env_file, secret, launchtype, subnet td = action.get_task_definition(task) td.set_commands(**{key: value for (key, value) in command}) td.set_environment(env, exclusive_env, env_file) - td.set_docker_labels(dockerlabels, exclusive_dockerlabels) + td.set_docker_labels(docker_label, exclusive_docker_labels) td.set_secrets(secret) if diff: From 379813ee2849d69140c67c59e1e381cb7613ca38 Mon Sep 17 00:00:00 2001 From: Anthony Eadicicco Date: Wed, 28 Jul 2021 15:30:21 -0400 Subject: [PATCH 03/15] Reorder @click options --- ecs_deploy/cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index 7d29d5f..2d97927 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -39,6 +39,7 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: ') @click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file') @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): ') +@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('-u', '--ulimit', type=(str, str, int, int), multiple=True, help='Adds or changes a ulimit variable in the container description (Not available for Fargate): ') @click.option('--system-control', type=(str, str, str), multiple=True, help='Adds or changes a system control variable in the container description (Not available for Fargate): ') @click.option('-p', '--port', type=(str, int, int), multiple=True, help='Adds or changes a port mappings in the container description (Not available for Fargate): ') @@ -64,6 +65,7 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--rollback/--no-rollback', default=False, help='Rollback to previous revision, if deployment failed (default: --no-rollback)') @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') +@click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') @click.option('--sleep-time', default=1, type=int, help='Amount of seconds to wait between each check of the service (default: 1)') @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, which services should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') @@ -74,8 +76,6 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.') @click.option('--add-container', type=str, multiple=True, required=False, help='Add a placeholder container in the task definition.') @click.option('--remove-container', type=str, multiple=True, required=False, help='Remove a container from the task definition.') -@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') -@click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, privileged, essential, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, slack_service_match='.*'): """ Redeploy or modify a service. @@ -175,6 +175,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem @click.option('--privileged', type=(str, bool), multiple=True, help='Overwrites the memory reservation value for a container: ') @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: ') @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): ') +@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('-u', '--ulimit', type=(str, str, int, int), multiple=True, help='Adds or changes a ulimit variable in the container description (Not available for Fargate): ') @click.option('--system-control', type=(str, str, str), multiple=True, help='Adds or changes a system control variable in the container description (Not available for Fargate): ') @click.option('-p', '--port', type=(str, int, int), multiple=True, help='Adds or changes a port mappings in the container description (Not available for Fargate): ') @@ -198,6 +199,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem @click.option('--rollback/--no-rollback', default=False, help='Rollback to previous revision, if deployment failed (default: --no-rollback)') @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') +@click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, deployments of which crons should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') @click.option('--exclusive-ulimits', is_flag=True, default=False, help='Set the given ulimits exclusively and remove all other pre-existing ulimits from all containers') @@ -205,8 +207,6 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem @click.option('--exclusive-ports', is_flag=True, default=False, help='Set the given port mappings exclusively and remove all other pre-existing port mappings from all containers') @click.option('--exclusive-mounts', is_flag=True, default=False, help='Set the given mount points exclusively and remove all other pre-existing mount points from all containers') @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.') -@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') -@click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, privileged, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, diff, deregister, rollback, exclusive_env, exclusive_secrets, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, docker_label, exclusive_docker_labels): """ Update a scheduled task. @@ -280,6 +280,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: ') @click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file') @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): ') +@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('-r', '--role', type=str, help='Sets the task\'s role ARN: ') @click.option('--region', help='AWS region (e.g. eu-central-1)') @click.option('--access-key-id', help='AWS access key id') @@ -288,9 +289,8 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio @click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition') @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') -@click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)') -@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') +@click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)') def update(task, image, tag, command, env, env_file, secret, role, region, access_key_id, secret_access_key, profile, diff, exclusive_env, exclusive_secrets, deregister, docker_label, exclusive_docker_labels): """ Update a task definition. @@ -377,6 +377,7 @@ def scale(cluster, service, desired_count, access_key_id, secret_access_key, reg @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: ') @click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file') @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): ') +@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('--launchtype', type=click.Choice([LAUNCH_TYPE_EC2, LAUNCH_TYPE_FARGATE]), default=LAUNCH_TYPE_EC2, help='ECS Launch type (default: EC2)') @click.option('--subnet', type=str, multiple=True, help='A subnet ID to launch the task within. Required for launch type FARGATE (multiple values possible)') @click.option('--securitygroup', type=str, multiple=True, help='A security group ID to launch the task within. Required for launch type FARGATE (multiple values possible)') @@ -387,9 +388,8 @@ def scale(cluster, service, desired_count, access_key_id, secret_access_key, reg @click.option('--secret-access-key', help='AWS secret access key') @click.option('--profile', help='AWS configuration profile name') @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') -@click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition') -@click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') +@click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition') def run(cluster, task, count, command, env, env_file, secret, launchtype, subnet, securitygroup, public_ip, platform_version, region, access_key_id, secret_access_key, profile, exclusive_env, diff, docker_label, exclusive_docker_labels): """ Run a one-off task. From b02d1432f405f4c9c45a695096b6ce828eb00290 Mon Sep 17 00:00:00 2001 From: Anthony Eadicicco Date: Wed, 28 Jul 2021 16:11:14 -0400 Subject: [PATCH 04/15] dockerlabels is a dict, not a list of dicts --- ecs_deploy/ecs.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index 16abd59..c04427a 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -636,8 +636,7 @@ def set_docker_labels(self, dockerlabel_list, exclusive=False): ) def apply_docker_labels(self, container, new_dockerlabels, exclusive=False): - dockerlabels = container.get('dockerLabels', {}) - old_dockerlabels = {label['name']: label['value'] for label in dockerlabels} + old_dockerlabels = container.get('dockerLabels', {}) if exclusive is True: merged = new_dockerlabels @@ -656,9 +655,7 @@ def apply_docker_labels(self, container, new_dockerlabels, exclusive=False): ) self._diff.append(diff) - container[u'dockerLabels'] = { - l: merged[l] for l in merged - } + container[u'dockerLabels'] = {l: merged[l] for l in merged} def set_secrets(self, secrets_list, exclusive=False): secrets = defaultdict(dict) From def289e1e6ffb15f8225ced1d148d603f4d1151c Mon Sep 17 00:00:00 2001 From: Anthony Eadicicco Date: Wed, 28 Jul 2021 16:31:07 -0400 Subject: [PATCH 05/15] Add ecs.py tests --- tests/test_ecs.py | 50 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/test_ecs.py b/tests/test_ecs.py index 90a2a5e..b6dcf99 100644 --- a/tests/test_ecs.py +++ b/tests/test_ecs.py @@ -31,13 +31,14 @@ {u'name': u'webserver', u'image': u'webserver:123', u'command': u'run', u'environment': ({"name": "foo", "value": "bar"}, {"name": "lorem", "value": "ipsum"}, {"name": "empty", "value": ""}), u'secrets': ({"name": "baz", "valueFrom": "qux"}, {"name": "dolor", "valueFrom": "sit"}), + u'dockerLabels': {"foo": "bar", "lorem": "ipsum", "empty": ""}, u'logConfiguration': {}, u'ulimits': [{'name': 'memlock', 'softLimit': 256, 'hardLimit': 256}], u'systemControls': [{'namespace': 'net.core.somaxconn', 'value': '511'}], u'portMappings': [{'containerPort': 8080, 'hostPort': 8080}], u'mountPoints': [{'sourceVolume': 'volume', 'containerPath': '/container/path', 'readOnly': False}]}, {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': (), - u'logConfiguration': {}, + u'logConfiguration': {}, u'dockerLabels': {}, u'ulimits': [{'name': 'memlock', 'softLimit': 256, 'hardLimit': 256}], u'systemControls': [{'namespace': 'net.core.somaxconn', 'value': '511'}], u'portMappings': [{'containerPort': 8080, 'hostPort': 8080}], @@ -53,13 +54,14 @@ {u'name': u'webserver', u'image': u'webserver:123', u'command': u'run', u'environment': ({"name": "foo", "value": "bar"}, {"name": "lorem", "value": "ipsum"}, {"name": "empty", "value": ""}), u'secrets': ({"name": "baz", "valueFrom": "qux"}, {"name": "dolor", "valueFrom": "sit"}), + u'dockerLabels': {"foo": "bar", "lorem": "ipsum", "empty": ""}, u'logConfiguration': {}, u'ulimits': [{'name': 'memlock', 'softLimit': 256, 'hardLimit': 256}], u'systemControls': [{'namespace': 'net.core.somaxconn', 'value': '511'}], u'portMappings': [{'containerPort': 8080, 'hostPort': 8080}], u'mountPoints': [{'sourceVolume': 'volume', 'containerPath': '/container/path', 'readOnly': False}]}, {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': (), - u'logConfiguration': {}, + u'logConfiguration': {}, u'dockerLabels': {}, u'ulimits': [{'name': 'memlock', 'softLimit': 256, 'hardLimit': 256}], u'systemControls': [{'namespace': 'net.core.somaxconn', 'value': '511'}], u'portMappings': [{'containerPort': 8080, 'hostPort': 8080}], @@ -74,7 +76,8 @@ TASK_DEFINITION_CONTAINERS_3 = [ {u'name': u'webserver', u'image': u'webserver:456', u'command': u'execute', u'environment': ({"name": "foo", "value": "foobar"}, {"name": "newvar", "value": "new value"}), - u'secrets': ({"name": "baz", "valueFrom": "foobaz"}, {"name": "dolor", "valueFrom": "loremdolor"})}, + u'secrets': ({"name": "baz", "valueFrom": "foobaz"}, {"name": "dolor", "valueFrom": "loremdolor"}), + u'dockerLabels': {"foo": "foobar", "newlabel": "new value"}}, {u'name': u'application', u'image': u'application:123', u'command': u'run', u'environment': ()} ] TASK_DEFINITION_ROLE_ARN_3 = u'arn:test:another-role:1' @@ -669,6 +672,31 @@ def test_task_set_environment_exclusively(task_definition): assert {'name': 'new-var', 'value': 'new-value'} in task_definition.containers[1]['environment'] +def test_task_set_docker_labels(task_definition): + assert len(task_definition.containers[0]['dockerLabels']) == 3 + + task_definition.set_docker_labels(((u'webserver', u'foo', u'baz'), (u'webserver', u'some-name', u'some-value'))) + + assert len(task_definition.containers[0]['dockerLabels']) == 4 + + assert 'foo' in task_definition.containers[0]['dockerLabels'] + assert 'lorem' in task_definition.containers[0]['dockerLabels'] + assert 'some-name' in task_definition.containers[0]['dockerLabels'] + +def test_task_set_docker_label_exclusively(task_definition): + assert len(task_definition.containers[0]['dockerLabels']) == 3 + assert len(task_definition.containers[1]['dockerLabels']) == 0 + + task_definition.set_docker_labels(((u'application', u'foo', u'baz'), (u'application', u'new-var', u'new-value')), exclusive=True) + + assert len(task_definition.containers[0]['dockerLabels']) == 0 + assert len(task_definition.containers[1]['dockerLabels']) == 2 + + assert task_definition.containers[0]['dockerLabels'] == {} + assert 'foo' in task_definition.containers[1]['dockerLabels'] + assert 'new-var' in task_definition.containers[1]['dockerLabels'] + + def test_task_set_secrets_exclusively(task_definition): assert len(task_definition.containers[0]['secrets']) == 2 @@ -900,6 +928,15 @@ def test_task_get_overrides_with_environment(task_definition): assert dict(name='foo', value='baz') in overrides[0]['environment'] +def test_task_get_overrides_with_docker_labels(task_definition): + task_definition.set_docker_labels((('webserver', 'foo', 'baz'),)) + overrides = task_definition.get_overrides() + assert len(overrides) == 1 + assert overrides[0]['name'] == 'webserver' + #assert 'foo' in overrides[0]['dockerLabels'] + assert overrides[0]['dockerLabels']['foo'] == 'baz' + + def test_task_get_overrides_with_secrets(task_definition): task_definition.set_secrets((('webserver', 'foo', 'baz'),)) overrides = task_definition.get_overrides() @@ -946,6 +983,13 @@ def test_task_get_overrides_environment(task_definition): assert environment[0] == dict(name='foo', value='bar') +def test_task_get_overrides_docker_labels(task_definition): + dockerlabels = task_definition.get_overrides_docker_labels(dict(foo='bar')) + assert isinstance(dockerlabels, dict) + assert len(dockerlabels) == 1 + assert dockerlabels['foo'] == 'bar' + + def test_task_get_overrides_secrets(task_definition): secrets = task_definition.get_overrides_secrets(dict(foo='bar')) assert isinstance(secrets, list) From 1f1d84a1f9cbfe2c95cec3ca2c83b210e557fc4e Mon Sep 17 00:00:00 2001 From: Anthony Eadicicco Date: Wed, 28 Jul 2021 23:28:59 -0400 Subject: [PATCH 06/15] Add CLI tests --- tests/test_cli.py | 254 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2f447b0..6e94ef3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -357,6 +357,115 @@ def test_deploy_exclusive_environment(get_client, runner): assert u'Successfully changed task definition to: test-task:2' in result.output assert u'Deployment successful' in result.output +@patch('ecs_deploy.cli.get_client') +def test_deploy_one_new_docker_laberl(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, + '-d', 'application', 'foo', 'bar', + '-d', 'webserver', 'foo', 'baz')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Deploying based on task definition: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "foo" of container "application" to: "bar"' in result.output + assert u'Changed dockerLabel "foo" of container "webserver" to: "baz"' in result.output + assert u'Changed dockerLabel "lorem" of container "webserver" to: "ipsum"' not in result.output + assert u'Successfully created revision: 2' in result.output + assert u'Successfully deregistered revision: 1' in result.output + assert u'Successfully changed task definition to: test-task:2' in result.output + assert u'Deployment successful' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_deploy_change_docker_label_empty_string(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-d', 'application', 'foo', '')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Deploying based on task definition: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "foo" of container "application" to: ""' in result.output + assert u'Successfully created revision: 2' in result.output + assert u'Successfully deregistered revision: 1' in result.output + assert u'Successfully changed task definition to: test-task:2' in result.output + assert u'Deployment successful' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_deploy_new_empty_docker_label(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-d', 'application', 'new', '')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Deploying based on task definition: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "new" of container "application" to: ""' in result.output + assert u'Successfully created revision: 2' in result.output + assert u'Successfully deregistered revision: 1' in result.output + assert u'Successfully changed task definition to: test-task:2' in result.output + assert u'Deployment successful' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_deploy_empty_docker_label_again(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-d', 'webserver', 'empty', '')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Deploying based on task definition: test-task:1" in result.output + assert u"Updating task definition" not in result.output + assert u'Changed dockerLabel' not in result.output + assert u'Successfully created revision: 2' in result.output + assert u'Successfully deregistered revision: 1' in result.output + assert u'Successfully changed task definition to: test-task:2' in result.output + assert u'Deployment successful' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_deploy_previously_empty_docker_label_with_value(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-d', 'webserver', 'empty', 'not-empty')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Deploying based on task definition: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "empty" of container "webserver" to: "not-empty"' in result.output + assert u'Successfully created revision: 2' in result.output + assert u'Successfully deregistered revision: 1' in result.output + assert u'Successfully changed task definition to: test-task:2' in result.output + assert u'Deployment successful' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_deploy_exclusive_docker_label(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-d', 'webserver', 'new-label', 'new-value', '--exclusive-docker-labels')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Deploying based on task definition: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "new-label" of container "webserver" to: "new-value"' in result.output + + assert u'Removed dockerLabel "foo" of container "webserver"' in result.output + assert u'Removed dockerLabel "lorem" of container "webserver"' in result.output + + assert u'Successfully created revision: 2' in result.output + assert u'Successfully deregistered revision: 1' in result.output + assert u'Successfully changed task definition to: test-task:2' in result.output + assert u'Deployment successful' in result.output + @patch('ecs_deploy.cli.get_client') def test_deploy_exclusive_secret(get_client, runner): @@ -419,6 +528,23 @@ def test_deploy_without_changing_environment_value(get_client, runner): assert u'Deployment successful' in result.output +@patch('ecs_deploy.cli.get_client') +def test_deploy_without_changing_docker_labels(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '-d', 'webserver', 'foo', 'bar')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Deploying based on task definition: test-task:1" in result.output + assert u"Updating task definition" not in result.output + assert u'Changed dockerLabel' not in result.output + assert u'Successfully created revision: 2' in result.output + assert u'Successfully deregistered revision: 1' in result.output + assert u'Successfully changed task definition to: test-task:2' in result.output + assert u'Deployment successful' in result.output + + @patch('ecs_deploy.cli.get_client') def test_deploy_without_changing_secrets_value(get_client, runner): get_client.return_value = EcsTestClient('acces_key', 'secret_key') @@ -756,6 +882,21 @@ def test_run_task_with_environment_var(get_client, runner): assert u"- arn:lorem:ipsum" in result.output +@patch('ecs_deploy.cli.get_client') +def test_run_task_with_docker_label(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.run, (CLUSTER_NAME, 'test-task', '2', '-d', 'application', 'foo', 'bar')) + + assert not result.exception + assert result.exit_code == 0 + + assert u"Using task definition: test-task" in result.output + assert u'Changed dockerLabel "foo" of container "application" to: "bar"' in result.output + assert u"Successfully started 2 instances of task: test-task:2" in result.output + assert u"- arn:foo:bar" in result.output + assert u"- arn:lorem:ipsum" in result.output + + @patch('ecs_deploy.cli.get_client') def test_run_task_without_diff(get_client, runner): get_client.return_value = EcsTestClient('acces_key', 'secret_key') @@ -1016,6 +1157,98 @@ def test_update_task_exclusive_environment(get_client, runner): assert u'Successfully created revision: 2' in result.output +@patch('ecs_deploy.cli.get_client') +def test_update_task_one_new_docker_label(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_1, + '-d', 'application', 'foo', 'bar', + '-d', 'webserver', 'foo', 'baz')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Update task definition based on: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "foo" of container "application" to: "bar"' in result.output + assert u'Changed dockerLabel "foo" of container "webserver" to: "baz"' in result.output + assert u'Changed dockerLabel "lorem" of container "webserver" to: "ipsum"' not in result.output + assert u'Successfully created revision: 2' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_update_task_change_docker_label_empty_string(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_1, '-d', 'application', 'foo', '')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Update task definition based on: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "foo" of container "application" to: ""' in result.output + assert u'Successfully created revision: 2' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_update_task_new_empty_docker_label(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_1, '-d', 'application', 'new', '')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Update task definition based on: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "new" of container "application" to: ""' in result.output + assert u'Successfully created revision: 2' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_update_task_empty_docker_label_again(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_1, '-d', 'webserver', 'empty', '')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Update task definition based on: test-task:1" in result.output + assert u"Updating task definition" not in result.output + assert u'Changed dockerLabel' not in result.output + assert u'Successfully created revision: 2' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_update_task_previously_empty_docker_label_with_value(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_1, '-d', 'webserver', 'empty', 'not-empty')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Update task definition based on: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "empty" of container "webserver" to: "not-empty"' in result.output + assert u'Successfully created revision: 2' in result.output + + +@patch('ecs_deploy.cli.get_client') +def test_update_task_exclusive_docker_labels(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_1, '-d', 'webserver', 'new-label', 'new-value', '--exclusive-docker-labels')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Update task definition based on: test-task:1" in result.output + assert u"Updating task definition" in result.output + assert u'Changed dockerLabel "new-label" of container "webserver" to: "new-value"' in result.output + + assert u'Removed dockerLabel "foo" of container "webserver"' in result.output + assert u'Removed dockerLabel "lorem" of container "webserver"' in result.output + + assert u'Successfully created revision: 2' in result.output + + @patch('ecs_deploy.cli.get_client') def test_update_task_exclusive_secret(get_client, runner): get_client.return_value = EcsTestClient('acces_key', 'secret_key') @@ -1068,6 +1301,20 @@ def test_update_task_without_changing_environment_value(get_client, runner): assert u'Successfully created revision: 2' in result.output +@patch('ecs_deploy.cli.get_client') +def test_update_task_without_changing_docker_labels(get_client, runner): + get_client.return_value = EcsTestClient('acces_key', 'secret_key') + result = runner.invoke(cli.update, (TASK_DEFINITION_ARN_1, '-d', 'webserver', 'foo', 'bar')) + + assert result.exit_code == 0 + assert not result.exception + + assert u"Update task definition based on: test-task:1" in result.output + assert u"Updating task definition" not in result.output + assert u'Changed dockerLabel' not in result.output + assert u'Successfully created revision: 2' in result.output + + @patch('ecs_deploy.cli.get_client') def test_update_task_without_changing_secrets_value(get_client, runner): get_client.return_value = EcsTestClient('acces_key', 'secret_key') @@ -1139,6 +1386,11 @@ def test_diff(get_client, runner): assert '+ "foobar"' in result.output assert 'remove: containers.webserver.environment' in result.output assert '- empty: ' in result.output + assert 'change: containers.webserver.dockerLabels.foo' in result.output + assert '- "bar"' in result.output + assert '+ "foobar"' in result.output + assert 'remove: containers.webserver.dockerLabels' in result.output + assert '- empty: ' in result.output assert 'change: containers.webserver.secrets.baz' in result.output assert '- "qux"' in result.output assert '+ "foobaz"' in result.output @@ -1153,6 +1405,8 @@ def test_diff(get_client, runner): assert '+ "arn:test:another-role:1"' in result.output assert 'add: containers.webserver.environment' in result.output assert '+ newvar: "new value"' in result.output + assert 'add: containers.webserver.dockerLabel' in result.output + assert '+ newlabel: "new value"' in result.output @patch('ecs_deploy.cli.get_client') From 4de4be5e984e0d15a273ebe56d38d889f866ebed Mon Sep 17 00:00:00 2001 From: Anthony Eadicicco Date: Wed, 28 Jul 2021 23:46:26 -0400 Subject: [PATCH 07/15] Fix broken diff test Also directly copy dockerlabels dict instead of manually --- ecs_deploy/ecs.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ecs_deploy/ecs.py b/ecs_deploy/ecs.py index c04427a..8ba5661 100644 --- a/ecs_deploy/ecs.py +++ b/ecs_deploy/ecs.py @@ -336,12 +336,10 @@ def diff_raw(self, task_b): containers_b[container].get('secrets', {})} for container in containers_a: - containers_a[container]['dockerLabels'] = {l['name']: l['value'] for l in - containers_a[container].get('dockerLabels', {})} + containers_a[container]['dockerLabels'] = containers_a[container].get('dockerLabels', {}).copy() for container in containers_b: - containers_b[container]['dockerLabels'] = {l['name']: l['value'] for l in - containers_b[container].get('dockerLabels', {})} + containers_b[container]['dockerLabels'] = containers_b[container].get('dockerLabels', {}).copy() composite_a = { 'containers': containers_a, @@ -409,7 +407,7 @@ def get_overrides_secrets(secrets): @staticmethod def get_overrides_docker_labels(dockerlabels): - return {l: dockerlabels[l] for l in dockerlabels} + return dockerlabels.copy() def set_images(self, tag=None, **images): self.validate_container_options(**images) @@ -655,7 +653,7 @@ def apply_docker_labels(self, container, new_dockerlabels, exclusive=False): ) self._diff.append(diff) - container[u'dockerLabels'] = {l: merged[l] for l in merged} + container[u'dockerLabels'] = merged.copy() def set_secrets(self, secrets_list, exclusive=False): secrets = defaultdict(dict) From 0fa2b8aaa60e80141de2080d683d559f7814da4c Mon Sep 17 00:00:00 2001 From: Anthony Eadicicco Date: Wed, 28 Jul 2021 23:55:04 -0400 Subject: [PATCH 08/15] Update README to document docker labels --- README.rst | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index ce1eb96..743fad0 100644 --- a/README.rst +++ b/README.rst @@ -85,7 +85,7 @@ Currently the following actions are supported: deploy ====== -Redeploy a service either without any modifications or with a new image, environment variable and/or command definition. +Redeploy a service either without any modifications or with a new image, environment variable, docker label, and/or command definition. scale ===== @@ -93,12 +93,12 @@ Scale a service up or down and change the number of running tasks. run === -Run a one-off task based on an existing task-definition and optionally override command and/or environment variables. +Run a one-off task based on an existing task-definition and optionally override command, environment variables and/or docker labels. update ====== Update a task definition by creating a new revision to set a new image, -environment variable and/or command definition, etc. +environment variable, docker label, and/or command definition, etc. cron (scheduled task) ===================== @@ -230,6 +230,36 @@ To reset all existing secrets (secret environment variables) of a task definitio This will remove **all other** existing secret environment variables of **all containers** of the task definition, except for the new secret variable `NEW_SECRET` with the value coming from the AWS Parameter Store with the name "KEY_OF_SECRET_IN_PARAMETER_STORE" in the webserver container. + +Set an docker label +=================== +To add a new or adjust an existing docker labels of a specific container, run the following command:: + + $ ecs deploy my-cluster my-service -d webserver somelabel somevalue + +This will modify the **webserver** container definition and add or overwrite the docker label "somelabel" with the value "somevalue". This way you can add new or adjust already existing docker labels. + + +Adjust multiple docker labels +============================= +You can add or change multiple docker labels at once, by adding the `-d` (or `--docker-label`) options several times:: + + $ ecs deploy my-cluster my-service -d webserver somelabel somevalue -d webserver otherlabel othervalue -d app applabel appvalue + +This will modify the definition **of two containers**. +The **webserver**'s docker label "somelabel" will be set to "somevalue" and the label "otherlabel" to "othervalue". +The **app**'s docker label "applabel" will be set to "appvalue". + + +Set docker labels exclusively, remove all other pre-existing docker labels +========================================================================== +To reset all existing docker labels of a task definition, use the flag ``--exclusive-docker-labels`` :: + + $ ecs deploy my-cluster my-service -d webserver somelabel somevalue --exclusive-docker-labels + +This will remove **all other** existing docker labels of **all containers** of the task definition, except for the label "somelabel" with the value "somevalue" in the webserver container. + + Modify a command ================ To change the command of a specific container, run the following command:: From a96f5d5fffcbf0dd90fcd9a2db9cde8748898d15 Mon Sep 17 00:00:00 2001 From: Anthony Eadicicco Date: Thu, 29 Jul 2021 09:46:57 -0400 Subject: [PATCH 09/15] Fix README typo --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 743fad0..0c7d113 100644 --- a/README.rst +++ b/README.rst @@ -231,7 +231,7 @@ To reset all existing secrets (secret environment variables) of a task definitio This will remove **all other** existing secret environment variables of **all containers** of the task definition, except for the new secret variable `NEW_SECRET` with the value coming from the AWS Parameter Store with the name "KEY_OF_SECRET_IN_PARAMETER_STORE" in the webserver container. -Set an docker label +Set a docker label =================== To add a new or adjust an existing docker labels of a specific container, run the following command:: From d983881732bf777abc8b8c4c066b9d75d82b235e Mon Sep 17 00:00:00 2001 From: Fabian Fuelling Date: Tue, 3 Aug 2021 09:59:08 +0200 Subject: [PATCH 10/15] Update README.rst Move CI from travis-ci.org to travis-ci.com --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ce1eb96..714a967 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ ECS Deploy ---------- -.. image:: https://travis-ci.org/fabfuel/ecs-deploy.svg?branch=develop - :target: https://travis-ci.org/fabfuel/ecs-deploy +.. image:: https://travis-ci.com/fabfuel/ecs-deploy.svg?branch=develop + :target: https://travis-ci.com/github/fabfuel/ecs-deploy .. image:: https://scrutinizer-ci.com/g/fabfuel/ecs-deploy/badges/coverage.png?b=develop :target: https://scrutinizer-ci.com/g/fabfuel/ecs-deploy From c7781b104d38a59c1d37b194ebd7b366693cab51 Mon Sep 17 00:00:00 2001 From: Michael Mayer Date: Mon, 26 Jul 2021 16:32:56 -0700 Subject: [PATCH 11/15] Add environmentFiles parameter --- ecs_deploy/cli.py | 23 +++++++++++++++------- ecs_deploy/ecs.py | 49 +++++++++++++++++++++++++++++++++++++++++++++++ tests/test_cli.py | 16 ++++++++++++++++ tests/test_ecs.py | 22 +++++++++++++++++++++ 4 files changed, 103 insertions(+), 7 deletions(-) diff --git a/ecs_deploy/cli.py b/ecs_deploy/cli.py index 2d97927..2db2690 100644 --- a/ecs_deploy/cli.py +++ b/ecs_deploy/cli.py @@ -38,6 +38,7 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--essential', type=(str, bool), multiple=True, help='Overwrites the essential value for a container: ') @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: ') @click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file') +@click.option('--s3-env-file', type=(str, str), multiple=True, required=False, help='Location of .env-file in S3') @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): ') @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: ') @click.option('-u', '--ulimit', type=(str, str, int, int), multiple=True, help='Adds or changes a ulimit variable in the container description (Not available for Fargate): ') @@ -66,6 +67,7 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers') @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers') @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers') +@click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers') @click.option('--sleep-time', default=1, type=int, help='Amount of seconds to wait between each check of the service (default: 1)') @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL') @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, which services should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH') @@ -76,7 +78,7 @@ def get_client(access_key_id, secret_access_key, region, profile): @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.') @click.option('--add-container', type=str, multiple=True, required=False, help='Add a placeholder container in the task definition.') @click.option('--remove-container', type=str, multiple=True, required=False, help='Remove a container from the task definition.') -def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, privileged, essential, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, slack_service_match='.*'): +def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, privileged, essential, env, env_file, s3_env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, slack_service_match='.*'): """ Redeploy or modify a service. @@ -107,6 +109,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem td.set_essential(**{key: value for (key, value) in essential}) td.set_environment(env, exclusive_env, env_file) td.set_docker_labels(docker_label, exclusive_docker_labels) + td.set_s3_env_file(s3_env_file, exclusive_s3_env_file) td.set_secrets(secret, exclusive_secrets) td.set_ulimits(ulimit, exclusive_ulimits) td.set_system_controls(system_control, exclusive_system_controls) @@ -182,6 +185,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem @click.option('-m', '--mount', type=(str, str, str), multiple=True, help='Adds or changes a mount points in the container description (Not available for Fargate): ') @click.option('-l', '--log', type=(str, str, str, str), multiple=True, help='Adds or changes a log configuration in the container description (Not available for Fargate):