Merge pull request #177 from mjmayer/add_environmentfiles_parameter

Add environmentfiles parameter

ecs_deploy/cli.py

@@ -38,6 +38,7 @@ def get_client(access_key_id, secret_access_key, region, profile):
 @click.option('--essential', type=(str, bool), multiple=True, help='Overwrites the essential value for a container: <container> <essential>')
 @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: <container> <name> <value>')
 @click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file')
+@click.option('--s3-env-file', type=(str, str), multiple=True, required=False, help='Location of .env-file in S3 in ARN format (eg arn:aws:s3:::/bucket_name/object_name')
 @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): <container> <name> <parameter name>')
 @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: <container> <name> <value>')
 @click.option('-u', '--ulimit', type=(str, str, int, int), multiple=True, help='Adds or changes a ulimit variable in the container description (Not available for Fargate): <container> <ulimit name> <softlimit value> <hardlimit value>')
@@ -66,6 +67,7 @@ def get_client(access_key_id, secret_access_key, region, profile):
 @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers')
 @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers')
+@click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers')
 @click.option('--sleep-time', default=1, type=int, help='Amount of seconds to wait between each check of the service (default: 1)')
 @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL')
 @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, which services should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH')
@@ -76,7 +78,7 @@ def get_client(access_key_id, secret_access_key, region, profile):
 @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.')
 @click.option('--add-container', type=str, multiple=True, required=False, help='Add a placeholder container in the task definition.')
 @click.option('--remove-container', type=str, multiple=True, required=False, help='Remove a container from the task definition.')
-def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, privileged, essential, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, slack_service_match='.*'):
+def deploy(cluster, service, tag, image, command, health_check, cpu, memory, memoryreservation, privileged, essential, env, env_file, s3_env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, task, region, access_key_id, secret_access_key, profile, timeout, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, ignore_warnings, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, sleep_time, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, add_container, remove_container, slack_url, docker_label, exclusive_docker_labels, slack_service_match='.*'):
     """
     Redeploy or modify a service.
 
@@ -107,6 +109,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem
         td.set_essential(**{key: value for (key, value) in essential})
         td.set_environment(env, exclusive_env, env_file)
         td.set_docker_labels(docker_label, exclusive_docker_labels)
+        td.set_s3_env_file(s3_env_file, exclusive_s3_env_file)
         td.set_secrets(secret, exclusive_secrets)
         td.set_ulimits(ulimit, exclusive_ulimits)
         td.set_system_controls(system_control, exclusive_system_controls)
@@ -182,6 +185,7 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem
 @click.option('-m', '--mount', type=(str, str, str), multiple=True, help='Adds or changes a mount points in the container description (Not available for Fargate): <container> <container port value> <host port value>')
 @click.option('-l', '--log', type=(str, str, str, str), multiple=True, help='Adds or changes a log configuration in the container description (Not available for Fargate): <container> <log driver> <option name> <option value>')
 @click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file')
+@click.option('--s3-env-file', type=(str, str), multiple=True, required=False, help='Location of .env-file in S3 in ARN format (eg arn:aws:s3:::/bucket_name/object_name')
 @click.option('-r', '--role', type=str, help='Sets the task\'s role ARN: <task role ARN>')
 @click.option('-x', '--execution-role', type=str, help='Sets the execution\'s role ARN: <execution role ARN>')
 @click.option('--region', help='AWS region (e.g. eu-central-1)')
@@ -200,14 +204,15 @@ def deploy(cluster, service, tag, image, command, health_check, cpu, memory, mem
 @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers')
 @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers')
+@click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers')
 @click.option('--slack-url', required=False, help='Webhook URL of the Slack integration. Can also be defined via environment variable SLACK_URL')
 @click.option('--slack-service-match', default=".*", required=False, help='A regular expression for defining, deployments of which crons should be notified. (default: .* =all). Can also be defined via environment variable SLACK_SERVICE_MATCH')
 @click.option('--exclusive-ulimits', is_flag=True, default=False, help='Set the given ulimits exclusively and remove all other pre-existing ulimits from all containers')
 @click.option('--exclusive-system-controls', is_flag=True, default=False, help='Set the given system controls exclusively and remove all other pre-existing system controls from all containers')
 @click.option('--exclusive-ports', is_flag=True, default=False, help='Set the given port mappings exclusively and remove all other pre-existing port mappings from all containers')
 @click.option('--exclusive-mounts', is_flag=True, default=False, help='Set the given mount points exclusively and remove all other pre-existing mount points from all containers')
 @click.option('--volume', type=(str, str), multiple=True, required=False, help='Set volume mapping from host to container in the task definition.')
-def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, privileged, env, env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, diff, deregister, rollback, exclusive_env, exclusive_secrets, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, docker_label, exclusive_docker_labels):
+def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservation, privileged, env, env_file, s3_env_file, secret, ulimit, system_control, port, mount, log, role, execution_role, region, access_key_id, secret_access_key, newrelic_apikey, newrelic_appid, newrelic_region, newrelic_revision, comment, user, profile, diff, deregister, rollback, exclusive_env, exclusive_secrets, exclusive_s3_env_file, slack_url, slack_service_match, exclusive_ulimits, exclusive_system_controls, exclusive_ports, exclusive_mounts, volume, docker_label, exclusive_docker_labels):
     """
     Update a scheduled task.
 
@@ -231,6 +236,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio
         td.set_privileged(**{key: value for (key, value) in privileged})
         td.set_environment(env, exclusive_env, env_file)
         td.set_docker_labels(docker_label, exclusive_docker_labels)
+        td.set_s3_env_file(s3_env_file, exclusive_s3_env_file)
         td.set_secrets(secret, exclusive_secrets)
         td.set_ulimits(ulimit, exclusive_ulimits)
         td.set_system_controls(system_control, exclusive_system_controls)
@@ -279,6 +285,7 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio
 @click.option('-c', '--command', type=(str, str), multiple=True, help='Overwrites the command in a container: <container> <command>')
 @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: <container> <name> <value>')
 @click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file')
+@click.option('--s3-env-file', type=(str, str), multiple=True, required=False, help='Location of .env-file in S3 in ARN format (eg arn:aws:s3:::/bucket_name/object_name')
 @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): <container> <name> <parameter name>')
 @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: <container> <name> <value>')
 @click.option('-r', '--role', type=str, help='Sets the task\'s role ARN: <task role ARN>')
@@ -290,8 +297,9 @@ def cron(cluster, task, rule, image, tag, command, cpu, memory, memoryreservatio
 @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--exclusive-secrets', is_flag=True, default=False, help='Set the given secrets exclusively and remove all other pre-existing secrets from all containers')
 @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers')
+@click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers')
 @click.option('--deregister/--no-deregister', default=True, help='Deregister or keep the old task definition (default: --deregister)')
-def update(task, image, tag, command, env, env_file, secret, role, region, access_key_id, secret_access_key, profile, diff, exclusive_env, exclusive_secrets, deregister, docker_label, exclusive_docker_labels):
+def update(task, image, tag, command, env, env_file, s3_env_file, secret, role, region, access_key_id, secret_access_key, profile, diff, exclusive_env, exclusive_s3_env_file, exclusive_secrets, deregister, docker_label, exclusive_docker_labels):
     """
     Update a task definition.
 
@@ -310,6 +318,7 @@ def update(task, image, tag, command, env, env_file, secret, role, region, acces
         td.set_environment(env, exclusive_env, env_file)
         td.set_docker_labels(docker_label, exclusive_docker_labels)
         td.set_secrets(secret, exclusive_secrets)
+        td.set_s3_env_file(s3_env_file, exclusive_s3_env_file)
         td.set_role_arn(role)
 
         if diff:
@@ -376,6 +385,7 @@ def scale(cluster, service, desired_count, access_key_id, secret_access_key, reg
 @click.option('-c', '--command', type=(str, str), multiple=True, help='Overwrites the command in a container: <container> <command>')
 @click.option('-e', '--env', type=(str, str, str), multiple=True, help='Adds or changes an environment variable: <container> <name> <value>')
 @click.option('--env-file', type=(str, str), default=((None, None),), multiple=True, required=False, help='Load environment variables from .env-file')
+@click.option('--s3-env-file', type=(str, str), multiple=True, required=False, help='Location of .env-file in S3 in ARN format (eg arn:aws:s3:::/bucket_name/object_name')
 @click.option('-s', '--secret', type=(str, str, str), multiple=True, help='Adds or changes a secret environment variable from the AWS Parameter Store (Not available for Fargate): <container> <name> <parameter name>')
 @click.option('-d', '--docker-label', type=(str, str, str), multiple=True, help='Adds or changes a docker label: <container> <name> <value>')
 @click.option('--launchtype', type=click.Choice([LAUNCH_TYPE_EC2, LAUNCH_TYPE_FARGATE]), default=LAUNCH_TYPE_EC2, help='ECS Launch type (default: EC2)')
@@ -389,8 +399,9 @@ def scale(cluster, service, desired_count, access_key_id, secret_access_key, reg
 @click.option('--profile', help='AWS configuration profile name')
 @click.option('--exclusive-env', is_flag=True, default=False, help='Set the given environment variables exclusively and remove all other pre-existing env variables from all containers')
 @click.option('--exclusive-docker-labels', is_flag=True, default=False, help='Set the given docker labels exclusively and remove all other pre-existing docker-labels from all containers')
+@click.option('--exclusive-s3-env-file', is_flag=True, default=False, help='Set the given s3 env files exclusively and remove all other pre-existing s3 env files from all containers')
 @click.option('--diff/--no-diff', default=True, help='Print what values were changed in the task definition')
-def run(cluster, task, count, command, env, env_file, secret, launchtype, subnet, securitygroup, public_ip, platform_version, region, access_key_id, secret_access_key, profile, exclusive_env, diff, docker_label, exclusive_docker_labels):
+def run(cluster, task, count, command, env, env_file, s3_env_file, secret, launchtype, subnet, securitygroup, public_ip, platform_version, region, access_key_id, secret_access_key, profile, exclusive_env, exclusive_s3_env_file, diff, docker_label, exclusive_docker_labels):
     """
     Run a one-off task.
 
@@ -407,6 +418,7 @@ def run(cluster, task, count, command, env, env_file, secret, launchtype, subnet
         td.set_commands(**{key: value for (key, value) in command})
         td.set_environment(env, exclusive_env, env_file)
         td.set_docker_labels(docker_label, exclusive_docker_labels)
+        td.set_s3_env_file(s3_env_file, exclusive_s3_env_file)
         td.set_secrets(secret)
 
         if diff:

ecs_deploy/ecs.py

@@ -654,6 +654,57 @@ def apply_docker_labels(self, container, new_dockerlabels, exclusive=False):
         self._diff.append(diff)
 
         container[u'dockerLabels'] = merged.copy()
+    def set_s3_env_file(self, s3_env_files, exclusive=False):
+        # environmentFiles in task definition https://docs.aws.amazon.com/AmazonECS/latest/developerguide/taskdef-envfiles.html
+        s3_files_by_container = defaultdict(list)
+        if s3_env_files:
+            multiple_s3_env_files = any(isinstance(i, tuple) for i in s3_env_files)
+            if not multiple_s3_env_files:
+                s3_files_by_container[s3_env_files[0]] = {s3_env_files[1]}
+            if multiple_s3_env_files:
+                for s3_file in s3_env_files:
+                    if s3_files_by_container[s3_file[0]]:
+                        s3_files_by_container[s3_file[0]].add(s3_file[1])
+                        break
+                    s3_files_by_container[s3_file[0]] = {s3_file[1]}
+
+        for container in self.containers:
+            if container[u'name'] in s3_files_by_container:
+                self.apply_s3_env_file(
+                    container=container,
+                    new_s3_env_file=s3_files_by_container[container[u'name']],
+                    exclusive=exclusive
+                )
+            elif exclusive is True:
+                self.apply_s3_env_file(
+                    container=container,
+                    new_s3_env_file={},
+                    exclusive=exclusive
+                )
+
+    def apply_s3_env_file(self, container, new_s3_env_file, exclusive=False):
+        s3_env_file = container.get('environmentFiles', {})
+        old_s3_env_file = {env_file['value'] for env_file in s3_env_file}
+
+        if exclusive is True:
+            merged = new_s3_env_file
+        else:
+            merged = old_s3_env_file.copy()
+            merged.update(new_s3_env_file)
+
+        if old_s3_env_file == merged:
+            return
+
+        diff = EcsTaskDefinitionDiff(
+            container=container[u'name'],
+            field=u'environmentFiles',
+            value=merged,
+            old_value=old_s3_env_file
+        )
+        self._diff.append(diff)
+        container[u'environmentFiles'] = [
+            {"value": e, "type": "s3"} for e in merged
+        ]
 
     def set_secrets(self, secrets_list, exclusive=False):
         secrets = defaultdict(dict)

tests/test_cli.py

@@ -334,6 +334,22 @@ def test_deploy_previously_empty_environment_variable_with_value(get_client, run
     assert u'Successfully changed task definition to: test-task:2' in result.output
     assert u'Deployment successful' in result.output
 
+@patch('ecs_deploy.cli.get_client')
+def test_deploy_s3_env_file_with_previous_value(get_client, runner):
+    get_client.return_value = EcsTestClient('acces_key', 'secret_key')
+    result = runner.invoke(cli.deploy, (CLUSTER_NAME, SERVICE_NAME, '--s3-env-file', 'webserver', 'arn:aws:s3:::centerfun/.env', '--s3-env-file', 'webserver', 'arn:aws:s3:::stormzone/.env'))
+
+    assert result.exit_code == 0
+    assert not result.exception
+
+    assert u"Deploying based on task definition: test-task:1" in result.output
+    assert u"Updating task definition" in result.output
+    assert u'Changed environmentFiles of container "webserver" to: "{\'arn:aws:s3:::stormzone/.env\', \'arn:aws:s3:::coolBuckets/dev/.env\', \'arn:aws:s3:::myS3bucket/myApp/.env\', \'arn:aws:s3:::centerfun/.env\'}" (was: "{\'arn:aws:s3:::coolBuckets/dev/.env\', \'arn:aws:s3:::myS3bucket/myApp/.env\'}")'
+    assert u'Successfully created revision: 2' in result.output
+    assert u'Successfully deregistered revision: 1' in result.output
+    assert u'Successfully changed task definition to: test-task:2' in result.output
+    assert u'Deployment successful' in result.output
+
 
 @patch('ecs_deploy.cli.get_client')
 def test_deploy_exclusive_environment(get_client, runner):