Merge pull request #97 from aws-samples/master

release update

docs/core-env/create-custom-compute-resources.md

@@ -23,7 +23,7 @@ The longer these scripts / customizations take to complete, the longer it will
 be before your instance is ready for work.
 
 Launch Templates are capable of pre-configuring a lot of EC2 instance options.
-Since this will be working with AWS Batch, which already does a lot of automatic 
+Since this will be working with AWS Batch, which already does a lot of automatic
 instance configuration on its own, you only need to supply the `UserData`
 script below:
 
@@ -37,23 +37,25 @@ Content-Type: text/cloud-config; charset="us-ascii"
 packages:
 - jq
 - btrfs-progs
-- python27-pip
 - sed
 - wget
+- unzip
 # add more package names here if you need them
 
 runcmd:
-- pip install -U awscli boto3
+- curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip"
+- unzip -q /tmp/awscliv2.zip -d /tmp
+- /tmp/aws/install
 - cd /opt && wget https://aws-genomics-workflows.s3.amazonaws.com/artifacts/aws-ebs-autoscale.tgz && tar -xzf aws-ebs-autoscale.tgz
 - sh /opt/ebs-autoscale/bin/init-ebs-autoscale.sh /scratch /dev/sdc  2>&1 > /var/log/init-ebs-autoscale.log
 # you can add more commands here if you have additional provisioning steps
 
 --==BOUNDARY==--
 ```
 
-The above will add an `ebs-autoscale` daemon to an instance.  By default it will 
-add a 20GB EBS volume to the logical volume mounted at `/scratch`.  
-If you want this volume to be larger initially, you can specify a bigger one 
+The above will add an `ebs-autoscale` daemon to an instance.  By default it will
+add a 20GB EBS volume to the logical volume mounted at `/scratch`.
+If you want this volume to be larger initially, you can specify a bigger one
 mapped to `/dev/sdc`  the Launch Template.
 
 !!! note
@@ -153,11 +155,11 @@ You should get something like the following as a response:
 ```json
 {
     "LaunchTemplate": {
-        "LatestVersionNumber": 1, 
-        "LaunchTemplateId": "lt-0123456789abcdef0", 
-        "LaunchTemplateName": "genomics-workflow-template", 
-        "DefaultVersionNumber": 1, 
-        "CreatedBy": "arn:aws:iam::123456789012:user/alice", 
+        "LatestVersionNumber": 1,
+        "LaunchTemplateId": "lt-0123456789abcdef0",
+        "LaunchTemplateName": "genomics-workflow-template",
+        "DefaultVersionNumber": 1,
+        "CreatedBy": "arn:aws:iam::123456789012:user/alice",
         "CreateTime": "2019-01-01T00:00:00.000Z"
     }
 }
@@ -166,8 +168,8 @@ You should get something like the following as a response:
 ## Custom AMIs
 
 A slightly more involved method for customizing an instance is
-to create a new AMI based on the ECS Optimized AMI.  This is good if you have 
-a lot of customization to do - lots of software to install and/or need large 
+to create a new AMI based on the ECS Optimized AMI.  This is good if you have
+a lot of customization to do - lots of software to install and/or need large
 datasets preloaded that will be needed by all your jobs.
 
 You can learn more about how to [create your own AMIs in the EC2 userguide](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html).

src/templates/aws-genomics-launch-template.template.yaml

@@ -1,10 +1,10 @@
 AWSTemplateFormatVersion: "2010-09-09"
 Description: >-
-  (WWPS-GLS-WF-LT) Creates an EC2 Launch Template for AWS Batch based 
+  (WWPS-GLS-WF-LT) Creates an EC2 Launch Template for AWS Batch based
   genomics workflows
 
 Mappings:
-  ScratchMountPointMap: 
+  ScratchMountPointMap:
     step-functions:
       mountpoint: "/var/lib/docker"
     cromwell:
@@ -33,7 +33,7 @@ Mappings:
         - sed -i 's+OPTIONS=.*+OPTIONS="--storage-driver btrfs"+g' /etc/sysconfig/docker-storage
         - cp -au /var/lib/docker.bk/* /var/lib/docker
         - cd /opt && wget $artifactRootUrl/aws-ecs-additions.tgz && tar -xzf aws-ecs-additions.tgz
-        - sh /opt/ecs-additions/ecs-additions-step-functions.sh 
+        - sh /opt/ecs-additions/ecs-additions-step-functions.sh
         - service docker start
         - start ecs
 
@@ -57,10 +57,10 @@ Mappings:
         - sed -i 's+OPTIONS=.*+OPTIONS="--storage-driver btrfs"+g' /etc/sysconfig/docker-storage
         - cp -au /var/lib/docker.bk/* /var/lib/docker
         - cd /opt && wget $artifactRootUrl/aws-ecs-additions.tgz && tar -xzf aws-ecs-additions.tgz
-        - sh /opt/ecs-additions/ecs-additions-nextflow.sh 
+        - sh /opt/ecs-additions/ecs-additions-nextflow.sh
         - service docker start
         - start ecs
-        
+
 Parameters:
   LaunchTemplateNamePrefix:
     Type: String
@@ -119,19 +119,19 @@ Resources:
           - Ebs:
               DeleteOnTermination: True
               VolumeSize: 50
-              VolumeType: gp2 
+              VolumeType: gp2
             DeviceName: /dev/xvda
           - Ebs:
               Encrypted: True
               DeleteOnTermination: True
               VolumeSize: !Ref DockerImageAndMetadataVolumeSize
-              VolumeType: gp2 
+              VolumeType: gp2
             DeviceName: /dev/xvdcz
           - Ebs:
               Encrypted: True
               DeleteOnTermination: True
               VolumeSize: !Ref ScratchVolumeSize
-              VolumeType: gp2 
+              VolumeType: gp2
             DeviceName: /dev/sdc
         UserData:
           Fn::Base64: !Sub
@@ -145,14 +145,16 @@ Resources:
               packages:
               - jq
               - btrfs-progs
-              - python27-pip
               - sed
               - wget
               - git
               - amazon-ssm-agent
+              - unzip
 
               runcmd:
-              - pip install -U awscli boto3
+              - curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip"
+              - unzip -q /tmp/awscliv2.zip -d /tmp
+              - /tmp/aws/install
               - export scratchPath="${ScratchMountPoint}"
               - export artifactRootUrl="${ArtifactRootUrl}"
               - start amazon-ssm-agent
@@ -161,7 +163,7 @@ Resources:
 
               --==BOUNDARY==--
             - ECSAdditions:
-                Fn::FindInMap: 
+                Fn::FindInMap:
                   - ECSAdditionsMap
                   - !Ref WorkflowOrchestrator
                   - additions

src/templates/nextflow/nextflow-resources.template.yaml

@@ -41,12 +41,12 @@ Parameters:
     Description: >-
       S3 Bucket used to store input and output data for the workflow.
       This should bucket should already exist.
-    
+
   S3NextflowBucketName:
     Type: String
     Description: >-
       S3 Bucket used to store Nextflow metadata (session cache, logs, and intermediate results)
-    
+
   ExistingBucket:
     Type: String
     Description: >-
@@ -55,7 +55,7 @@ Parameters:
       - Yes
       - No
     Default: No
-  
+
   S3NextflowPrefix:
     Type: String
     Description: >-
@@ -69,32 +69,32 @@ Parameters:
       (Optional) Folder in the Nextflow metadata bucket (under the {Nextflow Prefix} if needed)
       for session cache and logs.
     Default: logs
-  
+
   S3WorkDirPrefix:
     Type: String
     Description: >-
       (Optional) Folder in the Nextflow metadata bucket (under the {Nextflow Prefix} if needed)
       that contains workflow intermediate results
     Default: runs
-  
+
   NextflowContainerImage:
     Type: String
     Description: >-
       (Optional) Container image for nextflow with custom entrypoint for config and workflow
-      script staging. (Example, "<dockerhub-user>/nextflow:latest").  
-      Provide this if you have a specific version of nextflow you want to use, otherwise a 
-      container will be built using the latest version. 
+      script staging. (Example, "<dockerhub-user>/nextflow:latest").
+      Provide this if you have a specific version of nextflow you want to use, otherwise a
+      container will be built using the latest version.
 
   BatchDefaultJobQueue:
     Type: String
     Description: >-
       ARN of the Batch Job Queue to use by default for workflow tasks.
-  
+
   BatchHighPriorityJobQueue:
     Type: String
     Description: >-
       ARN of the Batch Job Queue to use for high priority workflow tasks.
-  
+
   TemplateRootUrl:
     Type: String
     Description: >-
@@ -109,7 +109,7 @@ Conditions:
     Fn::Equals:
       - !Ref S3NextflowBucketName
       - !Ref S3DataBucketName
-  
+
   BuildNextflowContainer:
     Fn::Equals:
       - !Ref NextflowContainerImage
@@ -134,7 +134,7 @@ Resources:
           - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
       Tags: !FindInMap ["TagMap", "default", "tags"]
-    
+
   ContainerBuildNextflow:
     Type: AWS::CloudFormation::Stack
     Condition: BuildNextflowContainer
@@ -148,7 +148,7 @@ Resources:
         ProjectBuildSpecFile: ./src/containers/buildspec-nextflow.yml
         CreateBatchJobDefinition: "No"
       Tags: !FindInMap ["TagMap", "default", "tags"]
-  
+
   IAMNextflowJobRole:
     Type: AWS::IAM::Role
     Properties:
@@ -164,9 +164,9 @@ Resources:
                 Action:
                   - "batch:List*"
                   - "batch:Describe*"
-              
+
               # only permit access (job submission) to the queues and compute environments
-              # configured to run nextflow    
+              # configured to run nextflow
               - Sid: "BatchWriteAccessAllowJobSubmission"
                 Effect: Allow
                 Resource:
@@ -175,12 +175,13 @@ Resources:
                   - arn:aws:batch:*:*:job-definition/nf-*:*
                 Action:
                   - "batch:*Job"
-              
+
               # nextflow needs to be able to create job definitions
               # these are prefixed with "nf-"
               - Sid: "BatchWriteAccessAllowJobDefinition"
                 Effect: Allow
                 Resource:
+                  - arn:aws:batch:*:*:job-definition/nf-*
                   - arn:aws:batch:*:*:job-definition/nf-*:*
                 Action:
                   - "batch:*JobDefinition"
@@ -219,7 +220,7 @@ Resources:
     Type: AWS::Batch::JobDefinition
     Properties:
       Type: container
-      ContainerProperties: 
+      ContainerProperties:
         Memory: 1024
         JobRoleArn: !GetAtt IAMNextflowJobRole.Arn
         Vcpus: 2
@@ -232,24 +233,24 @@ Resources:
           - Name: "NF_JOB_QUEUE"
             Value: !Ref BatchDefaultJobQueue
           - Name: "NF_LOGSDIR"
-            Value: 
-              Fn::Join: 
+            Value:
+              Fn::Join:
                 - "/"
                 - - Fn::If:
                       - DataBucketIsNextflowBucket
                       - !Join ["/", [!Sub "s3://${S3NextflowBucketName}", !Ref S3NextflowPrefix]]
                       - !Sub "s3://${S3NextflowBucketName}"
                   - !Ref S3LogsDirPrefix
           - Name: "NF_WORKDIR"
-            Value: 
-              Fn::Join: 
+            Value:
+              Fn::Join:
                 - "/"
                 - - Fn::If:
                       - DataBucketIsNextflowBucket
                       - !Join ["/", [!Sub "s3://${S3NextflowBucketName}", !Ref S3NextflowPrefix]]
                       - !Sub "s3://${S3NextflowBucketName}"
                   - !Ref S3WorkDirPrefix
-              
+
       JobDefinitionName: nextflow
 
 
@@ -262,7 +263,7 @@ Outputs:
         - NextflowBucketDoesNotExist
         - !Ref S3NextflowBucket
         - !Ref S3NextflowBucketName
-  
+
   LogsDir:
     Description: >-
       S3 URI where nextflow session cache and logs are stored.
@@ -279,7 +280,7 @@ Outputs:
     Description: >-
       S3 URI where workflow intermediate results are stored.
     Value:
-      Fn::Join: 
+      Fn::Join:
         - "/"
         - - Fn::If:
               - DataBucketIsNextflowBucket
@@ -300,9 +301,9 @@ Outputs:
     Description: >-
       Batch Job Definition that creates a nextflow head node for running workflows
     Value: !Ref BatchNextflowJobDefinition
-  
+
   NextflowJobRole:
     Description: >-
       IAM Role that allows the nextflow head node job access to S3 and Batch
     Value: !GetAtt IAMNextflowJobRole.Arn
-...
+...