diff --git a/docs/core-env/create-custom-compute-resources.md b/docs/core-env/create-custom-compute-resources.md index ed9e2bca1..45a8ad109 100644 --- a/docs/core-env/create-custom-compute-resources.md +++ b/docs/core-env/create-custom-compute-resources.md @@ -23,7 +23,7 @@ The longer these scripts / customizations take to complete, the longer it will be before your instance is ready for work. Launch Templates are capable of pre-configuring a lot of EC2 instance options. -Since this will be working with AWS Batch, which already does a lot of automatic +Since this will be working with AWS Batch, which already does a lot of automatic instance configuration on its own, you only need to supply the `UserData` script below: @@ -37,13 +37,15 @@ Content-Type: text/cloud-config; charset="us-ascii" packages: - jq - btrfs-progs -- python27-pip - sed - wget +- unzip # add more package names here if you need them runcmd: -- pip install -U awscli boto3 +- curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" +- unzip -q /tmp/awscliv2.zip -d /tmp +- /tmp/aws/install - cd /opt && wget https://aws-genomics-workflows.s3.amazonaws.com/artifacts/aws-ebs-autoscale.tgz && tar -xzf aws-ebs-autoscale.tgz - sh /opt/ebs-autoscale/bin/init-ebs-autoscale.sh /scratch /dev/sdc 2>&1 > /var/log/init-ebs-autoscale.log # you can add more commands here if you have additional provisioning steps @@ -51,9 +53,9 @@ runcmd: --==BOUNDARY==-- ``` -The above will add an `ebs-autoscale` daemon to an instance. By default it will -add a 20GB EBS volume to the logical volume mounted at `/scratch`. -If you want this volume to be larger initially, you can specify a bigger one +The above will add an `ebs-autoscale` daemon to an instance. By default it will +add a 20GB EBS volume to the logical volume mounted at `/scratch`. +If you want this volume to be larger initially, you can specify a bigger one mapped to `/dev/sdc` the Launch Template. !!! note @@ -153,11 +155,11 @@ You should get something like the following as a response: ```json { "LaunchTemplate": { - "LatestVersionNumber": 1, - "LaunchTemplateId": "lt-0123456789abcdef0", - "LaunchTemplateName": "genomics-workflow-template", - "DefaultVersionNumber": 1, - "CreatedBy": "arn:aws:iam::123456789012:user/alice", + "LatestVersionNumber": 1, + "LaunchTemplateId": "lt-0123456789abcdef0", + "LaunchTemplateName": "genomics-workflow-template", + "DefaultVersionNumber": 1, + "CreatedBy": "arn:aws:iam::123456789012:user/alice", "CreateTime": "2019-01-01T00:00:00.000Z" } } @@ -166,8 +168,8 @@ You should get something like the following as a response: ## Custom AMIs A slightly more involved method for customizing an instance is -to create a new AMI based on the ECS Optimized AMI. This is good if you have -a lot of customization to do - lots of software to install and/or need large +to create a new AMI based on the ECS Optimized AMI. This is good if you have +a lot of customization to do - lots of software to install and/or need large datasets preloaded that will be needed by all your jobs. You can learn more about how to [create your own AMIs in the EC2 userguide](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html). diff --git a/src/templates/aws-genomics-launch-template.template.yaml b/src/templates/aws-genomics-launch-template.template.yaml index 7948b1bff..ca94ebf3b 100644 --- a/src/templates/aws-genomics-launch-template.template.yaml +++ b/src/templates/aws-genomics-launch-template.template.yaml @@ -1,10 +1,10 @@ AWSTemplateFormatVersion: "2010-09-09" Description: >- - (WWPS-GLS-WF-LT) Creates an EC2 Launch Template for AWS Batch based + (WWPS-GLS-WF-LT) Creates an EC2 Launch Template for AWS Batch based genomics workflows Mappings: - ScratchMountPointMap: + ScratchMountPointMap: step-functions: mountpoint: "/var/lib/docker" cromwell: @@ -33,7 +33,7 @@ Mappings: - sed -i 's+OPTIONS=.*+OPTIONS="--storage-driver btrfs"+g' /etc/sysconfig/docker-storage - cp -au /var/lib/docker.bk/* /var/lib/docker - cd /opt && wget $artifactRootUrl/aws-ecs-additions.tgz && tar -xzf aws-ecs-additions.tgz - - sh /opt/ecs-additions/ecs-additions-step-functions.sh + - sh /opt/ecs-additions/ecs-additions-step-functions.sh - service docker start - start ecs @@ -57,10 +57,10 @@ Mappings: - sed -i 's+OPTIONS=.*+OPTIONS="--storage-driver btrfs"+g' /etc/sysconfig/docker-storage - cp -au /var/lib/docker.bk/* /var/lib/docker - cd /opt && wget $artifactRootUrl/aws-ecs-additions.tgz && tar -xzf aws-ecs-additions.tgz - - sh /opt/ecs-additions/ecs-additions-nextflow.sh + - sh /opt/ecs-additions/ecs-additions-nextflow.sh - service docker start - start ecs - + Parameters: LaunchTemplateNamePrefix: Type: String @@ -119,19 +119,19 @@ Resources: - Ebs: DeleteOnTermination: True VolumeSize: 50 - VolumeType: gp2 + VolumeType: gp2 DeviceName: /dev/xvda - Ebs: Encrypted: True DeleteOnTermination: True VolumeSize: !Ref DockerImageAndMetadataVolumeSize - VolumeType: gp2 + VolumeType: gp2 DeviceName: /dev/xvdcz - Ebs: Encrypted: True DeleteOnTermination: True VolumeSize: !Ref ScratchVolumeSize - VolumeType: gp2 + VolumeType: gp2 DeviceName: /dev/sdc UserData: Fn::Base64: !Sub @@ -145,14 +145,16 @@ Resources: packages: - jq - btrfs-progs - - python27-pip - sed - wget - git - amazon-ssm-agent + - unzip runcmd: - - pip install -U awscli boto3 + - curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip" + - unzip -q /tmp/awscliv2.zip -d /tmp + - /tmp/aws/install - export scratchPath="${ScratchMountPoint}" - export artifactRootUrl="${ArtifactRootUrl}" - start amazon-ssm-agent @@ -161,7 +163,7 @@ Resources: --==BOUNDARY==-- - ECSAdditions: - Fn::FindInMap: + Fn::FindInMap: - ECSAdditionsMap - !Ref WorkflowOrchestrator - additions diff --git a/src/templates/nextflow/nextflow-resources.template.yaml b/src/templates/nextflow/nextflow-resources.template.yaml index 2a03742d6..c36977efa 100644 --- a/src/templates/nextflow/nextflow-resources.template.yaml +++ b/src/templates/nextflow/nextflow-resources.template.yaml @@ -41,12 +41,12 @@ Parameters: Description: >- S3 Bucket used to store input and output data for the workflow. This should bucket should already exist. - + S3NextflowBucketName: Type: String Description: >- S3 Bucket used to store Nextflow metadata (session cache, logs, and intermediate results) - + ExistingBucket: Type: String Description: >- @@ -55,7 +55,7 @@ Parameters: - Yes - No Default: No - + S3NextflowPrefix: Type: String Description: >- @@ -69,32 +69,32 @@ Parameters: (Optional) Folder in the Nextflow metadata bucket (under the {Nextflow Prefix} if needed) for session cache and logs. Default: logs - + S3WorkDirPrefix: Type: String Description: >- (Optional) Folder in the Nextflow metadata bucket (under the {Nextflow Prefix} if needed) that contains workflow intermediate results Default: runs - + NextflowContainerImage: Type: String Description: >- (Optional) Container image for nextflow with custom entrypoint for config and workflow - script staging. (Example, "/nextflow:latest"). - Provide this if you have a specific version of nextflow you want to use, otherwise a - container will be built using the latest version. + script staging. (Example, "/nextflow:latest"). + Provide this if you have a specific version of nextflow you want to use, otherwise a + container will be built using the latest version. BatchDefaultJobQueue: Type: String Description: >- ARN of the Batch Job Queue to use by default for workflow tasks. - + BatchHighPriorityJobQueue: Type: String Description: >- ARN of the Batch Job Queue to use for high priority workflow tasks. - + TemplateRootUrl: Type: String Description: >- @@ -109,7 +109,7 @@ Conditions: Fn::Equals: - !Ref S3NextflowBucketName - !Ref S3DataBucketName - + BuildNextflowContainer: Fn::Equals: - !Ref NextflowContainerImage @@ -134,7 +134,7 @@ Resources: - ServerSideEncryptionByDefault: SSEAlgorithm: AES256 Tags: !FindInMap ["TagMap", "default", "tags"] - + ContainerBuildNextflow: Type: AWS::CloudFormation::Stack Condition: BuildNextflowContainer @@ -148,7 +148,7 @@ Resources: ProjectBuildSpecFile: ./src/containers/buildspec-nextflow.yml CreateBatchJobDefinition: "No" Tags: !FindInMap ["TagMap", "default", "tags"] - + IAMNextflowJobRole: Type: AWS::IAM::Role Properties: @@ -164,9 +164,9 @@ Resources: Action: - "batch:List*" - "batch:Describe*" - + # only permit access (job submission) to the queues and compute environments - # configured to run nextflow + # configured to run nextflow - Sid: "BatchWriteAccessAllowJobSubmission" Effect: Allow Resource: @@ -175,12 +175,13 @@ Resources: - arn:aws:batch:*:*:job-definition/nf-*:* Action: - "batch:*Job" - + # nextflow needs to be able to create job definitions # these are prefixed with "nf-" - Sid: "BatchWriteAccessAllowJobDefinition" Effect: Allow Resource: + - arn:aws:batch:*:*:job-definition/nf-* - arn:aws:batch:*:*:job-definition/nf-*:* Action: - "batch:*JobDefinition" @@ -219,7 +220,7 @@ Resources: Type: AWS::Batch::JobDefinition Properties: Type: container - ContainerProperties: + ContainerProperties: Memory: 1024 JobRoleArn: !GetAtt IAMNextflowJobRole.Arn Vcpus: 2 @@ -232,8 +233,8 @@ Resources: - Name: "NF_JOB_QUEUE" Value: !Ref BatchDefaultJobQueue - Name: "NF_LOGSDIR" - Value: - Fn::Join: + Value: + Fn::Join: - "/" - - Fn::If: - DataBucketIsNextflowBucket @@ -241,15 +242,15 @@ Resources: - !Sub "s3://${S3NextflowBucketName}" - !Ref S3LogsDirPrefix - Name: "NF_WORKDIR" - Value: - Fn::Join: + Value: + Fn::Join: - "/" - - Fn::If: - DataBucketIsNextflowBucket - !Join ["/", [!Sub "s3://${S3NextflowBucketName}", !Ref S3NextflowPrefix]] - !Sub "s3://${S3NextflowBucketName}" - !Ref S3WorkDirPrefix - + JobDefinitionName: nextflow @@ -262,7 +263,7 @@ Outputs: - NextflowBucketDoesNotExist - !Ref S3NextflowBucket - !Ref S3NextflowBucketName - + LogsDir: Description: >- S3 URI where nextflow session cache and logs are stored. @@ -279,7 +280,7 @@ Outputs: Description: >- S3 URI where workflow intermediate results are stored. Value: - Fn::Join: + Fn::Join: - "/" - - Fn::If: - DataBucketIsNextflowBucket @@ -300,9 +301,9 @@ Outputs: Description: >- Batch Job Definition that creates a nextflow head node for running workflows Value: !Ref BatchNextflowJobDefinition - + NextflowJobRole: Description: >- IAM Role that allows the nextflow head node job access to S3 and Batch Value: !GetAtt IAMNextflowJobRole.Arn -... \ No newline at end of file +...