Add deployment configuration for running background job worker

[aymanizz]

remove hardcoded account id in policy document
fix resource arn for s3 policy

Generated Cloudformation Template Diff

diff --git a/cloudformation-main.yml b/cloudformation-sync.yml
--- a/cloudformation-main.yml
+++ b/cloudformation-sync.yml
@@ -436,7 +436,7 @@ Resources:
                   - s3-object-lambda:*
                 Effect: Allow
                 Resource:
-                  - arn:aws:s3:::bucket_name
+                  - arn:aws:s3:::bucket_name/*
                 Sid: S3BucketAccess
               - Action:
                   - iam:ListPolicies
@@ -446,12 +446,18 @@ Resources:
                 Effect: Allow
                 Resource: '*'
                 Sid: DataSciSharingIAMAccess
-              - Action: iam:*
+              - Action:
+                  - iam:*
                 Effect: Allow
                 Resource:
-                  - arn:aws:iam::<<redacted account id>>:policy/ckan-smdh/datasci-sharing/*
-                  - arn:aws:iam::<<redacted account id>>:group/ckan-smdh/datasci-sharing/*
+                  - arn:aws:iam::123456789012:policy/ckan-smdh/datasci-sharing/*
+                  - arn:aws:iam::123456789012:group/ckan-smdh/datasci-sharing/*
                 Sid: DataSciSharingIAMManagedResourcesActions
+              - Action:
+                  - sqs:*
+                Effect: Allow
+                Resource: arn:aws:sqs::123456789012:sync_queue_name
+                Sid: SQSResourceManagement
             Version: 2012-10-17
           PolicyName: CkanPolicy
         - PolicyDocument:
@@ -540,6 +546,387 @@ Resources:
         - Ref: CkaninternalNetwork
       SubnetId: subnet-2
     Type: AWS::EFS::MountTarget
+  CkanworkerService:
+    DependsOn:
+      - RedisService
+      - SolrService
+      - CkanstorageNFSMountTargetOnSubnet0
+      - CkanstorageNFSMountTargetOnSubnet1
+      - CkanstorageNFSMountTargetOnSubnet2
+    Properties:
+      Cluster:
+        Fn::GetAtt:
+          - Cluster
+          - Arn
+      DeploymentConfiguration:
+        MaximumPercent: 100
+        MinimumHealthyPercent: 0
+      DeploymentController:
+        Type: ECS
+      DesiredCount: 1
+      LaunchType: FARGATE
+      NetworkConfiguration:
+        AwsvpcConfiguration:
+          AssignPublicIp: ENABLED
+          SecurityGroups:
+            - Ref: CkaninternalNetwork
+          Subnets:
+            - subnet-0
+            - subnet-1
+            - subnet-2
+      PlatformVersion: 1.4.0
+      PropagateTags: SERVICE
+      SchedulingStrategy: REPLICA
+      ServiceRegistries:
+        - RegistryArn:
+            Fn::GetAtt:
+              - CkanworkerServiceDiscoveryEntry
+              - Arn
+      Tags:
+        - Key: com.docker.compose.project
+          Value: docker-ckan
+        - Key: com.docker.compose.service
+          Value: ckan-worker
+      TaskDefinition:
+        Ref: CkanworkerTaskDefinition
+    Type: AWS::ECS::Service
+  CkanworkerServiceDiscoveryEntry:
+    Properties:
+      Description: '"ckan-worker" service discovery entry in Cloud Map'
+      DnsConfig:
+        DnsRecords:
+          - TTL: 60
+            Type: A
+        RoutingPolicy: MULTIVALUE
+      HealthCheckCustomConfig:
+        FailureThreshold: 1
+      Name: ckan-worker
+      NamespaceId:
+        Ref: CloudMap
+    Type: AWS::ServiceDiscovery::Service
+  CkanworkerTaskDefinition:
+    Properties:
+      ContainerDefinitions:
+        - Command:
+            - eu-west-2.compute.internal
+            - docker-ckan.local
+          Essential: false
+          Image: docker/ecs-searchdomain-sidecar:1.0
+          LogConfiguration:
+            LogDriver: awslogs
+            Options:
+              awslogs-group:
+                Ref: LogGroup
+              awslogs-region:
+                Ref: AWS::Region
+              awslogs-stream-prefix: docker-ckan
+          Name: Ckanworker_ResolvConf_InitContainer
+        - DependsOn:
+            - Condition: SUCCESS
+              ContainerName: Ckanworker_ResolvConf_InitContainer
+          Environment:
           # ...
+          Essential: true
+          HealthCheck:
+            Command:
+              - CMD
+              - /srv/app/probe_worker.sh
+          Image: <<redacted account id>>.dkr.ecr.eu-west-2.amazonaws.com/mdepckan/ckan-deploy:latest@sha256:1
+          LinuxParameters: {}
+          LogConfiguration:
+            LogDriver: awslogs
+            Options:
+              awslogs-group:
+                Ref: LogGroup
+              awslogs-region:
+                Ref: AWS::Region
+              awslogs-stream-prefix: docker-ckan
+          MountPoints:
+            - ContainerPath: /var/lib/ckan
+              SourceVolume: ckan_storage
+          Name: ckan-worker
+          Ulimits:
+            - HardLimit: 20000
+              Name: nofile
+              SoftLimit: 20000
+      Cpu: "4096"
+      ExecutionRoleArn:
+        Ref: CkanworkerTaskExecutionRole
+      Family: docker-ckan-ckan-worker
+      Memory: "16384"
+      NetworkMode: awsvpc
+      RequiresCompatibilities:
+        - FARGATE
+      TaskRoleArn:
+        Ref: CkanworkerTaskRole
+      Volumes:
+        - EFSVolumeConfiguration:
+            AuthorizationConfig:
+              AccessPointId:
+                Ref: CkanstorageAccessPoint
+              IAM: ENABLED
+            FilesystemId: fs-0
+            TransitEncryption: ENABLED
+          Name: ckan_storage
+    Type: AWS::ECS::TaskDefinition
+  CkanworkerTaskExecutionRole:
   # ...
+  CkanworkerTaskRole:
+    Properties:
+      AssumeRolePolicyDocument:
+        Statement:
+          - Action:
+              - sts:AssumeRole
+            Condition: {}
+            Effect: Allow
+            Principal:
+              Service: ecs-tasks.amazonaws.com
+        Version: 2012-10-17
+      Policies:
+        - PolicyDocument:
+            Statement:
+              - Action:
+                  - s3:*
+                  - s3-object-lambda:*
+                Effect: Allow
+                Resource:
+                  - arn:aws:s3:::bucket_name/*
+                Sid: S3BucketAccess
+              - Action:
+                  - iam:ListPolicies
+                  - iam:ListRoles
+                  - iam:ListUsers
+                  - iam:ListGroups
+                Effect: Allow
+                Resource: '*'
+                Sid: DataSciSharingIAMAccess
+              - Action:
+                  - iam:*
+                Effect: Allow
+                Resource:
+                  - arn:aws:iam::123456789012:policy/ckan-smdh/datasci-sharing/*
+                  - arn:aws:iam::123456789012:group/ckan-smdh/datasci-sharing/*
+                Sid: DataSciSharingIAMManagedResourcesActions
+              - Action:
+                  - sqs:*
+                Effect: Allow
+                Resource: arn:aws:sqs::123456789012:sync_queue_name
+                Sid: SQSResourceManagement
+            Version: 2012-10-17
+          PolicyName: CkanworkerPolicy
+        - PolicyDocument:
+            Statement:
+              - Action:
+                  - elasticfilesystem:ClientMount
+                  - elasticfilesystem:ClientWrite
+                  - elasticfilesystem:ClientRootAccess
+                Condition:
+                  StringEquals:
+                    elasticfilesystem:AccessPointArn:
+                      Ref: CkanstorageAccessPoint
+                Effect: Allow
+                Principal: {}
+                Resource:
+                  - arn:aws:elasticfilesystem:eu-west-2:<<redacted account id>>:file-system/fs-0
+            Version: 2012-10-17
+          PolicyName: CkanworkerCkanstorageVolumeMountPolicy
+      Tags:
       # ...
+    Type: AWS::IAM::Role
   CloudMap:
     Properties:
       Description: Service Map for Docker Compose project docker-ckan