serverless.yaml

service:
  name: serverless-blockstream

plugins:
  - serverless-pseudo-parameters
  - serverless-dynamodb-autoscaling
  - serverless-webpack

custom:
  webpackIncludeModules: true # enable auto-packing of external modules
  region: ${opt:region, self:provider.region}
  stage: ${opt:stage, self:provider.stage}
  prefix: ${self:service}-${self:custom.stage}
  newBlockQueueName: ${self:custom.prefix}-new-blocks.fifo
  logFirehoseQueue: ${self:custom.prefix}-log-firehose.fifo
  transactionFirehoseQueue: ${self:custom.prefix}-tx-firehose.fifo
  getAbisLambdaName: ${self:custom.prefix}-get-abis
  blockReadyTopicName: ${self:custom.prefix}-block-ready
  blockProcessedTopicName: ${self:custom.prefix}-block-processed
  srcNodeUrl: ${opt:srcNodeUrl, env:srcNodeUrl}
  networkId: ${opt:networkId, env:networkId} # id of the network from which the stack pulls data, validated against the node
  numBlocksDelay: ${opt:numBlocksDelay, env:numBlocksDelay, "0"} # how many blocks we wait to send an event
  blockDataTtlMs: ${opt:blockDataTtlMs, env:blockDataTtlMs, "604800000"} # how long the data stays in dynamo before being removed
  logLevel: ${opt:logLevel, env:logLevel, "info"} # log level of the lambdas
  rewindBlockLookback: ${opt:rewindBlockLookback, env:rewindBlockLookback, "25"} # log level of the lambdas
  baseDynamoDDBCapacity: ${opt:baseDynamoDDBCapacity, env:baseDynamoDDBCapacity, "2"}
  maxDynamoDDBCapacity: ${opt:maxDynamoDDBCapacity, env:maxDynamoDDBCapacity, "100"}
  dynamoTargetUtilitzation: ${opt:dynamoTargetUtilitzation, env:dynamoTargetUtilitzation, "0.6"}
  etherscanApiUrl: ${opt:etherscanApiUrl, env:etherscanApiUrl}
  etherscanApiKey: ${opt:etherscanApiKey, env:etherscanApiKey}
  alertNotificationEmail: "aws-alerts+${self:custom.prefix}@ethercast.io" # where cloudwatch alarms alert to
  capacities:
    - table: BlocksTable  # DynamoDB Resource
      read:
        minimum: ${self:custom.baseDynamoDDBCapacity}        # Minimum read capacity
        maximum: ${self:custom.maxDynamoDDBCapacity}      # Maximum read capacity
        usage: ${self:custom.dynamoTargetUtilitzation}        # Targeted usage percentage
      write:
        minimum: ${self:custom.baseDynamoDDBCapacity}        # Minimum write capacity
        maximum: ${self:custom.maxDynamoDDBCapacity}      # Maximum write capacity
        usage: ${self:custom.dynamoTargetUtilitzation}        # Targeted usage percentage
    - table: AbiTable  # DynamoDB Resource
      read:
        minimum: 10        # Minimum read capacity
        maximum: ${self:custom.maxDynamoDDBCapacity}      # Maximum read capacity
        usage: ${self:custom.dynamoTargetUtilitzation}        # Targeted usage percentage
      write:
        minimum: 10        # Minimum write capacity
        maximum: ${self:custom.maxDynamoDDBCapacity}      # Maximum write capacity
        usage: ${self:custom.dynamoTargetUtilitzation}        # Targeted usage percentage
    - table: BlockStreamStateTable  # DynamoDB Resource
      read:
        minimum: ${self:custom.baseDynamoDDBCapacity}        # Minimum read capacity
        maximum: ${self:custom.maxDynamoDDBCapacity}      # Maximum read capacity
        usage: ${self:custom.dynamoTargetUtilitzation}        # Targeted usage percentage
      write:
        minimum: ${self:custom.baseDynamoDDBCapacity}        # Minimum write capacity
        maximum: ${self:custom.maxDynamoDDBCapacity}      # Maximum write capacity
        usage: ${self:custom.dynamoTargetUtilitzation}        # Targeted usage percentage

provider:
  name: aws
  logRetentionInDays: 7
  runtime: nodejs6.10
  environment: # service wide environment variables
    BLOCKSTREAM_STATE_TABLE: ${self:custom.prefix}-blockstream-state # the table containing the state of blockstream
    BLOCKS_TABLE: ${self:custom.prefix}-blocks # the table containing full blocks and their logs (compressed)
    ABI_TABLE: ${self:custom.prefix}-abis # the table containing contract abis
    BLOCK_READY_TOPIC_NAME: ${self:custom.blockReadyTopicName}
    BLOCK_PROCESSED_TOPIC_NAME: ${self:custom.blockProcessedTopicName}
    NEW_BLOCK_QUEUE_NAME: ${self:custom.newBlockQueueName}  # Queue that receives new blocks
    LOG_FIREHOSE_QUEUE_NAME: ${self:custom.logFirehoseQueue} # Queue that receives log events
    TRANSACTION_FIREHOSE_QUEUE_NAME: ${self:custom.transactionFirehoseQueue} # Queue that receives transactions
    SRC_NODE_URL: ${self:custom.srcNodeUrl} # the ethereum node URL that is used
    NUM_BLOCKS_DELAY: ${self:custom.numBlocksDelay} # how many blocks behind the chain we wait to send logs
    BLOCK_DATA_TTL_MS: ${self:custom.blockDataTtlMs}
    LOG_LEVEL: ${self:custom.logLevel}
    NETWORK_ID: ${self:custom.networkId}
    REWIND_BLOCK_LOOKBACK: ${self:custom.rewindBlockLookback}
    ETHERSCAN_API_URL: ${self:custom.etherscanApiUrl}
    ETHERSCAN_API_KEY: ${self:custom.etherscanApiKey}

  iamRoleStatements: # permissions for all of your functions can be set here
    - Effect: Allow
      Action: # Gives permission to DynamoDB tables in a specific region
        - dynamodb:Query
        - dynamodb:GetItem
        - dynamodb:PutItem
        - dynamodb:UpdateItem
        - dynamodb:BatchWriteItem
        - dynamodb:BatchGetItem
      Resource:
        - "arn:aws:dynamodb:#{AWS::Region}:#{AWS::AccountId}:table/${self:provider.environment.BLOCKSTREAM_STATE_TABLE}"
        - "arn:aws:dynamodb:#{AWS::Region}:#{AWS::AccountId}:table/${self:provider.environment.BLOCKS_TABLE}"
        - "arn:aws:dynamodb:#{AWS::Region}:#{AWS::AccountId}:table/${self:provider.environment.ABI_TABLE}"
    - Effect: Allow
      Action: # Gives permission to push logs to the SQS queue
        - sqs:SendMessage
        - sqs:SendMessageBatch
        - sqs:ReceiveMessage
        - sqs:DeleteMessage
        - sqs:GetQueueUrl
      Resource:
        - "arn:aws:sqs:#{AWS::Region}:#{AWS::AccountId}:${self:custom.newBlockQueueName}"
    - Effect: Allow
      Action: # Gives permission to push logs to the SQS queue
        - sqs:SendMessage
        - sqs:SendMessageBatch
        - sqs:GetQueueUrl
      Resource:
        - "arn:aws:sqs:#{AWS::Region}:#{AWS::AccountId}:${self:custom.logFirehoseQueue}"
        - "arn:aws:sqs:#{AWS::Region}:#{AWS::AccountId}:${self:custom.transactionFirehoseQueue}"
    - Effect: Allow
      Action: # Gives permission to notify and find these topic arns
        - sns:Publish
        - sns:CreateTopic
      Resource:
        - "arn:aws:sns:#{AWS::Region}:#{AWS::AccountId}:${self:custom.blockReadyTopicName}"
        - "arn:aws:sns:#{AWS::Region}:#{AWS::AccountId}:${self:custom.blockProcessedTopicName}"


functions:
  poll-for-blocks:
    handler: src/poll-for-blocks.start
    memorySize: 128
    timeout: 60
    reservedConcurrency: 1
    events:
      - schedule:
          rate: rate(1 minute)
          enabled: true

  drain-block-queue:
    handler: src/drain-block-queue.start
    memorySize: 1024
    reservedConcurrency: 1
    timeout: 300
    events:
      - sns:
          arn:
            Ref: "BlockReadyTopic"
          topicName: ${self:custom.blockReadyTopicName}

  get-abis:
    handler: src/get-abis.handle
    name: ${self:custom.getAbisLambdaName}

resources:
  Description: The block streaming stack that populates the log and transaction queues
  Outputs:
    LogFirehoseQueueName:
      Description: The name of the log firehose SQS queue to which all incoming logs events are sent
      Value:
        Fn::GetAtt:
          - "LogFirehoseQueue"
          - "QueueName"
      Export:
        Name: ${self:custom.prefix}-log-firehose-queue-name
    TransactionFirehoseQueueName:
      Description: The name of the transaction firehose SQS queue to which all incoming transactions are sent
      Value:
        Fn::GetAtt:
          - "TransactionFirehoseQueue"
          - "QueueName"
      Export:
        Name: ${self:custom.prefix}-tx-firehose-queue-name
    BlockProcessedTopicArn:
      Description: Arn of the topic notified when a block is ready
      Value:
        Ref: "BlockProcessedTopic"
      Export:
        Name: ${self:custom.prefix}-block-processed-topic-arn

  Resources:
    BlockReadyTopic: # this topic is notified when a block is available for processing
      Type: AWS::SNS::Topic
      Properties:
        DisplayName: ${self:custom.blockReadyTopicName}
        TopicName: ${self:custom.blockReadyTopicName}

    BlockProcessedTopic: # this topic is notified when a block is finished processing
      Type: AWS::SNS::Topic
      Properties:
        DisplayName: ${self:custom.blockProcessedTopicName}
        TopicName: ${self:custom.blockProcessedTopicName}

    BlockStreamStateTable: # This table is throttled and should not autoscale
      Type: AWS::DynamoDB::Table
      Properties:
        TableName: ${self:provider.environment.BLOCKSTREAM_STATE_TABLE}
        AttributeDefinitions:
          - AttributeName: networkId
            AttributeType: N
        KeySchema:
          - AttributeName: networkId
            KeyType: HASH
        ProvisionedThroughput:
          ReadCapacityUnits: ${self:custom.baseDynamoDDBCapacity}
          WriteCapacityUnits: ${self:custom.baseDynamoDDBCapacity}

    BlocksTable: # This table's autoscaling is to handle downtimes or node switching. When it needs to catch up, it costs more.
      Type: AWS::DynamoDB::Table
      Properties:
        TableName: ${self:provider.environment.BLOCKS_TABLE}
        AttributeDefinitions:
          - AttributeName: hash
            AttributeType: S
          - AttributeName: number
            AttributeType: S
        KeySchema:
          - AttributeName: hash
            KeyType: HASH
          - AttributeName: number
            KeyType: RANGE
        ProvisionedThroughput:
          ReadCapacityUnits: ${self:custom.baseDynamoDDBCapacity}
          WriteCapacityUnits: ${self:custom.baseDynamoDDBCapacity}
        TimeToLiveSpecification:
          AttributeName: ttl
          Enabled: true

    AbiTable: # This table's autoscaling is to handle downtimes or node switching. When it needs to catch up, it costs more.
      Type: AWS::DynamoDB::Table
      Properties:
        TableName: ${self:provider.environment.ABI_TABLE}
        AttributeDefinitions:
          - AttributeName: address
            AttributeType: S
        KeySchema:
          - AttributeName: address
            KeyType: HASH
        ProvisionedThroughput:
          ReadCapacityUnits: 10
          WriteCapacityUnits: 10
        TimeToLiveSpecification:
          AttributeName: ttl
          Enabled: true

    NewBlockQueue:
      Type: AWS::SQS::Queue
      Properties:
        QueueName: ${self:custom.newBlockQueueName}
        FifoQueue: true
        MessageRetentionPeriod: 1209600
        VisibilityTimeout: 60

    TransactionFirehoseQueue:
      Type: AWS::SQS::Queue
      Properties:
        QueueName: ${self:custom.transactionFirehoseQueue}
        FifoQueue: true
        MessageRetentionPeriod: 1209600
        VisibilityTimeout: 60

    LogFirehoseQueue:
      Type: AWS::SQS::Queue
      Properties:
        QueueName: ${self:custom.logFirehoseQueue}
        FifoQueue: true
        MessageRetentionPeriod: 1209600
        VisibilityTimeout: 60

    AlarmTopic:
      Type: AWS::SNS::Topic
      Properties:
        Subscription:
          - Endpoint: ${self:custom.alertNotificationEmail}
            Protocol: email

    ProcessBlocksAlarm:
      Type: AWS::CloudWatch::Alarm
      Properties:
        AlarmDescription: "Blocks are not being removed from the queue in < 3 minutes"
        Namespace: "AWS/SQS"
        MetricName: ApproximateAgeOfOldestMessage
        Dimensions:
          - Name: QueueName
            Value: ${self:custom.newBlockQueueName}
        Statistic: Maximum
        Period: 60
        Threshold: 180
        EvaluationPeriods: 3
        ComparisonOperator: GreaterThanThreshold
        AlarmActions:
          - Ref: AlarmTopic
        OKActions:
          - Ref: AlarmTopic
        TreatMissingData: missing

    GenerateBlocksAlarm:
      Type: AWS::CloudWatch::Alarm
      Properties:
        AlarmDescription: "New blocks are not being reported"
        Namespace: "AWS/SQS"
        MetricName: NumberOfMessagesSent
        Dimensions:
          - Name: QueueName
            Value: ${self:custom.newBlockQueueName}
        Statistic: Sum
        Period: 300
        Threshold: 0
        EvaluationPeriods: 3
        ComparisonOperator: LessThanOrEqualToThreshold
        AlarmActions:
          - Ref: AlarmTopic
        OKActions:
          - Ref: AlarmTopic
        TreatMissingData: breaching

    ProcessLogsAlarm:
      Type: AWS::CloudWatch::Alarm
      Properties:
        AlarmDescription: "Logs are not being removed from the queue in < 3 minutes"
        Namespace: "AWS/SQS"
        MetricName: ApproximateAgeOfOldestMessage
        Dimensions:
          - Name: QueueName
            Value: ${self:custom.logFirehoseQueue}
        Statistic: Maximum
        Period: 60
        Threshold: 180
        EvaluationPeriods: 3
        ComparisonOperator: GreaterThanOrEqualToThreshold
        AlarmActions:
          - Ref: AlarmTopic
        OKActions:
          - Ref: AlarmTopic
        TreatMissingData: missing

    GenerateLogsAlarm:
      Type: AWS::CloudWatch::Alarm
      Properties:
        AlarmDescription: "Logs are not being added to the queue"
        Namespace: "AWS/SQS"
        MetricName: NumberOfMessagesSent
        Dimensions:
          - Name: QueueName
            Value: ${self:custom.logFirehoseQueue}
        Statistic: Sum
        Period: 300
        Threshold: 0
        EvaluationPeriods: 3
        ComparisonOperator: LessThanOrEqualToThreshold
        AlarmActions:
          - Ref: AlarmTopic
        OKActions:
          - Ref: AlarmTopic
        TreatMissingData: breaching

    ProcessTransactionsAlarm:
      Type: AWS::CloudWatch::Alarm
      Properties:
        AlarmDescription: "Transactions are not being removed from the queue in < 3 minutes"
        Namespace: "AWS/SQS"
        MetricName: ApproximateAgeOfOldestMessage
        Dimensions:
          - Name: QueueName
            Value: ${self:custom.transactionFirehoseQueue}
        Statistic: Maximum
        Period: 60
        Threshold: 180
        EvaluationPeriods: 3
        ComparisonOperator: GreaterThanOrEqualToThreshold
        AlarmActions:
          - Ref: AlarmTopic
        OKActions:
          - Ref: AlarmTopic
        TreatMissingData: missing

    GenerateTransactionsAlarm:
      Type: AWS::CloudWatch::Alarm
      Properties:
        AlarmDescription: "Transactions are not being sent to the queue"
        Namespace: "AWS/SQS"
        MetricName: NumberOfMessagesSent
        Dimensions:
          - Name: QueueName
            Value: ${self:custom.transactionFirehoseQueue}
        Statistic: Sum
        Period: 300
        Threshold: 0
        EvaluationPeriods: 3
        ComparisonOperator: LessThanOrEqualToThreshold
        AlarmActions:
          - Ref: AlarmTopic
        OKActions:
          - Ref: AlarmTopic
        TreatMissingData: breaching