First steps to lambda deploy

lgsf/aws_lambda/fixtures/sqs-message-der.json

@@ -0,0 +1,20 @@
+{
+  "Records": [
+    {
+      "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78",
+      "receiptHandle": "MessageReceiptHandle",
+      "body": "{\"scraper_type\": \"councillors\",\"council\": \"DER\"}",
+    "attributes": {
+        "ApproximateReceiveCount": "1",
+        "SentTimestamp": "1523232000000",
+        "SenderId": "123456789012",
+        "ApproximateFirstReceiveTimestamp": "1523232000001"
+      },
+      "messageAttributes": {},
+      "md5OfBody": "7b270e59b47ff90a553787216d55d91d",
+      "eventSource": "aws:sqs",
+      "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue",
+      "awsRegion": "eu-west-2"
+    }
+  ]
+}

lgsf/aws_lambda/fixtures/sqs-message-sfk.json

@@ -0,0 +1,20 @@
+{
+  "Records": [
+    {
+      "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78",
+      "receiptHandle": "MessageReceiptHandle",
+      "body": "{\"scraper_type\": \"councillors\",\"council\": \"SFK\"}",
+      "attributes": {
+        "ApproximateReceiveCount": "1",
+        "SentTimestamp": "1523232000000",
+        "SenderId": "123456789012",
+        "ApproximateFirstReceiveTimestamp": "1523232000001"
+      },
+      "messageAttributes": {},
+      "md5OfBody": "7b270e59b47ff90a553787216d55d91d",
+      "eventSource": "aws:sqs",
+      "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue",
+      "awsRegion": "eu-west-2"
+    }
+  ]
+}

lgsf/aws_lambda/fixtures/sqs-message-wlv.json

@@ -0,0 +1,20 @@
+{
+  "Records": [
+    {
+      "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78",
+      "receiptHandle": "MessageReceiptHandle",
+      "body": "{\"scraper_type\": \"councillors\",\"council\": \"WLV\"}",
+      "attributes": {
+        "ApproximateReceiveCount": "1",
+        "SentTimestamp": "1523232000000",
+        "SenderId": "123456789012",
+        "ApproximateFirstReceiveTimestamp": "1523232000001"
+      },
+      "messageAttributes": {},
+      "md5OfBody": "7b270e59b47ff90a553787216d55d91d",
+      "eventSource": "aws:sqs",
+      "eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue",
+      "awsRegion": "eu-west-2"
+    }
+  ]
+}

lgsf/aws_lambda/handlers.py

@@ -9,6 +9,22 @@
 from lgsf.path_utils import load_scraper
 
 
+def scraper_worker_handler(event, context):
+    print(f"EVENT: {event}")
+    message = json.loads(event["Records"][0]["body"])
+    council = message["council"]
+    command_name = message["scraper_type"]
+    scraper_cls = load_scraper(council, command_name)
+    options = {"council": council, "verbose": True, "aws_lambda": True}
+    console = Console(file=sys.stdout)
+    scraper = scraper_cls(options, console)
+    try:
+        scraper.run()
+    except Exception as e:
+        scraper.console.log(e)
+        scraper.delete_branch()
+
+
 def queue_builder_handler(event, context):
     councillors_command = Command(argv=["", "--all-councils"], stdout=sys.stdout)
     councillors_command.options = {"all_councils": True, "exclude_missing": True}

sam-template.yaml

@@ -39,6 +39,22 @@ Resources:
       RepositoryName: CouncillorsRepo
       RepositoryDescription: Repository for scraped councillor data.
 
+  ScraperWorkerFunction:
+    Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
+    Properties:
+      CodeUri: .
+      Handler: lgsf.aws_lambda.handlers.scraper_worker_handler
+      Runtime: python3.8
+      Events:
+        SQSEvent:
+          Type: SQS
+          Properties:
+            Queue: !GetAtt SQSScraperQueue.Arn
+            BatchSize: 1
+            Enabled: true
+      Role: !Sub "arn:aws:iam::${AWS::AccountId}:role/LGSFLambdaExecutionRole"
+
+
 
 Outputs:
   # Find out more about other implicit resources you can reference within SAM