Migrating to Chalice Cron

.github/workflows/dry_run.yml

@@ -31,4 +31,4 @@ jobs:
         run: |
           curl -sSL https://install.python-poetry.org | python3 -
           poetry install
-          poetry run python3 slowzones.py --dry-run
+          poetry run python3 mbta-slowzone-bot/chalicelib/slowzones.py --dry-run

.gitignore

@@ -109,6 +109,7 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+.envrc
 
 # Spyder project settings
 .spyderproject

devops/deploy.sh

@@ -0,0 +1,62 @@
+#!/bin/bash -x
+
+if [[ -z "$DD_API_KEY" ]]; then
+    echo "Must provide DD_API_KEY in environment" 1>&2
+    exit 1
+fi
+
+if [[ -z "$ACCESS_KEY" || -z "$ACCESS_SECRET" || -z "$CONSUMER_KEY" || -z "$CONSUMER_SECRET" ]]; then
+    echo "Must provide ACCESS_KEY, ACCESS_SECRET, CONSUMER_KEY and CONSUMER_SECRET in environment" 1>&2
+    exit 1
+fi
+
+if [[ -z "$MASTODON_CLIENT_KEY" || -z "$MASTODON_CLIENT_SECRET" || -z "$MASTODON_ACCESS_TOKEN" ]]; then
+    echo "Must provide MASTODON_CLIENT_KEY, MASTODON_CLIENT_SECRET and MASTODON_ACCESS_TOKEN in environment" 1>&2
+    exit 1
+fi
+
+STACK_NAME=mbta-slowzone-bot
+BUCKET=mbta-slowzone-bot
+
+# Identify the version and commit of the current deploy
+GIT_VERSION=`git describe --tags --always`
+GIT_SHA=`git rev-parse HEAD`
+echo "Deploying version $GIT_VERSION | $GIT_SHA"
+
+# Adding some datadog tags to get better data
+DD_TAGS="git.commit.sha:$GIT_SHA,git.repository_url:github.com/transitmatters/mbta-slow-zone-bot"
+DD_GIT_REPOSITORY_URL="github.com/transitmatters/mbta-slow-zone-bot"
+DD_GIT_COMMIT_SHA="$GIT_SHA"
+
+poetry export -f requirements.txt --output mbta-slowzone-bot/requirements.txt --without-hashes
+
+pushd mbta-slowzone-bot/
+
+poetry run chalice package --stage prod --merge-template .chalice/resources.json cfn/
+
+# Shrink the deployment package for the lambda layer https://stackoverflow.com/a/69355796
+echo "Shrinking the deployment package for the lambda layer"
+
+source ../devops/helpers.sh
+shrink
+
+# Check package size before deploying
+maximumsize=79100000
+actualsize=$(wc -c <"cfn/layer-deployment.zip")
+if [ $actualsize -ge $maximumsize ]; then
+    echo ""
+    echo "layer-deployment.zip is over $maximumsize bytes. Shrink the package further to be able to deploy"
+    exit 1
+fi
+
+aws s3 cp stations.json s3://$BUCKET
+aws cloudformation package --template-file cfn/sam.json --s3-bucket $BUCKET --output-template-file cfn/packaged.yaml
+aws cloudformation deploy --template-file cfn/packaged.yaml --stack-name $STACK_NAME \
+    --capabilities CAPABILITY_NAMED_IAM --no-fail-on-empty-changeset \
+    --parameter-overrides DDApiKey=$DD_API_KEY GitVersion=$GIT_VERSION DDTags=$DD_TAGS \
+        TwitterAccessKey=$ACCESS_KEY TwitterAccessSecret=$ACCESS_SECRET \
+        TwitterConsumerKey=$CONSUMER_KEY TwitterConsumerSecret=$CONSUMER_SECRET \
+        MastodonClientKey=$MASTODON_CLIENT_KEY \
+        MastodonClientSecret=$MASTODON_CLIENT_SECRET \
+        MastodonAccessToken=$MASTODON_ACCESS_TOKEN \ 
+        SlackWebhookUrl=$SLOW_ZONE_BOT_SLACK_WEBHOOK_URL

devops/helpers.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Shrink the deployment package for the lambda layer https://stackoverflow.com/a/69355796
+function shrink {
+    zip -d -qq cfn/layer-deployment.zip '*/__pycache__/*'
+    zip -d -qq cfn/layer-deployment.zip '**/*.pyc'
+    zip -d -qq cfn/layer-deployment.zip '**/LICENSE*'
+    zip -d -qq cfn/layer-deployment.zip '**/AUTHOR*'
+    zip -d -qq cfn/layer-deployment.zip '**/NOTICE*'
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Africa/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Asia/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Atlantic/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Arctic/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Antarctica/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Australia/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Brazil/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Chile/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Europe/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Indian/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/tzdata/zoneinfo/Mexico/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/examples/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/a*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/b*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/co*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/ch*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/cu*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/d*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/e*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/f*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/g*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/h*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/i*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/j*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/k*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/l*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/m*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/n*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/o*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/p*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/q*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/r*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/sa*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/se*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/sn*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/sm*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/ss*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/st*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/su*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/t*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/u*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/v*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/w*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/x*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/y*/*
+    zip -d -qq cfn/layer-deployment.zip python/lib/**/site-packages/boto*/data/z*/*
+}

mbta-slowzone-bot/.chalice/config.json

@@ -0,0 +1,29 @@
+{
+  "version": "2.0",
+  "app_name": "mbta-slowzone-bot",
+  "automatic_layer": true,
+  "stages": {
+    "prod": {
+      "api_gateway_stage": "api",
+      "autogen_policy": false,
+      "environment_variables": {
+        "DD_SITE": "datadoghq.com",
+        "DD_ENV": "prod",
+        "DD_SERVICE": "mbta-slowzone-bot",
+        "DD_TRACE_ENABLED": "true",
+        "DD_PROFILING_ENABLED": "true"
+      },
+      "tags": {
+        "service": "mbta-slowzone-bot",
+        "env": "prod"
+      },
+      "lambda_functions": {
+        "process_daily_lamp": {
+          "iam_policy_file": "policy-slowzone-bot.json",
+          "lambda_timeout": 1000,
+          "lambda_memory_size": 1024
+        }
+      }
+    }
+  }
+}

mbta-slowzone-bot/.chalice/policy-slowzone-bot.json

@@ -0,0 +1,19 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Effect": "Allow",
+      "Resource": "arn:*:logs:*:*:*"
+    },
+    {
+      "Action": ["s3:*"],
+      "Effect": "Allow",
+      "Resource": ["arn:aws:s3:::mbta-slowzone-bot/*"]
+    }
+  ]
+}

mbta-slowzone-bot/.chalice/resources.json

@@ -0,0 +1,48 @@
+{
+  "Parameters": {
+    "DDApiKey": {
+      "Type": "String",
+      "Description": "Datadog API key."
+    },
+    "DDTags": {
+      "Type": "String",
+      "Description": "Additional Datadog Tags"
+    },
+    "GitVersion": {
+      "Type": "String",
+      "Description": "Current Git Id"
+    },
+    "TwitterAccessKey": {
+      "Type": "String",
+      "Description": "Twitter Access Key"
+    },
+    "TwitterAccessSecret": {
+      "Type": "String",
+      "Description": "Twitter Access Secret"
+    },
+    "TwitterConsumerKey": {
+      "Type": "String",
+      "Description": "Twitter Consumer Key"
+    },
+    "TwitterConsumerSecret": {
+      "Type": "String",
+      "Description": "Twitter Consumer Secret"
+    },
+    "MastodonClientKey": {
+      "Type": "String",
+      "Description": "Mastodon Access Token"
+    },
+    "MastodonClientSecret": {
+      "Type": "String",
+      "Description": "Mastodon Access Secret"
+    },
+    "MastodonAccessToken": {
+      "Type": "String",
+      "Description": "Mastodon Access Token"
+    },
+    "SlackWebhookUrl": {
+      "Type": "String",
+      "Description": "Slack Webhook URL"
+    }
+  }
+}

mbta-slowzone-bot/app.py

@@ -0,0 +1,16 @@
+from chalice import Chalice, Cron, ConvertToMiddleware
+from datadog_lambda.wrapper import datadog_lambda_wrapper
+from chalicelib import (
+    slowzones,
+)
+
+app = Chalice(app_name="mbta-slowzone-bot")
+
+app.register_middleware(ConvertToMiddleware(datadog_lambda_wrapper))
+
+
+# 13:30 UTC -> 8:30/9:30am ET every day.
+@app.schedule(Cron(30, 13, "*", "*", "?", "*"))
+def run_slowzone_bot(event):
+    # TODO: Run as production once tested
+    slowzones.run(dry_run=True, debug=True) 

mbta-slowzone-bot/chalicelib/clients.py

@@ -0,0 +1,27 @@
+import tweepy
+import mastodon as mastodon
+import os
+from chalicelib.constants import (
+    ACCESS_KEY,
+    ACCESS_SECRET,
+    CONSUMER_KEY,
+    CONSUMER_SECRET,
+    MASTODON_CLIENT_KEY,
+    MASTODON_CLIENT_SECRET,
+    MASTODON_ACCESS_TOKEN,
+)
+
+twitter_client = api = tweepy.Client(
+    bearer_token=os.environ.get("BEARER_TOKEN"),
+    access_token=ACCESS_KEY,
+    access_token_secret=ACCESS_SECRET,
+    consumer_key=CONSUMER_KEY,
+    consumer_secret=CONSUMER_SECRET,
+)
+
+mastodon_client = mastodon.Mastodon(
+    api_base_url="https://better.boston",
+    client_id=MASTODON_CLIENT_KEY,
+    client_secret=MASTODON_CLIENT_SECRET,
+    access_token=MASTODON_ACCESS_TOKEN,
+)

mbta-slowzone-bot/chalicelib/constants.py

@@ -0,0 +1,13 @@
+
+import os
+
+
+ACCESS_KEY = os.environ.get("ACCESS_KEY")
+ACCESS_SECRET = os.environ.get("ACCESS_SECRET")
+CONSUMER_KEY = os.environ.get("CONSUMER_KEY")
+CONSUMER_SECRET = os.environ.get("CONSUMER_SECRET")
+MASTODON_CLIENT_KEY = os.environ.get("MASTODON_CLIENT_KEY")
+MASTODON_CLIENT_SECRET = os.environ.get("MASTODON_CLIENT_SECRET")
+MASTODON_ACCESS_TOKEN = os.environ.get("MASTODON_ACCESS_TOKEN")
+
+SLOW_ZONE_BOT_SLACK_WEBHOOK_URL = os.environ.get("SLOW_ZONE_BOT_SLACK_WEBHOOK_URL")

mbta-slowzone-bot/chalicelib/domains/dry.py

@@ -0,0 +1,42 @@
+from chalicelib.output import format_fixed_slow_zone, format_new_slow_zone, format_updated_slow_zone
+import logging
+import sys
+
+
+def send_new_slow_zone_dry(sz):
+    for line in sz:
+        logging.debug(f"line {line} in slow zone {sz}")
+        for z in line:
+            logging.debug(f"z {z} in line {line}")
+            try:
+                output = format_new_slow_zone(z)
+                logging.info("\n" + output)
+            except Exception as e:
+                logging.error(f"Error formatting output for new slow zone {z}: {e}")
+                sys.exit(1)
+
+
+def send_fixed_slow_zone_dry(sz):
+    for line in sz:
+        logging.debug(f"line {line} in slow zone {sz}")
+        for z in line:
+            logging.debug(f"z {z} in line {line}")
+            try:
+                output = format_fixed_slow_zone(z)
+                logging.info("\n" + output)
+            except Exception as e:
+                logging.error(f"Error formatting output for fixed slow zone {z}: {e}")
+                sys.exit(1)
+
+
+def send_updated_slow_zone_dry(sz):
+    for line in sz:
+        logging.debug(f"line {line} in slow zone {sz}")
+        for z in line:
+            logging.debug(f"z {z} in line {line}")
+            try:
+                output = format_updated_slow_zone(z)
+                logging.info("\n" + output)
+            except Exception as e:
+                logging.error(f"Error formatting output for updated slow zone {z}: {e}")
+                sys.exit(1)

mbta-slowzone-bot/chalicelib/domains/mastodon.py

@@ -0,0 +1,33 @@
+from mastodon import Mastodon
+from chalicelib.output import chunks, format_fixed_slow_zone, format_new_slow_zone, format_updated_slow_zone
+import logging
+
+
+def send_toot_threads(tt_map, client: Mastodon):
+    for map in tt_map:
+        split_map = map.split("\n")
+        chunked_map = list(chunks(split_map, 7))
+        output = "\n".join(list(chunked_map)[0])
+        toot = client.status_post(status=output)
+        logging.info(f"toot: {toot}")
+
+
+def send_new_slow_zone_toots(sz, client: Mastodon):
+    for line in sz:
+        for z in line:
+            output = format_new_slow_zone(z)
+            client.status_post(status=output)
+
+
+def send_fixed_slow_zone_toots(sz, client: Mastodon):
+    for line in sz:
+        for z in line:
+            output = format_fixed_slow_zone(z)
+            client.status_post(status=output)
+
+
+def send_updated_slow_zone_toots(sz, client: Mastodon):
+    for line in sz:
+        for z in line:
+            output = format_updated_slow_zone(z)
+            client.status_post(status=output)