diff --git a/.gitignore b/.gitignore index ef73e00..91cc371 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .env.local **/__pycache__ .python-version -.temp \ No newline at end of file +.temp +feeds diff --git a/ingestor/app.py b/ingestor/app.py index 6dc644d..1555479 100644 --- a/ingestor/app.py +++ b/ingestor/app.py @@ -104,13 +104,6 @@ def update_delivered_trip_metrics_yesterday(event): daily_speeds.update_daily_table(two_days_ago) -# 7am UTC -> 2/3am ET -@app.schedule(Cron(0, 7, "*", "*", "?", "*")) -def update_gtfs(event): - today = date.today() - gtfs.ingest_gtfs_feeds_to_dynamo_and_s3(date_range=(today, today)) - - # 7:10am UTC -> 2:10/3:10am ET every day @app.schedule(Cron(10, 7, "*", "*", "?", "*")) def update_ridership(event): @@ -129,6 +122,14 @@ def update_time_predictions(event): predictions.update_predictions() +# 8:00am UTC -> 3:00/4:00am ET every day +@app.schedule(Cron(0, 8, "*", "*", "?", "*")) +def update_gtfs(event): + today = datetime.now() + last_week = (today - timedelta(days=7)).date() + gtfs.ingest_gtfs_feeds_to_dynamo_and_s3(date_range=(last_week, today.date())) + + # 4:40am UTC -> 2:40/3:40am ET every day @app.schedule(Cron(40, 7, "*", "*", "?", "*")) def update_trip_metrics(event): diff --git a/ingestor/chalicelib/s3.py b/ingestor/chalicelib/s3.py index 320b493..527f965 100644 --- a/ingestor/chalicelib/s3.py +++ b/ingestor/chalicelib/s3.py @@ -55,7 +55,7 @@ def ls(bucket, prefix): return all_keys -def clear_cf_cache(distribution, keys): +def clear_cf_cache(distribution: str, keys: list[str]): cloudfront.create_invalidation( DistributionId=distribution, InvalidationBatch={ diff --git a/ingestor/chalicelib/trip_metrics/backfill.py b/ingestor/chalicelib/trip_metrics/backfill.py index e457467..7c5ae37 100644 --- a/ingestor/chalicelib/trip_metrics/backfill.py +++ b/ingestor/chalicelib/trip_metrics/backfill.py @@ -3,8 +3,8 @@ from .ingest import ingest_trip_metrics, get_date_ranges -START_DATE = date(2023, 8, 29) -END_DATE = date(2023, 8, 30) +START_DATE = date(2024, 3, 1) +END_DATE = date(2024, 5, 14) MAX_RANGE_SIZE = 90 if __name__ == "__main__":