From bd885d6f75e3b1921e34599e2965a1058ab6b0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Melissa=20Weber=20Mendon=C3=A7a?= Date: Fri, 7 Jun 2024 07:55:40 -0700 Subject: [PATCH] Add notebook with NYC MTA example (#197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add notebook with NYC MTA example Signed-off-by: Melissa Weber Mendonça * Addressing reviewer comments Signed-off-by: Melissa Weber Mendonça * Improved output, text and suggestions from reviewers Signed-off-by: Melissa Weber Mendonça * Update stop_id Signed-off-by: Melissa Weber Mendonça * Add counter for southbound trains Signed-off-by: Melissa Weber Mendonça * Fix typos in notebook examples Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --------- Signed-off-by: Melissa Weber Mendonça Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> Co-authored-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- docs/wiki/references/Examples.md | 4 + examples/07_end_to_end/README.md | 1 + examples/07_end_to_end/mta.ipynb | 783 ++++++++++++++++++ examples/07_end_to_end/seismic_waveform.ipynb | 2 +- 4 files changed, 789 insertions(+), 1 deletion(-) create mode 100644 examples/07_end_to_end/mta.ipynb diff --git a/docs/wiki/references/Examples.md b/docs/wiki/references/Examples.md index 41229319..0ad65ddb 100644 --- a/docs/wiki/references/Examples.md +++ b/docs/wiki/references/Examples.md @@ -5,3 +5,7 @@ - Grid of examples with short descriptions - Instructions for how to run the examples. --> + +# Examples + +To see a collection of examples that demonstrate how to use CSP, visit the [examples folder](https://github.com/Point72/csp/tree/main/examples). diff --git a/examples/07_end_to_end/README.md b/examples/07_end_to_end/README.md index ae83b01e..d063f512 100644 --- a/examples/07_end_to_end/README.md +++ b/examples/07_end_to_end/README.md @@ -1,3 +1,4 @@ # End to End Examples - [Seismic Data with obspy](./seismic_waveform.ipynb) +- [MTA Subway Data](./mta.ipynb) diff --git a/examples/07_end_to_end/mta.ipynb b/examples/07_end_to_end/mta.ipynb new file mode 100644 index 00000000..65eb7b2d --- /dev/null +++ b/examples/07_end_to_end/mta.ipynb @@ -0,0 +1,783 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f4d848d1-dd58-4b75-9662-377cb66b5be9", + "metadata": {}, + "source": [ + "# Using CSP to analyze MTA data" + ] + }, + { + "cell_type": "markdown", + "id": "6906efcb-324a-4bc1-b910-6d8905c0adb0", + "metadata": {}, + "source": [ + "The NYC Metropolitan Transportation Authority provides an [API for developers](https://api.mta.info), and we'll explore the MTA's realtime [GTFS-rt](https://developers.google.com/transit/gtfs-realtime) transportation data feed." + ] + }, + { + "cell_type": "markdown", + "id": "dacee7cb-d803-4eee-b135-c9ffd6ad2888", + "metadata": {}, + "source": [ + "In order to deal with the GTFS-rt data, we'll use the [nyct-gtfs](https://pypi.org/project/nyct-gtfs/) library, available from PyPI through\n", + "\n", + "```\n", + "pip install nyct-gtfs\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "dcc5655d-3c45-4f96-842e-bb5f1b7d2127", + "metadata": {}, + "source": [ + "The MTA feed can be inspected as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "9409305a-a84a-493f-8eeb-9a7c1c2c81eb", + "metadata": {}, + "outputs": [], + "source": [ + "from nyct_gtfs import NYCTFeed\n", + "\n", + "# Load the realtime feed from the MTA site for lines 1-7 and S\n", + "# (note that the api_key argument is required, but can be empty)\n", + "feed = NYCTFeed(\"https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs\", api_key=\"\")" + ] + }, + { + "cell_type": "markdown", + "id": "8b628b32-6483-4305-8bb7-5cc10f23ff79", + "metadata": {}, + "source": [ + "> **Note:** There is a website with official documentation for the MTA API at https://new.mta.info/developers, including instructions on how to access the feed for other lines. However, you may not have access to this website depending on your geographical location. " + ] + }, + { + "cell_type": "markdown", + "id": "18e067c5-3f57-43a5-a744-a3bfa213c69b", + "metadata": {}, + "source": [ + "Let's explore the data first. `feed` is a `nyct_gtfs.feed.NYCTFeed` instance, with the most important methods being the following:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "349a3179-f49d-4be8-8f65-9599579695b8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[0;31mSignature:\u001b[0m \u001b[0mfeed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrefresh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mDocstring:\u001b[0m Reload this object's feed information from the MTA API\n", + "\u001b[0;31mFile:\u001b[0m ~/micromamba/envs/csp-dev/lib/python3.11/site-packages/nyct_gtfs/feed.py\n", + "\u001b[0;31mType:\u001b[0m method" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "feed.refresh?" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4d2d4de5-2a72-4c68-9618-106bda47c005", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[0;31mSignature:\u001b[0m\n", + "\u001b[0mfeed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfilter_trips\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mline_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mtravel_direction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mtrain_assigned\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0munderway\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mshape_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mheaded_for_stop_id\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mupdated_after\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m \u001b[0mhas_delay_alert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\n", + "\u001b[0;34m\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mDocstring:\u001b[0m\n", + "Get the list of subway trips from the GTFS-realtime feed, optionally filtering based on one or more parameters.\n", + "\n", + "If more than one filter is specified, only trips that match all filters will be returned.\n", + "\n", + ":param line_id: A line identifier str (or list of strs) such as \"1\", \"A\", \"GS\", or \"FS\"\n", + ":param travel_direction: A travel direction str, either \"N\" for North or \"S\" for South (see `Trip.direction`)\n", + ":param train_assigned: A boolean that is True iff a train has been assigned to this trip\n", + ":param underway: A boolean that is True iff a train has begun this trip\n", + ":param shape_id: A str (or list of strs) representing the shape id (i.e. \"1..S03R\") (see `Trip.shape_id`)\n", + ":param headed_for_stop_id: A str (or list of strs) representing a stop id(s) that this trip must be heading to\n", + ":param updated_after: A datetime.datetime, trips whose most recent update is before this time are excluded\n", + " (note, specifying this option always excludes trains not underway - since only trains\n", + " that are underway provide position updates)\n", + ":param has_delay_alert: A boolean that is True iff a train currently has a delay alert published\n", + ":return: A list of `Trip` objects\n", + "\u001b[0;31mFile:\u001b[0m ~/micromamba/envs/csp-dev/lib/python3.11/site-packages/nyct_gtfs/feed.py\n", + "\u001b[0;31mType:\u001b[0m method" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "feed.filter_trips?" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "9d89289d-ee10-4bee-a37d-0863fde7d1f9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\u001b[0;31mType:\u001b[0m property\n", + "\u001b[0;31mString form:\u001b[0m \n", + "\u001b[0;31mDocstring:\u001b[0m Get the list of subway trips from the GTFS-realtime feed. Returns a list of `Trip` objects" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "feed.trips?" + ] + }, + { + "cell_type": "markdown", + "id": "3bf0fe58-ced8-49a7-84bb-09b9abdb41ee", + "metadata": {}, + "source": [ + "In our case, we will (for simplicity) filter the trips to collect information only about the 1, 2 and 3 trains, and we will start with trains going through 34 St-Penn Station (identified with stop IDs 128S or 128N):" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "429163be-6a02-4a14-b1f5-5f5016213af0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{\"110450_1..S03R\", STOPPED_AT 127S @16:01:09},\n", + " {\"110900_1..S03R\", INCOMING_AT 125S @16:00:45},\n", + " {\"111150_1..S03R\", INCOMING_AT 122S @16:01:03},\n", + " {\"111550_1..S03R\", IN_TRANSIT_TO 119S @16:01:06},\n", + " {\"112000_1..S03R\", IN_TRANSIT_TO 116S @16:01:09},\n", + " {\"112600_1..N03R\", STOPPED_AT 128N @16:00:58},\n", + " {\"112700_1..S03R\", STOPPED_AT 113S @16:01:09},\n", + " {\"112950_1..S03R\", STOPPED_AT 111S @16:00:57},\n", + " {\"113000_1..N03R\", STOPPED_AT 132N @16:00:29},\n", + " {\"113400_1..S03R\", IN_TRANSIT_TO 110S @16:01:05},\n", + " {\"113400_1..N03R\", IN_TRANSIT_TO 134N @16:01:09},\n", + " {\"113700_1..S03R\", STOPPED_AT 106S @16:00:29},\n", + " {\"113800_1..N03R\", IN_TRANSIT_TO 137N @16:01:05},\n", + " {\"113950_1..S03R\", STOPPED_AT 104S @16:01:07},\n", + " {\"109550_2..S01R\", IN_TRANSIT_TO 120S @16:01:10},\n", + " {\"110150_2..S01R\", IN_TRANSIT_TO 227S @16:01:08},\n", + " {\"110400_2..N01R\", IN_TRANSIT_TO 128N @16:01:10},\n", + " {\"110950_2..S01R\", STOPPED_AT 220S @16:00:35},\n", + " {\"111000_2..N01R\", STOPPED_AT 228N @16:00:29},\n", + " {\"111450_2..S01R\", STOPPED_AT 217S @16:01:08},\n", + " {\"111800_2..N01R\", INCOMING_AT 233N @16:01:10},\n", + " {\"112350_2..S01R\", STOPPED_AT 214S @16:00:19},\n", + " {\"112700_2..N01R\", IN_TRANSIT_TO 238N @16:00:56},\n", + " {\"113100_2..S01R\", STOPPED_AT 210S @16:01:02},\n", + " {\"113250_2..N01R\", IN_TRANSIT_TO 241N @16:01:08},\n", + " {\"113500_2..S01R\", STOPPED_AT 206S @16:00:32},\n", + " {\"114000_2..N01R\", INCOMING_AT 245N @16:01:08},\n", + " {\"110000_3..N01R\", IN_TRANSIT_TO 132N @16:01:03},\n", + " {\"110950_3..N01R\", STOPPED_AT 230N @16:00:24},\n", + " {\"111750_3..N01R\", STOPPED_AT 235N @16:00:31},\n", + " {\"111900_3..S01R\", STOPPED_AT 128S @16:00:33},\n", + " {\"112550_3..N01R\", IN_TRANSIT_TO 239N @16:01:07},\n", + " {\"113000_3..S01R\", IN_TRANSIT_TO 120S @16:01:02},\n", + " {\"113200_3..N01R\", STOPPED_AT 251N @16:00:22},\n", + " {\"113700_3..S01R\", IN_TRANSIT_TO 225S @16:01:00},\n", + " {\"114000_3..N01R\", STOPPED_AT 256N @16:00:43}]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# This cell can be run multiple times, and data will be refreshed every 30s\n", + "feed.refresh()\n", + "trains = feed.filter_trips(underway=True, headed_for_stop_id=['128S', '128N'])\n", + "trains" + ] + }, + { + "cell_type": "markdown", + "id": "e52dd161-b878-4d6f-b4a0-bda3a6f35967", + "metadata": {}, + "source": [ + "We can also show this data in a human-readable way." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "071a82d9-9da6-484f-951c-2c0bbc26f0f6", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Southbound 1 to South Ferry, departed origin 18:24:30, Currently STOPPED_AT Times Sq-42 St, last update at 16:01:09\n", + "Southbound 1 to South Ferry, departed origin 18:29:00, Currently INCOMING_AT 59 St-Columbus Circle, last update at 16:00:45\n", + "Southbound 1 to South Ferry, departed origin 18:31:30, Currently INCOMING_AT 79 St, last update at 16:01:03\n", + "Southbound 1 to South Ferry, departed origin 18:35:30, Currently IN_TRANSIT_TO 103 St, last update at 16:01:06\n", + "Southbound 1 to South Ferry, departed origin 18:40:00, Currently IN_TRANSIT_TO 125 St, last update at 16:01:09\n", + "Northbound 1 to Van Cortlandt Park-242 St, departed origin 18:46:00, Currently STOPPED_AT 34 St-Penn Station, last update at 16:00:58\n", + "Southbound 1 to South Ferry, departed origin 18:47:00, Currently STOPPED_AT 157 St, last update at 16:01:09\n", + "Southbound 1 to South Ferry, departed origin 18:49:30, Currently STOPPED_AT 181 St, last update at 16:00:57\n", + "Northbound 1 to Van Cortlandt Park-242 St, departed origin 18:50:00, Currently STOPPED_AT 14 St, last update at 16:00:29\n", + "Southbound 1 to South Ferry, departed origin 18:54:00, Currently IN_TRANSIT_TO 191 St, last update at 16:01:05\n", + "Northbound 1 to Van Cortlandt Park-242 St, departed origin 18:54:00, Currently IN_TRANSIT_TO Houston St, last update at 16:01:09\n", + "Southbound 1 to South Ferry, departed origin 18:57:00, Currently STOPPED_AT Marble Hill-225 St, last update at 16:00:29\n", + "Northbound 1 to Van Cortlandt Park-242 St, departed origin 18:58:00, Currently IN_TRANSIT_TO Chambers St, last update at 16:01:05\n", + "Southbound 1 to South Ferry, departed origin 18:59:30, Currently STOPPED_AT 231 St, last update at 16:01:07\n", + "Southbound 2 to Flatbush Av-Brooklyn College, departed origin 18:15:30, Currently IN_TRANSIT_TO 96 St, last update at 16:01:10\n", + "Southbound 2 to Flatbush Av-Brooklyn College, departed origin 18:21:30, Currently IN_TRANSIT_TO Central Park North (110 St), last update at 16:01:08\n", + "Northbound 2 to Wakefield-241 St, departed origin 18:24:00, Currently IN_TRANSIT_TO 34 St-Penn Station, last update at 16:01:10\n", + "Southbound 2 to Flatbush Av-Brooklyn College, departed origin 18:29:30, Currently STOPPED_AT Jackson Av, last update at 16:00:35\n", + "Northbound 2 to Wakefield-241 St, departed origin 18:30:00, Currently STOPPED_AT Park Place, last update at 16:00:29\n", + "Southbound 2 to Flatbush Av-Brooklyn College, departed origin 18:34:30, Currently STOPPED_AT Simpson St, last update at 16:01:08\n", + "Northbound 2 to Wakefield-241 St, departed origin 18:38:00, Currently INCOMING_AT Hoyt St, last update at 16:01:10\n", + "Southbound 2 to Flatbush Av-Brooklyn College, departed origin 18:43:30, Currently STOPPED_AT West Farms Sq-E Tremont Av, last update at 16:00:19\n", + "Northbound 2 to Wakefield-241 St, departed origin 18:47:00, Currently IN_TRANSIT_TO Eastern Pkwy-Brooklyn Museum, last update at 16:00:56\n", + "Southbound 2 to Flatbush Av-Brooklyn College, departed origin 18:51:00, Currently STOPPED_AT Allerton Av, last update at 16:01:02\n", + "Northbound 2 to Wakefield-241 St, departed origin 18:52:30, Currently IN_TRANSIT_TO President St-Medgar Evers College, last update at 16:01:08\n", + "Southbound 2 to Flatbush Av-Brooklyn College, departed origin 18:55:00, Currently STOPPED_AT 225 St, last update at 16:00:32\n", + "Northbound 2 to Wakefield-241 St, departed origin 19:00:00, Currently INCOMING_AT Beverly Rd, last update at 16:01:08\n", + "Northbound 3 to Harlem-148 St, departed origin 18:20:00, Currently IN_TRANSIT_TO 14 St, last update at 16:01:03\n", + "Northbound 3 to Harlem-148 St, departed origin 18:29:30, Currently STOPPED_AT Wall St, last update at 16:00:24\n", + "Northbound 3 to Harlem-148 St, departed origin 18:37:30, Currently STOPPED_AT Atlantic Av-Barclays Ctr, last update at 16:00:31\n", + "Southbound 3 to New Lots Av, departed origin 18:39:00, Currently STOPPED_AT 34 St-Penn Station, last update at 16:00:33\n", + "Northbound 3 to Harlem-148 St, departed origin 18:45:30, Currently IN_TRANSIT_TO Franklin Av-Medgar Evers College, last update at 16:01:07\n", + "Southbound 3 to New Lots Av, departed origin 18:50:00, Currently IN_TRANSIT_TO 96 St, last update at 16:01:02\n", + "Northbound 3 to Harlem-148 St, departed origin 18:52:00, Currently STOPPED_AT Sutter Av-Rutland Rd, last update at 16:00:22\n", + "Southbound 3 to New Lots Av, departed origin 18:57:00, Currently IN_TRANSIT_TO 125 St, last update at 16:01:00\n", + "Northbound 3 to Harlem-148 St, departed origin 19:00:00, Currently STOPPED_AT Van Siclen Av, last update at 16:00:43\n" + ] + } + ], + "source": [ + "for train in trains:\n", + " print(train)" + ] + }, + { + "cell_type": "markdown", + "id": "18cc16fa-686b-4b27-be0d-0015ab28ddcc", + "metadata": {}, + "source": [ + "We can now check for the times when trains will pass through 34St-Penn Station." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "ace17da1-7c91-469f-9fab-6dd163516057", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Station | Line | Direction | Arrival time\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:03:09\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:06:55\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:12:03\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:17:06\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:21:22\n", + "34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:01:28\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:25:09\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:28:27\n", + "34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:04:59\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:31:05\n", + "34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:09:43\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:34:59\n", + "34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:13:05\n", + "34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:37:07\n", + "34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:10:49\n", + "34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:14:31\n", + "34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:03:05\n", + "34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:26:35\n", + "34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:09:29\n", + "34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:31:08\n", + "34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:21:28\n", + "34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:36:19\n", + "34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:29:56\n", + "34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:44:02\n", + "34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:34:28\n", + "34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:50:02\n", + "34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:39:48\n", + "34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:08:06\n", + "34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:12:54\n", + "34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:23:01\n", + "34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:01:03\n", + "34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:31:10\n", + "34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:12:52\n", + "34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:38:22\n", + "34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:17:51\n", + "34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:45:43\n" + ] + } + ], + "source": [ + "trains_at_penn = []\n", + "print(\"Station | Line | Direction | Arrival time\")\n", + "for train in trains:\n", + " for update in train.stop_time_updates:\n", + " if update.stop_id in ['128S', '128N']:\n", + " print(f\"{update.stop_name} | {train.route_id} | {train.headsign_text} | {update.arrival}\")\n", + " trains_at_penn.append((train, update))" + ] + }, + { + "cell_type": "markdown", + "id": "8190647e-4d45-4ab5-930c-4c1696d8c125", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "5931527f-a501-4ee1-a9a4-be9f7436e85a", + "metadata": {}, + "source": [ + "## Using CSP to ingest and analyze the data" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "1bd20d3e-a594-4c6d-a56b-b352b09f11d3", + "metadata": {}, + "source": [ + "When using CSP to ingest and analyze this data, we start with a graph representing the operations we want to perform. [CSP Graphs](https://github.com/Point72/csp/wiki/CSP-Graph) are composed of some number of \"input\" adapters, a set of connected calculation \"nodes\" and at the end sent off to \"output\" adapters. For simplicity, we'll build a graph that will show trains passing through 34 St-Penn Station.\n", + "\n", + "There are two types of [Input Adapters](https://github.com/Point72/csp/wiki/5.-Adapters): Historical (aka Simulated) adapters and Realtime Adapters. Historical adapters are used to feed in historical timeseries data into the graph. Realtime Adapters are used to feed in live event data, generally created from external sources on separate threads.\n", + "\n", + "As you may have guessed, in our case we need to use a [Realtime adapter](https://github.com/Point72/csp/wiki/Write-Realtime-Input-Adapters), which will ingest the data and periodically refresh it.\n", + "\n", + "In CSP terminology, a single adapter corresponds to a single timeseries edge in the graph. When writing realtime adapters, you will need to implement a \"push\" adapter, which will get data from a separate thread that drives external events and \"pushes\" them into the engine as they occur. For this, [we will need \"graph building time\" and \"runtime\" versions of your adapter](https://github.com/Point72/csp/wiki/Write-Realtime-Input-Adapters#pushinputadapter---python). \n", + "\n", + "> Once the graph is constructed, `csp.graph` code is no longer needed. Once the\n", + "> graph is run, only inputs, `csp.nodes` and outputs will be active as data flows\n", + "> through the graph, driven by input ticks.\n", + "\n", + "In our case, \"ticks\" correspond to feed refreshes, and we'll observe this data being updated every 30s. We will read 3 minutes of data for the purposes of this demonstration." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f808ff5b-2735-4b58-bd29-054dabdc6620", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Start of graph building\n", + "End of graph building\n", + "FetchTrainDataAdapter::start\n", + "----------------------------------------------\n", + "2024-06-06 23:01:21.452959: refreshing MTA feed\n", + "----------------------------------------------\n", + " Station | Line | Direction | Arrival time\n", + "2024-06-06 23:01:21.990072 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:03:09 | Southbound train count: 1\n", + "2024-06-06 23:01:21.990361 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:06:55 | Southbound train count: 2\n", + "2024-06-06 23:01:21.990415 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:12:03 | Southbound train count: 3\n", + "2024-06-06 23:01:21.990812 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:17:06 | Southbound train count: 4\n", + "2024-06-06 23:01:21.991054 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:21:22 | Southbound train count: 5\n", + "2024-06-06 23:01:21.991102 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:01:28 | Southbound train count: 5\n", + "2024-06-06 23:01:21.991136 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:25:09 | Southbound train count: 6\n", + "2024-06-06 23:01:21.991177 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:28:27 | Southbound train count: 7\n", + "2024-06-06 23:01:21.991215 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:04:59 | Southbound train count: 7\n", + "2024-06-06 23:01:21.991247 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:31:05 | Southbound train count: 8\n", + "2024-06-06 23:01:21.991284 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:09:43 | Southbound train count: 8\n", + "2024-06-06 23:01:21.991314 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:34:59 | Southbound train count: 9\n", + "2024-06-06 23:01:21.991352 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:13:05 | Southbound train count: 9\n", + "2024-06-06 23:01:21.991388 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:37:07 | Southbound train count: 10\n", + "2024-06-06 23:01:21.991426 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:10:49 | Southbound train count: 11\n", + "2024-06-06 23:01:21.991464 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:14:31 | Southbound train count: 12\n", + "2024-06-06 23:01:21.991500 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:03:05 | Southbound train count: 12\n", + "2024-06-06 23:01:21.991544 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:26:35 | Southbound train count: 13\n", + "2024-06-06 23:01:21.991583 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:09:29 | Southbound train count: 13\n", + "2024-06-06 23:01:21.991614 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:31:08 | Southbound train count: 14\n", + "2024-06-06 23:01:21.991651 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:21:28 | Southbound train count: 14\n", + "2024-06-06 23:01:21.991683 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:36:19 | Southbound train count: 15\n", + "2024-06-06 23:01:21.991719 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:29:56 | Southbound train count: 15\n", + "2024-06-06 23:01:21.991750 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:44:02 | Southbound train count: 16\n", + "2024-06-06 23:01:21.991786 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:34:28 | Southbound train count: 16\n", + "2024-06-06 23:01:21.991818 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:50:02 | Southbound train count: 17\n", + "2024-06-06 23:01:21.991857 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:39:48 | Southbound train count: 17\n", + "2024-06-06 23:01:21.991889 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:08:06 | Southbound train count: 17\n", + "2024-06-06 23:01:21.991922 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:12:54 | Southbound train count: 17\n", + "2024-06-06 23:01:21.991955 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:23:01 | Southbound train count: 17\n", + "2024-06-06 23:01:21.991987 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:01:03 | Southbound train count: 18\n", + "2024-06-06 23:01:21.992026 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:31:10 | Southbound train count: 18\n", + "2024-06-06 23:01:21.992057 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:12:52 | Southbound train count: 19\n", + "2024-06-06 23:01:21.992095 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:38:22 | Southbound train count: 19\n", + "2024-06-06 23:01:21.992127 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:17:51 | Southbound train count: 20\n", + "2024-06-06 23:01:21.992165 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:45:43 | Southbound train count: 20\n", + "----------------------------------------------\n", + "2024-06-06 23:01:51.990188: refreshing MTA feed\n", + "----------------------------------------------\n", + " Station | Line | Direction | Arrival time\n", + "2024-06-06 23:01:52.403371 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:03:09 | Southbound train count: 1\n", + "2024-06-06 23:01:52.403534 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:07:21 | Southbound train count: 2\n", + "2024-06-06 23:01:52.403565 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:12:07 | Southbound train count: 3\n", + "2024-06-06 23:01:52.403582 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:17:14 | Southbound train count: 4\n", + "2024-06-06 23:01:52.403598 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:21:17 | Southbound train count: 5\n", + "2024-06-06 23:01:52.403613 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:01:28 | Southbound train count: 5\n", + "2024-06-06 23:01:52.403628 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:25:09 | Southbound train count: 6\n", + "2024-06-06 23:01:52.403646 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:28:27 | Southbound train count: 7\n", + "2024-06-06 23:01:52.403662 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:05:10 | Southbound train count: 7\n", + "2024-06-06 23:01:52.403677 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:31:05 | Southbound train count: 8\n", + "2024-06-06 23:01:52.403692 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:08:53 | Southbound train count: 8\n", + "2024-06-06 23:01:52.403706 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:35:11 | Southbound train count: 9\n", + "2024-06-06 23:01:52.403721 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:13:06 | Southbound train count: 9\n", + "2024-06-06 23:01:52.403734 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:37:07 | Southbound train count: 10\n", + "2024-06-06 23:01:52.403749 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:09:53 | Southbound train count: 11\n", + "2024-06-06 23:01:52.403764 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:14:23 | Southbound train count: 12\n", + "2024-06-06 23:01:52.403778 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:03:05 | Southbound train count: 12\n", + "2024-06-06 23:01:52.403794 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:26:53 | Southbound train count: 13\n", + "2024-06-06 23:01:52.403810 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:09:49 | Southbound train count: 13\n", + "2024-06-06 23:01:52.403825 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:31:08 | Southbound train count: 14\n", + "2024-06-06 23:01:52.403840 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:21:01 | Southbound train count: 14\n", + "2024-06-06 23:01:52.403854 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:36:43 | Southbound train count: 15\n", + "2024-06-06 23:01:52.403869 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:29:56 | Southbound train count: 15\n", + "2024-06-06 23:01:52.403882 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:44:02 | Southbound train count: 16\n", + "2024-06-06 23:01:52.403898 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:34:09 | Southbound train count: 16\n", + "2024-06-06 23:01:52.403913 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:50:19 | Southbound train count: 17\n", + "2024-06-06 23:01:52.403928 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:39:49 | Southbound train count: 17\n", + "2024-06-06 23:01:52.403941 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:08:06 | Southbound train count: 17\n", + "2024-06-06 23:01:52.403955 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:13:06 | Southbound train count: 17\n", + "2024-06-06 23:01:52.403970 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:23:17 | Southbound train count: 17\n", + "2024-06-06 23:01:52.403983 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:31:10 | Southbound train count: 17\n", + "2024-06-06 23:01:52.403997 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:12:52 | Southbound train count: 18\n", + "2024-06-06 23:01:52.404012 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:38:41 | Southbound train count: 18\n", + "2024-06-06 23:01:52.404026 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:17:17 | Southbound train count: 19\n", + "2024-06-06 23:01:52.404041 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:46:02 | Southbound train count: 19\n", + "----------------------------------------------\n", + "2024-06-06 23:02:22.403432: refreshing MTA feed\n", + "----------------------------------------------\n", + " Station | Line | Direction | Arrival time\n", + "2024-06-06 23:02:22.845570 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:03:09 | Southbound train count: 1\n", + "2024-06-06 23:02:22.845745 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:07:21 | Southbound train count: 2\n", + "2024-06-06 23:02:22.845778 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:12:07 | Southbound train count: 3\n", + "2024-06-06 23:02:22.845796 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:17:11 | Southbound train count: 4\n", + "2024-06-06 23:02:22.845821 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:21:12 | Southbound train count: 5\n", + "2024-06-06 23:02:22.845837 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:25:26 | Southbound train count: 6\n", + "2024-06-06 23:02:22.845853 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:28:47 | Southbound train count: 7\n", + "2024-06-06 23:02:22.845868 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:05:10 | Southbound train count: 7\n", + "2024-06-06 23:02:22.845884 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:30:47 | Southbound train count: 8\n", + "2024-06-06 23:02:22.845900 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:08:53 | Southbound train count: 8\n", + "2024-06-06 23:02:22.845914 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:35:01 | Southbound train count: 9\n", + "2024-06-06 23:02:22.845930 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:13:06 | Southbound train count: 9\n", + "2024-06-06 23:02:22.845944 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:37:17 | Southbound train count: 10\n", + "2024-06-06 23:02:22.845961 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:09:53 | Southbound train count: 11\n", + "2024-06-06 23:02:22.845977 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:14:23 | Southbound train count: 12\n", + "2024-06-06 23:02:22.845992 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:02:18 | Southbound train count: 12\n", + "2024-06-06 23:02:22.846008 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:26:52 | Southbound train count: 13\n", + "2024-06-06 23:02:22.846025 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:09:52 | Southbound train count: 13\n", + "2024-06-06 23:02:22.846040 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:31:34 | Southbound train count: 14\n", + "2024-06-06 23:02:22.846055 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:21:01 | Southbound train count: 14\n", + "2024-06-06 23:02:22.846068 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:36:11 | Southbound train count: 15\n", + "2024-06-06 23:02:22.846084 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:29:22 | Southbound train count: 15\n", + "2024-06-06 23:02:22.846098 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:44:13 | Southbound train count: 16\n", + "2024-06-06 23:02:22.846114 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:34:09 | Southbound train count: 16\n", + "2024-06-06 23:02:22.846128 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:50:14 | Southbound train count: 17\n", + "2024-06-06 23:02:22.846142 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:40:03 | Southbound train count: 17\n", + "2024-06-06 23:02:22.846156 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:08:06 | Southbound train count: 17\n", + "2024-06-06 23:02:22.846170 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:13:06 | Southbound train count: 17\n", + "2024-06-06 23:02:22.846184 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:22:54 | Southbound train count: 17\n", + "2024-06-06 23:02:22.846198 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:31:39 | Southbound train count: 17\n", + "2024-06-06 23:02:22.846212 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:12:52 | Southbound train count: 18\n", + "2024-06-06 23:02:22.846228 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:38:41 | Southbound train count: 18\n", + "2024-06-06 23:02:22.846241 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:17:13 | Southbound train count: 19\n", + "2024-06-06 23:02:22.846256 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:45:56 | Southbound train count: 19\n", + "----------------------------------------------\n", + "2024-06-06 23:02:52.845751: refreshing MTA feed\n", + "----------------------------------------------\n", + " Station | Line | Direction | Arrival time\n", + "2024-06-06 23:02:53.166756 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:03:51 | Southbound train count: 1\n", + "2024-06-06 23:02:53.166986 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:07:41 | Southbound train count: 2\n", + "2024-06-06 23:02:53.167039 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:12:28 | Southbound train count: 3\n", + "2024-06-06 23:02:53.167089 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:17:26 | Southbound train count: 4\n", + "2024-06-06 23:02:53.167132 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:21:26 | Southbound train count: 5\n", + "2024-06-06 23:02:53.167173 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:25:26 | Southbound train count: 6\n", + "2024-06-06 23:02:53.167214 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:28:47 | Southbound train count: 7\n", + "2024-06-06 23:02:53.167255 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:05:29 | Southbound train count: 7\n", + "2024-06-06 23:02:53.167304 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:31:00 | Southbound train count: 8\n", + "2024-06-06 23:02:53.167348 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:09:17 | Southbound train count: 8\n", + "2024-06-06 23:02:53.167381 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:35:01 | Southbound train count: 9\n", + "2024-06-06 23:02:53.167423 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:13:06 | Southbound train count: 9\n", + "2024-06-06 23:02:53.167458 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:36:56 | Southbound train count: 10\n", + "2024-06-06 23:02:53.167510 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:17:24 | Southbound train count: 10\n", + "2024-06-06 23:02:53.167544 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:09:53 | Southbound train count: 11\n", + "2024-06-06 23:02:53.167583 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:14:43 | Southbound train count: 12\n", + "2024-06-06 23:02:53.167622 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:26:06 | Southbound train count: 13\n", + "2024-06-06 23:02:53.167661 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:09:52 | Southbound train count: 13\n", + "2024-06-06 23:02:53.167695 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:31:22 | Southbound train count: 14\n", + "2024-06-06 23:02:53.167736 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:21:20 | Southbound train count: 14\n", + "2024-06-06 23:02:53.167770 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:36:13 | Southbound train count: 15\n", + "2024-06-06 23:02:53.167811 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:29:38 | Southbound train count: 15\n", + "2024-06-06 23:02:53.167845 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:43:37 | Southbound train count: 16\n", + "2024-06-06 23:02:53.167886 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:34:33 | Southbound train count: 16\n", + "2024-06-06 23:02:53.167920 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:50:28 | Southbound train count: 17\n", + "2024-06-06 23:02:53.167960 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:40:03 | Southbound train count: 17\n", + "2024-06-06 23:02:53.167994 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:08:06 | Southbound train count: 17\n", + "2024-06-06 23:02:53.168031 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:13:06 | Southbound train count: 17\n", + "2024-06-06 23:02:53.168067 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:22:54 | Southbound train count: 17\n", + "2024-06-06 23:02:53.168101 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:31:39 | Southbound train count: 17\n", + "2024-06-06 23:02:53.168136 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:12:52 | Southbound train count: 18\n", + "2024-06-06 23:02:53.168177 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:38:19 | Southbound train count: 18\n", + "2024-06-06 23:02:53.168212 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:17:28 | Southbound train count: 19\n", + "2024-06-06 23:02:53.168252 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:46:05 | Southbound train count: 19\n", + "----------------------------------------------\n", + "2024-06-06 23:03:23.166894: refreshing MTA feed\n", + "----------------------------------------------\n", + " Station | Line | Direction | Arrival time\n", + "2024-06-06 23:03:23.576893 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:03:29 | Southbound train count: 1\n", + "2024-06-06 23:03:23.577081 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:06:59 | Southbound train count: 2\n", + "2024-06-06 23:03:23.577111 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:12:19 | Southbound train count: 3\n", + "2024-06-06 23:03:23.577129 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:17:26 | Southbound train count: 4\n", + "2024-06-06 23:03:23.577144 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:21:26 | Southbound train count: 5\n", + "2024-06-06 23:03:23.577159 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:24:58 | Southbound train count: 6\n", + "2024-06-06 23:03:23.577176 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:28:34 | Southbound train count: 7\n", + "2024-06-06 23:03:23.577192 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:05:21 | Southbound train count: 7\n", + "2024-06-06 23:03:23.577208 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:31:00 | Southbound train count: 8\n", + "2024-06-06 23:03:23.577223 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:08:55 | Southbound train count: 8\n", + "2024-06-06 23:03:23.577236 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:35:18 | Southbound train count: 9\n", + "2024-06-06 23:03:23.577251 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:13:40 | Southbound train count: 9\n", + "2024-06-06 23:03:23.577263 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:37:07 | Southbound train count: 10\n", + "2024-06-06 23:03:23.577277 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:17:24 | Southbound train count: 10\n", + "2024-06-06 23:03:23.577291 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:10:43 | Southbound train count: 11\n", + "2024-06-06 23:03:23.577307 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:14:43 | Southbound train count: 12\n", + "2024-06-06 23:03:23.577322 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:26:06 | Southbound train count: 13\n", + "2024-06-06 23:03:23.577336 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:09:52 | Southbound train count: 13\n", + "2024-06-06 23:03:23.577349 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:31:22 | Southbound train count: 14\n", + "2024-06-06 23:03:23.577366 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:21:00 | Southbound train count: 14\n", + "2024-06-06 23:03:23.577380 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:36:30 | Southbound train count: 15\n", + "2024-06-06 23:03:23.577395 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:29:38 | Southbound train count: 15\n", + "2024-06-06 23:03:23.577408 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:43:41 | Southbound train count: 16\n", + "2024-06-06 23:03:23.577422 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:34:33 | Southbound train count: 16\n", + "2024-06-06 23:03:23.577435 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:50:12 | Southbound train count: 17\n", + "2024-06-06 23:03:23.577448 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:39:46 | Southbound train count: 17\n", + "2024-06-06 23:03:23.577461 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:08:06 | Southbound train count: 17\n", + "2024-06-06 23:03:23.577475 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:13:59 | Southbound train count: 17\n", + "2024-06-06 23:03:23.577489 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:22:54 | Southbound train count: 17\n", + "2024-06-06 23:03:23.577503 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:32:10 | Southbound train count: 17\n", + "2024-06-06 23:03:23.577516 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:12:52 | Southbound train count: 18\n", + "2024-06-06 23:03:23.577531 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:38:16 | Southbound train count: 18\n", + "2024-06-06 23:03:23.577545 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:17:19 | Southbound train count: 19\n", + "2024-06-06 23:03:23.577560 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:46:05 | Southbound train count: 19\n", + "2024-06-06 23:03:23.577573 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:24:45 | Southbound train count: 20\n", + "----------------------------------------------\n", + "2024-06-06 23:03:53.576969: refreshing MTA feed\n", + "----------------------------------------------\n", + " Station | Line | Direction | Arrival time\n", + "2024-06-06 23:03:54.047143 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:03:29 | Southbound train count: 1\n", + "2024-06-06 23:03:54.047321 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:06:59 | Southbound train count: 2\n", + "2024-06-06 23:03:54.047355 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:12:14 | Southbound train count: 3\n", + "2024-06-06 23:03:54.047377 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:17:02 | Southbound train count: 4\n", + "2024-06-06 23:03:54.047396 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:20:32 | Southbound train count: 5\n", + "2024-06-06 23:03:54.047415 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:24:58 | Southbound train count: 6\n", + "2024-06-06 23:03:54.047433 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:28:34 | Southbound train count: 7\n", + "2024-06-06 23:03:54.047451 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:05:21 | Southbound train count: 7\n", + "2024-06-06 23:03:54.047469 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:30:48 | Southbound train count: 8\n", + "2024-06-06 23:03:54.047488 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:09:10 | Southbound train count: 8\n", + "2024-06-06 23:03:54.047506 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:35:21 | Southbound train count: 9\n", + "2024-06-06 23:03:54.047523 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:13:40 | Southbound train count: 9\n", + "2024-06-06 23:03:54.047538 :: 34 St-Penn Station | 1 | South Ferry | 2024-06-06 16:37:07 | Southbound train count: 10\n", + "2024-06-06 23:03:54.047556 :: 34 St-Penn Station | 1 | Van Cortlandt Park-242 St | 2024-06-06 16:17:21 | Southbound train count: 10\n", + "2024-06-06 23:03:54.047571 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:10:43 | Southbound train count: 11\n", + "2024-06-06 23:03:54.047590 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:14:43 | Southbound train count: 12\n", + "2024-06-06 23:03:54.047607 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:26:25 | Southbound train count: 13\n", + "2024-06-06 23:03:54.047625 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:10:47 | Southbound train count: 13\n", + "2024-06-06 23:03:54.047645 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:31:16 | Southbound train count: 14\n", + "2024-06-06 23:03:54.047664 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:21:00 | Southbound train count: 14\n", + "2024-06-06 23:03:54.047680 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:36:30 | Southbound train count: 15\n", + "2024-06-06 23:03:54.047697 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:29:32 | Southbound train count: 15\n", + "2024-06-06 23:03:54.047713 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:43:57 | Southbound train count: 16\n", + "2024-06-06 23:03:54.047730 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:34:33 | Southbound train count: 16\n", + "2024-06-06 23:03:54.047746 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:50:12 | Southbound train count: 17\n", + "2024-06-06 23:03:54.047763 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:40:02 | Southbound train count: 17\n", + "2024-06-06 23:03:54.047779 :: 34 St-Penn Station | 2 | Flatbush Av-Brooklyn College | 2024-06-06 16:57:57 | Southbound train count: 18\n", + "2024-06-06 23:03:54.047796 :: 34 St-Penn Station | 2 | Wakefield-241 St | 2024-06-06 16:45:30 | Southbound train count: 18\n", + "2024-06-06 23:03:54.047811 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:06:57 | Southbound train count: 18\n", + "2024-06-06 23:03:54.047828 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:13:59 | Southbound train count: 18\n", + "2024-06-06 23:03:54.047844 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:23:46 | Southbound train count: 18\n", + "2024-06-06 23:03:54.047860 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:32:10 | Southbound train count: 18\n", + "2024-06-06 23:03:54.047876 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:12:57 | Southbound train count: 19\n", + "2024-06-06 23:03:54.047895 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:38:37 | Southbound train count: 19\n", + "2024-06-06 23:03:54.047911 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:17:14 | Southbound train count: 20\n", + "2024-06-06 23:03:54.047928 :: 34 St-Penn Station | 3 | Harlem-148 St | 2024-06-06 16:46:02 | Southbound train count: 20\n", + "2024-06-06 23:03:54.047944 :: 34 St-Penn Station | 3 | New Lots Av | 2024-06-06 16:23:38 | Southbound train count: 21\n", + "FetchTrainDataAdapter::stop\n", + "Done.\n" + ] + } + ], + "source": [ + "import csp\n", + "from csp.impl.pushadapter import PushInputAdapter\n", + "from csp.impl.wiring import py_push_adapter_def\n", + "\n", + "import nyct_gtfs\n", + "\n", + "import os\n", + "import time\n", + "import threading\n", + "from datetime import datetime, timedelta\n", + "\n", + "\n", + "class Event(csp.Struct):\n", + " train: nyct_gtfs.trip.Trip\n", + " update: nyct_gtfs.stop_time_update.StopTimeUpdate\n", + " arrival: datetime\n", + " direction: str\n", + "\n", + "# Create a runtime implementation of the adapter\n", + "class FetchTrainDataAdapter(PushInputAdapter):\n", + " def __init__(self, interval, stations):\n", + " self._interval = interval\n", + " self._thread = None\n", + " self._running = False\n", + " self._stations = stations\n", + "\n", + " def start(self, starttime, endtime):\n", + " print(\"FetchTrainDataAdapter::start\")\n", + " self._running = True\n", + " self._thread = threading.Thread(target=self._run)\n", + " self._thread.start()\n", + "\n", + " def stop(self):\n", + " print(\"FetchTrainDataAdapter::stop\")\n", + " if self._running:\n", + " self._running = False\n", + " self._thread.join()\n", + "\n", + " def _run(self):\n", + " # This is where we will read and process the real-time data feed\n", + " feed = nyct_gtfs.NYCTFeed(\"https://api-endpoint.mta.info/Dataservice/mtagtfsfeeds/nyct%2Fgtfs\", api_key=\"\")\n", + "\n", + " while self._running:\n", + " print(\"----------------------------------------------\")\n", + " print(f\"{datetime.utcnow()}: refreshing MTA feed\")\n", + " print(\"----------------------------------------------\")\n", + " print(\" Station | Line | Direction | Arrival time\")\n", + " feed.refresh()\n", + " trains = feed.filter_trips(underway=True, headed_for_stop_id=self._stations)\n", + " # tick whenever feed is refreshed\n", + " for train in trains:\n", + " for update in train.stop_time_updates:\n", + " if update.stop_id in self._stations:\n", + " self.push_tick(Event(train=train, update=update, direction=train.direction, arrival=update.arrival))\n", + " time.sleep(self._interval.total_seconds())\n", + "\n", + "# Create the graph-time representation of our adapter\n", + "FetchTrainData = py_push_adapter_def(\"FetchTrainData\", FetchTrainDataAdapter, csp.ts[Event], interval=timedelta, stations=list)\n", + "\n", + "@csp.node\n", + "def pretty_print(train_data: csp.ts[Event], count: csp.ts[float]) -> csp.ts[str]:\n", + " message = f\" {train_data.update.stop_name} | {train_data.train.route_id} | {train_data.train.headsign_text} | {train_data.update.arrival} | Southbound train count: {int(count)}\"\n", + " return message\n", + "\n", + "@csp.graph\n", + "def mta_graph():\n", + " print(\"Start of graph building\")\n", + " stations = ['128S', '128N']\n", + " interval = timedelta(seconds=30)\n", + " trains_at_penn = FetchTrainData(interval, stations=stations)\n", + " # trains_at_penn is an edge that can be processed through a node.\n", + " # Select all southbound trains going through Penn Station\n", + " south_trains = csp.filter(trains_at_penn.direction == \"S\", trains_at_penn)\n", + " # Convert timestamps to unique float values\n", + " timestamp = csp.apply(south_trains.arrival, datetime.timestamp, float)\n", + " # Count the number of unique entries in this timeseries block (reset every 30 seconds)\n", + " count = csp.stats.count(csp.stats.unique(timestamp), interval=timedelta(seconds=30), min_window=timedelta(seconds=1))\n", + " result = pretty_print(trains_at_penn, count)\n", + " csp.print(\":\", result)\n", + " print(\"End of graph building\")\n", + "\n", + "start = datetime.utcnow()\n", + "end = start + timedelta(minutes=3)\n", + "csp.run(mta_graph, starttime=start, realtime=True, endtime=end)\n", + "print(\"Done.\")" + ] + }, + { + "cell_type": "markdown", + "id": "1a871048-7437-4827-871d-97976d7b93d1", + "metadata": {}, + "source": [ + "---" + ] + }, + { + "cell_type": "markdown", + "id": "580d6aad-3797-42e9-a60c-85477a6b3839", + "metadata": {}, + "source": [ + "### References\n", + "\n", + "* https://erikbern.com/2016/04/04/nyc-subway-math\n", + "* https://erikbern.com/2016/07/09/waiting-time-math.html\n", + "* https://pypi.org/project/nyct-gtfs/\n", + "* https://api.mta.info/#/landing\n", + "* https://developers.google.com/transit/gtfs-realtime\n", + "* https://github.com/Point72/csp/blob/main/examples/4_writing_adapters/e_14_user_adapters_03_pushinput.py\n", + "* https://github.com/Point72/csp/wiki/5.-Adapters#realtime-adapters" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/07_end_to_end/seismic_waveform.ipynb b/examples/07_end_to_end/seismic_waveform.ipynb index 1b7fd055..9ee2e29b 100644 --- a/examples/07_end_to_end/seismic_waveform.ipynb +++ b/examples/07_end_to_end/seismic_waveform.ipynb @@ -920,7 +920,7 @@ " self._endtime = None\n", " \n", " def start(self, starttime, endtime):\n", - " \"\"\"start wil get called at the start of the engine run. At this point one would start up the realtime data source / spawn the driving \n", + " \"\"\"start will get called at the start of the engine run. At this point one would start up the realtime data source / spawn the driving \n", " thread(s) and subscribe to the needed data\"\"\"\n", " self._running = True\n", " self._starttime = starttime\n",