Skip to content

Commit

Permalink
Shows gobble data in the dashboard (#943)
Browse files Browse the repository at this point in the history
* Shows gobble data in the dashboard

Only shows gobble data when manually navigating to a bus url with a date
past the current maximum

* Fixing rapid parsing

* Fixing format for headways

* Updating based on feedback
  • Loading branch information
devinmatte authored Jan 30, 2024
1 parent 9402fa0 commit 5721452
Show file tree
Hide file tree
Showing 15 changed files with 191 additions and 56 deletions.
6 changes: 3 additions & 3 deletions common/components/notices/BusDataNotice.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import React from 'react';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import { faMagnifyingGlassChart } from '@fortawesome/free-solid-svg-icons';
import { faLocationCrosshairs } from '@fortawesome/free-solid-svg-icons';
import { useDelimitatedRoute } from '../../utils/router';

export const BusDataNotice: React.FC = () => {
Expand All @@ -9,8 +9,8 @@ export const BusDataNotice: React.FC = () => {
if (line === 'line-bus' || linePath === 'bus') {
return (
<div className={'flex items-center'}>
<FontAwesomeIcon icon={faMagnifyingGlassChart} size={'lg'} />
<div className={'m-3 text-sm italic'}>
<FontAwesomeIcon icon={faLocationCrosshairs} size={'lg'} />
<div className={'mx-3 my-2 text-sm italic'}>
<p>
Due to data collection issues, bus data is not guaranteed to be complete for any stop or
date.
Expand Down
61 changes: 61 additions & 0 deletions common/components/notices/GobbleDataNotice.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import React from 'react';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import dayjs from 'dayjs';
import { faChartSimple } from '@fortawesome/free-solid-svg-icons';
import Link from 'next/link';
import classNames from 'classnames';
import { useDelimitatedRoute } from '../../utils/router';
import { BUS_MAX_DAY } from '../../constants/dates';
import { lineColorTextHover } from '../../styles/general';

export const GobbleDataNotice: React.FC = () => {
const {
line,
linePath,
query: { date, startDate, endDate },
} = useDelimitatedRoute();

const isStartDateAfterBusMaxDay =
(startDate !== undefined && dayjs(startDate).isAfter(BUS_MAX_DAY)) ||
(date !== undefined && dayjs(date).isAfter(BUS_MAX_DAY));
const isEndDateAfterBusMaxDay = endDate !== undefined && dayjs(endDate).isAfter(BUS_MAX_DAY);

if (
(line === 'line-bus' || linePath === 'bus') &&
(isStartDateAfterBusMaxDay || isEndDateAfterBusMaxDay)
) {
return (
<div className={'flex items-center'}>
<FontAwesomeIcon icon={faChartSimple} size={'lg'} />
<div className={'mx-3 my-2 text-sm italic'}>
<p>
Data shown here is collected by TransitMatters using the{' '}
<Link
href="https://www.mbta.com/developers/v3-api/streaming"
rel="noopener noreferrer"
target="_blank"
className={classNames(lineColorTextHover[line ?? 'DEFAULT'])}
>
MBTA's streaming API
</Link>
. Unlike other data sources we show, this data is not cleaned or filtered in any way
before display. Innacuracies may be present.
</p>
<p>
Official MBTA data will be shown when available. Technical details of our data
collection can be found{' '}
<Link
href="https://github.com/transitmatters/gobble"
rel="noopener noreferrer"
target="_blank"
className={classNames(lineColorTextHover[line ?? 'DEFAULT'])}
>
here
</Link>
</p>
</div>
</div>
);
}
return null;
};
2 changes: 1 addition & 1 deletion common/components/notices/SameDayNotice.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export const SameDayNotice: React.FC = () => {
return (
<div className={'flex items-center'}>
<FontAwesomeIcon icon={faCalendarDay} size={'lg'} />
<div className={'mx-3 my-1 text-sm italic'}>
<div className={'mx-3 my-2 text-sm italic'}>
<p>
Due to data not being cleaned yet, today's data may not be fully accurate and may look
messy.
Expand Down
2 changes: 1 addition & 1 deletion common/components/notices/TerminusNotice.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export const TerminusNotice: React.FC<TerminusNoticeProps> = ({ toStation, fromS
return (
<div className={'flex items-center'}>
<FontAwesomeIcon icon={faMagnifyingGlassChart} size={'lg'} />
<div className={'mx-3 my-1 text-sm italic'}>
<div className={'mx-3 my-2 text-sm italic'}>
<p>
Due to data collection issues at terminus stations, data is not guaranteed to be
complete.
Expand Down
2 changes: 0 additions & 2 deletions common/components/widgets/WidgetPage.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import classNames from 'classnames';
import React from 'react';
import { BusDataNotice } from '../notices/BusDataNotice';

interface WidgetPageProps {
children?: React.ReactNode;
Expand All @@ -10,7 +9,6 @@ export const WidgetPage: React.FC<WidgetPageProps> = ({ children }) => {
return (
<div className={classNames('flex w-full flex-1 flex-col items-center gap-y-2 lg:p-0')}>
{children}
<BusDataNotice />
</div>
);
};
2 changes: 1 addition & 1 deletion common/constants/dates.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ const OVERVIEW_TRAIN_MIN_DATE = '2016-02-01';
const TRAIN_MIN_DATE = '2016-01-15';
const BUS_MIN_DATE = '2018-08-01';
export const BUS_MAX_DATE = '2023-12-31';
const BUS_MAX_DAY = dayjs(BUS_MAX_DATE);
export const BUS_MAX_DAY = dayjs(BUS_MAX_DATE);
export const BUS_MAX_DATE_MINUS_ONE_WEEK = dayjs(BUS_MAX_DATE)
.subtract(7, 'days')
.format(DATE_FORMAT);
Expand Down
3 changes: 1 addition & 2 deletions modules/ridership/RidershipDetails.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ export function RidershipDetails() {
} = useDelimitatedRoute();
const config = SPEED_RANGE_PARAM_MAP.week;
const lineId = getRidershipLineId(line, busRoute);
const lineOrRoute = busRoute ? `line-${busRoute.replaceAll('/', '')}` : line;
const enabled = Boolean(startDate && endDate && lineId);

const ridership = useRidershipData(
Expand All @@ -29,7 +28,7 @@ export function RidershipDetails() {
},
enabled
);
const ridershipDataReady = !ridership.isError && startDate && endDate && lineOrRoute && line;
const ridershipDataReady = !ridership.isError && startDate && endDate && line;

return (
<PageWrapper pageTitle={'Ridership'}>
Expand Down
3 changes: 1 addition & 2 deletions modules/ridership/RidershipWidget.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,12 @@ export const RidershipWidget: React.FC = () => {
const endDate = TODAY_STRING;
const config = getSpeedGraphConfig(dayjs(startDate), dayjs(endDate));
const lineId = getRidershipLineId(line, query.busRoute);
const lineOrRoute = query.busRoute ? `line-${query.busRoute.replaceAll('/', '')}` : line;
const ridership = useRidershipData({
line_id: lineId,
start_date: startDate,
end_date: endDate,
});
const serviceReady = !ridership.isError && lineId && line && lineOrRoute;
const serviceReady = !ridership.isError && lineId && line;

return (
<WidgetDiv>
Expand Down
10 changes: 8 additions & 2 deletions modules/tripexplorer/TripExplorer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import { ChartPageDiv } from '../../common/components/charts/ChartPageDiv';
import { Layout } from '../../common/layouts/layoutTypes';
import { useDelimitatedRoute } from '../../common/utils/router';
import { getParentStationForStopId } from '../../common/utils/stations';
import { BusDataNotice } from '../../common/components/notices/BusDataNotice';
import { GobbleDataNotice } from '../../common/components/notices/GobbleDataNotice';
import { useAlertStore } from './AlertStore';
import { TripGraphs } from './TripGraphs';

Expand Down Expand Up @@ -37,8 +39,12 @@ export const TripExplorer = () => {
<ChartPageDiv>
{alertsForModal?.length ? <AlertNotice /> : null}
<TripGraphs fromStation={fromStation} toStation={toStation} />
<SameDayNotice />
<TerminusNotice toStation={toStation} fromStation={fromStation} />
<div>
<GobbleDataNotice />
<BusDataNotice />
<SameDayNotice />
<TerminusNotice toStation={toStation} fromStation={fromStation} />
</div>
</ChartPageDiv>
</PageWrapper>
);
Expand Down
42 changes: 35 additions & 7 deletions server/chalicelib/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,16 +28,27 @@ def upload(key, bytes, compress=True):
s3.put_object(Bucket=BUCKET, Key=key, Body=bytes)


def is_bus(stop_id):
def is_bus(stop_id: str):
return ("-0-" in stop_id) or ("-1-" in stop_id)


def download_one_event_file(date, stop_id):
def get_live_folder(stop_id: str):
if is_bus(stop_id):
return "daily-bus-data"
else:
return "daily-rapid-data"


def download_one_event_file(date, stop_id: str, use_live_data=False):
"""As advertised: single event file from s3"""
year, month = date.year, date.month
year, month, day = date.year, date.month, date.day

folder = "monthly-bus-data" if is_bus(stop_id) else "monthly-data"
key = f"Events/{folder}/{stop_id}/Year={year}/Month={month}/events.csv.gz"
if use_live_data:
folder = get_live_folder(stop_id)
key = f"Events-live/{folder}/{stop_id}/Year={year}/Month={month}/Day={day}/events.csv.gz"
else:
folder = "monthly-bus-data" if is_bus(stop_id) else "monthly-data"
key = f"Events/{folder}/{stop_id}/Year={year}/Month={month}/events.csv.gz"

# Download events from S3
try:
Expand All @@ -47,6 +58,8 @@ def download_one_event_file(date, stop_id):
if ex.response["Error"]["Code"] == "NoSuchKey":
# raise Exception(f"Data not available on S3 for key {key} ") from None
print(f"WARNING: No data available on S3 for key: {key}")
if not use_live_data and is_bus(stop_id):
return download_one_event_file(date, stop_id, use_live_data=True)
return []
else:
raise
Expand All @@ -67,8 +80,23 @@ def parallel_download_events(datestop):
return download_one_event_file(date, stop)


def download_events(sdate, edate, stops):
datestops = itertools.product(parallel.month_range(sdate, edate), stops)
def download_events(sdate, edate, stops: list):
# This used to be month_range but updated to date_range to support live ranges
# If something breaks, this may be why
datestops = itertools.product(parallel.date_range(sdate, edate), stops)
result = parallel_download_events(datestops)
result = filter(lambda row: sdate.strftime("%Y-%m-%d") <= row["service_date"] <= edate.strftime("%Y-%m-%d"), result)
return sorted(result, key=lambda row: row["event_time"])


def get_all_s3_objects(s3, **base_kwargs):
continuation_token = None
while True:
list_kwargs = dict(MaxKeys=1000, **base_kwargs)
if continuation_token:
list_kwargs["ContinuationToken"] = continuation_token
response = s3.list_objects_v2(**list_kwargs)
yield from response.get("Contents", [])
if not response.get("IsTruncated"): # At the end of the list?
break
continuation_token = response.get("NextContinuationToken")
8 changes: 4 additions & 4 deletions server/chalicelib/s3_historical.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def unique_everseen(iterable, key=None):
yield element


def dwells(stop_ids, sdate, edate):
def dwells(stop_ids: list, sdate, edate):
rows_by_time = s3.download_events(sdate, edate, stop_ids)

dwells = []
Expand All @@ -59,7 +59,7 @@ def dwells(stop_ids, sdate, edate):
return dwells


def headways(stop_ids, sdate, edate):
def headways(stop_ids: list, sdate, edate):
rows_by_time = s3.download_events(sdate, edate, stop_ids)

only_departures = filter(lambda row: row["event_type"] in EVENT_DEPARTURE, rows_by_time)
Expand Down Expand Up @@ -88,7 +88,7 @@ def headways(stop_ids, sdate, edate):
{
"route_id": this["route_id"],
"direction": this["direction_id"],
"current_dep_dt": this["event_time"],
"current_dep_dt": date_utils.return_formatted_date(this_dt),
"headway_time_sec": headway_time_sec,
"benchmark_headway_time_sec": benchmark_headway,
}
Expand All @@ -97,7 +97,7 @@ def headways(stop_ids, sdate, edate):
return headways


def travel_times(stops_a, stops_b, sdate, edate):
def travel_times(stops_a: list, stops_b: list, sdate, edate):
rows_by_time_a = s3.download_events(sdate, edate, stops_a)
rows_by_time_b = s3.download_events(sdate, edate, stops_b)

Expand Down
1 change: 0 additions & 1 deletion server/rapid/process_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ def process_events(input_csv, outdir, nozip=False):
input_csv,
usecols=columns,
parse_dates=["service_date"],
infer_datetime_format=True,
dtype={
"route_id": "str",
"trip_id": "str",
Expand Down
2 changes: 1 addition & 1 deletion server/rapid/setup_rapid_input.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ wget -N -O data/input/2023.zip https://www.arcgis.com/sharing/rest/content/items

cd data/input
for i in `seq 2017 2023`; do
unzip -d $i $i.zip
unzip -o -d $i $i.zip
done

# The following years only have single csv files
Expand Down
Loading

0 comments on commit 5721452

Please sign in to comment.