Skip to content

Commit

Permalink
correct inaccurate NTD extract object name (#3464)
Browse files Browse the repository at this point in the history
* fix inaccurate NTD object extract name

* clarify description

* revise time zone name
  • Loading branch information
charlie-costanzo authored Sep 20, 2024
1 parent d3cffd6 commit c1f84bc
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 7 deletions.
2 changes: 1 addition & 1 deletion airflow/dags/sync_ntd_data_api/METADATA.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
description: "Scrape NTD endpoints from DOT API monthly"
schedule_interval: "0 11 1 * *" # 11am GMT first day of every month
schedule_interval: "0 11 1 * *" # 11am UTC first day of every month
tags:
- all_gusty_features
default_args:
Expand Down
2 changes: 1 addition & 1 deletion airflow/dags/sync_ntd_data_xlsx/METADATA.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
description: "Scrape tables from DOT Ridership XLSX file daily"
schedule_interval: "0 10 * * *" # 10am GMT every day
schedule_interval: "0 10 * * *" # 10am UTC every day
tags:
- all_gusty_features
default_args:
Expand Down
10 changes: 5 additions & 5 deletions airflow/plugins/operators/scrape_ntd_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def fetch_from_ntd_api(self):
raise


class CSVExtract(NtdDataProductAPIExtract):
class JSONExtract(NtdDataProductAPIExtract):
bucket = API_BUCKET


Expand All @@ -90,12 +90,12 @@ def __init__(
self.root_url = root_url
self.endpoint_id = endpoint_id
self.file_format = file_format
"""An operator that downloads all data from a NTD API
and saves it as one CSV file hive-partitioned by date in Google Cloud
"""An operator that extracts and saves JSON data from the NTD API
and saves it as one JSONL file, hive-partitioned by date in Google Cloud
"""

# Save CSV files to the bucket
self.extract = CSVExtract(
# Save JSONL files to the bucket
self.extract = JSONExtract(
year=self.year,
product=self.product + "/" + self.year,
root_url=self.root_url,
Expand Down

0 comments on commit c1f84bc

Please sign in to comment.