From 6c0c1eab1832ccdc3838496e71581c2c1c55951f Mon Sep 17 00:00:00 2001 From: Erika Pacheco Date: Wed, 18 Sep 2024 15:03:09 -0700 Subject: [PATCH 1/2] Add new NTD API and XLSX buckets to Airflow configuration to fix errors when running locally [#3403] --- airflow/docker-compose.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airflow/docker-compose.yaml b/airflow/docker-compose.yaml index 7bf3fa4090..bc83601db3 100644 --- a/airflow/docker-compose.yaml +++ b/airflow/docker-compose.yaml @@ -99,6 +99,10 @@ x-airflow-common: CALITP_BUCKET__PUBLISH: "gs://test-calitp-publish" CALITP_BUCKET__SENTRY_EVENTS: "gs://test-calitp-sentry" + CALITP_BUCKET__NTD_API_DATA_PRODUCTS: "gs://test-calitp-ntd-api-products" + CALITP_BUCKET__NTD_XLSX_DATA_PRODUCTS__RAW: "gs://test-calitp-ntd-xlsx-products-raw" + CALITP_BUCKET__NTD_XLSX_DATA_PRODUCTS__CLEAN: "gs://test-calitp-ntd-xlsx-products-clean" + DBT_TARGET: staging_service_account # TODO: this can be removed once we've confirmed it's no longer in Airtable From a9b9c659a95f2298a1fedffc25cc7e33ab5db734 Mon Sep 17 00:00:00 2001 From: Erika Pacheco Date: Mon, 16 Sep 2024 17:44:23 -0700 Subject: [PATCH 2/2] Create NTD external tables for XLSX Complete Monthly Ridership with Adjustments and Estimates Data [#3403] --- ...ments_and_estimates__calendar_year_upt.yml | 83 +++ ...ments_and_estimates__calendar_year_vrm.yml | 83 +++ ...with_adjustments_and_estimates__master.yml | 71 +++ ...ip_with_adjustments_and_estimates__upt.yml | 577 ++++++++++++++++++ ...justments_and_estimates__upt_estimates.yml | 33 + ...p_with_adjustments_and_estimates__voms.yml | 577 ++++++++++++++++++ ...ip_with_adjustments_and_estimates__vrh.yml | 577 ++++++++++++++++++ ...ip_with_adjustments_and_estimates__vrm.yml | 577 ++++++++++++++++++ ...justments_and_estimates__vrm_estimates.yml | 33 + 9 files changed, 2611 insertions(+) create mode 100644 airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_upt.yml create mode 100644 airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_vrm.yml create mode 100644 airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__master.yml create mode 100644 airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__upt.yml create mode 100644 airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__upt_estimates.yml create mode 100644 airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__voms.yml create mode 100644 airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrh.yml create mode 100644 airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm.yml create mode 100644 airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm_estimates.yml diff --git a/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_upt.yml b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_upt.yml new file mode 100644 index 0000000000..8fdd599151 --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_upt.yml @@ -0,0 +1,83 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_upt + LIMIT 1; +source_objects: + - "complete_monthly_ridership_with_adjustments_and_estimates/historical/calendar_year_upt/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_upt" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "complete_monthly_ridership_with_adjustments_and_estimates/historical/calendar_year_upt/" +schema_fields: + - name: ntd_id + type: NUMERIC + - name: legacy_ntd_id + type: STRING + - name: agency + type: STRING + - name: mode_type_of_service_status + type: STRING + - name: reporter_type + type: STRING + - name: uace_cd + type: NUMERIC + - name: uza_name + type: STRING + - name: mode + type: STRING + - name: tos + type: STRING + - name: _3_mode + type: STRING + - name: _2002 + type: FLOAT + - name: _2003 + type: FLOAT + - name: _2004 + type: FLOAT + - name: _2005 + type: FLOAT + - name: _2006 + type: FLOAT + - name: _2007 + type: FLOAT + - name: _2008 + type: FLOAT + - name: _2009 + type: FLOAT + - name: _2010 + type: FLOAT + - name: _2011 + type: FLOAT + - name: _2012 + type: FLOAT + - name: _2013 + type: FLOAT + - name: _2014 + type: FLOAT + - name: _2015 + type: FLOAT + - name: _2016 + type: FLOAT + - name: _2017 + type: FLOAT + - name: _2018 + type: FLOAT + - name: _2019 + type: FLOAT + - name: _2020 + type: FLOAT + - name: _2021 + type: FLOAT + - name: _2022 + type: FLOAT + - name: _2023 + type: FLOAT + - name: _2024 + type: FLOAT diff --git a/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_vrm.yml b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_vrm.yml new file mode 100644 index 0000000000..f0dc8b8264 --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_vrm.yml @@ -0,0 +1,83 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_vrm + LIMIT 1; +source_objects: + - "complete_monthly_ridership_with_adjustments_and_estimates/historical/calendar_year_vrm/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__calendar_year_vrm" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "complete_monthly_ridership_with_adjustments_and_estimates/historical/calendar_year_vrm/" +schema_fields: + - name: ntd_id + type: NUMERIC + - name: legacy_ntd_id + type: STRING + - name: agency + type: STRING + - name: mode_type_of_service_status + type: STRING + - name: reporter_type + type: STRING + - name: uace_cd + type: NUMERIC + - name: uza_name + type: STRING + - name: mode + type: STRING + - name: tos + type: STRING + - name: _3_mode + type: STRING + - name: _2002 + type: FLOAT + - name: _2003 + type: FLOAT + - name: _2004 + type: FLOAT + - name: _2005 + type: FLOAT + - name: _2006 + type: FLOAT + - name: _2007 + type: FLOAT + - name: _2008 + type: FLOAT + - name: _2009 + type: FLOAT + - name: _2010 + type: FLOAT + - name: _2011 + type: FLOAT + - name: _2012 + type: FLOAT + - name: _2013 + type: FLOAT + - name: _2014 + type: FLOAT + - name: _2015 + type: FLOAT + - name: _2016 + type: FLOAT + - name: _2017 + type: FLOAT + - name: _2018 + type: FLOAT + - name: _2019 + type: FLOAT + - name: _2020 + type: FLOAT + - name: _2021 + type: FLOAT + - name: _2022 + type: FLOAT + - name: _2023 + type: FLOAT + - name: _2024 + type: FLOAT diff --git a/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__master.yml b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__master.yml new file mode 100644 index 0000000000..f81a244e5f --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__master.yml @@ -0,0 +1,71 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__master + LIMIT 1; +source_objects: + - "complete_monthly_ridership_with_adjustments_and_estimates/historical/master/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__master" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "complete_monthly_ridership_with_adjustments_and_estimates/historical/master/" +schema_fields: + - name: ntd_id + type: NUMERIC + - name: legacy_ntd_id + type: STRING + - name: agency + type: STRING + - name: mode + type: STRING + - name: tos + type: STRING + - name: _3_mode + type: STRING + - name: mode_type_of_service_status + type: STRING + - name: reporter_type + type: STRING + - name: organization_type + type: STRING + - name: hq_city + type: STRING + - name: hq_state + type: STRING + - name: uace_cd + type: NUMERIC + - name: uza_name + type: STRING + - name: uza_sq_miles + type: NUMERIC + - name: uza_population + type: NUMERIC + - name: service_area_population + type: NUMERIC + - name: service_area_sq_miles + type: NUMERIC + - name: last_closed_report_year + type: NUMERIC + - name: last_closed_fy_end_month + type: NUMERIC + - name: last_closed_fy_end_year + type: NUMERIC + - name: passenger_miles_fy + type: FLOAT + - name: unlinked_passenger_trips_fy + type: FLOAT + - name: avg_trip_length_fy + type: FLOAT + - name: fares_fy + type: FLOAT + - name: operating_expenses_fy + type: FLOAT + - name: avg_cost_per_trip_fy + type: FLOAT + - name: avg_fares_per_trip_fy + type: FLOAT diff --git a/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__upt.yml b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__upt.yml new file mode 100644 index 0000000000..5e333a4f0a --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__upt.yml @@ -0,0 +1,577 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__upt + LIMIT 1; +source_objects: + - "complete_monthly_ridership_with_adjustments_and_estimates/historical/upt/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__upt" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "complete_monthly_ridership_with_adjustments_and_estimates/historical/upt/" +schema_fields: + - name: ntd_id + type: NUMERIC + - name: legacy_ntd_id + type: STRING + - name: agency + type: STRING + - name: mode_type_of_service_status + type: STRING + - name: reporter_type + type: STRING + - name: uace_cd + type: NUMERIC + - name: uza_name + type: STRING + - name: mode + type: STRING + - name: tos + type: STRING + - name: _3_mode + type: STRING + - name: _1_2002 + type: FLOAT + - name: _2_2002 + type: FLOAT + - name: _3_2002 + type: FLOAT + - name: _4_2002 + type: FLOAT + - name: _5_2002 + type: FLOAT + - name: _6_2002 + type: FLOAT + - name: _7_2002 + type: FLOAT + - name: _8_2002 + type: FLOAT + - name: _9_2002 + type: FLOAT + - name: _10_2002 + type: FLOAT + - name: _11_2002 + type: FLOAT + - name: _12_2002 + type: FLOAT + - name: _1_2003 + type: FLOAT + - name: _2_2003 + type: FLOAT + - name: _3_2003 + type: FLOAT + - name: _4_2003 + type: FLOAT + - name: _5_2003 + type: FLOAT + - name: _6_2003 + type: FLOAT + - name: _7_2003 + type: FLOAT + - name: _8_2003 + type: FLOAT + - name: _9_2003 + type: FLOAT + - name: _10_2003 + type: FLOAT + - name: _11_2003 + type: FLOAT + - name: _12_2003 + type: FLOAT + - name: _1_2004 + type: FLOAT + - name: _2_2004 + type: FLOAT + - name: _3_2004 + type: FLOAT + - name: _4_2004 + type: FLOAT + - name: _5_2004 + type: FLOAT + - name: _6_2004 + type: FLOAT + - name: _7_2004 + type: FLOAT + - name: _8_2004 + type: FLOAT + - name: _9_2004 + type: FLOAT + - name: _10_2004 + type: FLOAT + - name: _11_2004 + type: FLOAT + - name: _12_2004 + type: FLOAT + - name: _1_2005 + type: FLOAT + - name: _2_2005 + type: FLOAT + - name: _3_2005 + type: FLOAT + - name: _4_2005 + type: FLOAT + - name: _5_2005 + type: FLOAT + - name: _6_2005 + type: FLOAT + - name: _7_2005 + type: FLOAT + - name: _8_2005 + type: FLOAT + - name: _9_2005 + type: FLOAT + - name: _10_2005 + type: FLOAT + - name: _11_2005 + type: FLOAT + - name: _12_2005 + type: FLOAT + - name: _1_2006 + type: FLOAT + - name: _2_2006 + type: FLOAT + - name: _3_2006 + type: FLOAT + - name: _4_2006 + type: FLOAT + - name: _5_2006 + type: FLOAT + - name: _6_2006 + type: FLOAT + - name: _7_2006 + type: FLOAT + - name: _8_2006 + type: FLOAT + - name: _9_2006 + type: FLOAT + - name: _10_2006 + type: FLOAT + - name: _11_2006 + type: FLOAT + - name: _12_2006 + type: FLOAT + - name: _1_2007 + type: FLOAT + - name: _2_2007 + type: FLOAT + - name: _3_2007 + type: FLOAT + - name: _4_2007 + type: FLOAT + - name: _5_2007 + type: FLOAT + - name: _6_2007 + type: FLOAT + - name: _7_2007 + type: FLOAT + - name: _8_2007 + type: FLOAT + - name: _9_2007 + type: FLOAT + - name: _10_2007 + type: FLOAT + - name: _11_2007 + type: FLOAT + - name: _12_2007 + type: FLOAT + - name: _1_2008 + type: FLOAT + - name: _2_2008 + type: FLOAT + - name: _3_2008 + type: FLOAT + - name: _4_2008 + type: FLOAT + - name: _5_2008 + type: FLOAT + - name: _6_2008 + type: FLOAT + - name: _7_2008 + type: FLOAT + - name: _8_2008 + type: FLOAT + - name: _9_2008 + type: FLOAT + - name: _10_2008 + type: FLOAT + - name: _11_2008 + type: FLOAT + - name: _12_2008 + type: FLOAT + - name: _1_2009 + type: FLOAT + - name: _2_2009 + type: FLOAT + - name: _3_2009 + type: FLOAT + - name: _4_2009 + type: FLOAT + - name: _5_2009 + type: FLOAT + - name: _6_2009 + type: FLOAT + - name: _7_2009 + type: FLOAT + - name: _8_2009 + type: FLOAT + - name: _9_2009 + type: FLOAT + - name: _10_2009 + type: FLOAT + - name: _11_2009 + type: FLOAT + - name: _12_2009 + type: FLOAT + - name: _1_2010 + type: FLOAT + - name: _2_2010 + type: FLOAT + - name: _3_2010 + type: FLOAT + - name: _4_2010 + type: FLOAT + - name: _5_2010 + type: FLOAT + - name: _6_2010 + type: FLOAT + - name: _7_2010 + type: FLOAT + - name: _8_2010 + type: FLOAT + - name: _9_2010 + type: FLOAT + - name: _10_2010 + type: FLOAT + - name: _11_2010 + type: FLOAT + - name: _12_2010 + type: FLOAT + - name: _1_2011 + type: FLOAT + - name: _2_2011 + type: FLOAT + - name: _3_2011 + type: FLOAT + - name: _4_2011 + type: FLOAT + - name: _5_2011 + type: FLOAT + - name: _6_2011 + type: FLOAT + - name: _7_2011 + type: FLOAT + - name: _8_2011 + type: FLOAT + - name: _9_2011 + type: FLOAT + - name: _10_2011 + type: FLOAT + - name: _11_2011 + type: FLOAT + - name: _12_2011 + type: FLOAT + - name: _1_2012 + type: FLOAT + - name: _2_2012 + type: FLOAT + - name: _3_2012 + type: FLOAT + - name: _4_2012 + type: FLOAT + - name: _5_2012 + type: FLOAT + - name: _6_2012 + type: FLOAT + - name: _7_2012 + type: FLOAT + - name: _8_2012 + type: FLOAT + - name: _9_2012 + type: FLOAT + - name: _10_2012 + type: FLOAT + - name: _11_2012 + type: FLOAT + - name: _12_2012 + type: FLOAT + - name: _1_2013 + type: FLOAT + - name: _2_2013 + type: FLOAT + - name: _3_2013 + type: FLOAT + - name: _4_2013 + type: FLOAT + - name: _5_2013 + type: FLOAT + - name: _6_2013 + type: FLOAT + - name: _7_2013 + type: FLOAT + - name: _8_2013 + type: FLOAT + - name: _9_2013 + type: FLOAT + - name: _10_2013 + type: FLOAT + - name: _11_2013 + type: FLOAT + - name: _12_2013 + type: FLOAT + - name: _1_2014 + type: FLOAT + - name: _2_2014 + type: FLOAT + - name: _3_2014 + type: FLOAT + - name: _4_2014 + type: FLOAT + - name: _5_2014 + type: FLOAT + - name: _6_2014 + type: FLOAT + - name: _7_2014 + type: FLOAT + - name: _8_2014 + type: FLOAT + - name: _9_2014 + type: FLOAT + - name: _10_2014 + type: FLOAT + - name: _11_2014 + type: FLOAT + - name: _12_2014 + type: FLOAT + - name: _1_2015 + type: FLOAT + - name: _2_2015 + type: FLOAT + - name: _3_2015 + type: FLOAT + - name: _4_2015 + type: FLOAT + - name: _5_2015 + type: FLOAT + - name: _6_2015 + type: FLOAT + - name: _7_2015 + type: FLOAT + - name: _8_2015 + type: FLOAT + - name: _9_2015 + type: FLOAT + - name: _10_2015 + type: FLOAT + - name: _11_2015 + type: FLOAT + - name: _12_2015 + type: FLOAT + - name: _1_2016 + type: FLOAT + - name: _2_2016 + type: FLOAT + - name: _3_2016 + type: FLOAT + - name: _4_2016 + type: FLOAT + - name: _5_2016 + type: FLOAT + - name: _6_2016 + type: FLOAT + - name: _7_2016 + type: FLOAT + - name: _8_2016 + type: FLOAT + - name: _9_2016 + type: FLOAT + - name: _10_2016 + type: FLOAT + - name: _11_2016 + type: FLOAT + - name: _12_2016 + type: FLOAT + - name: _1_2017 + type: FLOAT + - name: _2_2017 + type: FLOAT + - name: _3_2017 + type: FLOAT + - name: _4_2017 + type: FLOAT + - name: _5_2017 + type: FLOAT + - name: _6_2017 + type: FLOAT + - name: _7_2017 + type: FLOAT + - name: _8_2017 + type: FLOAT + - name: _9_2017 + type: FLOAT + - name: _10_2017 + type: FLOAT + - name: _11_2017 + type: FLOAT + - name: _12_2017 + type: FLOAT + - name: _1_2018 + type: FLOAT + - name: _2_2018 + type: FLOAT + - name: _3_2018 + type: FLOAT + - name: _4_2018 + type: FLOAT + - name: _5_2018 + type: FLOAT + - name: _6_2018 + type: FLOAT + - name: _7_2018 + type: FLOAT + - name: _8_2018 + type: FLOAT + - name: _9_2018 + type: FLOAT + - name: _10_2018 + type: FLOAT + - name: _11_2018 + type: FLOAT + - name: _12_2018 + type: FLOAT + - name: _1_2019 + type: FLOAT + - name: _2_2019 + type: FLOAT + - name: _3_2019 + type: FLOAT + - name: _4_2019 + type: FLOAT + - name: _5_2019 + type: FLOAT + - name: _6_2019 + type: FLOAT + - name: _7_2019 + type: FLOAT + - name: _8_2019 + type: FLOAT + - name: _9_2019 + type: FLOAT + - name: _10_2019 + type: FLOAT + - name: _11_2019 + type: FLOAT + - name: _12_2019 + type: FLOAT + - name: _1_2020 + type: FLOAT + - name: _2_2020 + type: FLOAT + - name: _3_2020 + type: FLOAT + - name: _4_2020 + type: FLOAT + - name: _5_2020 + type: FLOAT + - name: _6_2020 + type: FLOAT + - name: _7_2020 + type: FLOAT + - name: _8_2020 + type: FLOAT + - name: _9_2020 + type: FLOAT + - name: _10_2020 + type: FLOAT + - name: _11_2020 + type: FLOAT + - name: _12_2020 + type: FLOAT + - name: _1_2021 + type: FLOAT + - name: _2_2021 + type: FLOAT + - name: _3_2021 + type: FLOAT + - name: _4_2021 + type: FLOAT + - name: _5_2021 + type: FLOAT + - name: _6_2021 + type: FLOAT + - name: _7_2021 + type: FLOAT + - name: _8_2021 + type: FLOAT + - name: _9_2021 + type: FLOAT + - name: _10_2021 + type: FLOAT + - name: _11_2021 + type: FLOAT + - name: _12_2021 + type: FLOAT + - name: _1_2022 + type: FLOAT + - name: _2_2022 + type: FLOAT + - name: _3_2022 + type: FLOAT + - name: _4_2022 + type: FLOAT + - name: _5_2022 + type: FLOAT + - name: _6_2022 + type: FLOAT + - name: _7_2022 + type: FLOAT + - name: _8_2022 + type: FLOAT + - name: _9_2022 + type: FLOAT + - name: _10_2022 + type: FLOAT + - name: _11_2022 + type: FLOAT + - name: _12_2022 + type: FLOAT + - name: _1_2023 + type: FLOAT + - name: _2_2023 + type: FLOAT + - name: _3_2023 + type: FLOAT + - name: _4_2023 + type: FLOAT + - name: _5_2023 + type: FLOAT + - name: _6_2023 + type: FLOAT + - name: _7_2023 + type: FLOAT + - name: _8_2023 + type: FLOAT + - name: _9_2023 + type: FLOAT + - name: _10_2023 + type: FLOAT + - name: _11_2023 + type: FLOAT + - name: _12_2023 + type: FLOAT + - name: _1_2024 + type: FLOAT + - name: _2_2024 + type: FLOAT + - name: _3_2024 + type: FLOAT + - name: _4_2024 + type: FLOAT + - name: _5_2024 + type: FLOAT + - name: _6_2024 + type: FLOAT diff --git a/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__upt_estimates.yml b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__upt_estimates.yml new file mode 100644 index 0000000000..f78cd89b80 --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__upt_estimates.yml @@ -0,0 +1,33 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__upt_estimates + LIMIT 1; +source_objects: + - "complete_monthly_ridership_with_adjustments_and_estimates/historical/upt_estimates/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__upt_estimates" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "complete_monthly_ridership_with_adjustments_and_estimates/historical/upt_estimates/" +schema_fields: + - name: top_150 + type: STRING + - name: ntd_id + type: NUMERIC + - name: agency + type: STRING + - name: mode + type: STRING + - name: tos + type: STRING + - name: month + type: STRING + - name: year + type: NUMERIC + - name: estimated_upt + type: NUMERIC diff --git a/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__voms.yml b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__voms.yml new file mode 100644 index 0000000000..5283c286b1 --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__voms.yml @@ -0,0 +1,577 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__voms + LIMIT 1; +source_objects: + - "complete_monthly_ridership_with_adjustments_and_estimates/historical/voms/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__voms" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "complete_monthly_ridership_with_adjustments_and_estimates/historical/voms/" +schema_fields: + - name: ntd_id + type: NUMERIC + - name: legacy_ntd_id + type: STRING + - name: agency + type: STRING + - name: mode_type_of_service_status + type: STRING + - name: reporter_type + type: STRING + - name: uace_cd + type: NUMERIC + - name: uza_name + type: STRING + - name: mode + type: STRING + - name: tos + type: STRING + - name: _3_mode + type: STRING + - name: _1_2002 + type: NUMERIC + - name: _2_2002 + type: NUMERIC + - name: _3_2002 + type: NUMERIC + - name: _4_2002 + type: NUMERIC + - name: _5_2002 + type: NUMERIC + - name: _6_2002 + type: NUMERIC + - name: _7_2002 + type: NUMERIC + - name: _8_2002 + type: NUMERIC + - name: _9_2002 + type: NUMERIC + - name: _10_2002 + type: NUMERIC + - name: _11_2002 + type: NUMERIC + - name: _12_2002 + type: NUMERIC + - name: _1_2003 + type: NUMERIC + - name: _2_2003 + type: NUMERIC + - name: _3_2003 + type: NUMERIC + - name: _4_2003 + type: NUMERIC + - name: _5_2003 + type: NUMERIC + - name: _6_2003 + type: NUMERIC + - name: _7_2003 + type: NUMERIC + - name: _8_2003 + type: NUMERIC + - name: _9_2003 + type: NUMERIC + - name: _10_2003 + type: NUMERIC + - name: _11_2003 + type: NUMERIC + - name: _12_2003 + type: NUMERIC + - name: _1_2004 + type: NUMERIC + - name: _2_2004 + type: NUMERIC + - name: _3_2004 + type: NUMERIC + - name: _4_2004 + type: NUMERIC + - name: _5_2004 + type: NUMERIC + - name: _6_2004 + type: NUMERIC + - name: _7_2004 + type: NUMERIC + - name: _8_2004 + type: NUMERIC + - name: _9_2004 + type: NUMERIC + - name: _10_2004 + type: NUMERIC + - name: _11_2004 + type: NUMERIC + - name: _12_2004 + type: NUMERIC + - name: _1_2005 + type: NUMERIC + - name: _2_2005 + type: NUMERIC + - name: _3_2005 + type: NUMERIC + - name: _4_2005 + type: NUMERIC + - name: _5_2005 + type: NUMERIC + - name: _6_2005 + type: NUMERIC + - name: _7_2005 + type: NUMERIC + - name: _8_2005 + type: NUMERIC + - name: _9_2005 + type: NUMERIC + - name: _10_2005 + type: NUMERIC + - name: _11_2005 + type: NUMERIC + - name: _12_2005 + type: NUMERIC + - name: _1_2006 + type: NUMERIC + - name: _2_2006 + type: NUMERIC + - name: _3_2006 + type: NUMERIC + - name: _4_2006 + type: NUMERIC + - name: _5_2006 + type: NUMERIC + - name: _6_2006 + type: NUMERIC + - name: _7_2006 + type: NUMERIC + - name: _8_2006 + type: NUMERIC + - name: _9_2006 + type: NUMERIC + - name: _10_2006 + type: NUMERIC + - name: _11_2006 + type: NUMERIC + - name: _12_2006 + type: NUMERIC + - name: _1_2007 + type: NUMERIC + - name: _2_2007 + type: NUMERIC + - name: _3_2007 + type: NUMERIC + - name: _4_2007 + type: NUMERIC + - name: _5_2007 + type: NUMERIC + - name: _6_2007 + type: NUMERIC + - name: _7_2007 + type: NUMERIC + - name: _8_2007 + type: NUMERIC + - name: _9_2007 + type: NUMERIC + - name: _10_2007 + type: NUMERIC + - name: _11_2007 + type: NUMERIC + - name: _12_2007 + type: NUMERIC + - name: _1_2008 + type: NUMERIC + - name: _2_2008 + type: NUMERIC + - name: _3_2008 + type: NUMERIC + - name: _4_2008 + type: NUMERIC + - name: _5_2008 + type: NUMERIC + - name: _6_2008 + type: NUMERIC + - name: _7_2008 + type: NUMERIC + - name: _8_2008 + type: NUMERIC + - name: _9_2008 + type: NUMERIC + - name: _10_2008 + type: NUMERIC + - name: _11_2008 + type: NUMERIC + - name: _12_2008 + type: NUMERIC + - name: _1_2009 + type: NUMERIC + - name: _2_2009 + type: NUMERIC + - name: _3_2009 + type: NUMERIC + - name: _4_2009 + type: NUMERIC + - name: _5_2009 + type: NUMERIC + - name: _6_2009 + type: NUMERIC + - name: _7_2009 + type: NUMERIC + - name: _8_2009 + type: NUMERIC + - name: _9_2009 + type: NUMERIC + - name: _10_2009 + type: NUMERIC + - name: _11_2009 + type: NUMERIC + - name: _12_2009 + type: NUMERIC + - name: _1_2010 + type: NUMERIC + - name: _2_2010 + type: NUMERIC + - name: _3_2010 + type: NUMERIC + - name: _4_2010 + type: NUMERIC + - name: _5_2010 + type: NUMERIC + - name: _6_2010 + type: NUMERIC + - name: _7_2010 + type: NUMERIC + - name: _8_2010 + type: NUMERIC + - name: _9_2010 + type: NUMERIC + - name: _10_2010 + type: NUMERIC + - name: _11_2010 + type: NUMERIC + - name: _12_2010 + type: NUMERIC + - name: _1_2011 + type: NUMERIC + - name: _2_2011 + type: NUMERIC + - name: _3_2011 + type: NUMERIC + - name: _4_2011 + type: NUMERIC + - name: _5_2011 + type: NUMERIC + - name: _6_2011 + type: NUMERIC + - name: _7_2011 + type: NUMERIC + - name: _8_2011 + type: NUMERIC + - name: _9_2011 + type: NUMERIC + - name: _10_2011 + type: NUMERIC + - name: _11_2011 + type: NUMERIC + - name: _12_2011 + type: NUMERIC + - name: _1_2012 + type: NUMERIC + - name: _2_2012 + type: NUMERIC + - name: _3_2012 + type: NUMERIC + - name: _4_2012 + type: NUMERIC + - name: _5_2012 + type: NUMERIC + - name: _6_2012 + type: NUMERIC + - name: _7_2012 + type: NUMERIC + - name: _8_2012 + type: NUMERIC + - name: _9_2012 + type: NUMERIC + - name: _10_2012 + type: NUMERIC + - name: _11_2012 + type: NUMERIC + - name: _12_2012 + type: NUMERIC + - name: _1_2013 + type: NUMERIC + - name: _2_2013 + type: NUMERIC + - name: _3_2013 + type: NUMERIC + - name: _4_2013 + type: NUMERIC + - name: _5_2013 + type: NUMERIC + - name: _6_2013 + type: NUMERIC + - name: _7_2013 + type: NUMERIC + - name: _8_2013 + type: NUMERIC + - name: _9_2013 + type: NUMERIC + - name: _10_2013 + type: NUMERIC + - name: _11_2013 + type: NUMERIC + - name: _12_2013 + type: NUMERIC + - name: _1_2014 + type: NUMERIC + - name: _2_2014 + type: NUMERIC + - name: _3_2014 + type: NUMERIC + - name: _4_2014 + type: NUMERIC + - name: _5_2014 + type: NUMERIC + - name: _6_2014 + type: NUMERIC + - name: _7_2014 + type: NUMERIC + - name: _8_2014 + type: NUMERIC + - name: _9_2014 + type: NUMERIC + - name: _10_2014 + type: NUMERIC + - name: _11_2014 + type: NUMERIC + - name: _12_2014 + type: NUMERIC + - name: _1_2015 + type: NUMERIC + - name: _2_2015 + type: NUMERIC + - name: _3_2015 + type: NUMERIC + - name: _4_2015 + type: NUMERIC + - name: _5_2015 + type: NUMERIC + - name: _6_2015 + type: NUMERIC + - name: _7_2015 + type: NUMERIC + - name: _8_2015 + type: NUMERIC + - name: _9_2015 + type: NUMERIC + - name: _10_2015 + type: NUMERIC + - name: _11_2015 + type: NUMERIC + - name: _12_2015 + type: NUMERIC + - name: _1_2016 + type: NUMERIC + - name: _2_2016 + type: NUMERIC + - name: _3_2016 + type: NUMERIC + - name: _4_2016 + type: NUMERIC + - name: _5_2016 + type: NUMERIC + - name: _6_2016 + type: NUMERIC + - name: _7_2016 + type: NUMERIC + - name: _8_2016 + type: NUMERIC + - name: _9_2016 + type: NUMERIC + - name: _10_2016 + type: NUMERIC + - name: _11_2016 + type: NUMERIC + - name: _12_2016 + type: NUMERIC + - name: _1_2017 + type: NUMERIC + - name: _2_2017 + type: NUMERIC + - name: _3_2017 + type: NUMERIC + - name: _4_2017 + type: NUMERIC + - name: _5_2017 + type: NUMERIC + - name: _6_2017 + type: NUMERIC + - name: _7_2017 + type: NUMERIC + - name: _8_2017 + type: NUMERIC + - name: _9_2017 + type: NUMERIC + - name: _10_2017 + type: NUMERIC + - name: _11_2017 + type: NUMERIC + - name: _12_2017 + type: NUMERIC + - name: _1_2018 + type: NUMERIC + - name: _2_2018 + type: NUMERIC + - name: _3_2018 + type: NUMERIC + - name: _4_2018 + type: NUMERIC + - name: _5_2018 + type: NUMERIC + - name: _6_2018 + type: NUMERIC + - name: _7_2018 + type: NUMERIC + - name: _8_2018 + type: NUMERIC + - name: _9_2018 + type: NUMERIC + - name: _10_2018 + type: NUMERIC + - name: _11_2018 + type: NUMERIC + - name: _12_2018 + type: NUMERIC + - name: _1_2019 + type: NUMERIC + - name: _2_2019 + type: NUMERIC + - name: _3_2019 + type: NUMERIC + - name: _4_2019 + type: NUMERIC + - name: _5_2019 + type: NUMERIC + - name: _6_2019 + type: NUMERIC + - name: _7_2019 + type: NUMERIC + - name: _8_2019 + type: NUMERIC + - name: _9_2019 + type: NUMERIC + - name: _10_2019 + type: NUMERIC + - name: _11_2019 + type: NUMERIC + - name: _12_2019 + type: NUMERIC + - name: _1_2020 + type: NUMERIC + - name: _2_2020 + type: NUMERIC + - name: _3_2020 + type: NUMERIC + - name: _4_2020 + type: NUMERIC + - name: _5_2020 + type: NUMERIC + - name: _6_2020 + type: NUMERIC + - name: _7_2020 + type: NUMERIC + - name: _8_2020 + type: NUMERIC + - name: _9_2020 + type: NUMERIC + - name: _10_2020 + type: NUMERIC + - name: _11_2020 + type: NUMERIC + - name: _12_2020 + type: NUMERIC + - name: _1_2021 + type: NUMERIC + - name: _2_2021 + type: NUMERIC + - name: _3_2021 + type: NUMERIC + - name: _4_2021 + type: NUMERIC + - name: _5_2021 + type: NUMERIC + - name: _6_2021 + type: NUMERIC + - name: _7_2021 + type: NUMERIC + - name: _8_2021 + type: NUMERIC + - name: _9_2021 + type: NUMERIC + - name: _10_2021 + type: NUMERIC + - name: _11_2021 + type: NUMERIC + - name: _12_2021 + type: NUMERIC + - name: _1_2022 + type: NUMERIC + - name: _2_2022 + type: NUMERIC + - name: _3_2022 + type: NUMERIC + - name: _4_2022 + type: NUMERIC + - name: _5_2022 + type: NUMERIC + - name: _6_2022 + type: NUMERIC + - name: _7_2022 + type: NUMERIC + - name: _8_2022 + type: NUMERIC + - name: _9_2022 + type: NUMERIC + - name: _10_2022 + type: NUMERIC + - name: _11_2022 + type: NUMERIC + - name: _12_2022 + type: NUMERIC + - name: _1_2023 + type: NUMERIC + - name: _2_2023 + type: NUMERIC + - name: _3_2023 + type: NUMERIC + - name: _4_2023 + type: NUMERIC + - name: _5_2023 + type: NUMERIC + - name: _6_2023 + type: NUMERIC + - name: _7_2023 + type: NUMERIC + - name: _8_2023 + type: NUMERIC + - name: _9_2023 + type: NUMERIC + - name: _10_2023 + type: NUMERIC + - name: _11_2023 + type: NUMERIC + - name: _12_2023 + type: NUMERIC + - name: _1_2024 + type: NUMERIC + - name: _2_2024 + type: NUMERIC + - name: _3_2024 + type: NUMERIC + - name: _4_2024 + type: NUMERIC + - name: _5_2024 + type: NUMERIC + - name: _6_2024 + type: NUMERIC diff --git a/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrh.yml b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrh.yml new file mode 100644 index 0000000000..872650de3f --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrh.yml @@ -0,0 +1,577 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__vrh + LIMIT 1; +source_objects: + - "complete_monthly_ridership_with_adjustments_and_estimates/historical/vrh/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__vrh" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "complete_monthly_ridership_with_adjustments_and_estimates/historical/vrh/" +schema_fields: + - name: ntd_id + type: NUMERIC + - name: legacy_ntd_id + type: STRING + - name: agency + type: STRING + - name: mode_type_of_service_status + type: STRING + - name: reporter_type + type: STRING + - name: uace_cd + type: NUMERIC + - name: uza_name + type: STRING + - name: mode + type: STRING + - name: tos + type: STRING + - name: _3_mode + type: STRING + - name: _1_2002 + type: NUMERIC + - name: _2_2002 + type: NUMERIC + - name: _3_2002 + type: NUMERIC + - name: _4_2002 + type: NUMERIC + - name: _5_2002 + type: NUMERIC + - name: _6_2002 + type: NUMERIC + - name: _7_2002 + type: NUMERIC + - name: _8_2002 + type: NUMERIC + - name: _9_2002 + type: NUMERIC + - name: _10_2002 + type: NUMERIC + - name: _11_2002 + type: NUMERIC + - name: _12_2002 + type: NUMERIC + - name: _1_2003 + type: NUMERIC + - name: _2_2003 + type: NUMERIC + - name: _3_2003 + type: NUMERIC + - name: _4_2003 + type: NUMERIC + - name: _5_2003 + type: NUMERIC + - name: _6_2003 + type: NUMERIC + - name: _7_2003 + type: NUMERIC + - name: _8_2003 + type: NUMERIC + - name: _9_2003 + type: NUMERIC + - name: _10_2003 + type: NUMERIC + - name: _11_2003 + type: NUMERIC + - name: _12_2003 + type: NUMERIC + - name: _1_2004 + type: NUMERIC + - name: _2_2004 + type: NUMERIC + - name: _3_2004 + type: NUMERIC + - name: _4_2004 + type: NUMERIC + - name: _5_2004 + type: NUMERIC + - name: _6_2004 + type: NUMERIC + - name: _7_2004 + type: NUMERIC + - name: _8_2004 + type: NUMERIC + - name: _9_2004 + type: NUMERIC + - name: _10_2004 + type: NUMERIC + - name: _11_2004 + type: NUMERIC + - name: _12_2004 + type: NUMERIC + - name: _1_2005 + type: NUMERIC + - name: _2_2005 + type: NUMERIC + - name: _3_2005 + type: NUMERIC + - name: _4_2005 + type: NUMERIC + - name: _5_2005 + type: NUMERIC + - name: _6_2005 + type: NUMERIC + - name: _7_2005 + type: NUMERIC + - name: _8_2005 + type: NUMERIC + - name: _9_2005 + type: NUMERIC + - name: _10_2005 + type: NUMERIC + - name: _11_2005 + type: NUMERIC + - name: _12_2005 + type: NUMERIC + - name: _1_2006 + type: NUMERIC + - name: _2_2006 + type: NUMERIC + - name: _3_2006 + type: NUMERIC + - name: _4_2006 + type: NUMERIC + - name: _5_2006 + type: NUMERIC + - name: _6_2006 + type: NUMERIC + - name: _7_2006 + type: NUMERIC + - name: _8_2006 + type: NUMERIC + - name: _9_2006 + type: NUMERIC + - name: _10_2006 + type: NUMERIC + - name: _11_2006 + type: NUMERIC + - name: _12_2006 + type: NUMERIC + - name: _1_2007 + type: NUMERIC + - name: _2_2007 + type: NUMERIC + - name: _3_2007 + type: NUMERIC + - name: _4_2007 + type: NUMERIC + - name: _5_2007 + type: NUMERIC + - name: _6_2007 + type: NUMERIC + - name: _7_2007 + type: NUMERIC + - name: _8_2007 + type: NUMERIC + - name: _9_2007 + type: NUMERIC + - name: _10_2007 + type: NUMERIC + - name: _11_2007 + type: NUMERIC + - name: _12_2007 + type: NUMERIC + - name: _1_2008 + type: NUMERIC + - name: _2_2008 + type: NUMERIC + - name: _3_2008 + type: NUMERIC + - name: _4_2008 + type: NUMERIC + - name: _5_2008 + type: NUMERIC + - name: _6_2008 + type: NUMERIC + - name: _7_2008 + type: NUMERIC + - name: _8_2008 + type: NUMERIC + - name: _9_2008 + type: NUMERIC + - name: _10_2008 + type: NUMERIC + - name: _11_2008 + type: NUMERIC + - name: _12_2008 + type: NUMERIC + - name: _1_2009 + type: NUMERIC + - name: _2_2009 + type: NUMERIC + - name: _3_2009 + type: NUMERIC + - name: _4_2009 + type: NUMERIC + - name: _5_2009 + type: NUMERIC + - name: _6_2009 + type: NUMERIC + - name: _7_2009 + type: NUMERIC + - name: _8_2009 + type: NUMERIC + - name: _9_2009 + type: NUMERIC + - name: _10_2009 + type: NUMERIC + - name: _11_2009 + type: NUMERIC + - name: _12_2009 + type: NUMERIC + - name: _1_2010 + type: NUMERIC + - name: _2_2010 + type: NUMERIC + - name: _3_2010 + type: NUMERIC + - name: _4_2010 + type: NUMERIC + - name: _5_2010 + type: NUMERIC + - name: _6_2010 + type: NUMERIC + - name: _7_2010 + type: NUMERIC + - name: _8_2010 + type: NUMERIC + - name: _9_2010 + type: NUMERIC + - name: _10_2010 + type: NUMERIC + - name: _11_2010 + type: NUMERIC + - name: _12_2010 + type: NUMERIC + - name: _1_2011 + type: NUMERIC + - name: _2_2011 + type: NUMERIC + - name: _3_2011 + type: NUMERIC + - name: _4_2011 + type: NUMERIC + - name: _5_2011 + type: NUMERIC + - name: _6_2011 + type: NUMERIC + - name: _7_2011 + type: NUMERIC + - name: _8_2011 + type: NUMERIC + - name: _9_2011 + type: NUMERIC + - name: _10_2011 + type: NUMERIC + - name: _11_2011 + type: NUMERIC + - name: _12_2011 + type: NUMERIC + - name: _1_2012 + type: NUMERIC + - name: _2_2012 + type: NUMERIC + - name: _3_2012 + type: NUMERIC + - name: _4_2012 + type: NUMERIC + - name: _5_2012 + type: NUMERIC + - name: _6_2012 + type: NUMERIC + - name: _7_2012 + type: NUMERIC + - name: _8_2012 + type: NUMERIC + - name: _9_2012 + type: NUMERIC + - name: _10_2012 + type: NUMERIC + - name: _11_2012 + type: NUMERIC + - name: _12_2012 + type: NUMERIC + - name: _1_2013 + type: NUMERIC + - name: _2_2013 + type: NUMERIC + - name: _3_2013 + type: NUMERIC + - name: _4_2013 + type: NUMERIC + - name: _5_2013 + type: NUMERIC + - name: _6_2013 + type: NUMERIC + - name: _7_2013 + type: NUMERIC + - name: _8_2013 + type: NUMERIC + - name: _9_2013 + type: NUMERIC + - name: _10_2013 + type: NUMERIC + - name: _11_2013 + type: NUMERIC + - name: _12_2013 + type: NUMERIC + - name: _1_2014 + type: NUMERIC + - name: _2_2014 + type: NUMERIC + - name: _3_2014 + type: NUMERIC + - name: _4_2014 + type: NUMERIC + - name: _5_2014 + type: NUMERIC + - name: _6_2014 + type: NUMERIC + - name: _7_2014 + type: NUMERIC + - name: _8_2014 + type: NUMERIC + - name: _9_2014 + type: NUMERIC + - name: _10_2014 + type: NUMERIC + - name: _11_2014 + type: NUMERIC + - name: _12_2014 + type: NUMERIC + - name: _1_2015 + type: NUMERIC + - name: _2_2015 + type: NUMERIC + - name: _3_2015 + type: NUMERIC + - name: _4_2015 + type: NUMERIC + - name: _5_2015 + type: NUMERIC + - name: _6_2015 + type: NUMERIC + - name: _7_2015 + type: NUMERIC + - name: _8_2015 + type: NUMERIC + - name: _9_2015 + type: NUMERIC + - name: _10_2015 + type: NUMERIC + - name: _11_2015 + type: NUMERIC + - name: _12_2015 + type: NUMERIC + - name: _1_2016 + type: NUMERIC + - name: _2_2016 + type: NUMERIC + - name: _3_2016 + type: NUMERIC + - name: _4_2016 + type: NUMERIC + - name: _5_2016 + type: NUMERIC + - name: _6_2016 + type: NUMERIC + - name: _7_2016 + type: NUMERIC + - name: _8_2016 + type: NUMERIC + - name: _9_2016 + type: NUMERIC + - name: _10_2016 + type: NUMERIC + - name: _11_2016 + type: NUMERIC + - name: _12_2016 + type: NUMERIC + - name: _1_2017 + type: NUMERIC + - name: _2_2017 + type: NUMERIC + - name: _3_2017 + type: NUMERIC + - name: _4_2017 + type: NUMERIC + - name: _5_2017 + type: NUMERIC + - name: _6_2017 + type: NUMERIC + - name: _7_2017 + type: NUMERIC + - name: _8_2017 + type: NUMERIC + - name: _9_2017 + type: NUMERIC + - name: _10_2017 + type: NUMERIC + - name: _11_2017 + type: NUMERIC + - name: _12_2017 + type: NUMERIC + - name: _1_2018 + type: NUMERIC + - name: _2_2018 + type: NUMERIC + - name: _3_2018 + type: NUMERIC + - name: _4_2018 + type: NUMERIC + - name: _5_2018 + type: NUMERIC + - name: _6_2018 + type: NUMERIC + - name: _7_2018 + type: NUMERIC + - name: _8_2018 + type: NUMERIC + - name: _9_2018 + type: NUMERIC + - name: _10_2018 + type: NUMERIC + - name: _11_2018 + type: NUMERIC + - name: _12_2018 + type: NUMERIC + - name: _1_2019 + type: NUMERIC + - name: _2_2019 + type: NUMERIC + - name: _3_2019 + type: NUMERIC + - name: _4_2019 + type: NUMERIC + - name: _5_2019 + type: NUMERIC + - name: _6_2019 + type: NUMERIC + - name: _7_2019 + type: NUMERIC + - name: _8_2019 + type: NUMERIC + - name: _9_2019 + type: NUMERIC + - name: _10_2019 + type: NUMERIC + - name: _11_2019 + type: NUMERIC + - name: _12_2019 + type: NUMERIC + - name: _1_2020 + type: NUMERIC + - name: _2_2020 + type: NUMERIC + - name: _3_2020 + type: NUMERIC + - name: _4_2020 + type: NUMERIC + - name: _5_2020 + type: NUMERIC + - name: _6_2020 + type: NUMERIC + - name: _7_2020 + type: NUMERIC + - name: _8_2020 + type: NUMERIC + - name: _9_2020 + type: NUMERIC + - name: _10_2020 + type: NUMERIC + - name: _11_2020 + type: NUMERIC + - name: _12_2020 + type: NUMERIC + - name: _1_2021 + type: NUMERIC + - name: _2_2021 + type: NUMERIC + - name: _3_2021 + type: NUMERIC + - name: _4_2021 + type: NUMERIC + - name: _5_2021 + type: NUMERIC + - name: _6_2021 + type: NUMERIC + - name: _7_2021 + type: NUMERIC + - name: _8_2021 + type: NUMERIC + - name: _9_2021 + type: NUMERIC + - name: _10_2021 + type: NUMERIC + - name: _11_2021 + type: NUMERIC + - name: _12_2021 + type: NUMERIC + - name: _1_2022 + type: NUMERIC + - name: _2_2022 + type: NUMERIC + - name: _3_2022 + type: NUMERIC + - name: _4_2022 + type: NUMERIC + - name: _5_2022 + type: NUMERIC + - name: _6_2022 + type: NUMERIC + - name: _7_2022 + type: NUMERIC + - name: _8_2022 + type: NUMERIC + - name: _9_2022 + type: NUMERIC + - name: _10_2022 + type: NUMERIC + - name: _11_2022 + type: NUMERIC + - name: _12_2022 + type: NUMERIC + - name: _1_2023 + type: NUMERIC + - name: _2_2023 + type: NUMERIC + - name: _3_2023 + type: NUMERIC + - name: _4_2023 + type: NUMERIC + - name: _5_2023 + type: NUMERIC + - name: _6_2023 + type: NUMERIC + - name: _7_2023 + type: NUMERIC + - name: _8_2023 + type: NUMERIC + - name: _9_2023 + type: NUMERIC + - name: _10_2023 + type: NUMERIC + - name: _11_2023 + type: NUMERIC + - name: _12_2023 + type: NUMERIC + - name: _1_2024 + type: NUMERIC + - name: _2_2024 + type: NUMERIC + - name: _3_2024 + type: NUMERIC + - name: _4_2024 + type: NUMERIC + - name: _5_2024 + type: NUMERIC + - name: _6_2024 + type: NUMERIC diff --git a/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm.yml b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm.yml new file mode 100644 index 0000000000..115b8f22a8 --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm.yml @@ -0,0 +1,577 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm + LIMIT 1; +source_objects: + - "complete_monthly_ridership_with_adjustments_and_estimates/historical/vrm/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "complete_monthly_ridership_with_adjustments_and_estimates/historical/vrm/" +schema_fields: + - name: ntd_id + type: NUMERIC + - name: legacy_ntd_id + type: STRING + - name: agency + type: STRING + - name: mode_type_of_service_status + type: STRING + - name: reporter_type + type: STRING + - name: uace_cd + type: NUMERIC + - name: uza_name + type: STRING + - name: mode + type: STRING + - name: tos + type: STRING + - name: _3_mode + type: STRING + - name: _1_2002 + type: NUMERIC + - name: _2_2002 + type: NUMERIC + - name: _3_2002 + type: NUMERIC + - name: _4_2002 + type: NUMERIC + - name: _5_2002 + type: NUMERIC + - name: _6_2002 + type: NUMERIC + - name: _7_2002 + type: NUMERIC + - name: _8_2002 + type: NUMERIC + - name: _9_2002 + type: NUMERIC + - name: _10_2002 + type: NUMERIC + - name: _11_2002 + type: NUMERIC + - name: _12_2002 + type: NUMERIC + - name: _1_2003 + type: NUMERIC + - name: _2_2003 + type: NUMERIC + - name: _3_2003 + type: NUMERIC + - name: _4_2003 + type: NUMERIC + - name: _5_2003 + type: NUMERIC + - name: _6_2003 + type: NUMERIC + - name: _7_2003 + type: NUMERIC + - name: _8_2003 + type: NUMERIC + - name: _9_2003 + type: NUMERIC + - name: _10_2003 + type: NUMERIC + - name: _11_2003 + type: NUMERIC + - name: _12_2003 + type: NUMERIC + - name: _1_2004 + type: NUMERIC + - name: _2_2004 + type: NUMERIC + - name: _3_2004 + type: NUMERIC + - name: _4_2004 + type: NUMERIC + - name: _5_2004 + type: NUMERIC + - name: _6_2004 + type: NUMERIC + - name: _7_2004 + type: NUMERIC + - name: _8_2004 + type: NUMERIC + - name: _9_2004 + type: NUMERIC + - name: _10_2004 + type: NUMERIC + - name: _11_2004 + type: NUMERIC + - name: _12_2004 + type: NUMERIC + - name: _1_2005 + type: NUMERIC + - name: _2_2005 + type: NUMERIC + - name: _3_2005 + type: NUMERIC + - name: _4_2005 + type: NUMERIC + - name: _5_2005 + type: NUMERIC + - name: _6_2005 + type: NUMERIC + - name: _7_2005 + type: NUMERIC + - name: _8_2005 + type: NUMERIC + - name: _9_2005 + type: NUMERIC + - name: _10_2005 + type: NUMERIC + - name: _11_2005 + type: NUMERIC + - name: _12_2005 + type: NUMERIC + - name: _1_2006 + type: NUMERIC + - name: _2_2006 + type: NUMERIC + - name: _3_2006 + type: NUMERIC + - name: _4_2006 + type: NUMERIC + - name: _5_2006 + type: NUMERIC + - name: _6_2006 + type: NUMERIC + - name: _7_2006 + type: NUMERIC + - name: _8_2006 + type: NUMERIC + - name: _9_2006 + type: NUMERIC + - name: _10_2006 + type: NUMERIC + - name: _11_2006 + type: NUMERIC + - name: _12_2006 + type: NUMERIC + - name: _1_2007 + type: NUMERIC + - name: _2_2007 + type: NUMERIC + - name: _3_2007 + type: NUMERIC + - name: _4_2007 + type: NUMERIC + - name: _5_2007 + type: NUMERIC + - name: _6_2007 + type: NUMERIC + - name: _7_2007 + type: NUMERIC + - name: _8_2007 + type: NUMERIC + - name: _9_2007 + type: NUMERIC + - name: _10_2007 + type: NUMERIC + - name: _11_2007 + type: NUMERIC + - name: _12_2007 + type: NUMERIC + - name: _1_2008 + type: NUMERIC + - name: _2_2008 + type: NUMERIC + - name: _3_2008 + type: NUMERIC + - name: _4_2008 + type: NUMERIC + - name: _5_2008 + type: NUMERIC + - name: _6_2008 + type: NUMERIC + - name: _7_2008 + type: NUMERIC + - name: _8_2008 + type: NUMERIC + - name: _9_2008 + type: NUMERIC + - name: _10_2008 + type: NUMERIC + - name: _11_2008 + type: NUMERIC + - name: _12_2008 + type: NUMERIC + - name: _1_2009 + type: NUMERIC + - name: _2_2009 + type: NUMERIC + - name: _3_2009 + type: NUMERIC + - name: _4_2009 + type: NUMERIC + - name: _5_2009 + type: NUMERIC + - name: _6_2009 + type: NUMERIC + - name: _7_2009 + type: NUMERIC + - name: _8_2009 + type: NUMERIC + - name: _9_2009 + type: NUMERIC + - name: _10_2009 + type: NUMERIC + - name: _11_2009 + type: NUMERIC + - name: _12_2009 + type: NUMERIC + - name: _1_2010 + type: NUMERIC + - name: _2_2010 + type: NUMERIC + - name: _3_2010 + type: NUMERIC + - name: _4_2010 + type: NUMERIC + - name: _5_2010 + type: NUMERIC + - name: _6_2010 + type: NUMERIC + - name: _7_2010 + type: NUMERIC + - name: _8_2010 + type: NUMERIC + - name: _9_2010 + type: NUMERIC + - name: _10_2010 + type: NUMERIC + - name: _11_2010 + type: NUMERIC + - name: _12_2010 + type: NUMERIC + - name: _1_2011 + type: NUMERIC + - name: _2_2011 + type: NUMERIC + - name: _3_2011 + type: NUMERIC + - name: _4_2011 + type: NUMERIC + - name: _5_2011 + type: NUMERIC + - name: _6_2011 + type: NUMERIC + - name: _7_2011 + type: NUMERIC + - name: _8_2011 + type: NUMERIC + - name: _9_2011 + type: NUMERIC + - name: _10_2011 + type: NUMERIC + - name: _11_2011 + type: NUMERIC + - name: _12_2011 + type: NUMERIC + - name: _1_2012 + type: NUMERIC + - name: _2_2012 + type: NUMERIC + - name: _3_2012 + type: NUMERIC + - name: _4_2012 + type: NUMERIC + - name: _5_2012 + type: NUMERIC + - name: _6_2012 + type: NUMERIC + - name: _7_2012 + type: NUMERIC + - name: _8_2012 + type: NUMERIC + - name: _9_2012 + type: NUMERIC + - name: _10_2012 + type: NUMERIC + - name: _11_2012 + type: NUMERIC + - name: _12_2012 + type: NUMERIC + - name: _1_2013 + type: NUMERIC + - name: _2_2013 + type: NUMERIC + - name: _3_2013 + type: NUMERIC + - name: _4_2013 + type: NUMERIC + - name: _5_2013 + type: NUMERIC + - name: _6_2013 + type: NUMERIC + - name: _7_2013 + type: NUMERIC + - name: _8_2013 + type: NUMERIC + - name: _9_2013 + type: NUMERIC + - name: _10_2013 + type: NUMERIC + - name: _11_2013 + type: NUMERIC + - name: _12_2013 + type: NUMERIC + - name: _1_2014 + type: NUMERIC + - name: _2_2014 + type: NUMERIC + - name: _3_2014 + type: NUMERIC + - name: _4_2014 + type: NUMERIC + - name: _5_2014 + type: NUMERIC + - name: _6_2014 + type: NUMERIC + - name: _7_2014 + type: NUMERIC + - name: _8_2014 + type: NUMERIC + - name: _9_2014 + type: NUMERIC + - name: _10_2014 + type: NUMERIC + - name: _11_2014 + type: NUMERIC + - name: _12_2014 + type: NUMERIC + - name: _1_2015 + type: NUMERIC + - name: _2_2015 + type: NUMERIC + - name: _3_2015 + type: NUMERIC + - name: _4_2015 + type: NUMERIC + - name: _5_2015 + type: NUMERIC + - name: _6_2015 + type: NUMERIC + - name: _7_2015 + type: NUMERIC + - name: _8_2015 + type: NUMERIC + - name: _9_2015 + type: NUMERIC + - name: _10_2015 + type: NUMERIC + - name: _11_2015 + type: NUMERIC + - name: _12_2015 + type: NUMERIC + - name: _1_2016 + type: NUMERIC + - name: _2_2016 + type: NUMERIC + - name: _3_2016 + type: NUMERIC + - name: _4_2016 + type: NUMERIC + - name: _5_2016 + type: NUMERIC + - name: _6_2016 + type: NUMERIC + - name: _7_2016 + type: NUMERIC + - name: _8_2016 + type: NUMERIC + - name: _9_2016 + type: NUMERIC + - name: _10_2016 + type: NUMERIC + - name: _11_2016 + type: NUMERIC + - name: _12_2016 + type: NUMERIC + - name: _1_2017 + type: NUMERIC + - name: _2_2017 + type: NUMERIC + - name: _3_2017 + type: NUMERIC + - name: _4_2017 + type: NUMERIC + - name: _5_2017 + type: NUMERIC + - name: _6_2017 + type: NUMERIC + - name: _7_2017 + type: NUMERIC + - name: _8_2017 + type: NUMERIC + - name: _9_2017 + type: NUMERIC + - name: _10_2017 + type: NUMERIC + - name: _11_2017 + type: NUMERIC + - name: _12_2017 + type: NUMERIC + - name: _1_2018 + type: NUMERIC + - name: _2_2018 + type: NUMERIC + - name: _3_2018 + type: NUMERIC + - name: _4_2018 + type: NUMERIC + - name: _5_2018 + type: NUMERIC + - name: _6_2018 + type: NUMERIC + - name: _7_2018 + type: NUMERIC + - name: _8_2018 + type: NUMERIC + - name: _9_2018 + type: NUMERIC + - name: _10_2018 + type: NUMERIC + - name: _11_2018 + type: NUMERIC + - name: _12_2018 + type: NUMERIC + - name: _1_2019 + type: NUMERIC + - name: _2_2019 + type: NUMERIC + - name: _3_2019 + type: NUMERIC + - name: _4_2019 + type: NUMERIC + - name: _5_2019 + type: NUMERIC + - name: _6_2019 + type: NUMERIC + - name: _7_2019 + type: NUMERIC + - name: _8_2019 + type: NUMERIC + - name: _9_2019 + type: NUMERIC + - name: _10_2019 + type: NUMERIC + - name: _11_2019 + type: NUMERIC + - name: _12_2019 + type: NUMERIC + - name: _1_2020 + type: NUMERIC + - name: _2_2020 + type: NUMERIC + - name: _3_2020 + type: NUMERIC + - name: _4_2020 + type: NUMERIC + - name: _5_2020 + type: NUMERIC + - name: _6_2020 + type: NUMERIC + - name: _7_2020 + type: NUMERIC + - name: _8_2020 + type: NUMERIC + - name: _9_2020 + type: NUMERIC + - name: _10_2020 + type: NUMERIC + - name: _11_2020 + type: NUMERIC + - name: _12_2020 + type: NUMERIC + - name: _1_2021 + type: NUMERIC + - name: _2_2021 + type: NUMERIC + - name: _3_2021 + type: NUMERIC + - name: _4_2021 + type: NUMERIC + - name: _5_2021 + type: NUMERIC + - name: _6_2021 + type: NUMERIC + - name: _7_2021 + type: NUMERIC + - name: _8_2021 + type: NUMERIC + - name: _9_2021 + type: NUMERIC + - name: _10_2021 + type: NUMERIC + - name: _11_2021 + type: NUMERIC + - name: _12_2021 + type: NUMERIC + - name: _1_2022 + type: NUMERIC + - name: _2_2022 + type: NUMERIC + - name: _3_2022 + type: NUMERIC + - name: _4_2022 + type: NUMERIC + - name: _5_2022 + type: NUMERIC + - name: _6_2022 + type: NUMERIC + - name: _7_2022 + type: NUMERIC + - name: _8_2022 + type: NUMERIC + - name: _9_2022 + type: NUMERIC + - name: _10_2022 + type: NUMERIC + - name: _11_2022 + type: NUMERIC + - name: _12_2022 + type: NUMERIC + - name: _1_2023 + type: NUMERIC + - name: _2_2023 + type: NUMERIC + - name: _3_2023 + type: NUMERIC + - name: _4_2023 + type: NUMERIC + - name: _5_2023 + type: NUMERIC + - name: _6_2023 + type: NUMERIC + - name: _7_2023 + type: NUMERIC + - name: _8_2023 + type: NUMERIC + - name: _9_2023 + type: NUMERIC + - name: _10_2023 + type: NUMERIC + - name: _11_2023 + type: NUMERIC + - name: _12_2023 + type: NUMERIC + - name: _1_2024 + type: NUMERIC + - name: _2_2024 + type: NUMERIC + - name: _3_2024 + type: NUMERIC + - name: _4_2024 + type: NUMERIC + - name: _5_2024 + type: NUMERIC + - name: _6_2024 + type: NUMERIC diff --git a/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm_estimates.yml b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm_estimates.yml new file mode 100644 index 0000000000..34a3fa3fee --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm_estimates.yml @@ -0,0 +1,33 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm_estimates + LIMIT 1; +source_objects: + - "complete_monthly_ridership_with_adjustments_and_estimates/historical/vrm_estimates/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__ridership.historical__complete_monthly_ridership_with_adjustments_and_estimates__vrm_estimates" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "complete_monthly_ridership_with_adjustments_and_estimates/historical/vrm_estimates/" +schema_fields: + - name: top_150 + type: STRING + - name: ntd_id + type: NUMERIC + - name: agency + type: STRING + - name: mode + type: STRING + - name: tos + type: STRING + - name: month + type: STRING + - name: year + type: NUMERIC + - name: estimated_vrm + type: NUMERIC