Merge pull request #604 from ibi-group/dev

Release

__tests__/end-to-end.js

@@ -885,7 +885,7 @@ describe('end-to-end', () => {
       await click('[data-test-id="confirm-create-user-button"]')
 
       // wait for user to be saved
-      await wait(2000, 'for user to be created')
+      await wait(30000, 'for user to be created')
 
       // filter users
       await filterUsers(testUserSlug)
@@ -2471,6 +2471,11 @@ describe('end-to-end', () => {
           await page.keyboard.press('Tab')
           await page.keyboard.press('Enter')
 
+          // trip short name
+          await page.keyboard.type('test-trip-short-name')
+          await page.keyboard.press('Tab')
+          await page.keyboard.press('Enter')
+
           // Laurel Dr arrival
           await page.keyboard.type('1234')
           await page.keyboard.press('Tab')

__tests__/test-utils/mock-data/manager.js

@@ -139,6 +139,7 @@ export const mockFeedWithVersion = {
   retrievalMethod: 'MANUALLY_UPLOADED',
   s3Url: null,
   snapshotVersion: null,
+  transformRules: [],
   url: 'http://mdtrip.org/googletransit/AnnapolisTransit/google_transit.zip',
   user: null
 }
@@ -160,6 +161,7 @@ export const mockFeedWithoutVersion = {
   retrievalMethod: 'FETCHED_AUTOMATICALLY',
   s3Url: null,
   snapshotVersion: null,
+  transformRules: [],
   url: null,
   user: null
 }

docs/index.md

@@ -1,4 +1,4 @@
-# IBI Transit Data Tools
+# IBI Transit Data Tools (TRANSIT-data-tools)
 
 The IBI Transit Data Tools suite provides web-based tools for creating, managing, evaluating, and publishing transit data, specifically data stored in the General Transit Feed Specification (GTFS) format.
 

docs/user/managing-projects-feeds.md

@@ -52,10 +52,35 @@ Feed Versions are created from the main Feed Source profile page. There are thre
 
 3. **Import From the GTFS Editor**: Select "From snapshot" from the `+ Create new version` dropdown. The list of snapshots should now be visible showing any available snapshots of the feed in the Editor. Select the desired snapshot by clicking the "Publish" button to publish the snapshot as a new version.
 
+4. **Service Period Merge** (certain Data Tools configurations only): If a Feed Source has two or more Feed Versions, a new Feed Version can be created by merging two versions representing a transit agency's service over different time periods. While viewing a particular Feed Version, click `Merge with version` (underneath the map view) to select which past version you would like to merge with.
+
+5. **Regional Merge**: For Projects that contain multiple Feed Sources across a region, it can be useful to merge multiple transit agencies together into a combined GTFS feed for the entire region. While viewing a Project's list of Feed Sources, click `Actions > Merge all` to produce a combined GTFS file for all Feed Sources.
+
 **Note:** when uploading or fetching a feed, and the file being uploaded or fetched is not different from the latest version, no new Feed Version will be created.
 
-## Loading Feed Versions into Editor
+## Feed Transformations
+
+Data Tools now supports **Feed Transformations**, which can apply a set of changes automatically to each new GTFS feed loaded into the system. This is especially helpful for applying repeatable changes to or inserting supplementary files into GTFS feeds that need to be enhanced. It provides a critical stopgap to improve GTFS coming out of systems that cannot be modified (e.g., scheduling software that has a rigid export format). The different transformation types currently supported are:
+
+1. **Replace File From Version** - any file in an incoming GTFS (e.g., feed_info.txt) can be overwritten or inserted with a file extracted from a previously loaded feed version.
+
+2. **Replace File From String** - any file in an incoming GTFS (e.g., feed_info.txt) can be overwritten or inserted with user-defined CSV data.
+
+### Configuring Feed Transformations
+
+Follow the steps below to configure Feed Transformations for an existing Feed Source:
+
+1. Select a Feed Source and click `Settings > Feed Transformations`.
+2. From here, click `Add transformation` to begin creating a new set of rules for incoming GTFS feeds. Your first ruleset will automatically apply to GTFS that is fetched automatically *and* manually uploaded, but this can be changed for each ruleset to apply to any of the retrieval methods listed in [Creating Feed Versions](#creating-feed-versions).
+3. Click `Add step to transformation` to select a transformation type and fill in the required fields for each type. Multiple transformations can be specified and each will be applied to the incoming GTFS file in the order that they are defined.
+
+![screenshot](../img/configure-feed-transformations.png)
+
+### Viewing the Feed Transformation Summary
+
+Once the Feed Transformations have all been configured, you can import a new GTFS file using one of the retrieval methods you have specified for your ruleset. Once the GTFS file has been processed, a summary of the transformations is available by hovering over the wrench icon that is visible in the Feed Version panel.
 
+![screenshot](../img/feed-transformation-summary.png)
 
 
 ## Viewing and Managing Feed Versions

gtfs.yml

@@ -629,9 +629,28 @@
     inputType: DROPDOWN
     bulkEditEnabled: true
     options:
-    - value: 'USD'
-    - value: 'EUR'
-    - value: 'GBP'
+    - value: USD
+      text: US dollar (USD)
+    - value: AUD
+      text: Australian dollar (AUD)
+    - value: CAD
+      text: Canadian dollar (CAD)
+    - value: CHF
+      text: Swiss franc (CHF)
+    - value: CNH
+      text: Chinese renminbi (CNH)
+    - value: EUR
+      text: Euro (EUR)
+    - value: GBP
+      text: Pound sterling (GBP)
+    - value: JPY
+      text: Japanese yen (JPY)
+    - value: MXN
+      text: Mexican peso (MXN)
+    - value: NZD
+      text: New Zealand dollar (NZD)
+    - value: SEK
+      text: Swedish krona (SEK)
     columnWidth: 12
     helpContent: "The currency_type field defines the currency used to pay the fare. Please use the ISO 4217 alphabetical currency codes which can be found at the following URL:http://en.wikipedia.org/wiki/ISO_4217."
   - name: "payment_method"
@@ -776,6 +795,7 @@
 
 - id: scheduleexception
   name: (none)
+  datatools: true
   helpContent: Conveyal-specific table for classifying schedule exceptions.
   fields:
   - name: name

i18n/english.yml

@@ -228,6 +228,23 @@ components:
     upload: Upload
     versions: Versions
     viewPublic: View public page
+  FeedTransformationDescriptions:
+    general:
+      fileDefined: below text
+      filePlaceholder: '[choose file]'
+      tablePlaceholder: '[choose table]'
+      table: table
+      version: version
+      versionPlaceholder: '[choose version]'
+    DeleteRecordsTransformation:
+      label: Delete records from %tablePlaceholder%
+      name: Delete records transformation
+    ReplaceFileFromStringTransformation:
+      label: Replace %tablePlaceholder% from %filePlaceholder%
+      name: Replace file from string transformation
+    ReplaceFileFromVersionTransformation:
+      label: Replace %tablePlaceholder% from %versionPlaceholder%
+      name: Replace file from version transformation
   FeedVersionNavigator:
     confirmDelete: Are you sure you want to delete this version? This cannot be undone.
     confirmLoad: 'This will override all active GTFS Editor data for this Feed Source with the data from this version. If there is unsaved work in the Editor you want to keep, you must snapshot the current Editor data first. Are you sure you want to continue?'

lib/common/actions/index.js

@@ -11,7 +11,7 @@ export function createVoidPayloadAction (type: string) {
   return () => ({ type })
 }
 
-export function secureFetch (url: string, method: string = 'get', payload: any, raw: boolean = false, isJSON: boolean = true, actionOnFail: any): any {
+export function secureFetch (url: string, method: string = 'get', payload?: any, raw: boolean = false, isJSON: boolean = true, actionOnFail?: string): any {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     function consoleError (message) {
       console.error(`Error making ${method} request to ${url}: `, message)
@@ -95,7 +95,7 @@ export function fetchGraphQL ({
 }: {
   errorMessage?: string,
   query: string,
-  variables: any
+  variables?: {[key: string]: string | number | Array<string>}
 }): any {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     const body = {

lib/common/components/JobMonitor.js

@@ -7,6 +7,7 @@ import {Link} from 'react-router'
 
 import {removeRetiredJob} from '../../manager/actions/status'
 import SidebarPopover from './SidebarPopover'
+import {getConfigProperty} from '../util/config'
 
 import type {ServerJob} from '../../types'
 import type {JobStatusState} from '../../types/reducers'
@@ -136,6 +137,7 @@ class RetiredJob extends Component<{
   render () {
     const {job} = this.props
     const path = this._getPathToObject()
+    const supportEmail = getConfigProperty('application.support_email')
     return (
       <li className='job-container'>
         <div style={{ float: 'left' }}>
@@ -189,12 +191,15 @@ class RetiredJob extends Component<{
                           <Icon type='bug' /> Oh no! Looks like an error has occurred.
                         </span>
                       }>
-                      <p>
-                      To submit an error report email a screenshot of your browser
-                      window, the following text (current URL and error details),
-                      and a detailed description of the steps you followed
-                      to <a href='mailto:[email protected]'>[email protected]</a>.
-                      </p>
+                      {supportEmail
+                        ? <p>
+                          To submit an error report email a screenshot of your browser
+                          window, the following text (current URL and error details),
+                          and a detailed description of the steps you followed
+                          to <a href={`mailto:${supportEmail}`}>{supportEmail}</a>.
+                        </p>
+                        : null
+                      }
                       <p>{window.location.href}</p>
                       <span style={{whiteSpace: 'pre', fontSize: 'xx-small'}}>
                         {job.status.exceptionDetails}

lib/common/constants/index.js

@@ -1,4 +1,5 @@
 // @flow
+
 const SECURE: string = 'secure/'
 export const API_PREFIX: string = `/api/manager/`
 export const SECURE_API_PREFIX: string = `${API_PREFIX}${SECURE}`
@@ -9,3 +10,18 @@ export const DEFAULT_DESCRIPTION = 'A command center for managing, editing, vali
 export const DEFAULT_LOGO = 'https://d2tyb7byn1fef9.cloudfront.net/ibi_group_black-512x512.png'
 export const DEFAULT_LOGO_SMALL = 'https://d2tyb7byn1fef9.cloudfront.net/ibi_group-128x128.png'
 export const DEFAULT_TITLE = 'Data Tools'
+
+export const RETRIEVAL_METHODS = Object.freeze({
+  MANUALLY_UPLOADED: 'MANUALLY_UPLOADED',
+  FETCHED_AUTOMATICALLY: 'FETCHED_AUTOMATICALLY',
+  PRODUCED_IN_HOUSE: 'PRODUCED_IN_HOUSE',
+  SERVICE_PERIOD_MERGE: 'SERVICE_PERIOD_MERGE',
+  REGIONAL_MERGE: 'REGIONAL_MERGE',
+  VERSION_CLONE: 'VERSION_CLONE'
+})
+
+export const FEED_TRANSFORMATION_TYPES = Object.freeze({
+  DELETE_RECORDS: 'DeleteRecordsTransformation',
+  REPLACE_FILE_FROM_VERSION: 'ReplaceFileFromVersionTransformation',
+  REPLACE_FILE_FROM_STRING: 'ReplaceFileFromStringTransformation'
+})

lib/common/util/config.js

@@ -37,7 +37,7 @@ export function getGtfsPlusSpec (): Array<GtfsSpecTable> {
   const CONFIG: DataToolsConfig = window.DT_CONFIG
   const GTFS_PLUS_SPEC = CONFIG.specifications.gtfsplus
   if (!GTFS_PLUS_SPEC) throw new Error('GTFS+ yml configuration file is not defined!')
-  return GTFS_PLUS_SPEC
+  return GTFS_PLUS_SPEC.sort((table1, table2) => table1.name.localeCompare(table2.name))
 }
 
 /**
@@ -58,7 +58,7 @@ export function getComponentMessages (
       return message
     } else {
       if (logWarning) console.warn(`Couldn't find message entry for ${componentName}.${path}`)
-      return `{${path}}`
+      return message || `{${path}}`
     }
   }
 }

lib/editor/actions/trip.js

@@ -92,6 +92,7 @@ export function fetchTripsForCalendar (
 ) {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     const namespace = getEditorNamespace(feedId, getState())
+    if (!namespace) throw new Error('Editor namespace is undefined!')
     // This fetches patterns on the pattern_id field (rather than ID) because
     // pattern_id is needed to join on the nested trips table
     const query = `query ($namespace: String, $pattern_id: [String], $service_id: [String]) {
@@ -277,6 +278,7 @@ export function fetchCalendarTripCountsForPattern (
 ) {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     const namespace = getEditorNamespace(feedId, getState())
+    if (!namespace) throw new Error('Editor namespace is undefined!')
     // This fetches patterns on the pattern_id field (rather than ID) because
     // pattern_id is needed to join on the nested trips table
     const query = `query ($namespace: String, $pattern_id: String) {
@@ -304,6 +306,7 @@ export function fetchCalendarTripCountsForPattern (
 export function fetchTripCounts (feedId: string) {
   return function (dispatch: dispatchFn, getState: getStateFn) {
     const namespace = getEditorNamespace(feedId, getState())
+    if (!namespace) throw new Error('Editor namespace is undefined!')
     // This fetches patterns on the pattern_id field (rather than ID) because
     // pattern_id is needed to join on the nested trips table
     const query = `query ($namespace: String) {

lib/editor/selectors/timetable.js

@@ -49,6 +49,13 @@ export const getTimetableColumns = createSelector(
         key: 'tripHeadsign',
         type: 'TEXT',
         placeholder: 'Destination via Transfer Center'
+      },
+      {
+        name: 'Trip Short Name',
+        width: 120,
+        key: 'tripShortName',
+        type: 'TEXT',
+        placeholder: '801'
       }
     ]
     if (pattern && pattern.patternStops) {

lib/gtfsplus/components/GtfsPlusVersionSummary.js

@@ -1,14 +1,24 @@
 // @flow
 
+import moment from 'moment'
 import React, {Component} from 'react'
-import {Panel, Row, Col, Table, Button, ButtonToolbar, Glyphicon, Alert} from 'react-bootstrap'
+import {
+  Alert,
+  Button,
+  ButtonToolbar,
+  Col,
+  Glyphicon,
+  Panel,
+  Row,
+  Table
+} from 'react-bootstrap'
 import {browserHistory, Link} from 'react-router'
-import moment from 'moment'
 
-import {getGtfsPlusSpec} from '../../common/util/config'
 import * as gtfsPlusActions from '../actions/gtfsplus'
 
+import {getGtfsPlusSpec} from '../../common/util/config'
 import type {Props as ContainerProps} from '../containers/ActiveGtfsPlusVersionSummary'
+import type {GtfsPlusValidationIssue} from '../../types'
 import type {GtfsPlusReducerState, ManagerUserState} from '../../types/reducers'
 
 type Props = ContainerProps & {
@@ -51,7 +61,7 @@ export default class GtfsPlusVersionSummary extends Component<Props, State> {
     return issuesForTable[tableId].length.toLocaleString()
   }
 
-  _getTableLevelIssues = (tableId: string) => {
+  _getTableLevelIssues = (tableId: string): ?Array<GtfsPlusValidationIssue> => {
     const {issuesForTable} = this.props
     if (!issuesForTable) return null
     if (!(tableId in issuesForTable)) return null
@@ -214,7 +224,7 @@ export default class GtfsPlusVersionSummary extends Component<Props, State> {
                               {tableLevelIssues.length} critical table issue(s):
                               <ul>
                                 {tableLevelIssues.map((issue, i) =>
-                                  <li key={i}>{issue.description}</li>)}
+                                  <li key={i}>{issue.fieldName}: {issue.description}</li>)}
                               </ul>
                             </small>
                             : null

lib/manager/actions/__tests__/__snapshots__/projects.js.snap

@@ -67,6 +67,7 @@ Object {
                 "retrievalMethod": "MANUALLY_UPLOADED",
                 "s3Url": null,
                 "snapshotVersion": null,
+                "transformRules": Array [],
                 "url": "http://mdtrip.org/googletransit/AnnapolisTransit/google_transit.zip",
                 "user": null,
               },
@@ -162,6 +163,7 @@ Object {
           "retrievalMethod": "MANUALLY_UPLOADED",
           "s3Url": null,
           "snapshotVersion": null,
+          "transformRules": Array [],
           "url": "http://mdtrip.org/googletransit/AnnapolisTransit/google_transit.zip",
           "user": null,
         },