Submission status via websockets

cypress/fixtures/2021-FRONTENDTESTBANK9999-no-edits.txt

@@ -0,0 +1,2 @@
+1|FRONTENDTESTBANK9999|2021|4|Mr. Smug Pockets|555-555-5555|[email protected]|1234 Hocus Potato Way|Tatertown|UT|84096|9|1|53-1111111|FRONTENDTESTBANK9999
+2|FRONTENDTESTBANK9999|FRONTENDTESTBANK9999JAJZMZSDXF8A57HP1HJZQOZ66|20200613|3|2|2|2|3|218910|1|20210213|1234 Hocus Potato Way|Tatertown|NM|14755|35003|35003976400|1||||||4||||||2|3|1||||||||8||||||||1|4|2|4|2|3|75|8888|100|0|NA|3|2|8888|8888|9||9||10|||||NA|NA|NA|NA|NA|NA|32|NA|NA|256|29|2|2|2|2|NA|2|2|4|NA|2|2|53535|3||||||1||||||2|2|2

nginx/nginx.conf

@@ -23,7 +23,7 @@ http {
   add_header Strict-Transport-Security 'max-age=31536000; includeSubDomains; preload';
 
   # CSP
-  add_header Content-Security-Policy "default-src 'self' blob:; script-src 'self' 'unsafe-inline' blob: data: https://tagmanager.google.com https://www.googletagmanager.com https://www.google-analytics.com https://*.cfpb.gov https://www.consumerfinance.gov; img-src 'self' blob: data: https://www.google-analytics.com https://raw.githubusercontent.com; style-src 'self' 'unsafe-inline'; font-src 'self' data:; object-src 'none'; frame-src 'self' https://www.youtube.com/ https://ffiec.cfpb.gov/; connect-src 'self' https://*.cfpb.gov https://www.consumerfinance.gov https://raw.githubusercontent.com https://ffiec-api.cfpb.gov https://ffiec.cfpb.gov https://*.mapbox.com https://www.google-analytics.com https://s3.amazonaws.com;";
+  add_header Content-Security-Policy "default-src 'self' blob:; script-src 'self' 'unsafe-inline' blob: data: https://tagmanager.google.com https://www.googletagmanager.com https://www.google-analytics.com https://*.cfpb.gov https://www.consumerfinance.gov; img-src 'self' blob: data: https://www.google-analytics.com https://raw.githubusercontent.com; style-src 'self' 'unsafe-inline'; font-src 'self' data:; object-src 'none'; frame-src 'self' https://www.youtube.com/ https://ffiec.cfpb.gov/; connect-src 'self' ws://*.cfpb.gov wss://*.cfpb.gov https://*.cfpb.gov https://www.consumerfinance.gov https://raw.githubusercontent.com https://ffiec-api.cfpb.gov https://ffiec.cfpb.gov https://*.mapbox.com https://www.google-analytics.com https://s3.amazonaws.com;";
 
   # Restrict referrer
   add_header Referrer-Policy "strict-origin";
@@ -82,6 +82,16 @@ http {
       try_files $uri =404;
     }
 
+    # Pass Websocket Upgrade requests to the backend
+    location ~* /v2/filing/(.*)/progress$ {
+      proxy_http_version 1.1;
+      proxy_set_header Upgrade $http_upgrade;
+      proxy_set_header Connection "upgrade";
+      proxy_set_header Host $host;
+      proxy_read_timeout 86400s;
+      proxy_send_timeout 86400s;
+    }
+
     # Whitelisted extensions
     location ~* \.(html|css|js|json|png|jpg|svg|eot|ttf|woff|woff2|map|ico)$ {
       limit_except GET {

src/App.jsx

@@ -67,6 +67,8 @@ const App = () => {
         <Route component={NotFound} />
       </Switch>
       {showFooter && <Footer config={config} />}
+      {console.log(window.location)
+      }
     </AppContext.Provider>
   )
 }

src/filing/actions/fetchUpload.js

@@ -6,10 +6,12 @@ import receiveUpload from './receiveUpload.js'
 import hasHttpError from './hasHttpError.js'
 import receiveUploadError from './receiveUploadError.js'
 import { error } from '../utils/log.js'
+import requestProcessingProgress from './requestProcessingProgress'
 
 export default function fetchUpload(file) {
   return dispatch => {
     dispatch(requestUpload())
+    dispatch(requestProcessingProgress())
 
     const data = new FormData()
     data.append('file', file)

src/filing/actions/listenForProgress.js

@@ -0,0 +1,180 @@
+import fetchEdits from './fetchEdits.js'
+import receiveSubmission from './receiveSubmission.js'
+import receiveError from './receiveError.js'
+import hasHttpError from './hasHttpError.js'
+import { getLatestSubmission } from '../api/api.js'
+import requestProcessingProgress from './requestProcessingProgress'
+import receiveProcessingProgress from './receiveProcessingProgress'
+import { error } from '../utils/log.js'
+import {
+  SYNTACTICAL_VALIDITY_EDITS,
+  NO_MACRO_EDITS,
+  MACRO_EDITS,
+  UPLOADED
+} from '../constants/statusCodes.js'
+import * as AccessToken from '../../common/api/AccessToken.js'
+
+let keepSocketAlive
+
+// Extract completion percentage
+export const parseProgress = string => {
+  if (!string.match(/^InProgress/)) return string
+  return string.match(/\d{1,}/)[0] 
+}
+
+const shouldSkipKey = key => ['done', 'fetched'].indexOf(key) > -1
+
+/* Websocket Listener */
+export default function listenForProgress() {
+  return (dispatch) => {
+    if (!window.location.pathname.match('/upload'))
+      return Promise.resolve(null)
+
+    return getLatestSubmission()
+      .then((json) => {
+        return hasHttpError(json).then((hasError) => {
+          if (hasError) {
+            dispatch(receiveError(json))
+            throw new Error(json && `${json.status}: ${json.statusText}`)
+          }
+          console.log('- Getting latest submission JSON')
+          return dispatch(receiveSubmission(json))
+        })
+      })
+      .then((json) => {
+        if (!json) {
+          console.warn('-- No submission JSON found, skipping WS connection')
+
+          return
+        }
+
+        const { status, id } = json
+        const { lei, period, sequenceNumber } = id
+        const { year, quarter } = period
+        const { code } = status
+
+        if (code >= UPLOADED) {
+          console.log('- Opening websocket to listen for progress...')
+
+          // Open a websocket and listen for updates
+          const wsBaseUrl = process.env.REACT_APP_ENVIRONMENT === 'CI'
+            ? `${window.location.hostname}:8080` // `IP-ADDRESS:8080`
+            : `${window.location.host}/v2/filing`
+
+          const socketType = window.location.protocol == 'https:' ? 'wss' : 'ws'
+
+          const wsProgressUrl = quarter
+            ? `/institutions/${lei}/filings/${year}/quarter/${quarter}/submissions/${sequenceNumber}/progress`
+            : `/institutions/${lei}/filings/${year}/submissions/${sequenceNumber}/progress`
+
+          let socket
+
+          try {
+            console.log(`-- Attempting connection to ${socketType}://${wsBaseUrl}${wsProgressUrl}`)
+            document.cookie = 'X-Authorization-Token=' + AccessToken.get() + '; path=/';
+            socket = new WebSocket(`${socketType}://${wsBaseUrl}${wsProgressUrl}`)
+          } catch (e) {
+            console.log(`--- Connection to ${socketType}://${wsBaseUrl}${wsProgressUrl} failed!`)
+            error(e)
+            console.log('---')
+          }
+
+          socket.onopen = () => {
+            console.log('-- Socket open! Sending Bearer token and then listening for Progress...')
+            dispatch(requestProcessingProgress())
+
+            // Keep connection alive by pinging server every 60s
+            keepSocketAlive = setInterval(() => {
+              const timestamp = new Date().toLocaleString('en-US', {
+                timeZone: 'America/New_York',
+              })
+              socket.send(JSON.stringify({ keepAlive: `${timestamp} ET` }))
+            }, 60000)
+          }
+
+          // Listen for messages
+          socket.onmessage = (event) => {
+            const data = event.data && JSON.parse(event.data)[1]
+
+            if (!data) return
+
+            const uploadStatus = {
+              syntactical: parseProgress(data.syntactical),
+              quality: parseProgress(data.quality),
+              macro: parseProgress(data.macro),
+            }
+
+            // No Syntactical errors and all others Completed
+            uploadStatus.done =
+              !!uploadStatus.syntactical.match(/Error/) ||
+              Object.keys(uploadStatus).every((key) => {
+                if (shouldSkipKey(key)) return true
+                return uploadStatus[key].match(/^Completed/)
+              })
+
+            console.log('> Progress: ', uploadStatus)
+
+            // Update Submission for status messaging
+            getLatestSubmission().then((json) => {
+              return hasHttpError(json).then((hasError) => {
+                if (hasError) {
+                  dispatch(receiveError(json))
+                  throw new Error(json && `${json.status}: ${json.statusText}`)
+                }
+                return dispatch(receiveSubmission(json))
+              })
+            })
+
+            if (uploadStatus.done) {
+              console.log('<<< Closing Socket!')
+              socket.close(1000, 'Done Processing')
+
+              // Save status updates
+              dispatch(receiveProcessingProgress({ status: uploadStatus }))
+
+              const hasEdits = Object.keys(uploadStatus).some((key) => {
+                if (shouldSkipKey(key)) return false
+                return uploadStatus[key].match(/Error/)
+              })
+
+              if (hasEdits) return dispatch(fetchEdits())
+            }
+            else {
+              dispatch(receiveProcessingProgress({ status: uploadStatus }))
+            }
+          }
+
+          // TODO: Anything special on close?
+          socket.onclose = (event) => {
+            if (event.wasClean) {
+              console.log(
+                `[close] Connection closed cleanly, code=${event.code} reason=${event.reason}`
+              )
+            }
+            else {
+              // e.g. server process killed or network down
+              // event.code is usually 1006 in this case
+              console.log('[socket onclose] Connection died', event)
+            }
+
+            clearInterval(keepSocketAlive)
+          }
+
+          // TODO: What to do on websocket error?
+          socket.onerror = function (error) {
+            console.log(`[socket onerror] ${error.message}`, error)
+          }
+        }
+        else if (
+          // only get edits when we've reached a terminal edit state
+          code === SYNTACTICAL_VALIDITY_EDITS ||
+          code === NO_MACRO_EDITS ||
+          code === MACRO_EDITS) {
+          return dispatch(fetchEdits())
+        }
+      })
+      .catch((err) => {
+        error(err)
+      })
+  }
+}

src/filing/actions/pollForProgress.js

@@ -1,5 +1,6 @@
 import fetchEdits from './fetchEdits.js'
 import receiveSubmission from './receiveSubmission.js'
+import listenForProgress from './listenForProgress'
 import receiveError from './receiveError.js'
 import hasHttpError from './hasHttpError.js'
 import { getLatestSubmission } from '../api/api.js'
@@ -9,7 +10,9 @@ import {
   SYNTACTICAL_VALIDITY_EDITS,
   NO_MACRO_EDITS,
   MACRO_EDITS,
-  VALIDATED
+  UPLOADED,
+  VALIDATED,
+  VALIDATING,
 } from '../constants/statusCodes.js'
 
 export const makeDurationGetter = () => {
@@ -51,6 +54,11 @@ export default function pollForProgress() {
       .then(json => {
         if (!json) return
         const { code } = json.status
+
+        // Switch to Websocket if file is uploaded and still processing
+        if(code >= UPLOADED && code <= VALIDATING) 
+          return dispatch(listenForProgress())
+
         if (
           // continue polling until we reach a status that isn't processing
           code !== PARSED_WITH_ERRORS &&

src/filing/actions/receiveProcessingProgress.js

@@ -0,0 +1,10 @@
+import * as types from '../constants'
+
+export default function receiveProcessingProgress({ status }) {
+  return (dispatch) => {
+    return dispatch({
+      type: types.RECEIVE_PROCESSING_PROGRESS,
+      status
+    })
+  }
+}

src/filing/actions/requestProcessingProgress.js

@@ -0,0 +1,7 @@
+import * as types from '../constants'
+
+export default function requestProcessingProgress() {
+  return {
+    type: types.REQUEST_PROCESSING_PROGRESS
+  }
+}

src/filing/constants/index.js

@@ -82,4 +82,7 @@ export const RECEIVE_LATEST_SUBMISSION = 'RECEIVE_LATEST_SUBMISSION'
 
 export const RECEIVE_INSTITUTION_NOT_FOUND = 'RECEIVE_INSTITUTION_NOT_FOUND'
 
-export const RECEIVE_FILING_PAGE = 'RECEIVE_FILING_PAGE'
+export const RECEIVE_FILING_PAGE = 'RECEIVE_FILING_PAGE'
+
+export const REQUEST_PROCESSING_PROGRESS = 'REQUEST_PROCESSING_PROGRESS'
+export const RECEIVE_PROCESSING_PROGRESS = 'RECEIVE_PROCESSING_PROGRESS'

src/filing/index.js

@@ -24,11 +24,13 @@ if (process.env.NODE_ENV !== 'production') {
   // use redux dev tools, extension required
   // see https://github.com/zalmoxisus/redux-devtools-extension#installation
   const { composeWithDevTools } = require('redux-devtools-extension')
+  const composeEnhancers = composeWithDevTools({trace: true, traceLimit: 25 })
+
   store = createStore(
     combineReducers({
       app: appReducer
     }),
-    composeWithDevTools(applyMiddleware(...middleware))
+    composeEnhancers(applyMiddleware(...middleware))
   )
 } else {
   store = createStore(

src/filing/institutions/Progress.jsx

@@ -20,31 +20,31 @@ const navMap = {
       submission.status.code === PARSED_WITH_ERRORS || submission.isStalled,
     isCompleted: submission => submission.status.code > UPLOADED,
     errorText: 'upload error',
-    completedText: 'uploaded'
+    completedText: () => 'uploaded'
   },
   'syntactical & validity edits': {
     isErrored: submission => submission.status.code === SYNTACTICAL_VALIDITY_EDITS,
     isCompleted: submission => submission.status.code >= NO_SYNTACTICAL_VALIDITY_EDITS,
     errorText: 'syntactical & validity edits',
-    completedText: 'no syntactical & validity edits'
+    completedText: () => 'no syntactical & validity edits'
   },
   'quality edits': {
     isErrored: submission => submission.qualityExists && !submission.qualityVerified,
     isCompleted: submission => submission.status.code >= NO_QUALITY_EDITS && (!submission.qualityExists || submission.qualityVerified),
     errorText: 'quality edits'  ,
-    completedText: 'quality edits verified'
+    completedText: (submission) => submission.qualityExists ? 'quality edits verified' : 'no quality edits'
   },
   'macro quality edits': {
     isErrored: submission => submission.macroExists && !submission.macroVerified,
     isCompleted: submission => (submission.status.code > MACRO_EDITS || submission.status.code === NO_MACRO_EDITS) && (!submission.macroExists || submission.macroVerified),
     errorText: 'macro quality edits',
-    completedText: 'macro quality edits verified'
+    completedText: (submission) => submission.macroExists ? 'macro quality edits verified' : 'no macro edits'
   },
   submission: {
     isReachable: submission => submission.status.code >= VALIDATED || submission.status.code === NO_MACRO_EDITS ,
     isErrored: () => false,
     isCompleted: submission => submission.status.code === SIGNED,
-    completedText: 'submitted'
+    completedText: () => 'submitted'
   }
 }
 
@@ -58,7 +58,7 @@ const renderNavItem = (submission, name, i) => {
     renderedName = navItem.errorText
     navClass = 'error'
   } else if (completed) {
-    renderedName = navItem.completedText
+    renderedName = navItem.completedText(submission)
     navClass = 'complete'
   }
 

src/filing/institutions/ViewButton.jsx

@@ -30,7 +30,7 @@ const InstitutionViewButton = ({ submission, institution, filingPeriod, isClosed
     text = 'View upload progress'
   } else if (code === PARSED_WITH_ERRORS) {
     text = 'Review formatting errors'
-  } else if (code < NO_SYNTACTICAL_VALIDITY_EDITS) {
+  } else if (code < NO_MACRO_EDITS) {
     text = 'View progress'
   } else if (code > VALIDATING && code < VALIDATED && code !== NO_MACRO_EDITS) {
     text = 'Review edits'

src/filing/reducers/index.js

@@ -20,6 +20,7 @@ import redirecting from './redirecting.js'
 import latestSubmissions from './latestSubmissions'
 import refiling from './refiling'
 import filingPeriodOptions from './filingPeriodOptions'
+import processProgress from './processProgress'
 
 export default combineReducers({
   lei,
@@ -41,5 +42,6 @@ export default combineReducers({
   redirecting,
   latestSubmissions,
   refiling,
-  filingPeriodOptions
+  filingPeriodOptions,
+  processProgress
 })