WIP: Improve status code handling.

packages/model-runner/Makefile

@@ -37,8 +37,8 @@ release:
 run-local: build
 	rm -rf input log output export.zip
 	mkdir -p log input output
-	cp test/test-job.json input/inputFile.json
-	./bin/run-model test/test-job.json
+	jq .configuration test/test-job-mrc-ide-covidsim.json > input/inputFile.json
+	./bin/run-model test/test-job-mrc-ide-covidsim.json
 
 .PHONY: run-remote
 run-remote:

packages/model-runner/src/docker.ts

@@ -56,7 +56,7 @@ export async function pullImage(
 export async function runContainer(
   client: Dockerode,
   image: string
-): Promise<void> {
+): Promise<number> {
   // Setup the log file.
   const logFile = path.join(LOG_DIR, 'runner.log')
   const logWriteStream = fs.createWriteStream(logFile, {
@@ -82,18 +82,13 @@ export async function runContainer(
       },
       {},
       (err, result) => {
-        // TODO: Consider using dockerClient.modem.followProgress(stream, onFinished, onProgress)
-        // or some equivalent for result
         if (err) return reject(err)
-        logger.info(result)
-        // const output = result[0]
-        // const container = result[1]
         logger.info('Model exited with status code: %d', result.StatusCode)
-        // resolve(container.remove())
+        // result[1].remove()
         if (result.Error) {
           reject(result.Error)
         } else {
-          resolve()
+          resolve(result.StatusCode)
         }
       }
     )

packages/model-runner/src/main.ts

@@ -1,29 +1,31 @@
 import * as pino from 'pino'
 import * as path from 'path'
 import * as mkdirp from 'mkdirp'
-import { RunStatus, RequestInput } from '@covid-modeling/api'
+import { RequestInput, RunStatus } from '@covid-modeling/api'
 import { BlobStorage } from './blobstore'
 import { notifyUI } from './notify-ui'
 import { logger } from './logger'
 import {
-  LOG_DIR,
-  OUTPUT_DIR,
-  RUNNER_SHARED_SECRET,
   AZURE_STORAGE_ACCOUNT,
   AZURE_STORAGE_CONTAINER,
-  INPUT_DIR,
   HOST_WORK_DIR,
+  INPUT_DIR,
+  LOG_DIR,
+  OUTPUT_DIR,
+  RUNNER_SHARED_SECRET,
 } from './config'
 import * as Dockerode from 'dockerode'
 import * as fs from 'fs'
 import * as crypto from 'crypto'
 import { createExportZip } from './export'
 import * as docker from './docker'
-import { enforceRunnerInputSchema, enforceOutputSchema } from './schema'
+import { enforceOutputSchema, enforceRunnerInputSchema } from './schema'
+import { convertModelStatusCode, StatusCodes } from './status-codes'
 
 let inputID: string | number | null = null
 let callbackURL: string | null = null
 let modelSlug: string | null = null
+let statusCode: number = StatusCodes.Ok
 
 const handleRejection: NodeJS.UnhandledRejectionListener = err => {
   const finalLogger = pino.final(logger)
@@ -35,10 +37,10 @@ const handleRejection: NodeJS.UnhandledRejectionListener = err => {
       resultsLocation: '',
       exportLocation: '',
     }).finally(() => {
-      process.exit(1)
+      process.exit(StatusCodes.UnknownError)
     })
   } else {
-    process.exit(1)
+    process.exit(StatusCodes.UnknownError)
   }
 }
 
@@ -102,16 +104,29 @@ async function main() {
     logger.info(JSON.stringify(input.configuration))
 
     logger.info('Running container: %s', dockerImage)
-    await docker
-      .runContainer(dockerClient, dockerImage)
-      .then(data => {
-        // TODO: we seem to have lost the reference to the container
-        // logger.info('container %d removed', data)
+
+    statusCode = convertModelStatusCode(
+      await docker.runContainer(dockerClient, dockerImage).catch(err => {
+        logger.error(err)
+        return StatusCodes.DockerError
       })
-      .catch(err => logger.error(err))
+    )
 
     logger.info('Finished model run')
 
+    if (statusCode < 0) {
+      if (input.callbackURL) {
+        await notifyUI(input.callbackURL, RUNNER_SHARED_SECRET, input.id, {
+          modelSlug,
+          status: RunStatus.Failed,
+          resultsLocation: '',
+          exportLocation: '',
+          // TODO: Send back the status code or some other indicator of the type of failure.
+        })
+      }
+      process.exit(statusCode)
+    }
+
     logger.info('Creating export zip.')
     const exportZipFile = 'export.zip'
     await createExportZip([INPUT_DIR, OUTPUT_DIR], exportZipFile)
@@ -162,6 +177,7 @@ async function main() {
   } catch (err) {
     handleRejection(err, Promise.resolve())
   }
+  process.exit(statusCode)
 }
 
 function uniqueId(runInput: RequestInput, imageId: string): string {
@@ -183,7 +199,7 @@ Usage:
     `.trim()
   )
 
-  process.exit(1)
+  process.exit(StatusCodes.InvalidArguments)
 }
 
 main()

packages/model-runner/src/status-codes.ts

@@ -0,0 +1,10 @@
+export enum StatusCodes {
+  Ok = 0,
+  InvalidArguments = 1,
+  UnknownError = 2,
+  DockerError = 5,
+}
+
+export function convertModelStatusCode(code: number): number {
+  return -code
+}