SIMSBIOHUB-365: SIMS -> BIOHUB Intake Dataset

api/package.json

@@ -31,6 +31,7 @@
     "@elastic/elasticsearch": "~8.1.0",
     "adm-zip": "~0.5.5",
     "ajv": "~8.6.3",
+    "avj": "0.0.0",
     "aws-sdk": "^2.1391.0",
     "axios": "~0.21.4",
     "clamdjs": "~1.0.2",
@@ -41,6 +42,7 @@
     "fast-json-patch": "~3.1.1",
     "fast-xml-parser": "~4.1.3",
     "fastq": "^1.15.0",
+    "json-schema-traverse": "^1.0.0",
     "jsonpath-plus": "~7.2.0",
     "jsonwebtoken": "~8.5.1",
     "jwks-rsa": "~2.0.5",

api/src/database/db.test.ts

@@ -421,7 +421,7 @@ describe('db', () => {
       db.getServiceAccountDBConnection(SOURCE_SYSTEM['SIMS-SVC-4464']);
 
       expect(getDBConnectionStub).to.have.been.calledWith({
-        preferred_username: `service-account-${SOURCE_SYSTEM['SIMS-SVC-4464']}@${SYSTEM_IDENTITY_SOURCE.SYSTEM}`,
+        preferred_username: `service-account-${SOURCE_SYSTEM['SIMS-SVC-4464']}`,
         identity_provider: SYSTEM_IDENTITY_SOURCE.SYSTEM
       });
 

api/src/database/db.ts

@@ -421,7 +421,7 @@ export const getAPIUserDBConnection = (): IDBConnection => {
  */
 export const getServiceAccountDBConnection = (sourceSystem: SOURCE_SYSTEM): IDBConnection => {
   return getDBConnection({
-    preferred_username: `service-account-${sourceSystem}@${SYSTEM_IDENTITY_SOURCE.SYSTEM}`,
+    preferred_username: `service-account-${sourceSystem}`,
     identity_provider: SYSTEM_IDENTITY_SOURCE.SYSTEM
   });
 };

api/src/openapi/root-api-doc.ts

@@ -140,6 +140,34 @@ export const rootAPIDoc = {
             }
           }
         }
+      },
+      SubmissionFeature: {
+        title: 'BioHub Data Submission Feature',
+        type: 'object',
+        required: ['id', 'type', 'properties', 'features'],
+        properties: {
+          id: {
+            title: 'Unique id of the feature',
+            type: 'string'
+          },
+          type: {
+            title: 'Feature type',
+            type: 'string'
+          },
+          properties: {
+            title: 'Feature properties',
+            type: 'object',
+            properties: {}
+          },
+          features: {
+            title: 'Feature child features',
+            type: 'array',
+            items: {
+              $ref: '#/components/schemas/SubmissionFeature'
+            }
+          }
+        },
+        additionalProperties: false
       }
     }
   }

api/src/openapi/schemas/biohub-data-submission.ts

@@ -1,7 +1,7 @@
 export const BioHubDataSubmission = {
   title: 'BioHub Data Submission',
   type: 'object',
-  required: ['id', 'type', 'features'],
+  required: ['id', 'type', 'properties', 'features'],
   properties: {
     id: {
       title: 'Unique id of the submission',
@@ -19,40 +19,9 @@ export const BioHubDataSubmission = {
     features: {
       type: 'array',
       items: {
-        $ref: '#/$defs/Feature'
+        $ref: '#/components/schemas/SubmissionFeature'
       }
     }
   },
-  $defs: {
-    Feature: {
-      title: 'BioHub Data Submission Feature',
-      type: 'object',
-      required: ['id', 'type', 'properties', 'features'],
-      properties: {
-        id: {
-          title: 'Unique id of the feature',
-          type: 'string'
-        },
-        type: {
-          title: 'Feature type',
-          type: 'string'
-        },
-        properties: {
-          title: 'Feature properties',
-          type: 'object',
-          properties: {}
-        },
-        features: {
-          title: 'Feature child features',
-          type: 'array',
-          items: {
-            $ref: '#/$defs/Feature'
-          }
-        }
-      },
-      additionalProperties: false
-    },
-    additionalProperties: false
-  },
   additionalProperties: false
 };

api/src/paths/dataset/intake.ts

@@ -0,0 +1,137 @@
+import { RequestHandler } from 'express';
+import { Operation } from 'express-openapi';
+import { SOURCE_SYSTEM } from '../../constants/database';
+import { getServiceAccountDBConnection } from '../../database/db';
+import { HTTP400 } from '../../errors/http-error';
+import { defaultErrorResponses } from '../../openapi/schemas/http-responses';
+import { authorizeRequestHandler } from '../../request-handlers/security/authorization';
+import { SubmissionService } from '../../services/submission-service';
+import { ValidationService } from '../../services/validation-service';
+import { getKeycloakSource } from '../../utils/keycloak-utils';
+import { getLogger } from '../../utils/logger';
+
+const defaultLog = getLogger('paths/dataset/intake');
+
+export const POST: Operation = [
+  authorizeRequestHandler(() => {
+    return {
+      and: [
+        {
+          validServiceClientIDs: [SOURCE_SYSTEM['SIMS-SVC-4464']],
+          discriminator: 'ServiceClient'
+        }
+      ]
+    };
+  }),
+  datasetIntake()
+];
+
+POST.apiDoc = {
+  description: 'Submit dataset to BioHub',
+  tags: ['dataset'],
+  security: [
+    {
+      Bearer: []
+    }
+  ],
+  requestBody: {
+    content: {
+      'application/json': {
+        schema: {
+          title: 'BioHub Data Submission',
+          type: 'object',
+          required: ['id', 'type', 'properties', 'features'],
+          properties: {
+            id: {
+              title: 'Unique id of the submission',
+              type: 'string'
+            },
+            type: {
+              type: 'string',
+              enum: ['dataset']
+            },
+            properties: {
+              title: 'Dataset properties',
+              type: 'object',
+              properties: {}
+            },
+            features: {
+              type: 'array',
+              items: {
+                $ref: '#/components/schemas/SubmissionFeature'
+              },
+              additionalProperties: false
+            }
+          },
+          additionalProperties: false
+        }
+      }
+    }
+  },
+  responses: {
+    200: {
+      description: 'Submission OK',
+      content: {
+        'application/json': {
+          schema: {
+            type: 'object',
+            required: ['submission_id'],
+            properties: {
+              submission_id: {
+                type: 'integer'
+              }
+            }
+          }
+        }
+      }
+    },
+    ...defaultErrorResponses
+  }
+};
+
+export function datasetIntake(): RequestHandler {
+  return async (req, res) => {
+    const sourceSystem = getKeycloakSource(req['keycloak_token']);
+
+    if (!sourceSystem) {
+      throw new HTTP400('Failed to identify known submission source system', [
+        'token did not contain a clientId/azp or clientId/azp value is unknown'
+      ]);
+    }
+
+    const dataset = {
+      ...req.body,
+      properties: { ...req.body.properties, additionalInformation: req.body.properties.additionalInformation }
+    };
+    const id = req.body.id;
+
+    const connection = getServiceAccountDBConnection(sourceSystem);
+
+    try {
+      await connection.open();
+
+      const submissionService = new SubmissionService(connection);
+      const validationService = new ValidationService(connection);
+
+      // validate the dataset submission
+      if (!(await validationService.validateDatasetSubmission(dataset))) {
+        throw new HTTP400('Invalid dataset submission');
+      }
+
+      // insert the submission record
+      const response = await submissionService.insertSubmissionRecordWithPotentialConflict(id);
+
+      // insert each submission feature record
+      await submissionService.insertSubmissionFeatureRecords(response.submission_id, dataset.features);
+
+      await connection.commit();
+      res.status(200).json(response);
+    } catch (error) {
+      defaultLog.error({ label: 'datasetIntake', message: 'error', error });
+      await connection.rollback();
+      throw error;
+    } finally {
+      connection.release();
+    }
+  };
+}

api/src/repositories/submission-repository.test.ts

@@ -490,10 +490,7 @@ describe('SubmissionRepository', () => {
 
       const submissionRepository = new SubmissionRepository(mockDBConnection);
 
-      const response = await submissionRepository.insertSubmissionRecordWithPotentialConflict({
-        uuid: 'aaaa',
-        source_transform_id: 1
-      });
+      const response = await submissionRepository.insertSubmissionRecordWithPotentialConflict('aaaa');
 
       expect(response).to.eql({
         uuid: 'aaaa',
@@ -510,10 +507,7 @@ describe('SubmissionRepository', () => {
       const submissionRepository = new SubmissionRepository(mockDBConnection);
 
       try {
-        await submissionRepository.insertSubmissionRecordWithPotentialConflict({
-          uuid: 'bbbb',
-          source_transform_id: 3
-        });
+        await submissionRepository.insertSubmissionRecordWithPotentialConflict('aaaa');
         expect.fail();
       } catch (actualError) {
         expect((actualError as ApiExecuteSQLError).message).to.equal('Failed to get or insert submission record');