Remove source_id from submission.

Make uuid a source submitted field for submission. make source_id optional for submission_features.
bcgov · Jan 16, 2024 · 2f1ce3b · 2f1ce3b
1 parent 5acf91a
commit 2f1ce3b
Show file tree

Hide file tree

Showing 15 changed files with 26 additions and 69 deletions.
diff --git a/api/src/openapi/root-api-doc.ts b/api/src/openapi/root-api-doc.ts
@@ -144,7 +144,7 @@ export const rootAPIDoc = {
       SubmissionFeature: {
         title: 'BioHub Data Submission Feature',
         type: 'object',
-        required: ['id', 'type', 'properties', 'features'],
+        required: ['type', 'properties', 'features'],
         properties: {
           id: {
             description: 'Unique id of the feature',

diff --git a/api/src/paths/administrative/submission/published.test.ts b/api/src/paths/administrative/submission/published.test.ts
@@ -51,7 +51,6 @@ describe('getPublishedSubmissionsForAdmins', () => {
       {
         submission_id: 1,
         uuid: '123-456-789',
-        source_id: '321',
         system_user_id: 3,
         security_review_timestamp: '2023-12-12',
         submitted_timestamp: '2023-12-12',
@@ -71,7 +70,6 @@ describe('getPublishedSubmissionsForAdmins', () => {
       {
         submission_id: 2,
         uuid: '789-456-123',
-        source_id: '123',
         system_user_id: 3,
         security_review_timestamp: '2023-12-12',
         submitted_timestamp: '2023-12-12',

diff --git a/api/src/paths/administrative/submission/reviewed.test.ts b/api/src/paths/administrative/submission/reviewed.test.ts
@@ -51,7 +51,6 @@ describe('getReviewedSubmissionsForAdmins', () => {
       {
         submission_id: 1,
         uuid: '123-456-789',
-        source_id: '321',
         security_review_timestamp: '2023-12-12',
         submitted_timestamp: '2023-12-12',
         publish_timestamp: '2023-12-12',
@@ -71,7 +70,6 @@ describe('getReviewedSubmissionsForAdmins', () => {
       {
         submission_id: 2,
         uuid: '789-456-123',
-        source_id: '321',
         security_review_timestamp: '2023-12-12',
         submitted_timestamp: '2023-12-12',
         system_user_id: 3,

diff --git a/api/src/paths/administrative/submission/unreviewed.test.ts b/api/src/paths/administrative/submission/unreviewed.test.ts
@@ -51,7 +51,6 @@ describe('getUnreviewedSubmissionsForAdmins', () => {
       {
         submission_id: 1,
         uuid: '123-456-789',
-        source_id: '321',
         security_review_timestamp: null,
         submitted_timestamp: '2023-12-12',
         publish_timestamp: '2023-12-12',
@@ -71,7 +70,6 @@ describe('getUnreviewedSubmissionsForAdmins', () => {
       {
         submission_id: 2,
         uuid: '789-456-123',
-        source_id: '321',
         security_review_timestamp: null,
         submitted_timestamp: '2023-12-12',
         publish_timestamp: '2023-12-12',

diff --git a/api/src/paths/administrative/submission/{submissionId}/index.test.ts b/api/src/paths/administrative/submission/{submissionId}/index.test.ts
@@ -51,7 +51,6 @@ describe('patchSubmissionRecord', () => {
     const mockSubmissionRecord: SubmissionRecord = {
       submission_id: 3,
       uuid: '999-456-123',
-      source_id: '321',
       security_review_timestamp: '2023-12-12',
       publish_timestamp: '2023-12-12',
       submitted_timestamp: '2023-12-12',

diff --git a/api/src/paths/submission/intake.test.ts b/api/src/paths/submission/intake.test.ts
@@ -111,7 +111,6 @@ describe('intake', () => {
         .resolves({
           submission_id: submissionId,
           uuid: '123-456-789',
-          source_id: '123',
           security_review_timestamp: '2023-12-12',
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,

diff --git a/api/src/paths/submission/intake.ts b/api/src/paths/submission/intake.ts
@@ -44,6 +44,7 @@ POST.apiDoc = {
           properties: {
             id: {
               description: 'Unique id of the submission.',
+              format: 'uuid',
               type: 'string'
             },
             name: {
@@ -127,7 +128,7 @@ export function submissionIntake(): RequestHandler {
       ]);
     }
 
-    const submissionSourceId = req.body.id;
+    const submissionUuid = req.body.id;
     const submissionName = req.body.name;
     const submissionDescription = req.body.description;
 
@@ -149,7 +150,7 @@ export function submissionIntake(): RequestHandler {
 
       // insert the submission record
       const submissionRecord = await submissionService.insertSubmissionRecordWithPotentialConflict(
-        submissionSourceId,
+        submissionUuid,
         submissionName,
         submissionDescription,
         serviceClientSystemUser.system_user_id,

diff --git a/api/src/repositories/submission-repository.test.ts b/api/src/repositories/submission-repository.test.ts
@@ -916,7 +916,6 @@ describe('SubmissionRepository', () => {
         {
           submission_id: 1,
           uuid: '123-456-789',
-          source_id: '123',
           security_review_timestamp: null,
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -933,7 +932,6 @@ describe('SubmissionRepository', () => {
         {
           submission_id: 2,
           uuid: '789-456-123',
-          source_id: '456',
           security_review_timestamp: null,
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -971,7 +969,6 @@ describe('SubmissionRepository', () => {
         {
           submission_id: 1,
           uuid: '123-456-789',
-          source_id: '123',
           security_review_timestamp: '2023-12-12',
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -988,7 +985,6 @@ describe('SubmissionRepository', () => {
         {
           submission_id: 2,
           uuid: '789-456-123',
-          source_id: '321',
           security_review_timestamp: '2023-12-12',
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -1026,7 +1022,6 @@ describe('SubmissionRepository', () => {
         {
           submission_id: 1,
           uuid: '123-456-789',
-          source_id: '321',
           security_review_timestamp: '2023-12-12',
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -1043,7 +1038,6 @@ describe('SubmissionRepository', () => {
         {
           submission_id: 2,
           uuid: '789-456-123',
-          source_id: '456',
           security_review_timestamp: '2023-12-12',
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -1080,7 +1074,6 @@ describe('SubmissionRepository', () => {
         {
           submission_id: 1,
           uuid: '123-456-789',
-          source_id: '123',
           security_review_timestamp: '2023-12-12',
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -1098,7 +1091,6 @@ describe('SubmissionRepository', () => {
         {
           submission_id: 2,
           uuid: '789-456-123',
-          source_id: '456',
           security_review_timestamp: '2023-12-12',
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -1117,7 +1109,6 @@ describe('SubmissionRepository', () => {
         {
           submission_id: 3,
           uuid: '999-456-123',
-          source_id: '789',
           security_review_timestamp: '2023-12-12',
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -1381,7 +1372,6 @@ describe('SubmissionRepository', () => {
         const mockSubmissionRecord: SubmissionRecord = {
           submission_id: 1,
           uuid: '123-456-789',
-          source_id: '123',
           security_review_timestamp: '2023-12-12',
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -1418,7 +1408,6 @@ describe('SubmissionRepository', () => {
         const mockSubmissionRecord: SubmissionRecord = {
           submission_id: 1,
           uuid: '123-456-789',
-          source_id: '456',
           security_review_timestamp: null,
           submitted_timestamp: '2023-12-12',
           system_user_id: 3,
@@ -1701,7 +1690,6 @@ describe('SubmissionRepository', () => {
       const mockResponse: SubmissionRecordPublished = {
         submission_id: 1,
         uuid: 'string',
-        source_id: '123',
         security_review_timestamp: null,
         publish_timestamp: 'string',
         submitted_timestamp: 'string',

diff --git a/api/src/repositories/submission-repository.ts b/api/src/repositories/submission-repository.ts
@@ -22,7 +22,7 @@ export interface IDatasetsForReview {
 }
 
 export interface ISubmissionFeature {
-  id: string;
+  id: string | null;
   type: string;
   properties: Record<string, unknown>;
   features: ISubmissionFeature[];
@@ -77,7 +77,7 @@ export const SubmissionFeatureRecord = z.object({
   uuid: z.string(),
   submission_id: z.number(),
   feature_type_id: z.number(),
-  source_id: z.string(),
+  source_id: z.string().nullable(),
   data: z.record(z.any()),
   parent_submission_feature_id: z.number().nullable(),
   record_effective_date: z.string(),
@@ -256,7 +256,6 @@ export interface ISubmissionObservationRecord {
 export const SubmissionRecord = z.object({
   submission_id: z.number(),
   uuid: z.string(),
-  source_id: z.string(),
   security_review_timestamp: z.string().nullable(),
   submitted_timestamp: z.string(),
   system_user_id: z.number(),
@@ -370,30 +369,30 @@ export class SubmissionRepository extends BaseRepository {
   /**
    * Insert a new submission record.
    *
-   * @param {string} sourceId
+   * @param {string} uuid
    * @param {string} name
    * @param {string} description
    * @param {string} userIdentifier
    * @return {*}  {Promise<SubmissionRecord>}
    * @memberof SubmissionRepository
    */
   async insertSubmissionRecordWithPotentialConflict(
-    sourceId: string,
+    uuid: string,
     name: string,
     description: string,
     systemUserId: number,
     systemUserIdentifier: string
   ): Promise<SubmissionRecord> {
     const sqlStatement = SQL`
       INSERT INTO submission (
-        source_id,
+        uuid,
         submitted_timestamp,
         name,
         description,
         system_user_id,
         source_system
       ) VALUES (
-        ${sourceId},
+        ${uuid},
         now(),
         ${name},
         ${description},
@@ -420,15 +419,15 @@ export class SubmissionRepository extends BaseRepository {
    * Insert a new submission feature record.
    *
    * @param {number} submissionId
-   * @param {string} sourceId
+   * @param {(string | null)} featureSourceId
    * @param {string} featureTypeName
    * @param {ISubmissionFeature['properties']} featureProperties
    * @return {*}  {Promise<{ submission_feature_id: number }>}
    * @memberof SubmissionRepository
    */
   async insertSubmissionFeatureRecord(
     submissionId: number,
-    featureSourceId: string,
+    featureSourceId: string | null,
     featureTypeName: string,
     featureProperties: ISubmissionFeature['properties']
   ): Promise<{ submission_feature_id: number }> {
@@ -1220,7 +1219,7 @@ export class SubmissionRepository extends BaseRepository {
     const sqlStatement = SQL`
       WITH w_unique_submissions as (
         SELECT
-          DISTINCT ON (submission.source_id) submission.*,
+          DISTINCT ON (submission.uuid) submission.*,
           submission_feature.feature_type_id as root_feature_type_id,
           feature_type.name as root_feature_type_name,
           ${SECURITY_APPLIED_STATUS.PENDING} as security
@@ -1239,7 +1238,7 @@ export class SubmissionRepository extends BaseRepository {
         AND
           submission_feature.parent_submission_feature_id IS NULL
         ORDER BY
-          submission.source_id, submission.submission_id DESC
+          submission.uuid, submission.submission_id DESC
       )
       SELECT
         *
@@ -1263,7 +1262,7 @@ export class SubmissionRepository extends BaseRepository {
     const sqlStatement = SQL`
       WITH w_unique_submissions as (
         SELECT
-          DISTINCT ON (submission.source_id) submission.*,
+          DISTINCT ON (submission.uuid) submission.*,
           submission_feature.feature_type_id as root_feature_type_id,
           feature_type.name as root_feature_type_name,
           CASE
@@ -1297,7 +1296,7 @@ export class SubmissionRepository extends BaseRepository {
           submission_feature.feature_type_id,
           feature_type.name
         ORDER BY
-          submission.source_id, submission.submission_id DESC
+          submission.uuid, submission.submission_id DESC
       )
       SELECT
         *

diff --git a/api/src/services/artifact-service.ts b/api/src/services/artifact-service.ts
@@ -68,8 +68,7 @@ export class ArtifactService extends DBService {
     // Generate S3 key
     const artifactS3Key = generateSubmissionFeatureS3FileKey({
       submissionId: artifactFeatureSubmission.submission_id,
-      submissionFeatureId: artifactFeatureSubmission.submission_feature_id,
-      artifactId: artifactFeatureSubmission.source_id // TODO is this needed. The submission_feature_id should be enough.
+      submissionFeatureId: artifactFeatureSubmission.submission_feature_id
     });
 
     defaultLog.debug({ label: 'uploadSubmissionFeatureArtifact', message: 'S3 key', artifactS3Key });