SIMSBIOHUB-374: BioHub Feature Submission Schema (#209)

* Submission Feature Migrations and Test Seeding
Remove DB_SCHEMA usage in most places.
Remove DB Views
Add Mock test data seed (disabled by default)

.github/workflows/cleanClosedPR.yml

@@ -86,5 +86,5 @@ jobs:
         env:
           POD_SELECTOR: biohub-platform
         run: |
-          oc --namespace a0ec71-dev   get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found" $1}' | bash
-          oc --namespace a0ec71-tools get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found" $1}' | bash
+          oc --namespace a0ec71-dev   get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found " $1}' | bash
+          oc --namespace a0ec71-tools get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found " $1}' | bash

.github/workflows/deploy.yml

@@ -5,6 +5,8 @@ name: PR-Based Deploy on OpenShift
 on:
   pull_request:
     types: [opened, reopened, synchronize, ready_for_review]
+    branches-ignore:
+      - prod
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.number }}
@@ -41,9 +43,11 @@ jobs:
       - checkEnv
     outputs:
       paths_result: ${{ steps.skip_check.outputs.paths_result }}
-      # Set to `true` if the latest commit message contains `ignore-skip` anywhere in the message.
+      # Set to `true` if the latest commit message contains `ignore-skip` anywhere in the message OR the base branch
+      # is dev, test, or prod.
       # Used to disable duplicate action skipping, if needed.
-      ignore_skip: ${{ contains(steps.head_commit_message.outputs.commit_message, 'ignore-skip') }}
+      ignore_skip: ${{ contains(steps.head_commit_message.outputs.commit_message, 'ignore-skip') || 
+        github.head_ref == 'dev' || github.head_ref == 'test' || github.head_ref == 'prod' }}
     steps:
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@v5

Makefile

@@ -56,6 +56,14 @@ clean: ## Closes and cleans (removes) all project containers
 	@echo "==============================================="
 	@docker-compose -f docker-compose.yml down -v --rmi all --remove-orphans
 
+prune: ## Deletes ALL docker artifacts (even those not associated to this project)
+	@echo -n "Delete ALL docker artifacts? [y/n] " && read ans && [ $${ans:-n} = y ]
+	@echo "==============================================="
+	@echo "Make: prune - deleting all docker artifacts"
+	@echo "==============================================="
+	@docker system prune --all --volumes -f
+	@docker volume prune --all -f
+
 ## ------------------------------------------------------------------------------
 ## Build/Run Postgres DB Commands
 ## - Builds all of the BioHub postgres db projects (db, db_setup)
@@ -82,13 +90,15 @@ build-backend: ## Builds all backend containers
 	@echo "==============================================="
 	@echo "Make: build-backend - building backend images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup api queue
+	@docker-compose -f docker-compose.yml build db db_setup api
+	# @docker-compose -f docker-compose.yml build db db_setup api queue
 
 run-backend: ## Runs all backend containers
 	@echo "==============================================="
 	@echo "Make: run-backend - running backend images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup api queue
+	@docker-compose -f docker-compose.yml up -d db db_setup api
+	# @docker-compose -f docker-compose.yml up -d db db_setup api queue
 
 ## ------------------------------------------------------------------------------
 ## Build/Run Backend+Web Commands (backend + web frontend)
@@ -99,13 +109,15 @@ build-web: ## Builds all backend+web containers
 	@echo "==============================================="
 	@echo "Make: build-web - building web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml build db db_setup api queue app
+	@docker-compose -f docker-compose.yml build db db_setup api app
+	# @docker-compose -f docker-compose.yml build db db_setup api queue app
 
 run-web: ## Runs all backend+web containers
 	@echo "==============================================="
 	@echo "Make: run-web - running web images"
 	@echo "==============================================="
-	@docker-compose -f docker-compose.yml up -d db db_setup api queue app
+	@docker-compose -f docker-compose.yml up -d db db_setup api app
+	# @docker-compose -f docker-compose.yml up -d db db_setup api queue app
 
 ## ------------------------------------------------------------------------------
 ## Commands to shell into the target container

api/.pipeline/config.js

@@ -62,9 +62,9 @@ const phases = {
     env: 'build',
     tz: config.timezone.api,
     branch: branch,
-    cpuRequest: '100m',
+    cpuRequest: '50m',
     cpuLimit: '1250m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '3Gi'
   },
   dev: {
@@ -87,9 +87,10 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.dev,
     logLevel: 'debug',
-    cpuRequest: '100m',
+    nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
     cpuLimit: '500m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '2Gi',
     replicas: '1',
     replicasMax: (isStaticDeployment && '2') || '1'
@@ -114,9 +115,10 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.test,
     logLevel: 'info',
-    cpuRequest: '200m',
+    nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
     cpuLimit: '1000m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '2Gi',
     replicas: '2',
     replicasMax: '3'
@@ -141,9 +143,10 @@ const phases = {
     tz: config.timezone.api,
     sso: config.sso.prod,
     logLevel: 'info',
-    cpuRequest: '200m',
+    nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes)
+    cpuRequest: '50m',
     cpuLimit: '1000m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '2Gi',
     replicas: '2',
     replicasMax: '3'

api/.pipeline/lib/api.deploy.js

@@ -31,22 +31,30 @@ const apiDeploy = async (settings) => {
         HOST: phases[phase].host,
         CHANGE_ID: phases.build.changeId || changeId,
         APP_HOST: phases[phase].appHost,
+        // Node
         NODE_ENV: phases[phase].env || 'dev',
+        NODE_OPTIONS: phases[phase].nodeOptions,
+        // Elastic Search
         ELASTICSEARCH_URL: phases[phase].elasticsearchURL,
         ELASTICSEARCH_EML_INDEX: phases[phase].elasticsearchEmlIndex,
         ELASTICSEARCH_TAXONOMY_INDEX: phases[phase].elasticsearchTaxonomyIndex,
+        // S3 (Object Store)
         S3_KEY_PREFIX: phases[phase].s3KeyPrefix,
+        // Database
         TZ: phases[phase].tz,
+        DB_SERVICE_NAME: `${phases[phase].dbName}-postgresql${phases[phase].suffix}`,
+        // Keycloak
         KEYCLOAK_ADMIN_USERNAME: phases[phase].sso.adminUserName,
         KEYCLOAK_SECRET: phases[phase].sso.keycloakSecret,
         KEYCLOAK_SECRET_ADMIN_PASSWORD: phases[phase].sso.keycloakSecretAdminPassword,
-        DB_SERVICE_NAME: `${phases[phase].dbName}-postgresql${phases[phase].suffix}`,
         KEYCLOAK_HOST: phases[phase].sso.url,
         KEYCLOAK_CLIENT_ID: phases[phase].sso.clientId,
         KEYCLOAK_REALM: phases[phase].sso.realm,
         KEYCLOAK_INTEGRATION_ID: phases[phase].sso.integrationId,
         KEYCLOAK_API_HOST: phases[phase].sso.apiHost,
+        // Log Level
         LOG_LEVEL: phases[phase].logLevel || 'info',
+        // OPenshift Resources
         CPU_REQUEST: phases[phase].cpuRequest,
         CPU_LIMIT: phases[phase].cpuLimit,
         MEMORY_REQUEST: phases[phase].memoryRequest,

api/.pipeline/templates/api.dc.yaml

@@ -35,6 +35,7 @@ parameters:
     description: Application Environment type variable
     required: true
     value: 'dev'
+  - name: NODE_OPTIONS
   - name: ELASTICSEARCH_URL
     description: Platform Elasticsearch URL
     required: true
@@ -223,6 +224,8 @@ objects:
                   value: ${CHANGE_ID}
                 - name: NODE_ENV
                   value: ${NODE_ENV}
+                - name: NODE_OPTIONS
+                  value: ${NODE_OPTIONS}
                 - name: ELASTICSEARCH_URL
                   value: ${ELASTICSEARCH_URL}
                 - name: ELASTICSEARCH_EML_INDEX

api/src/openapi/schemas/biohub-data-submission.ts

@@ -0,0 +1,58 @@
+export const BioHubDataSubmission = {
+  title: 'BioHub Data Submission',
+  type: 'object',
+  required: ['id', 'type', 'features'],
+  properties: {
+    id: {
+      title: 'Unique id of the submission',
+      type: 'string'
+    },
+    type: {
+      type: 'string',
+      enum: ['dataset']
+    },
+    properties: {
+      title: 'Feature properties',
+      type: 'object',
+      properties: {}
+    },
+    features: {
+      type: 'array',
+      items: {
+        $ref: '#/$defs/Feature'
+      }
+    }
+  },
+  $defs: {
+    Feature: {
+      title: 'BioHub Data Submission Feature',
+      type: 'object',
+      required: ['id', 'type', 'properties', 'features'],
+      properties: {
+        id: {
+          title: 'Unique id of the feature',
+          type: 'string'
+        },
+        type: {
+          title: 'Feature type',
+          type: 'string'
+        },
+        properties: {
+          title: 'Feature properties',
+          type: 'object',
+          properties: {}
+        },
+        features: {
+          title: 'Feature child features',
+          type: 'array',
+          items: {
+            $ref: '#/$defs/Feature'
+          }
+        }
+      },
+      additionalProperties: false
+    },
+    additionalProperties: false
+  },
+  additionalProperties: false
+};

app/.pipeline/config.js

@@ -62,7 +62,7 @@ const phases = {
     tag: tag,
     env: 'build',
     branch: branch,
-    cpuRequest: '100m',
+    cpuRequest: '50m',
     cpuLimit: '1000m',
     memoryRequest: '512Mi',
     memoryLimit: '3Gi'
@@ -86,7 +86,7 @@ const phases = {
     sso: config.sso.dev,
     cpuRequest: '50m',
     cpuLimit: (isStaticDeployment && '300m') || '200m',
-    memoryRequest: '50Mi',
+    memoryRequest: '100Mi',
     memoryLimit: (isStaticDeployment && '300Mi') || '200Mi',
     replicas: '1',
     replicasMax: (isStaticDeployment && '2') || '1'
@@ -107,7 +107,7 @@ const phases = {
     maxUploadFileSize,
     env: 'test',
     sso: config.sso.test,
-    cpuRequest: '100m',
+    cpuRequest: '50m',
     cpuLimit: '400m',
     memoryRequest: '100Mi',
     memoryLimit: '400Mi',
@@ -130,7 +130,7 @@ const phases = {
     maxUploadFileSize,
     env: 'prod',
     sso: config.sso.prod,
-    cpuRequest: '100m',
+    cpuRequest: '50m',
     cpuLimit: '400m',
     memoryRequest: '100Mi',
     memoryLimit: '400Mi',

database/.eslintrc

@@ -16,6 +16,15 @@
     "@typescript-eslint/no-explicit-any": "off",
     "@typescript-eslint/explicit-module-boundary-types": "off",
     "@typescript-eslint/ban-types": ["error", { "types": { "object": false, "extendDefaults": true } }],
+    "@typescript-eslint/ban-ts-comment": [
+      "error",
+      {
+        "ts-expect-error": false,
+        "ts-ignore": false,
+        "ts-nocheck": false,
+        "ts-check": false
+      }
+    ],
     "no-var": "error"
   }
 }

database/.pipeline/config.js

@@ -80,7 +80,7 @@ const phases = {
     volumeCapacity: (isStaticDeployment && '3Gi') || '500Mi',
     cpuRequest: '50m',
     cpuLimit: '200m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '2Gi',
     replicas: '1'
   },
@@ -97,9 +97,9 @@ const phases = {
     tz: config.timezone.db,
     dbSetupDockerfilePath: dbSetupDockerfilePath,
     volumeCapacity: '3Gi',
-    cpuRequest: '100m',
+    cpuRequest: '50m',
     cpuLimit: '500m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '3Gi',
     replicas: '1'
   },
@@ -116,9 +116,9 @@ const phases = {
     tz: config.timezone.db,
     dbSetupDockerfilePath: dbSetupDockerfilePath,
     volumeCapacity: '5Gi',
-    cpuRequest: '100m',
+    cpuRequest: '50m',
     cpuLimit: '500m',
-    memoryRequest: '512Mi',
+    memoryRequest: '100Mi',
     memoryLimit: '3Gi',
     replicas: '1'
   }

database/.pipeline/lib/db.setup.deploy.js

@@ -69,7 +69,6 @@ const dbSetupDeploy = async (settings) => {
         NODE_ENV: phases[phase].env || 'dev',
         DB_SERVICE_NAME: dbName,
         DB_SCHEMA: 'biohub',
-        DB_SCHEMA_DAPI_V1: 'biohub_dapi_v1',
         IMAGE: dbSetupImageStream.image.dockerImageReference
       }
     })

database/.pipeline/templates/db.setup.dc.yaml

@@ -24,9 +24,6 @@ parameters:
   - name: DB_SCHEMA
     description: 'Database schema'
     required: true
-  - name: DB_SCHEMA_DAPI_V1
-    description: 'Database api v1 schema'
-    required: true
   - name: NODE_ENV
     description: Application Environment type variable
     required: true
@@ -96,8 +93,6 @@ objects:
               value: ${VERSION}
             - name: DB_SCHEMA
               value: ${DB_SCHEMA}
-            - name: DB_SCHEMA_DAPI_V1
-              value: ${DB_SCHEMA_DAPI_V1}
           imagePullPolicy: Always
       restartPolicy: Never
       activeDeadlineSeconds: 900