From 46daa6e4eccc4f65a0b5021f457fc05bc24d1219 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Wed, 15 Nov 2023 08:51:17 -0800 Subject: [PATCH 01/20] Submission Feature Migrations and Test Seeding Remove DB_SCHEMA usage in most places. Remove DB Views Add Mock test data seed (disabled by default) --- Makefile | 8 + api/package-lock.json | 121 ++---- database/.eslintrc | 9 + database/.pipeline/lib/db.setup.deploy.js | 1 - database/.pipeline/templates/db.setup.dc.yaml | 5 - database/package-lock.json | 112 +++++ database/package.json | 2 + .../src/migrations/20220225205948_release.ts | 33 +- ...2015_add_record_sims_eml_transformation.ts | 7 +- ...patial_transform_eml_project_boundaries.ts | 7 +- ...transform_eml_project_boundary_centroid.ts | 7 +- ...0_update_record_sims_eml_transformation.ts | 7 +- ...5_remove_unique_id_artifact_persecution.ts | 7 - .../20231109000001_feature_tables.ts | 269 ++++++++++++ .../20231109000002_search_tables.ts | 295 +++++++++++++ ...09000003_populate_feature_lookup_tables.ts | 94 +++++ database/src/seeds/01_db_system_users.ts | 5 +- ..._security_transform_eml_dwc_occurrences.ts | 7 +- .../src/seeds/03_dwc_spatial_transform.ts | 7 +- database/src/seeds/04_mock_test_data.ts | 387 ++++++++++++++++++ docker-compose.yml | 5 +- env_config/env.docker | 5 +- 22 files changed, 1230 insertions(+), 170 deletions(-) create mode 100644 database/src/migrations/20231109000001_feature_tables.ts create mode 100644 database/src/migrations/20231109000002_search_tables.ts create mode 100644 database/src/migrations/20231109000003_populate_feature_lookup_tables.ts create mode 100644 database/src/seeds/04_mock_test_data.ts diff --git a/Makefile b/Makefile index 1c603c9cc..8cc2ba40c 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,14 @@ clean: ## Closes and cleans (removes) all project containers @echo "===============================================" @docker-compose -f docker-compose.yml down -v --rmi all --remove-orphans +prune: ## Deletes ALL docker artifacts (even those not associated to this project) + @echo -n "Delete ALL docker artifacts? [y/n] " && read ans && [ $${ans:-n} = y ] + @echo "===============================================" + @echo "Make: prune - deleting all docker artifacts" + @echo "===============================================" + @docker system prune --all --volumes -f + @docker volume prune --all -f + ## ------------------------------------------------------------------------------ ## Build/Run Postgres DB Commands ## - Builds all of the BioHub postgres db projects (db, db_setup) diff --git a/api/package-lock.json b/api/package-lock.json index bc6de85dc..7e0745025 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -538,36 +538,6 @@ "strip-ansi": "^7.0.1" } }, - "string-width-cjs": { - "version": "npm:string-width@4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - } - } - }, "strip-ansi": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", @@ -576,21 +546,6 @@ "ansi-regex": "^6.0.1" } }, - "strip-ansi-cjs": { - "version": "npm:strip-ansi@6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - }, - "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - } - } - }, "wrap-ansi": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", @@ -600,54 +555,6 @@ "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } - }, - "wrap-ansi-cjs": { - "version": "npm:wrap-ansi@7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" - }, - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "requires": { - "color-convert": "^2.0.1" - } - }, - "emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" - }, - "string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - } - } } } }, @@ -8807,6 +8714,16 @@ "strip-ansi": "^6.0.1" } }, + "string-width-cjs": { + "version": "npm:string-width@4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, "string.prototype.padend": { "version": "3.1.4", "resolved": "https://registry.npmjs.org/string.prototype.padend/-/string.prototype.padend-3.1.4.tgz", @@ -8864,6 +8781,14 @@ "ansi-regex": "^5.0.1" } }, + "strip-ansi-cjs": { + "version": "npm:strip-ansi@6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, "strip-bom": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", @@ -9901,6 +9826,16 @@ "strip-ansi": "^6.0.0" } }, + "wrap-ansi-cjs": { + "version": "npm:wrap-ansi@7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/database/.eslintrc b/database/.eslintrc index 2a457fb10..8e93220f7 100644 --- a/database/.eslintrc +++ b/database/.eslintrc @@ -16,6 +16,15 @@ "@typescript-eslint/no-explicit-any": "off", "@typescript-eslint/explicit-module-boundary-types": "off", "@typescript-eslint/ban-types": ["error", { "types": { "object": false, "extendDefaults": true } }], + "@typescript-eslint/ban-ts-comment": [ + "error", + { + "ts-expect-error": false, + "ts-ignore": false, + "ts-nocheck": false, + "ts-check": false + } + ], "no-var": "error" } } diff --git a/database/.pipeline/lib/db.setup.deploy.js b/database/.pipeline/lib/db.setup.deploy.js index e59288377..ba97b6181 100644 --- a/database/.pipeline/lib/db.setup.deploy.js +++ b/database/.pipeline/lib/db.setup.deploy.js @@ -69,7 +69,6 @@ const dbSetupDeploy = async (settings) => { NODE_ENV: phases[phase].env || 'dev', DB_SERVICE_NAME: dbName, DB_SCHEMA: 'biohub', - DB_SCHEMA_DAPI_V1: 'biohub_dapi_v1', IMAGE: dbSetupImageStream.image.dockerImageReference } }) diff --git a/database/.pipeline/templates/db.setup.dc.yaml b/database/.pipeline/templates/db.setup.dc.yaml index f0dba10ca..2cfdb1b87 100644 --- a/database/.pipeline/templates/db.setup.dc.yaml +++ b/database/.pipeline/templates/db.setup.dc.yaml @@ -24,9 +24,6 @@ parameters: - name: DB_SCHEMA description: 'Database schema' required: true - - name: DB_SCHEMA_DAPI_V1 - description: 'Database api v1 schema' - required: true - name: NODE_ENV description: Application Environment type variable required: true @@ -96,8 +93,6 @@ objects: value: ${VERSION} - name: DB_SCHEMA value: ${DB_SCHEMA} - - name: DB_SCHEMA_DAPI_V1 - value: ${DB_SCHEMA_DAPI_V1} imagePullPolicy: Always restartPolicy: Never activeDeadlineSeconds: 900 diff --git a/database/package-lock.json b/database/package-lock.json index e88feb905..2148757cf 100644 --- a/database/package-lock.json +++ b/database/package-lock.json @@ -89,6 +89,12 @@ } } }, + "@faker-js/faker": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-8.2.0.tgz", + "integrity": "sha512-VacmzZqVxdWdf9y64lDOMZNDMM/FQdtM9IsaOPKOm2suYwEatb8VkdHqOzXcDnZbk7YDE2BmsJmy/2Hmkn563g==", + "dev": true + }, "@humanwhocodes/config-array": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.5.0.tgz", @@ -284,6 +290,16 @@ "eslint-visitor-keys": "^2.0.0" } }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "dev": true, + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, "acorn": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", @@ -506,6 +522,12 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true + }, "create-require": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", @@ -960,6 +982,16 @@ "is-callable": "^1.1.3" } }, + "from2": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", + "integrity": "sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==", + "dev": true, + "requires": { + "inherits": "^2.0.1", + "readable-stream": "^2.0.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -995,6 +1027,26 @@ "integrity": "sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==", "dev": true }, + "geojson-random": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/geojson-random/-/geojson-random-0.5.0.tgz", + "integrity": "sha512-a4j6KJCC/ZhufwiXMxuoXrJiOxwwBQ0Y0DEcGmytnSJC11AkWHSKSFOw/ovh6QFp8n9XIDXhQXAvArRCvi2Y4A==", + "dev": true, + "requires": { + "from2": "^2.1.0", + "geojson-stream": "0.1.0" + } + }, + "geojson-stream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/geojson-stream/-/geojson-stream-0.1.0.tgz", + "integrity": "sha512-svSg5fFXPaTiqzEBGXScA+nISaeC9rLvku2PH+wM5LToATUw2bLIrvls43ymnT9Xnp51nBPVyK9m4Af40KpJ7w==", + "dev": true, + "requires": { + "JSONStream": "^1.0.0", + "through": "^2.3.4" + } + }, "get-intrinsic": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", @@ -1381,6 +1433,12 @@ "call-bind": "^1.0.2" } }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", @@ -1421,6 +1479,12 @@ "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", "dev": true }, + "jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==", + "dev": true + }, "knex": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/knex/-/knex-2.4.2.tgz", @@ -1858,6 +1922,12 @@ "integrity": "sha512-R8o23sf5iVL/U71h9SFUdhdOEPsi3nm42FD/oDYIZ2PQa4TNWWuWecxln6jlIQzpZTDMUeO1NicJP6lLn2TtRw==", "dev": true }, + "process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true + }, "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", @@ -1898,6 +1968,21 @@ } } }, + "readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, "rechoir": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.8.0.tgz", @@ -1968,6 +2053,12 @@ "queue-microtask": "^1.2.2" } }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, "safe-regex-test": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", @@ -2157,6 +2248,15 @@ "es-abstract": "^1.20.4" } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -2236,6 +2336,12 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, "tildify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/tildify/-/tildify-2.0.0.tgz", @@ -2345,6 +2451,12 @@ "punycode": "^2.1.0" } }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, "v8-compile-cache": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz", diff --git a/database/package.json b/database/package.json index 9da4b4d40..c6a98ccd2 100644 --- a/database/package.json +++ b/database/package.json @@ -27,6 +27,7 @@ "typescript": "~4.1.6" }, "devDependencies": { + "@faker-js/faker": "^8.2.0", "@types/node": "~14.14.31", "@types/pg": "~7.14.4", "@typescript-eslint/eslint-plugin": "~4.33.0", @@ -34,6 +35,7 @@ "eslint": "~7.32.0", "eslint-config-prettier": "~6.15.0", "eslint-plugin-prettier": "~3.3.1", + "geojson-random": "^0.5.0", "npm-run-all": "~4.1.5", "prettier": "~2.3.2", "prettier-plugin-organize-imports": "~2.3.4", diff --git a/database/src/migrations/20220225205948_release.ts b/database/src/migrations/20220225205948_release.ts index 51e7f401f..35e527581 100644 --- a/database/src/migrations/20220225205948_release.ts +++ b/database/src/migrations/20220225205948_release.ts @@ -63,35 +63,21 @@ export async function up(knex: Knex): Promise { path.join(__dirname, DB_RELEASE, 'populate_persecution_or_harm.sql') ); - const vw_generated_dapi_views = fs.readFileSync(path.join(__dirname, DB_RELEASE, 'vw_generated_dapi_views.sql')); - await knex.raw(` -- set up spatial extensions ${create_spatial_extensions} -- set up biohub schema create schema if not exists biohub; + + -- setup postgres user GRANT ALL ON SCHEMA biohub TO postgres; - set search_path = biohub, public; - - -- setup biohub api schema - create schema if not exists biohub_dapi_v1; + set search_path = biohub, public, topology; -- setup api user create user ${DB_USER_API} password '${DB_USER_API_PASS}'; - alter schema biohub_dapi_v1 owner to ${DB_USER_API}; - - -- Grant rights on biohub_dapi_v1 to biohub_api user - grant all on schema biohub_dapi_v1 to ${DB_USER_API}; - grant all on schema biohub_dapi_v1 to postgres; - alter DEFAULT PRIVILEGES in SCHEMA biohub_dapi_v1 grant ALL on tables to ${DB_USER_API}; - alter DEFAULT PRIVILEGES in SCHEMA biohub_dapi_v1 grant ALL on tables to postgres; - - -- biohub grants - GRANT USAGE ON SCHEMA biohub TO ${DB_USER_API}; - ALTER DEFAULT PRIVILEGES IN SCHEMA biohub GRANT ALL ON TABLES TO ${DB_USER_API}; - - alter role ${DB_USER_API} set search_path to biohub_dapi_v1, biohub, public, topology; + GRANT ALL ON SCHEMA biohub TO ${DB_USER_API}; + alter role ${DB_USER_API} set search_path to biohub, public, topology; ${biohub_ddl} ${populate_user_identity_source} @@ -118,21 +104,14 @@ export async function up(knex: Knex): Promise { ${populate_persecution_or_harm_type} ${populate_persecution_or_harm} - -- create the views - set search_path = biohub_dapi_v1; - set role biohub_api; - ${vw_generated_dapi_views} - set role postgres; - set search_path = biohub; - grant execute on function biohub.api_set_context(_system_user_identifier system_user.user_identifier%type, _user_identity_source_name user_identity_source.name%type) to ${DB_USER_API}; + set search_path = biohub, public; `); } export async function down(knex: Knex): Promise { await knex.raw(` DROP SCHEMA IF EXISTS biohub CASCADE; - DROP SCHEMA IF EXISTS biohub_dapi_v1 CASCADE; DROP USER IF EXISTS ${DB_USER_API}; `); } diff --git a/database/src/migrations/20220602152015_add_record_sims_eml_transformation.ts b/database/src/migrations/20220602152015_add_record_sims_eml_transformation.ts index a6701b570..7205960bd 100644 --- a/database/src/migrations/20220602152015_add_record_sims_eml_transformation.ts +++ b/database/src/migrations/20220602152015_add_record_sims_eml_transformation.ts @@ -1,8 +1,5 @@ import { Knex } from 'knex'; -const DB_SCHEMA = process.env.DB_SCHEMA; -const DB_SCHEMA_DAPI_V1 = process.env.DB_SCHEMA_DAPI_V1; - /** * EML source Transform. * @@ -247,8 +244,8 @@ export async function up(knex: Knex): Promise { `; await knex.raw(` - SET SCHEMA '${DB_SCHEMA}'; - SET SEARCH_PATH = ${DB_SCHEMA}, ${DB_SCHEMA_DAPI_V1}; + SET SCHEMA 'biohub'; + SET SEARCH_PATH = 'biohub'; insert into source_transform ( system_user_id, diff --git a/database/src/migrations/20220602152016_spatial_transform_eml_project_boundaries.ts b/database/src/migrations/20220602152016_spatial_transform_eml_project_boundaries.ts index 69d2adfe8..23d4421e7 100644 --- a/database/src/migrations/20220602152016_spatial_transform_eml_project_boundaries.ts +++ b/database/src/migrations/20220602152016_spatial_transform_eml_project_boundaries.ts @@ -1,8 +1,5 @@ import { Knex } from 'knex'; -const DB_SCHEMA = process.env.DB_SCHEMA; -const DB_SCHEMA_DAPI_V1 = process.env.DB_SCHEMA_DAPI_V1; - /** * EML Spatial Transform. * @@ -186,8 +183,8 @@ export async function up(knex: Knex): Promise { `; await knex.raw(` - SET SCHEMA '${DB_SCHEMA}'; - SET SEARCH_PATH = ${DB_SCHEMA}, ${DB_SCHEMA_DAPI_V1}; + SET SCHEMA 'biohub'; + SET SEARCH_PATH = 'biohub'; insert into spatial_transform ( name, diff --git a/database/src/migrations/20220602152016_spatial_transform_eml_project_boundary_centroid.ts b/database/src/migrations/20220602152016_spatial_transform_eml_project_boundary_centroid.ts index 7377a04d0..402ce5842 100644 --- a/database/src/migrations/20220602152016_spatial_transform_eml_project_boundary_centroid.ts +++ b/database/src/migrations/20220602152016_spatial_transform_eml_project_boundary_centroid.ts @@ -1,8 +1,5 @@ import { Knex } from 'knex'; -const DB_SCHEMA = process.env.DB_SCHEMA; -const DB_SCHEMA_DAPI_V1 = process.env.DB_SCHEMA_DAPI_V1; - /** * EML Spatial Transform. * @@ -136,8 +133,8 @@ export async function up(knex: Knex): Promise { `; await knex.raw(` - SET SCHEMA '${DB_SCHEMA}'; - SET SEARCH_PATH = ${DB_SCHEMA}, ${DB_SCHEMA_DAPI_V1}; + SET SCHEMA 'biohub'; + SET SEARCH_PATH = 'biohub'; insert into spatial_transform ( name, diff --git a/database/src/migrations/20230518125700_update_record_sims_eml_transformation.ts b/database/src/migrations/20230518125700_update_record_sims_eml_transformation.ts index 4f788f8f2..f5ae282b3 100644 --- a/database/src/migrations/20230518125700_update_record_sims_eml_transformation.ts +++ b/database/src/migrations/20230518125700_update_record_sims_eml_transformation.ts @@ -1,8 +1,5 @@ import { Knex } from 'knex'; -const DB_SCHEMA = process.env.DB_SCHEMA; -const DB_SCHEMA_DAPI_V1 = process.env.DB_SCHEMA_DAPI_V1; - /** * EML source Transform. * @@ -264,8 +261,8 @@ export async function up(knex: Knex): Promise { project_type pt; `; await knex.raw(` - SET SCHEMA '${DB_SCHEMA}'; - SET SEARCH_PATH = ${DB_SCHEMA}, ${DB_SCHEMA_DAPI_V1}; + SET SCHEMA 'biohub'; + SET SEARCH_PATH = 'biohub'; update source_transform set metadata_transform=$transform$${transformSQL}$transform$ diff --git a/database/src/migrations/20230531012345_remove_unique_id_artifact_persecution.ts b/database/src/migrations/20230531012345_remove_unique_id_artifact_persecution.ts index 869eaedfa..86d39ede2 100644 --- a/database/src/migrations/20230531012345_remove_unique_id_artifact_persecution.ts +++ b/database/src/migrations/20230531012345_remove_unique_id_artifact_persecution.ts @@ -9,16 +9,9 @@ import { Knex } from 'knex'; */ export async function up(knex: Knex): Promise { await knex.raw(` - set search_path=biohub_dapi_v1; - - drop view if exists artifact_persecution; - set search_path=biohub; DROP INDEX if exists artifact_persecution_uk1; CREATE UNIQUE INDEX artifact_persecution_uk1 ON artifact_persecution(artifact_id, persecution_or_harm_id); - - set search_path=biohub_dapi_v1; - CREATE OR REPLACE VIEW artifact_persecution as SELECT * FROM biohub.artifact_persecution; `); } diff --git a/database/src/migrations/20231109000001_feature_tables.ts b/database/src/migrations/20231109000001_feature_tables.ts new file mode 100644 index 000000000..85a028bb8 --- /dev/null +++ b/database/src/migrations/20231109000001_feature_tables.ts @@ -0,0 +1,269 @@ +import { Knex } from 'knex'; + +/** + * Add tables: + * - submission_feature + * - feature_type + * - feature_type_property + * - feature_property + * - feature_property_type + * + * @export + * @param {Knex} knex + * @return {*} {Promise} + */ +export async function up(knex: Knex): Promise { + await knex.raw(` + ---------------------------------------------------------------------------------------- + -- Create tables + ---------------------------------------------------------------------------------------- + set search_path=biohub,public; + + CREATE TABLE submission_feature( + submission_feature_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + submission_id integer NOT NULL, + feature_type_id integer NOT NULL, + data jsonb NOT NULL, + parent_submission_feature_id integer, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT submission_feature_pk PRIMARY KEY (submission_feature_id) + ); + + COMMENT ON COLUMN submission_feature.submission_feature_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN submission_feature.submission_id IS 'Foreign key to the submission table.'; + COMMENT ON COLUMN submission_feature.feature_type_id IS 'Foreign key to the feature_type table.'; + COMMENT ON COLUMN submission_feature.data IS 'The json data of the submission_feature record.'; + COMMENT ON COLUMN submission_feature.parent_submission_feature_id IS 'Foreign key to the submission_feature table.'; + COMMENT ON COLUMN submission_feature.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN submission_feature.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN submission_feature.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN submission_feature.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN submission_feature.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN submission_feature.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN submission_feature.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE submission_feature IS 'A set of data for a specific feature of a submission.'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE feature_type( + feature_type_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + name varchar(50) NOT NULL, + display_name varchar(100) NOT NULL, + description varchar(3000), + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT feature_type_pk PRIMARY KEY (feature_type_id) + ); + + COMMENT ON COLUMN feature_type.feature_type_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN feature_type.name IS 'The name of the feature_type record.'; + COMMENT ON COLUMN feature_type.display_name IS 'The formatted name of the feature_type record.'; + COMMENT ON COLUMN feature_type.description IS 'The description of the feature_type record.'; + COMMENT ON COLUMN feature_type.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN feature_type.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN feature_type.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN feature_type.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN feature_type.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN feature_type.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN feature_type.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE feature_type IS 'Defines feature types.'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE feature_type_property( + feature_type_property_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + feature_type_id integer NOT NULL, + feature_property_id integer NOT NULL, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT feature_type_property_pk PRIMARY KEY (feature_type_property_id) + ); + + COMMENT ON COLUMN feature_type_property.feature_type_property_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN feature_type_property.feature_type_id IS 'Foreign key to the feature_type table.'; + COMMENT ON COLUMN feature_type_property.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN feature_type_property.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN feature_type_property.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN feature_type_property.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN feature_type_property.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN feature_type_property.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN feature_type_property.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN feature_type_property.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE feature_type_property IS 'A join table on feature type and feature_property. Defines which properties can be used by a given feature type.'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE feature_property( + feature_property_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + feature_property_type_id integer NOT NULL, + name varchar(50) NOT NULL, + display_name varchar(100) NOT NULL, + description varchar(3000), + parent_feature_property_id integer, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT feature_property_pk PRIMARY KEY (feature_property_id) + ); + + COMMENT ON COLUMN feature_property.feature_property_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN feature_property.feature_property_type_id IS 'Foreign key to the feature_property_type table.'; + COMMENT ON COLUMN feature_property.name IS 'The name of the feature_property record.'; + COMMENT ON COLUMN feature_property.display_name IS 'The formatted name of the feature_property record.'; + COMMENT ON COLUMN feature_property.description IS 'The description of the feature_property record.'; + COMMENT ON COLUMN feature_property.parent_feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN feature_property.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN feature_property.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN feature_property.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN feature_property.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN feature_property.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN feature_property.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN feature_property.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE feature_property IS 'Defines supported feature data properties.'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE feature_property_type( + feature_property_type_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + name varchar(100) NOT NULL, + description varchar(3000), + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT feature_property_type_pk PRIMARY KEY (feature_property_type_id) + ); + + COMMENT ON COLUMN feature_property_type.feature_property_type_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN feature_property_type.name IS 'The name of the feature_property_type record.'; + COMMENT ON COLUMN feature_property_type.description IS 'The description of the feature_property_type record.'; + COMMENT ON COLUMN feature_property_type.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN feature_property_type.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN feature_property_type.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN feature_property_type.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN feature_property_type.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN feature_property_type.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN feature_property_type.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE feature_property_type IS 'Defines supported feature data property types.'; + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: submission_feature + ---------------------------------------------------------------------------------------- + + -- Add foreign key constraint + ALTER TABLE submission_feature ADD CONSTRAINT submission_feature_fk1 + FOREIGN KEY (submission_id) + REFERENCES submission(submission_id); + + ALTER TABLE submission_feature ADD CONSTRAINT submission_feature_fk2 + FOREIGN KEY (feature_type_id) + REFERENCES feature_type(feature_type_id); + + ALTER TABLE submission_feature ADD CONSTRAINT submission_feature_fk3 + FOREIGN KEY (parent_submission_feature_id) + REFERENCES submission_feature(submission_feature_id); + + -- add indexes for foreign keys + CREATE INDEX submission_feature_idx1 ON submission_feature(submission_id); + + CREATE INDEX submission_feature_idx2 ON submission_feature(feature_type_id); + + CREATE INDEX submission_feature_idx3 ON submission_feature(submission_feature_id); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: feature_type + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) + CREATE UNIQUE INDEX feature_type_nuk1 ON feature_type(name, (record_end_date is NULL)) where record_end_date is null; + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: feature_type_property + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same feature_type_id, feature_property_id, and a NULL record_end_date) + CREATE UNIQUE INDEX feature_type_property_nuk1 ON feature_type_property(feature_type_id, feature_property_id, (record_end_date is NULL)) where record_end_date is null; + + -- Add foreign key constraint + ALTER TABLE feature_type_property ADD CONSTRAINT feature_type_property_fk1 + FOREIGN KEY (feature_type_id) + REFERENCES feature_type(feature_type_id); + + ALTER TABLE feature_type_property ADD CONSTRAINT feature_type_property_fk2 + FOREIGN KEY (feature_property_id) + REFERENCES feature_property(feature_property_id); + + -- add indexes for foreign keys + CREATE INDEX feature_type_property_idx1 ON feature_type_property(feature_type_id); + + CREATE INDEX feature_type_property_idx2 ON feature_type_property(feature_property_id); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: feature_property + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) + CREATE UNIQUE INDEX feature_property_nuk1 ON feature_property(name, (record_end_date is NULL)) where record_end_date is null; + + -- Add foreign key constraint + ALTER TABLE feature_property ADD CONSTRAINT feature_property_fk1 + FOREIGN KEY (feature_property_type_id) + REFERENCES feature_property_type(feature_property_type_id); + + -- add indexes for foreign keys + CREATE INDEX feature_property_idx1 ON feature_property(feature_property_type_id); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: feature_property_type + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) + CREATE UNIQUE INDEX feature_property_type_nuk1 ON feature_property_type(name, (record_end_date is NULL)) where record_end_date is null; + + ---------------------------------------------------------------------------------------- + -- Create audit and journal triggers + ---------------------------------------------------------------------------------------- + + create trigger audit_submission_feature before insert or update or delete on submission_feature for each row execute procedure tr_audit_trigger(); + create trigger journal_submission_feature after insert or update or delete on submission_feature for each row execute procedure tr_journal_trigger(); + + create trigger audit_feature_type before insert or update or delete on feature_type for each row execute procedure tr_audit_trigger(); + create trigger journal_feature_type after insert or update or delete on feature_type for each row execute procedure tr_journal_trigger(); + + create trigger audit_feature_type_property before insert or update or delete on feature_type_property for each row execute procedure tr_audit_trigger(); + create trigger journal_feature_type_property after insert or update or delete on feature_type_property for each row execute procedure tr_journal_trigger(); + + create trigger audit_feature_property before insert or update or delete on feature_property for each row execute procedure tr_audit_trigger(); + create trigger journal_feature_property after insert or update or delete on feature_property for each row execute procedure tr_journal_trigger(); + + create trigger audit_feature_property_type before insert or update or delete on feature_property_type for each row execute procedure tr_audit_trigger(); + create trigger journal_feature_property_type after insert or update or delete on feature_property_type for each row execute procedure tr_journal_trigger(); + `); +} + +export async function down(knex: Knex): Promise { + await knex.raw(``); +} diff --git a/database/src/migrations/20231109000002_search_tables.ts b/database/src/migrations/20231109000002_search_tables.ts new file mode 100644 index 000000000..a45cd6c43 --- /dev/null +++ b/database/src/migrations/20231109000002_search_tables.ts @@ -0,0 +1,295 @@ +import { Knex } from 'knex'; + +/** + * Add tables: + * - search_string + * - search_string + * - search_number + * - search_datetime + * - search_spatial + * - search_taxonomy + * + * @export + * @param {Knex} knex + * @return {*} {Promise} + */ +export async function up(knex: Knex): Promise { + await knex.raw(` + ---------------------------------------------------------------------------------------- + -- Create tables + ---------------------------------------------------------------------------------------- + set search_path=biohub,public; + + CREATE TABLE search_string( + search_string_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + submission_feature_id integer NOT NULL, + feature_property_id integer NOT NULL, + value varchar(250) NOT NULL, +-- record_effective_date date NOT NULL, +-- record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT search_string_pk PRIMARY KEY (search_string_id) + ); + + COMMENT ON COLUMN search_string.search_string_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN search_string.submission_feature_id IS 'Foreign key to the submission_feature table.'; + COMMENT ON COLUMN search_string.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN search_string.value IS 'The search value of the record.'; +-- COMMENT ON COLUMN search_string.record_effective_date IS 'Record level effective date.'; +-- COMMENT ON COLUMN search_string.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN search_string.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN search_string.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN search_string.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN search_string.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN search_string.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE search_string IS 'String search values'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE search_number( + search_number_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + submission_feature_id integer NOT NULL, + feature_property_id integer NOT NULL, + value numeric NOT NULL, +-- record_effective_date date NOT NULL, +-- record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT search_number_pk PRIMARY KEY (search_number_id) + ); + + COMMENT ON COLUMN search_number.search_number_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN search_number.submission_feature_id IS 'Foreign key to the submission_feature table.'; + COMMENT ON COLUMN search_number.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN search_number.value IS 'The search value of the record.'; +-- COMMENT ON COLUMN search_number.record_effective_date IS 'Record level effective date.'; +-- COMMENT ON COLUMN search_number.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN search_number.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN search_number.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN search_number.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN search_number.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN search_number.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE search_number IS 'Number search values'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE search_datetime( + search_datetime_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + submission_feature_id integer NOT NULL, + feature_property_id integer NOT NULL, + value timestamptz(6) NOT NULL, +-- record_effective_date date NOT NULL, +-- record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT search_datetime_pk PRIMARY KEY (search_datetime_id) + ); + + COMMENT ON COLUMN search_datetime.search_datetime_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN search_datetime.submission_feature_id IS 'Foreign key to the submission_feature table.'; + COMMENT ON COLUMN search_datetime.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN search_datetime.value IS 'The search value of the record.'; +-- COMMENT ON COLUMN search_datetime.record_effective_date IS 'Record level effective date.'; +-- COMMENT ON COLUMN search_datetime.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN search_datetime.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN search_datetime.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN search_datetime.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN search_datetime.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN search_datetime.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE search_datetime IS 'Datetime search values'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE search_spatial( + search_spatial_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + submission_feature_id integer NOT NULL, + feature_property_id integer NOT NULL, + value geometry NOT NULL, +-- record_effective_date date NOT NULL, +-- record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT search_spatial_pk PRIMARY KEY (search_spatial_id) + ); + + COMMENT ON COLUMN search_spatial.search_spatial_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN search_spatial.submission_feature_id IS 'Foreign key to the submission_feature table.'; + COMMENT ON COLUMN search_spatial.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN search_spatial.value IS 'The search value of the record.'; +-- COMMENT ON COLUMN search_spatial.record_effective_date IS 'Record level effective date.'; +-- COMMENT ON COLUMN search_spatial.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN search_spatial.create_date IS 'The spatial the record was created.'; + COMMENT ON COLUMN search_spatial.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN search_spatial.update_date IS 'The spatial the record was updated.'; + COMMENT ON COLUMN search_spatial.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN search_spatial.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE search_spatial IS 'Spatial search values'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE search_taxonomy( + search_taxonomy_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + submission_feature_id integer NOT NULL, + feature_property_id integer NOT NULL, + value numeric NOT NULL, +-- record_effective_date date NOT NULL, +-- record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT search_taxonomy_pk PRIMARY KEY (search_taxonomy_id) + ); + + COMMENT ON COLUMN search_taxonomy.search_taxonomy_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN search_taxonomy.submission_feature_id IS 'Foreign key to the submission_feature table.'; + COMMENT ON COLUMN search_taxonomy.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN search_taxonomy.value IS 'The search value of the record.'; +-- COMMENT ON COLUMN search_taxonomy.record_effective_date IS 'Record level effective date.'; +-- COMMENT ON COLUMN search_taxonomy.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN search_taxonomy.create_date IS 'The taxonomy the record was created.'; + COMMENT ON COLUMN search_taxonomy.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN search_taxonomy.update_date IS 'The taxonomy the record was updated.'; + COMMENT ON COLUMN search_taxonomy.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN search_taxonomy.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE search_taxonomy IS 'Taxonomy search values'; + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: search_string + ---------------------------------------------------------------------------------------- + + -- Add foreign key constraint + ALTER TABLE search_string ADD CONSTRAINT search_string_fk1 + FOREIGN KEY (submission_feature_id) + REFERENCES submission_feature(submission_feature_id); + + ALTER TABLE search_string ADD CONSTRAINT search_string_fk2 + FOREIGN KEY (feature_property_id) + REFERENCES feature_property(feature_property_id); + + -- add indexes for foreign keys + CREATE INDEX search_string_idx1 ON search_string(submission_feature_id); + + CREATE INDEX search_string_idx2 ON search_string(feature_property_id); + + CREATE INDEX search_string_idx3 ON search_string(value); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: search_number + ---------------------------------------------------------------------------------------- + + -- Add foreign key constraint + ALTER TABLE search_number ADD CONSTRAINT search_number_fk1 + FOREIGN KEY (submission_feature_id) + REFERENCES submission_feature(submission_feature_id); + + ALTER TABLE search_number ADD CONSTRAINT search_number_fk2 + FOREIGN KEY (feature_property_id) + REFERENCES feature_property(feature_property_id); + + -- add indexes for foreign keys + CREATE INDEX search_number_idx1 ON search_number(submission_feature_id); + + CREATE INDEX search_number_idx2 ON search_number(feature_property_id); + + CREATE INDEX search_number_idx3 ON search_number(value); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: search_datetime + ---------------------------------------------------------------------------------------- + + -- Add foreign key constraint + ALTER TABLE search_datetime ADD CONSTRAINT search_datetime_fk1 + FOREIGN KEY (submission_feature_id) + REFERENCES submission_feature(submission_feature_id); + + ALTER TABLE search_datetime ADD CONSTRAINT search_datetime_fk2 + FOREIGN KEY (feature_property_id) + REFERENCES feature_property(feature_property_id); + + -- add indexes for foreign keys + CREATE INDEX search_datetime_idx1 ON search_datetime(submission_feature_id); + + CREATE INDEX search_datetime_idx2 ON search_datetime(feature_property_id); + + CREATE INDEX search_datetime_idx3 ON search_datetime(value); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: search_spatial + ---------------------------------------------------------------------------------------- + + -- Add foreign key constraint + ALTER TABLE search_spatial ADD CONSTRAINT search_spatial_fk1 + FOREIGN KEY (submission_feature_id) + REFERENCES submission_feature(submission_feature_id); + + ALTER TABLE search_spatial ADD CONSTRAINT search_spatial_fk2 + FOREIGN KEY (feature_property_id) + REFERENCES feature_property(feature_property_id); + + -- add indexes for foreign keys + CREATE INDEX search_spatial_idx1 ON search_spatial(submission_feature_id); + + CREATE INDEX search_spatial_idx2 ON search_spatial(feature_property_id); + + -- add spatial index + CREATE INDEX search_spatial_idx3 ON search_spatial using GIST(value); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: search_taxonomy + ---------------------------------------------------------------------------------------- + + -- Add foreign key constraint + ALTER TABLE search_taxonomy ADD CONSTRAINT search_taxonomy_fk1 + FOREIGN KEY (submission_feature_id) + REFERENCES submission_feature(submission_feature_id); + + ALTER TABLE search_taxonomy ADD CONSTRAINT search_taxonomy_fk2 + FOREIGN KEY (feature_property_id) + REFERENCES feature_property(feature_property_id); + + -- add indexes for foreign keys + CREATE INDEX search_taxonomy_idx1 ON search_taxonomy(submission_feature_id); + + CREATE INDEX search_taxonomy_idx2 ON search_taxonomy(feature_property_id); + + CREATE INDEX search_taxonomy_idx3 ON search_taxonomy(value); + + ---------------------------------------------------------------------------------------- + -- Create audit and journal triggers + ---------------------------------------------------------------------------------------- + + create trigger audit_search_string before insert or update or delete on search_string for each row execute procedure tr_audit_trigger(); + create trigger journal_search_string after insert or update or delete on search_string for each row execute procedure tr_journal_trigger(); + + create trigger audit_search_number before insert or update or delete on search_number for each row execute procedure tr_audit_trigger(); + create trigger journal_search_number after insert or update or delete on search_number for each row execute procedure tr_journal_trigger(); + + create trigger audit_search_datetime before insert or update or delete on search_datetime for each row execute procedure tr_audit_trigger(); + create trigger journal_search_datetime after insert or update or delete on search_datetime for each row execute procedure tr_journal_trigger(); + + create trigger audit_search_spatial before insert or update or delete on search_spatial for each row execute procedure tr_audit_trigger(); + create trigger journal_search_spatial after insert or update or delete on search_spatial for each row execute procedure tr_journal_trigger(); + + create trigger audit_search_taxonomy before insert or update or delete on search_taxonomy for each row execute procedure tr_audit_trigger(); + create trigger journal_search_taxonomy after insert or update or delete on search_taxonomy for each row execute procedure tr_journal_trigger(); + `); +} + +export async function down(knex: Knex): Promise { + await knex.raw(``); +} diff --git a/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts b/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts new file mode 100644 index 000000000..31d920cdb --- /dev/null +++ b/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts @@ -0,0 +1,94 @@ +import { Knex } from 'knex'; + +/** + * Populate tables: + * - feature_property_type + * - feature_property + * - feature_type + * - feature_type_property + * + * @export + * @param {Knex} knex + * @return {*} {Promise} + */ +export async function up(knex: Knex): Promise { + await knex.raw(` + ---------------------------------------------------------------------------------------- + -- Create tables + ---------------------------------------------------------------------------------------- + set search_path=biohub,public; + + -- populate feature_property_type table + insert into feature_property_type (name, description, record_effective_date) values ('string', 'A text type', now()); + insert into feature_property_type (name, description, record_effective_date) values ('number', 'A number type', now()); + insert into feature_property_type (name, description, record_effective_date) values ('boolean', 'A boolean type', now()); + insert into feature_property_type (name, description, record_effective_date) values ('object', 'An object type', now()); + insert into feature_property_type (name, description, record_effective_date) values ('array', 'An array type', now()); + insert into feature_property_type (name, description, record_effective_date) values ('datetime', 'A datetime type (ISO 8601)', now()); + insert into feature_property_type (name, description, record_effective_date) values ('spatial', 'A spatial type', now()); + insert into feature_property_type (name, description, record_effective_date) values ('taxonomy', 'A taxonomic type', now()); + + -- populate feature_property table + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('name', 'Name', 'The name of the record', (select feature_property_type_id from feature_property_type where name = 'string'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('description', 'Description', 'The description of the record', (select feature_property_type_id from feature_property_type where name = 'string'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('taxonomy', 'Taxonomy', 'The taxonomy associated to the record', (select feature_property_type_id from feature_property_type where name = 'taxonomy'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('date_range', 'Date Range', 'A date range', (select feature_property_type_id from feature_property_type where name = 'object'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('start_date', 'Start Date', 'The start date of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), (select feature_property_id from feature_property where name = 'date_range'), now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('end_date', 'End Date', 'The end date of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), (select feature_property_id from feature_property where name = 'date_range'), now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('geometry', 'Geometry', 'The location of the record', (select feature_property_type_id from feature_property_type where name = 'spatial'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('count', 'Count', 'The count of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()); + + -- populate feature_type table + insert into feature_type (name, display_name, description, record_effective_date) values ('dataset', 'Dataset', 'A related collection of data (ie: survey)', now()); + insert into feature_type (name, display_name, description, record_effective_date) values ('sample_site', 'Sample Site', 'A location at which data was collected', now()); + insert into feature_type (name, display_name, description, record_effective_date) values ('sample_method', 'Sample Method', 'A method used to collect data', now()); + insert into feature_type (name, display_name, description, record_effective_date) values ('sample_period', 'Sample Period', 'A datetime period in which data was collected', now()); + insert into feature_type (name, display_name, description, record_effective_date) values ('observation', 'Observation', 'An observation record', now()); + insert into feature_type (name, display_name, description, record_effective_date) values ('animal', 'Animal', 'An individual animal record', now()); + insert into feature_type (name, display_name, description, record_effective_date) values ('telemetry', 'Telemetry ', 'A telemetry record', now()); + + -- populate feature_type_property table + -- feature_type: dataset + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'name'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'description'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'taxonomy'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'date_range'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'start_date'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'end_date'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'geometry'), now()); + + -- feature_type: sample_site + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_site'), (select feature_property_id from feature_property where name = 'name'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_site'), (select feature_property_id from feature_property where name = 'description'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_site'), (select feature_property_id from feature_property where name = 'geometry'), now()); + + -- feature_type: sample_method + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_method'), (select feature_property_id from feature_property where name = 'name'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_method'), (select feature_property_id from feature_property where name = 'description'), now()); + + -- feature_type: sample_period + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_period'), (select feature_property_id from feature_property where name = 'date_range'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_period'), (select feature_property_id from feature_property where name = 'start_date'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_period'), (select feature_property_id from feature_property where name = 'end_date'), now()); + + -- feature_type: observation + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'taxonomy'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'date_range'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'start_date'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'end_date'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'geometry'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'count'), now()); + + -- feature_type: animal + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'animal'), (select feature_property_id from feature_property where name = 'taxonomy'), now()); + + -- feature_type: telemetry + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'telemetry'), (select feature_property_id from feature_property where name = 'date_range'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'telemetry'), (select feature_property_id from feature_property where name = 'start_date'), now()); + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'telemetry'), (select feature_property_id from feature_property where name = 'end_date'), now()); + `); +} + +export async function down(knex: Knex): Promise { + await knex.raw(``); +} diff --git a/database/src/seeds/01_db_system_users.ts b/database/src/seeds/01_db_system_users.ts index cf37eb103..34cf5e6a7 100644 --- a/database/src/seeds/01_db_system_users.ts +++ b/database/src/seeds/01_db_system_users.ts @@ -1,6 +1,5 @@ import { Knex } from 'knex'; -const DB_SCHEMA = process.env.DB_SCHEMA; const DB_ADMIN = process.env.DB_ADMIN; export enum SYSTEM_IDENTITY_SOURCE { @@ -86,8 +85,8 @@ const systemUsers = [ */ export async function seed(knex: Knex): Promise { await knex.raw(` - set schema '${DB_SCHEMA}'; - set search_path = ${DB_SCHEMA}; + set schema 'biohub'; + set search_path = 'biohub'; `); for (const systemUser of systemUsers) { diff --git a/database/src/seeds/02_security_transform_eml_dwc_occurrences.ts b/database/src/seeds/02_security_transform_eml_dwc_occurrences.ts index db3d67eed..1b94dfce5 100644 --- a/database/src/seeds/02_security_transform_eml_dwc_occurrences.ts +++ b/database/src/seeds/02_security_transform_eml_dwc_occurrences.ts @@ -1,8 +1,5 @@ import { Knex } from 'knex'; -const DB_SCHEMA = process.env.DB_SCHEMA; -const DB_SCHEMA_DAPI_V1 = process.env.DB_SCHEMA_DAPI_V1; - /** * Add spatial transform * @@ -12,8 +9,8 @@ const DB_SCHEMA_DAPI_V1 = process.env.DB_SCHEMA_DAPI_V1; */ export async function seed(knex: Knex): Promise { await knex.raw(` - SET SCHEMA '${DB_SCHEMA}'; - SET SEARCH_PATH = ${DB_SCHEMA}, ${DB_SCHEMA_DAPI_V1}; + SET SCHEMA 'biohub'; + SET SEARCH_PATH = 'biohub'; `); const response = await knex.raw(checkTransformExists()); diff --git a/database/src/seeds/03_dwc_spatial_transform.ts b/database/src/seeds/03_dwc_spatial_transform.ts index a3f429ca6..fe20ee917 100644 --- a/database/src/seeds/03_dwc_spatial_transform.ts +++ b/database/src/seeds/03_dwc_spatial_transform.ts @@ -1,8 +1,5 @@ import { Knex } from 'knex'; -const DB_SCHEMA = process.env.DB_SCHEMA; -const DB_SCHEMA_DAPI_V1 = process.env.DB_SCHEMA_DAPI_V1; - /** * Add spatial transform * @@ -12,8 +9,8 @@ const DB_SCHEMA_DAPI_V1 = process.env.DB_SCHEMA_DAPI_V1; */ export async function seed(knex: Knex): Promise { await knex.raw(` - SET SCHEMA '${DB_SCHEMA}'; - SET SEARCH_PATH = ${DB_SCHEMA}, ${DB_SCHEMA_DAPI_V1}; + SET SCHEMA 'biohub'; + SET SEARCH_PATH = 'biohub'; `); const response = await knex.raw(checkTransformExists()); diff --git a/database/src/seeds/04_mock_test_data.ts b/database/src/seeds/04_mock_test_data.ts new file mode 100644 index 000000000..14280d405 --- /dev/null +++ b/database/src/seeds/04_mock_test_data.ts @@ -0,0 +1,387 @@ +import { faker } from '@faker-js/faker'; +// @ts-ignore ignore error over missing geojson-random declaration (.d.ts) file +import random from 'geojson-random'; +import { Knex } from 'knex'; + +// Disable mock data seeding by default, unless manually enabled +const ENABLED = false; + +/** + * Sample query for performance testing. + * + * -- Select feature_submissions on multiple conditions (AND) + * select * from submission_feature + * where feature_type_id = 1 and submission_feature_id in ( + * select distinct t1.submission_feature_id + * FROM search_string t1 + * WHERE EXISTS ( + * select 1 from search_string t3 where t3.submission_feature_id = t1.submission_feature_id and t3.value like '%cor%' + * ) and exists ( + * select 1 from search_string t4 where t4.submission_feature_id = t1.submission_feature_id and t4.value like '%arx%' + * ) and exists ( + * select 1 from search_number t5 where t5.submission_feature_id = t1.submission_feature_id and t5.value > 40 and t5.value < 50 + * ) and exists ( + * select 1 from search_taxonomy t6 where t6.submission_feature_id = t1.submission_feature_id and t6.value > 50000 and t6.value < 50100 + * ) and exists ( + * select 1 from search_datetime t7 where t7.submission_feature_id = t1.submission_feature_id and t7.value > '2023-08-01' and t7.value < '2024-04-01' and t7.feature_property_id = (select feature_property_id from feature_property where name = 'start_date') + * ) and exists ( + * select 1 from search_datetime t7 where t7.submission_feature_id = t1.submission_feature_id and t7.value > '2023-08-01' and t7.value < '2024-04-01' and t7.feature_property_id = (select feature_property_id from feature_property where name = 'end_date') + * ) and exists ( + * select 1 from search_spatial t8 where t8.submission_feature_id = t1.submission_feature_id and public.ST_INTERSECTS(t8.value, public.ST_GeomFromGeoJSON('{"coordinates":[[[-128.12596524778567,50.90095573861839],[-128.6951954392062,50.75063500834236],[-127.71373499792975,49.63640480052965],[-125.38308025753057,48.53083459202276],[-123.3647465830768,48.15806226354249],[-122.94623399379441,48.36504151433127],[-123.37439502763095,49.13209156231335],[-124.66835857611437,49.81654191782255],[-126.6572708981094,50.607171392416745],[-127.89342678974776,50.9888374217299],[-128.12596524778567,50.90095573861839]]],"type":"Polygon"}')) + * ) + * ); + */ + +/** + * Inserts mock submission/feature data, geared towards performance testing. + * + * @export + * @param {Knex} knex + * @return {*} {Promise} + */ +export async function seed(knex: Knex): Promise { + if (!ENABLED) { + return knex.raw(`SELECT null;`); // dummy query to appease knex + } + + await knex.raw(` + SET SCHEMA 'biohub'; + SET SEARCH_PATH = 'biohub','public'; + `); + + const numSubmissions = 10000; // Number of submissions (ie: surveys) + + for (let i = 0; i < numSubmissions; i++) { + await insertRecord(knex); + } +} + +/** + * Insert a single submission record, a single dataset record, and 50 observation records. + * + * @param {Knex} knex + */ +const insertRecord = async (knex: Knex) => { + // Submission (1) + const submission_id = await insertSubmissionRecord(knex); + + // Dataset (1) + const parent_submission_feature_id1 = await insertDatasetRecord(knex, { submission_id }); + + // Sample Sites (10) + for (let i = 0; i < 10; i++) { + const parent_submission_feature_id2 = await insertSampleSiteRecord(knex, { + submission_id, + parent_submission_feature_id: parent_submission_feature_id1 + }); + + // Animals (2 per sample site) + for (let i = 0; i < 2; i++) { + await insertAnimalRecord(knex, { submission_id, parent_submission_feature_id: parent_submission_feature_id2 }); + } + + // Observations (20 per sample site) + for (let i = 0; i < 20; i++) { + await insertObservationRecord(knex, { + submission_id, + parent_submission_feature_id: parent_submission_feature_id2 + }); + } + } +}; + +const insertSubmissionRecord = async (knex: Knex): Promise => { + const response = await knex.raw(`${insertSubmission()}`); + const submission_id = response.rows[0].submission_id; + + return submission_id; +}; + +const insertDatasetRecord = async (knex: Knex, options: { submission_id: number }): Promise => { + const response = await knex.raw( + `${insertSubmissionFeature({ + submission_id: options.submission_id, + parent_submission_feature_id: null, + feature_type: 'dataset' + })}` + ); + const submission_feature_id = response.rows[0].submission_feature_id; + + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + + await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); + await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); + await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); + + await knex.raw(`${insertSearchStartDatetime({ submission_feature_id })}`); + await knex.raw(`${insertSearchEndDatetime({ submission_feature_id })}`); + + await knex.raw(`${insertSpatialPolygon({ submission_feature_id })}`); + + return submission_feature_id; +}; + +const insertSampleSiteRecord = async ( + knex: Knex, + options: { submission_id: number; parent_submission_feature_id: number } +): Promise => { + const response = await knex.raw( + `${insertSubmissionFeature({ + submission_id: options.submission_id, + parent_submission_feature_id: options.parent_submission_feature_id, + feature_type: 'sample_site' + })}` + ); + const submission_feature_id = response.rows[0].submission_feature_id; + + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + + await knex.raw(`${insertSpatialPolygon({ submission_feature_id })}`); + + return submission_feature_id; +}; + +const insertObservationRecord = async ( + knex: Knex, + options: { submission_id: number; parent_submission_feature_id: number } +): Promise => { + const response = await knex.raw( + `${insertSubmissionFeature({ + submission_id: options.submission_id, + parent_submission_feature_id: options.parent_submission_feature_id, + feature_type: 'observation' + })}` + ); + const submission_feature_id = response.rows[0].submission_feature_id; + + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + + await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); + + await knex.raw(`${insertSearchStartDatetime({ submission_feature_id })}`); + await knex.raw(`${insertSearchEndDatetime({ submission_feature_id })}`); + + await knex.raw(`${insertSpatialPoint({ submission_feature_id })}`); + + return submission_feature_id; +}; + +const insertAnimalRecord = async ( + knex: Knex, + options: { submission_id: number; parent_submission_feature_id: number } +): Promise => { + const response = await knex.raw( + `${insertSubmissionFeature({ + submission_id: options.submission_id, + parent_submission_feature_id: options.parent_submission_feature_id, + feature_type: 'animal' + })}` + ); + const submission_feature_id = response.rows[0].submission_feature_id; + + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + await knex.raw(`${insertSearchString({ submission_feature_id })}`); + + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + await knex.raw(`${insertSearchNumber({ submission_feature_id })}`); + + await knex.raw(`${insertSearchTaxonomy({ submission_feature_id })}`); + + await knex.raw(`${insertSearchStartDatetime({ submission_feature_id })}`); + await knex.raw(`${insertSearchEndDatetime({ submission_feature_id })}`); + + await knex.raw(`${insertSpatialPoint({ submission_feature_id })}`); + + return submission_feature_id; +}; + +const insertSubmission = () => ` + INSERT INTO submission + ( + source_transform_id, + uuid + ) + values + ( + 1, + public.gen_random_uuid() + ) + RETURNING submission_id; +`; + +const insertSubmissionFeature = (options: { + submission_id: number; + parent_submission_feature_id: number | null; + feature_type: 'dataset' | 'sample_site' | 'observation' | 'animal'; +}) => ` + INSERT INTO submission_feature + ( + submission_id, + parent_submission_feature_id, + feature_type_id, + data, + record_effective_date + ) + values + ( + ${options.submission_id}, + ${options.parent_submission_feature_id}, + (select feature_type_id from feature_type where name = '${options.feature_type}'), + '{ + "name": "${faker.lorem.words(3)}" + }', + now() + ) + RETURNING submission_feature_id; +`; + +const insertSearchString = (options: { submission_feature_id: number }) => ` + INSERT INTO search_string + ( + submission_feature_id, + feature_property_id, + value + ) + values + ( + ${options.submission_feature_id}, + (select feature_property_id from feature_property where name = 'name'), + $$${faker.lorem.words(3)}$$ + ); +`; + +const insertSearchNumber = (options: { submission_feature_id: number }) => ` + INSERT INTO search_number + ( + submission_feature_id, + feature_property_id, + value + ) + values + ( + ${options.submission_feature_id}, + (select feature_property_id from feature_property where name = 'count'), + $$${faker.number.int({ min: 0, max: 100 })}$$ + ); +`; + +const insertSearchTaxonomy = (options: { submission_feature_id: number }) => ` + INSERT INTO search_taxonomy + ( + submission_feature_id, + feature_property_id, + value + ) + values + ( + ${options.submission_feature_id}, + (select feature_property_id from feature_property where name = 'taxonomy'), + $$${faker.number.int({ min: 10000, max: 99999 })}$$ + ); +`; + +const insertSearchStartDatetime = (options: { submission_feature_id: number }) => ` + INSERT INTO search_datetime + ( + submission_feature_id, + feature_property_id, + value + ) + values + ( + ${options.submission_feature_id}, + (select feature_property_id from feature_property where name = 'start_date'), + $$${faker.date.past().toISOString()}$$ + ); +`; + +const insertSearchEndDatetime = (options: { submission_feature_id: number }) => ` + INSERT INTO search_datetime + ( + submission_feature_id, + feature_property_id, + value + ) + values + ( + ${options.submission_feature_id}, + (select feature_property_id from feature_property where name = 'end_date'), + $$${faker.date.future().toISOString()}$$ + ); +`; + +const insertSpatialPolygon = (options: { submission_feature_id: number }) => + ` + INSERT INTO search_spatial + ( + submission_feature_id, + feature_property_id, + value + ) + values + ( + ${options.submission_feature_id}, + (select feature_property_id from feature_property where name = 'geometry'), + public.ST_GeomFromGeoJSON( + '${JSON.stringify( + random.polygon( + 1, // number of features in feature collection + randomIntFromInterval(4, 30), // number of coordinates + 1, // degrees freedom + [-135.878906, 48.617424, -114.433594, 60.664785] // bbox constraint + )['features'][0]['geometry'] + )}' + ) + ); +`; + +const insertSpatialPoint = (options: { submission_feature_id: number }) => + ` + INSERT INTO search_spatial + ( + submission_feature_id, + feature_property_id, + value + ) + values + ( + ${options.submission_feature_id}, + (select feature_property_id from feature_property where name = 'geometry'), + public.ST_GeomFromGeoJSON( + '${JSON.stringify( + random.point( + 1, // number of features in feature collection + [-135.878906, 48.617424, -114.433594, 60.664785] // bbox constraint + )['features'][0]['geometry'] + )}' + ) + ); +`; + +const randomIntFromInterval = (min: number, max: number) => { + return Math.floor(Math.random() * (max - min + 1) + min); +}; diff --git a/docker-compose.yml b/docker-compose.yml index 04eeededd..e10f3d9f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,6 +41,7 @@ services: - ${API_PORT}:${API_PORT} environment: - NODE_ENV=${NODE_ENV} + - NODE_OPTIONS=${API_NODE_OPTIONS} - ELASTICSEARCH_URL=${ELASTICSEARCH_URL} - ELASTICSEARCH_EML_INDEX=${ELASTICSEARCH_EML_INDEX} - ELASTICSEARCH_TAXONOMY_INDEX=${ELASTICSEARCH_TAXONOMY_INDEX} @@ -148,6 +149,7 @@ services: - ${APP_PORT}:${APP_PORT} environment: - NODE_ENV=${NODE_ENV} + - NODE_OPTIONS=${APP_NODE_OPTIONS} - REACT_APP_NODE_ENV=${NODE_ENV} - PORT=${APP_PORT} - REACT_APP_API_HOST=${API_HOST} @@ -181,7 +183,6 @@ services: - DB_SCHEMA=${DB_SCHEMA} - DB_USER_API=${DB_USER_API} - DB_USER_API_PASS=${DB_USER_API_PASS} - - DB_SCHEMA_DAPI_V1=${DB_SCHEMA_DAPI_V1} volumes: - /opt/app-root/src/node_modules # prevents local node_modules overriding container node_modules networks: @@ -208,7 +209,6 @@ services: - DB_SCHEMA=${DB_SCHEMA} - DB_USER_API=${DB_USER_API} - DB_USER_API_PASS=${DB_USER_API_PASS} - - DB_SCHEMA_DAPI_V1=${DB_SCHEMA_DAPI_V1} volumes: - /opt/app-root/src/node_modules # prevents local node_modules overriding container node_modules networks: @@ -235,7 +235,6 @@ services: - DB_SCHEMA=${DB_SCHEMA} - DB_USER_API=${DB_USER_API} - DB_USER_API_PASS=${DB_USER_API_PASS} - - DB_SCHEMA_DAPI_V1=${DB_SCHEMA_DAPI_V1} volumes: - /opt/app-root/src/node_modules # prevents local node_modules overriding container node_modules networks: diff --git a/env_config/env.docker b/env_config/env.docker index a2131e854..bcaecad14 100644 --- a/env_config/env.docker +++ b/env_config/env.docker @@ -21,6 +21,10 @@ # ------------------------------------------------------------------------------ NODE_ENV=local +# Dictates the max size of the heap (Mb). Should not be greater than 75% of total available memory. The default seems to be 2Gb. +API_NODE_OPTIONS=--max-old-space-size=4096 +APP_NODE_OPTIONS=--max-old-space-size=4096 + # ------------------------------------------------------------------------------ # App # ------------------------------------------------------------------------------ @@ -54,7 +58,6 @@ DB_USER_API_PASS=postgres DB_PORT=5432 DB_DATABASE=biohubbc DB_SCHEMA=biohub -DB_SCHEMA_DAPI_V1=biohub_dapi_v1 DB_TZ=America/Vancouver # ------------------------------------------------------------------------------ From 7d4ac94075c655509a943567788a22f84db3fa19 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 17 Nov 2023 13:33:43 -0800 Subject: [PATCH 02/20] Update db user permissions --- .../src/migrations/20220225205948_release.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/database/src/migrations/20220225205948_release.ts b/database/src/migrations/20220225205948_release.ts index 35e527581..0ff85661c 100644 --- a/database/src/migrations/20220225205948_release.ts +++ b/database/src/migrations/20220225205948_release.ts @@ -72,12 +72,25 @@ export async function up(knex: Knex): Promise { -- setup postgres user GRANT ALL ON SCHEMA biohub TO postgres; - set search_path = biohub, public, topology; + set search_path = biohub, public; - -- setup api user + -- setup biohub_api user create user ${DB_USER_API} password '${DB_USER_API_PASS}'; - GRANT ALL ON SCHEMA biohub TO ${DB_USER_API}; - alter role ${DB_USER_API} set search_path to biohub, public, topology; + -- GRANT ALL ON SCHEMA biohub TO ${DB_USER_API}; + -- GRANT ALL ON ALL TABLES IN SCHEMA biohub TO ${DB_USER_API}; + alter role ${DB_USER_API} set search_path to "$user", biohub, public; + + ALTER DEFAULT PRIVILEGES IN SCHEMA biohub, public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO biohub_api; + + ALTER DEFAULT PRIVILEGES IN SCHEMA biohub, public + GRANT EXECUTE ON FUNCTIONS TO biohub_api; + + ALTER DEFAULT PRIVILEGES IN SCHEMA biohub, public + GRANT USAGE ON TYPES TO biohub_api; + + ALTER DEFAULT PRIVILEGES IN SCHEMA biohub, public + GRANT USAGE, SELECT ON SEQUENCES TO biohub_api; ${biohub_ddl} ${populate_user_identity_source} @@ -93,7 +106,6 @@ export async function up(knex: Knex): Promise { ${create_sequences} -- populate look up tables - set search_path = biohub, public; ${populate_system_constants} ${populate_system_role} ${populate_system_metadata_constant} From 36116bed3101ee94522a72292412b34e6ad373db Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 17 Nov 2023 13:47:42 -0800 Subject: [PATCH 03/20] Add missing grant on schema --- database/src/migrations/20220225205948_release.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/database/src/migrations/20220225205948_release.ts b/database/src/migrations/20220225205948_release.ts index 0ff85661c..c53e0f07b 100644 --- a/database/src/migrations/20220225205948_release.ts +++ b/database/src/migrations/20220225205948_release.ts @@ -76,8 +76,7 @@ export async function up(knex: Knex): Promise { -- setup biohub_api user create user ${DB_USER_API} password '${DB_USER_API_PASS}'; - -- GRANT ALL ON SCHEMA biohub TO ${DB_USER_API}; - -- GRANT ALL ON ALL TABLES IN SCHEMA biohub TO ${DB_USER_API}; + GRANT USAGE ON SCHEMA biohub TO ${DB_USER_API}; alter role ${DB_USER_API} set search_path to "$user", biohub, public; ALTER DEFAULT PRIVILEGES IN SCHEMA biohub, public From bbaac45ea6c817786001eb1b27ffef4179864575 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 17 Nov 2023 15:57:41 -0800 Subject: [PATCH 04/20] Fix bad string --- .../migrations/20231109000003_populate_feature_lookup_tables.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts b/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts index 31d920cdb..b5b1f75ed 100644 --- a/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts +++ b/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts @@ -45,7 +45,7 @@ export async function up(knex: Knex): Promise { insert into feature_type (name, display_name, description, record_effective_date) values ('sample_period', 'Sample Period', 'A datetime period in which data was collected', now()); insert into feature_type (name, display_name, description, record_effective_date) values ('observation', 'Observation', 'An observation record', now()); insert into feature_type (name, display_name, description, record_effective_date) values ('animal', 'Animal', 'An individual animal record', now()); - insert into feature_type (name, display_name, description, record_effective_date) values ('telemetry', 'Telemetry ', 'A telemetry record', now()); + insert into feature_type (name, display_name, description, record_effective_date) values ('telemetry', 'Telemetry', 'A telemetry record', now()); -- populate feature_type_property table -- feature_type: dataset From 48b042ce32140d64d4d69092046d58f77cff400c Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Wed, 22 Nov 2023 15:31:40 -0800 Subject: [PATCH 05/20] Update feature/search tables. Add security tables/functions. --- .../openapi/schemas/biohub-data-submission.ts | 53 ++++ .../20231109000001_feature_tables.ts | 20 +- .../20231109000002_search_tables.ts | 32 +- ...09000003_populate_feature_lookup_tables.ts | 16 +- .../20231709000001_security_tables.ts | 274 ++++++++++++++++++ .../20231709000002_security_functions.ts | 118 ++++++++ 6 files changed, 469 insertions(+), 44 deletions(-) create mode 100644 api/src/openapi/schemas/biohub-data-submission.ts create mode 100644 database/src/migrations/20231709000001_security_tables.ts create mode 100644 database/src/migrations/20231709000002_security_functions.ts diff --git a/api/src/openapi/schemas/biohub-data-submission.ts b/api/src/openapi/schemas/biohub-data-submission.ts new file mode 100644 index 000000000..8e48b422c --- /dev/null +++ b/api/src/openapi/schemas/biohub-data-submission.ts @@ -0,0 +1,53 @@ +export const BioHubFeatureCollection = { + title: 'BioHub Data Submission', + type: 'object', + required: ['id', 'type', 'features'], + properties: { + id: { + title: 'Unique id of the submission', + type: 'string' + }, + type: { + type: 'string', + enum: ['Submission'] + }, + features: { + type: 'array', + items: { + $ref: '#/$defs/Feature' + } + } + }, + $defs: { + Feature: { + title: 'BioHub Data Submission Feature', + type: 'object', + required: ['id', 'type', 'properties', 'features'], + properties: { + id: { + title: 'Unique id of the feature', + type: 'string' + }, + type: { + title: 'Feature type', + type: 'string' + }, + properties: { + title: 'Feature properties', + type: 'object', + properties: {} + }, + features: { + title: 'Feature child features', + type: 'array', + items: { + $ref: '#/$defs/Feature' + } + } + }, + additionalProperties: false + }, + additionalProperties: false + }, + additionalProperties: false +}; diff --git a/database/src/migrations/20231109000001_feature_tables.ts b/database/src/migrations/20231109000001_feature_tables.ts index 85a028bb8..ec6a6f296 100644 --- a/database/src/migrations/20231109000001_feature_tables.ts +++ b/database/src/migrations/20231109000001_feature_tables.ts @@ -17,7 +17,7 @@ export async function up(knex: Knex): Promise { ---------------------------------------------------------------------------------------- -- Create tables ---------------------------------------------------------------------------------------- - set search_path=biohub,public; + set search_path=biohub,public; CREATE TABLE submission_feature( submission_feature_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), @@ -34,7 +34,7 @@ export async function up(knex: Knex): Promise { revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT submission_feature_pk PRIMARY KEY (submission_feature_id) ); - + COMMENT ON COLUMN submission_feature.submission_feature_id IS 'System generated surrogate primary key identifier.'; COMMENT ON COLUMN submission_feature.submission_id IS 'Foreign key to the submission table.'; COMMENT ON COLUMN submission_feature.feature_type_id IS 'Foreign key to the feature_type table.'; @@ -48,14 +48,14 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN submission_feature.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN submission_feature.revision_count IS 'Revision count used for concurrency control.'; COMMENT ON TABLE submission_feature IS 'A set of data for a specific feature of a submission.'; - + ---------------------------------------------------------------------------------------- CREATE TABLE feature_type( feature_type_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - name varchar(50) NOT NULL, + name varchar(100) NOT NULL, display_name varchar(100) NOT NULL, - description varchar(3000), + description varchar(500), record_effective_date date NOT NULL, record_end_date date, create_date timestamptz(6) DEFAULT now() NOT NULL, @@ -78,7 +78,7 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN feature_type.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN feature_type.revision_count IS 'Revision count used for concurrency control.'; COMMENT ON TABLE feature_type IS 'Defines feature types.'; - + ---------------------------------------------------------------------------------------- CREATE TABLE feature_type_property( @@ -106,15 +106,15 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN feature_type_property.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN feature_type_property.revision_count IS 'Revision count used for concurrency control.'; COMMENT ON TABLE feature_type_property IS 'A join table on feature type and feature_property. Defines which properties can be used by a given feature type.'; - + ---------------------------------------------------------------------------------------- CREATE TABLE feature_property( feature_property_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), feature_property_type_id integer NOT NULL, - name varchar(50) NOT NULL, + name varchar(100) NOT NULL, display_name varchar(100) NOT NULL, - description varchar(3000), + description varchar(500), parent_feature_property_id integer, record_effective_date date NOT NULL, record_end_date date, @@ -146,7 +146,7 @@ export async function up(knex: Knex): Promise { CREATE TABLE feature_property_type( feature_property_type_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), name varchar(100) NOT NULL, - description varchar(3000), + description varchar(500), record_effective_date date NOT NULL, record_end_date date, create_date timestamptz(6) DEFAULT now() NOT NULL, diff --git a/database/src/migrations/20231109000002_search_tables.ts b/database/src/migrations/20231109000002_search_tables.ts index a45cd6c43..353cac421 100644 --- a/database/src/migrations/20231109000002_search_tables.ts +++ b/database/src/migrations/20231109000002_search_tables.ts @@ -25,8 +25,6 @@ export async function up(knex: Knex): Promise { submission_feature_id integer NOT NULL, feature_property_id integer NOT NULL, value varchar(250) NOT NULL, --- record_effective_date date NOT NULL, --- record_end_date date, create_date timestamptz(6) DEFAULT now() NOT NULL, create_user integer NOT NULL, update_date timestamptz(6), @@ -34,20 +32,18 @@ export async function up(knex: Knex): Promise { revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT search_string_pk PRIMARY KEY (search_string_id) ); - + COMMENT ON COLUMN search_string.search_string_id IS 'System generated surrogate primary key identifier.'; COMMENT ON COLUMN search_string.submission_feature_id IS 'Foreign key to the submission_feature table.'; COMMENT ON COLUMN search_string.feature_property_id IS 'Foreign key to the feature_property table.'; COMMENT ON COLUMN search_string.value IS 'The search value of the record.'; --- COMMENT ON COLUMN search_string.record_effective_date IS 'Record level effective date.'; --- COMMENT ON COLUMN search_string.record_end_date IS 'Record level end date.'; COMMENT ON COLUMN search_string.create_date IS 'The datetime the record was created.'; COMMENT ON COLUMN search_string.create_user IS 'The id of the user who created the record as identified in the system user table.'; COMMENT ON COLUMN search_string.update_date IS 'The datetime the record was updated.'; COMMENT ON COLUMN search_string.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN search_string.revision_count IS 'Revision count used for concurrency control.'; COMMENT ON TABLE search_string IS 'String search values'; - + ---------------------------------------------------------------------------------------- CREATE TABLE search_number( @@ -55,8 +51,6 @@ export async function up(knex: Knex): Promise { submission_feature_id integer NOT NULL, feature_property_id integer NOT NULL, value numeric NOT NULL, --- record_effective_date date NOT NULL, --- record_end_date date, create_date timestamptz(6) DEFAULT now() NOT NULL, create_user integer NOT NULL, update_date timestamptz(6), @@ -64,13 +58,11 @@ export async function up(knex: Knex): Promise { revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT search_number_pk PRIMARY KEY (search_number_id) ); - + COMMENT ON COLUMN search_number.search_number_id IS 'System generated surrogate primary key identifier.'; COMMENT ON COLUMN search_number.submission_feature_id IS 'Foreign key to the submission_feature table.'; COMMENT ON COLUMN search_number.feature_property_id IS 'Foreign key to the feature_property table.'; COMMENT ON COLUMN search_number.value IS 'The search value of the record.'; --- COMMENT ON COLUMN search_number.record_effective_date IS 'Record level effective date.'; --- COMMENT ON COLUMN search_number.record_end_date IS 'Record level end date.'; COMMENT ON COLUMN search_number.create_date IS 'The datetime the record was created.'; COMMENT ON COLUMN search_number.create_user IS 'The id of the user who created the record as identified in the system user table.'; COMMENT ON COLUMN search_number.update_date IS 'The datetime the record was updated.'; @@ -85,8 +77,6 @@ export async function up(knex: Knex): Promise { submission_feature_id integer NOT NULL, feature_property_id integer NOT NULL, value timestamptz(6) NOT NULL, --- record_effective_date date NOT NULL, --- record_end_date date, create_date timestamptz(6) DEFAULT now() NOT NULL, create_user integer NOT NULL, update_date timestamptz(6), @@ -94,13 +84,11 @@ export async function up(knex: Knex): Promise { revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT search_datetime_pk PRIMARY KEY (search_datetime_id) ); - + COMMENT ON COLUMN search_datetime.search_datetime_id IS 'System generated surrogate primary key identifier.'; COMMENT ON COLUMN search_datetime.submission_feature_id IS 'Foreign key to the submission_feature table.'; COMMENT ON COLUMN search_datetime.feature_property_id IS 'Foreign key to the feature_property table.'; COMMENT ON COLUMN search_datetime.value IS 'The search value of the record.'; --- COMMENT ON COLUMN search_datetime.record_effective_date IS 'Record level effective date.'; --- COMMENT ON COLUMN search_datetime.record_end_date IS 'Record level end date.'; COMMENT ON COLUMN search_datetime.create_date IS 'The datetime the record was created.'; COMMENT ON COLUMN search_datetime.create_user IS 'The id of the user who created the record as identified in the system user table.'; COMMENT ON COLUMN search_datetime.update_date IS 'The datetime the record was updated.'; @@ -115,8 +103,6 @@ export async function up(knex: Knex): Promise { submission_feature_id integer NOT NULL, feature_property_id integer NOT NULL, value geometry NOT NULL, --- record_effective_date date NOT NULL, --- record_end_date date, create_date timestamptz(6) DEFAULT now() NOT NULL, create_user integer NOT NULL, update_date timestamptz(6), @@ -124,13 +110,11 @@ export async function up(knex: Knex): Promise { revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT search_spatial_pk PRIMARY KEY (search_spatial_id) ); - + COMMENT ON COLUMN search_spatial.search_spatial_id IS 'System generated surrogate primary key identifier.'; COMMENT ON COLUMN search_spatial.submission_feature_id IS 'Foreign key to the submission_feature table.'; COMMENT ON COLUMN search_spatial.feature_property_id IS 'Foreign key to the feature_property table.'; COMMENT ON COLUMN search_spatial.value IS 'The search value of the record.'; --- COMMENT ON COLUMN search_spatial.record_effective_date IS 'Record level effective date.'; --- COMMENT ON COLUMN search_spatial.record_end_date IS 'Record level end date.'; COMMENT ON COLUMN search_spatial.create_date IS 'The spatial the record was created.'; COMMENT ON COLUMN search_spatial.create_user IS 'The id of the user who created the record as identified in the system user table.'; COMMENT ON COLUMN search_spatial.update_date IS 'The spatial the record was updated.'; @@ -145,8 +129,6 @@ export async function up(knex: Knex): Promise { submission_feature_id integer NOT NULL, feature_property_id integer NOT NULL, value numeric NOT NULL, --- record_effective_date date NOT NULL, --- record_end_date date, create_date timestamptz(6) DEFAULT now() NOT NULL, create_user integer NOT NULL, update_date timestamptz(6), @@ -154,13 +136,11 @@ export async function up(knex: Knex): Promise { revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT search_taxonomy_pk PRIMARY KEY (search_taxonomy_id) ); - + COMMENT ON COLUMN search_taxonomy.search_taxonomy_id IS 'System generated surrogate primary key identifier.'; COMMENT ON COLUMN search_taxonomy.submission_feature_id IS 'Foreign key to the submission_feature table.'; COMMENT ON COLUMN search_taxonomy.feature_property_id IS 'Foreign key to the feature_property table.'; COMMENT ON COLUMN search_taxonomy.value IS 'The search value of the record.'; --- COMMENT ON COLUMN search_taxonomy.record_effective_date IS 'Record level effective date.'; --- COMMENT ON COLUMN search_taxonomy.record_end_date IS 'Record level end date.'; COMMENT ON COLUMN search_taxonomy.create_date IS 'The taxonomy the record was created.'; COMMENT ON COLUMN search_taxonomy.create_user IS 'The id of the user who created the record as identified in the system user table.'; COMMENT ON COLUMN search_taxonomy.update_date IS 'The taxonomy the record was updated.'; diff --git a/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts b/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts index b5b1f75ed..b00afd514 100644 --- a/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts +++ b/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts @@ -45,7 +45,7 @@ export async function up(knex: Knex): Promise { insert into feature_type (name, display_name, description, record_effective_date) values ('sample_period', 'Sample Period', 'A datetime period in which data was collected', now()); insert into feature_type (name, display_name, description, record_effective_date) values ('observation', 'Observation', 'An observation record', now()); insert into feature_type (name, display_name, description, record_effective_date) values ('animal', 'Animal', 'An individual animal record', now()); - insert into feature_type (name, display_name, description, record_effective_date) values ('telemetry', 'Telemetry', 'A telemetry record', now()); + insert into feature_type (name, display_name, description, record_effective_date) values ('telemetry', 'Telemetry', 'A telemetry record', now()); -- populate feature_type_property table -- feature_type: dataset @@ -56,21 +56,21 @@ export async function up(knex: Knex): Promise { insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'start_date'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'end_date'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'dataset'), (select feature_property_id from feature_property where name = 'geometry'), now()); - + -- feature_type: sample_site insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_site'), (select feature_property_id from feature_property where name = 'name'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_site'), (select feature_property_id from feature_property where name = 'description'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_site'), (select feature_property_id from feature_property where name = 'geometry'), now()); - + -- feature_type: sample_method insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_method'), (select feature_property_id from feature_property where name = 'name'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_method'), (select feature_property_id from feature_property where name = 'description'), now()); - + -- feature_type: sample_period insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_period'), (select feature_property_id from feature_property where name = 'date_range'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_period'), (select feature_property_id from feature_property where name = 'start_date'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'sample_period'), (select feature_property_id from feature_property where name = 'end_date'), now()); - + -- feature_type: observation insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'taxonomy'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'date_range'), now()); @@ -78,10 +78,10 @@ export async function up(knex: Knex): Promise { insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'end_date'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'geometry'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'observation'), (select feature_property_id from feature_property where name = 'count'), now()); - + -- feature_type: animal - insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'animal'), (select feature_property_id from feature_property where name = 'taxonomy'), now()); - + insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'animal'), (select feature_property_id from feature_property where name = 'taxonomy'), now()); + -- feature_type: telemetry insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'telemetry'), (select feature_property_id from feature_property where name = 'date_range'), now()); insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'telemetry'), (select feature_property_id from feature_property where name = 'start_date'), now()); diff --git a/database/src/migrations/20231709000001_security_tables.ts b/database/src/migrations/20231709000001_security_tables.ts new file mode 100644 index 000000000..dd43485d2 --- /dev/null +++ b/database/src/migrations/20231709000001_security_tables.ts @@ -0,0 +1,274 @@ +import { Knex } from 'knex'; + +/** + * Add tables: + * - submission_feature + * - feature_type + * - feature_type_property + * - feature_property + * - feature_property_type + * + * @export + * @param {Knex} knex + * @return {*} {Promise} + */ +export async function up(knex: Knex): Promise { + await knex.raw(` + ---------------------------------------------------------------------------------------- + -- Create tables + ---------------------------------------------------------------------------------------- + set search_path=biohub,public; + + CREATE TABLE security_rule( + security_rule_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + name varchar(100) NOT NULL, + description varchar(500), + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT security_rule_pk PRIMARY KEY (security_rule_id) + ); + + COMMENT ON COLUMN security_rule.security_rule_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN security_rule.name IS 'The name of the security_rule record.'; + COMMENT ON COLUMN security_rule.description IS 'The description of the security_rule record.'; + COMMENT ON COLUMN security_rule.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN security_rule.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN security_rule.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN security_rule.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN security_rule.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN security_rule.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN security_rule.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE security_rule IS 'A security rule.'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE security_string( + security_string_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + security_rule_id integer NOT NULL, + name varchar(100) NOT NULL, + description varchar(500), + feature_property_id integer NOT NULL, + value varchar(250) NOT NULL, + comparator varchar(50) NOT NULL, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT security_string_pk PRIMARY KEY (security_string_id) + ); + + COMMENT ON COLUMN security_string.security_string_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN security_string.security_rule_id IS 'Foreign key to the security_string table.'; + COMMENT ON COLUMN security_string.name IS 'The name of the security_string record.'; + COMMENT ON COLUMN security_string.description IS 'The description of the security_string record.'; + COMMENT ON COLUMN security_string.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN security_string.value IS 'The search value of the record.'; + COMMENT ON COLUMN security_string.comparator IS 'The comparison template.'; + COMMENT ON COLUMN security_string.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN security_string.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN security_string.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN security_string.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN security_string.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE security_string IS 'String search values'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE security_number( + security_number_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + security_rule_id integer NOT NULL, + name varchar(100) NOT NULL, + description varchar(500), + feature_property_id integer NOT NULL, + value numeric NOT NULL, + comparator varchar(50) NOT NULL, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT security_number_pk PRIMARY KEY (security_number_id) + ); + + COMMENT ON COLUMN security_number.security_number_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN security_number.security_rule_id IS 'Foreign key to the security_number table.'; + COMMENT ON COLUMN security_number.name IS 'The name of the security_number record.'; + COMMENT ON COLUMN security_number.description IS 'The description of the security_number record.'; + COMMENT ON COLUMN security_number.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN security_number.value IS 'The search value of the record.'; + COMMENT ON COLUMN security_number.comparator IS 'The comparison template.'; + COMMENT ON COLUMN security_number.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN security_number.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN security_number.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN security_number.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN security_number.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE security_number IS 'Numeric search values'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE security_datetime( + security_datetime_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + security_rule_id integer NOT NULL, + name varchar(100) NOT NULL, + description varchar(500), + feature_property_id integer NOT NULL, + value timestamptz(6) NOT NULL, + comparator varchar(50) NOT NULL, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT security_datetime_pk PRIMARY KEY (security_datetime_id) + ); + + COMMENT ON COLUMN security_datetime.security_datetime_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN security_datetime.security_rule_id IS 'Foreign key to the security_datetime table.'; + COMMENT ON COLUMN security_datetime.name IS 'The name of the security_datetime record.'; + COMMENT ON COLUMN security_datetime.description IS 'The description of the security_datetime record.'; + COMMENT ON COLUMN security_datetime.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN security_datetime.value IS 'The search value of the record.'; + COMMENT ON COLUMN security_datetime.comparator IS 'The comparison template.'; + COMMENT ON COLUMN security_datetime.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN security_datetime.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN security_datetime.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN security_datetime.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN security_datetime.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE security_datetime IS 'Numeric search values'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE security_spatial( + security_spatial_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + security_rule_id integer NOT NULL, + name varchar(100) NOT NULL, + description varchar(500), + feature_property_id integer NOT NULL, + value geometry NOT NULL, + comparator varchar(50) NOT NULL, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT security_spatial_pk PRIMARY KEY (security_spatial_id) + ); + + COMMENT ON COLUMN security_spatial.security_spatial_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN security_spatial.security_rule_id IS 'Foreign key to the security_spatial table.'; + COMMENT ON COLUMN security_spatial.name IS 'The name of the security_spatial record.'; + COMMENT ON COLUMN security_spatial.description IS 'The description of the security_spatial record.'; + COMMENT ON COLUMN security_spatial.feature_property_id IS 'Foreign key to the feature_property table.'; + COMMENT ON COLUMN security_spatial.value IS 'The search value of the record.'; + COMMENT ON COLUMN security_spatial.comparator IS 'The comparison template.'; + COMMENT ON COLUMN security_spatial.create_date IS 'The spatial the record was created.'; + COMMENT ON COLUMN security_spatial.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN security_spatial.update_date IS 'The spatial the record was updated.'; + COMMENT ON COLUMN security_spatial.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN security_spatial.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE security_spatial IS 'Numeric search values'; + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: security_rule + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) + CREATE UNIQUE INDEX security_rule_nuk1 ON security_rule(name, (record_end_date is NULL)) where record_end_date is null; + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: security_string + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) + CREATE UNIQUE INDEX security_string_nuk1 ON security_string(name, (record_end_date is NULL)) where record_end_date is null; + + -- Add foreign key constraint + ALTER TABLE security_string ADD CONSTRAINT security_string_fk1 + FOREIGN KEY (security_rule_id) + REFERENCES security_rule(security_rule_id); + + -- add indexes for foreign keys + CREATE INDEX security_string_idx1 ON security_string(security_rule_id); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: security_number + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) + CREATE UNIQUE INDEX security_number_nuk1 ON security_number(name, (record_end_date is NULL)) where record_end_date is null; + + -- Add foreign key constraint + ALTER TABLE security_number ADD CONSTRAINT security_number_fk1 + FOREIGN KEY (security_rule_id) + REFERENCES security_rule(security_rule_id); + + -- add indexes for foreign keys + CREATE INDEX security_number_idx1 ON security_number(security_rule_id); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: security_datetime + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) + CREATE UNIQUE INDEX security_datetime_nuk1 ON security_datetime(name, (record_end_date is NULL)) where record_end_date is null; + + -- Add foreign key constraint + ALTER TABLE security_datetime ADD CONSTRAINT security_datetime_fk1 + FOREIGN KEY (security_rule_id) + REFERENCES security_rule(security_rule_id); + + -- add indexes for foreign keys + CREATE INDEX security_datetime_idx1 ON security_datetime(security_rule_id); + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: security_spatial + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) + CREATE UNIQUE INDEX security_spatial_nuk1 ON security_spatial(name, (record_end_date is NULL)) where record_end_date is null; + + -- Add foreign key constraint + ALTER TABLE security_spatial ADD CONSTRAINT security_spatial_fk1 + FOREIGN KEY (security_rule_id) + REFERENCES security_rule(security_rule_id); + + -- add indexes for foreign keys + CREATE INDEX security_spatial_idx1 ON security_spatial(security_rule_id); + + ---------------------------------------------------------------------------------------- + -- Create audit and journal triggers + ---------------------------------------------------------------------------------------- + + create trigger audit_security_rule before insert or update or delete on security_rule for each row execute procedure tr_audit_trigger(); + create trigger journal_security_rule after insert or update or delete on security_rule for each row execute procedure tr_journal_trigger(); + + create trigger audit_security_string before insert or update or delete on security_string for each row execute procedure tr_audit_trigger(); + create trigger journal_security_string after insert or update or delete on security_string for each row execute procedure tr_journal_trigger(); + + create trigger audit_security_number before insert or update or delete on security_number for each row execute procedure tr_audit_trigger(); + create trigger journal_security_number after insert or update or delete on security_number for each row execute procedure tr_journal_trigger(); + + create trigger audit_security_datetime before insert or update or delete on security_datetime for each row execute procedure tr_audit_trigger(); + create trigger journal_security_datetime after insert or update or delete on security_datetime for each row execute procedure tr_journal_trigger(); + + create trigger audit_security_spatial before insert or update or delete on security_spatial for each row execute procedure tr_audit_trigger(); + create trigger journal_security_spatial after insert or update or delete on security_spatial for each row execute procedure tr_journal_trigger(); + `); +} + +export async function down(knex: Knex): Promise { + await knex.raw(``); +} diff --git a/database/src/migrations/20231709000002_security_functions.ts b/database/src/migrations/20231709000002_security_functions.ts new file mode 100644 index 000000000..ae00ec191 --- /dev/null +++ b/database/src/migrations/20231709000002_security_functions.ts @@ -0,0 +1,118 @@ +import { Knex } from 'knex'; + +/** + * Add functions: + * - evaluate_security_string_condition + * - evaluate_security_number_condition + * - evaluate_security_rule + * + * @export + * @param {Knex} knex + * @return {*} {Promise} + */ +export async function up(knex: Knex): Promise { + await knex.raw(` + ---------------------------------------------------------------------------------------- + -- Create functions + ---------------------------------------------------------------------------------------- + set search_path=biohub,public; + + -- Executes a single security_string condition against the provided submission_feature_id + -- Note: Returns true if the condition hit (applied). + CREATE OR REPLACE FUNCTION evaluate_security_string_condition(rule_name VARCHAR, submission_feature_id integer) + RETURNS TABLE (result boolean) + language plpgsql + SET client_min_messages = warning + AS $$ + DECLARE + comp TEXT; + val text; + BEGIN + SELECT security_string.comparator, security_string.value INTO comp, val FROM security_string WHERE name = evaluate_security_string_condition.rule_name; + + RETURN QUERY EXECUTE format('SELECT CASE WHEN EXISTS (SELECT 1 FROM search_string WHERE submission_feature_id = %s AND value %s ''%s'') THEN TRUE ELSE FALSE END', evaluate_security_string_condition.submission_feature_id, comp, val); + END; + $$; + + -- Executes a single security_number condition against the provided submission_feature_id + -- Note: Returns true if the condition hit (applied). + CREATE OR REPLACE FUNCTION evaluate_security_number_condition(rule_name VARCHAR, submission_feature_id integer) + RETURNS TABLE (result boolean) + language plpgsql + SET client_min_messages = warning + AS $$ + DECLARE + comparator TEXT; + value numeric; + BEGIN + SELECT security_number.comparator, security_number.value INTO comparator, value FROM security_number WHERE name = evaluate_security_number_condition.rule_name; + + RETURN QUERY EXECUTE format('SELECT CASE WHEN EXISTS (SELECT 1 FROM search_number WHERE submission_feature_id = %s AND value %s %s) THEN TRUE ELSE FALSE END', evaluate_security_number_condition.submission_feature_id, comparator, value); + END; + $$; + + -- Executes a single security_spatial condition against the provided submission_feature_id + -- Note: Returns true if the condition hit (applied). + -- Note: SRID of both geometries must be the same (prefer 4326) + CREATE OR REPLACE FUNCTION evaluate_security_spatial_condition(rule_name VARCHAR, submission_feature_id integer) + RETURNS TABLE (result BOOLEAN) + language plpgsql + set client_min_messages = warning + AS $$ + DECLARE + comparator TEXT; + value geometry; + BEGIN + SELECT security_spatial.comparator, security_spatial.value INTO comparator, value FROM security_spatial WHERE name = evaluate_security_spatial_condition.rule_name; + + RETURN QUERY EXECUTE FORMAT('SELECT CASE WHEN EXISTS (SELECT 1 FROM search_spatial WHERE submission_feature_id = %s AND ST_Intersects(value, ''%s'')) THEN TRUE ELSE FALSE END', evaluate_security_spatial_condition.submission_feature_id, value); + END; + $$; + + -- Executes all security rules against the provided submission_feature_id + -- Note: Returns a list of security rules and a boolean indicating if ALL conditions under that rule hit (applied). + CREATE OR REPLACE FUNCTION evaluate_security_rule(submission_feature_id integer) + RETURNS TABLE (result boolean, security_rule_id integer) + language plpgsql + set client_min_messages = warning + AS $$ + begin + return QUERY ( + select bool_and(r1.result) result, r1.security_rule_id from ( + (select bool_and(ex1.result) result, s1.security_rule_id from security_string s1, lateral evaluate_security_string_condition(s1.name, evaluate_security_rule.submission_feature_id) ex1 group by s1.security_rule_id) + union ALL + (select bool_and(ex2.result) result, s2.security_rule_id from security_number s2, lateral evaluate_security_number_condition(s2.name, evaluate_security_rule.submission_feature_id) ex2 group by s2.security_rule_id) + union ALL + (select bool_and(ex3.result) result, s3.security_rule_id from security_datetime s3, lateral evaluate_security_datetime_condition(s3.name, evaluate_security_rule.submission_feature_id) ex3 group by s3.security_rule_id) + union ALL + (select bool_and(ex3.result) result, s3.security_rule_id from security_spatial s3, lateral evaluate_security_spatial_condition(s3.name, evaluate_security_rule.submission_feature_id) ex3 group by s3.security_rule_id) + ) r1 group by r1.security_rule_id + ); + END; + $$; + + -- Executes all security rules against the provided submission_feature_id + -- Note: This is an alternate version to 'evaluate_security_rule' that includes the names of the conditions that hit/missed (applied/didn't apply). + CREATE OR REPLACE FUNCTION evaluate_security_rule_2(submission_feature_id integer) + RETURNS TABLE (result boolean, security_rule_id integer, security_condition_name varchar) + language plpgsql + set client_min_messages = warning + AS $$ + begin + return QUERY ( + (select bool_and(ex1.result) result, s1.security_rule_id, s1.name security_condition_name from security_string s1, lateral evaluate_security_string_condition(s1.name, evaluate_security_rule_2.submission_feature_id) ex1 group by s1.security_rule_id, s1.name) + union ALL + (select bool_and(ex2.result) result, s2.security_rule_id, s2.name security_condition_name from security_number s2, lateral evaluate_security_number_condition(s2.name, evaluate_security_rule_2.submission_feature_id) ex2 group by s2.security_rule_id, s2.name) + union ALL + (select bool_and(ex3.result) result, s3.security_rule_id, s3.name security_condition_name from security_datetime s3, lateral evaluate_security_datetime_condition(s3.name, evaluate_security_rule_2.submission_feature_id) ex3 group by s3.security_rule_id, s3.name) + union ALL + (select bool_and(ex3.result) result, s3.security_rule_id, s3.name security_condition_name from security_spatial s3, lateral evaluate_security_spatial_condition(s3.name, evaluate_security_rule_2.submission_feature_id) ex3 group by s3.security_rule_id, s3.name) + ); + END; + $$; + `); +} + +export async function down(knex: Knex): Promise { + await knex.raw(``); +} From bff94164f9d554f2c0bfc25a98f1b2ad038d2802 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Wed, 22 Nov 2023 15:35:23 -0800 Subject: [PATCH 06/20] Remove taxonomy search table, update taxonomy to be a number (taxon id) --- .../20231109000002_search_tables.ts | 49 ------------------- ...09000003_populate_feature_lookup_tables.ts | 23 +++++---- database/src/seeds/04_mock_test_data.ts | 2 +- 3 files changed, 12 insertions(+), 62 deletions(-) diff --git a/database/src/migrations/20231109000002_search_tables.ts b/database/src/migrations/20231109000002_search_tables.ts index 353cac421..8800d5b44 100644 --- a/database/src/migrations/20231109000002_search_tables.ts +++ b/database/src/migrations/20231109000002_search_tables.ts @@ -122,32 +122,6 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN search_spatial.revision_count IS 'Revision count used for concurrency control.'; COMMENT ON TABLE search_spatial IS 'Spatial search values'; - ---------------------------------------------------------------------------------------- - - CREATE TABLE search_taxonomy( - search_taxonomy_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - submission_feature_id integer NOT NULL, - feature_property_id integer NOT NULL, - value numeric NOT NULL, - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT search_taxonomy_pk PRIMARY KEY (search_taxonomy_id) - ); - - COMMENT ON COLUMN search_taxonomy.search_taxonomy_id IS 'System generated surrogate primary key identifier.'; - COMMENT ON COLUMN search_taxonomy.submission_feature_id IS 'Foreign key to the submission_feature table.'; - COMMENT ON COLUMN search_taxonomy.feature_property_id IS 'Foreign key to the feature_property table.'; - COMMENT ON COLUMN search_taxonomy.value IS 'The search value of the record.'; - COMMENT ON COLUMN search_taxonomy.create_date IS 'The taxonomy the record was created.'; - COMMENT ON COLUMN search_taxonomy.create_user IS 'The id of the user who created the record as identified in the system user table.'; - COMMENT ON COLUMN search_taxonomy.update_date IS 'The taxonomy the record was updated.'; - COMMENT ON COLUMN search_taxonomy.update_user IS 'The id of the user who updated the record as identified in the system user table.'; - COMMENT ON COLUMN search_taxonomy.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE search_taxonomy IS 'Taxonomy search values'; - ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: search_string ---------------------------------------------------------------------------------------- @@ -229,26 +203,6 @@ export async function up(knex: Knex): Promise { -- add spatial index CREATE INDEX search_spatial_idx3 ON search_spatial using GIST(value); - ---------------------------------------------------------------------------------------- - -- Create Indexes and Constraints for table: search_taxonomy - ---------------------------------------------------------------------------------------- - - -- Add foreign key constraint - ALTER TABLE search_taxonomy ADD CONSTRAINT search_taxonomy_fk1 - FOREIGN KEY (submission_feature_id) - REFERENCES submission_feature(submission_feature_id); - - ALTER TABLE search_taxonomy ADD CONSTRAINT search_taxonomy_fk2 - FOREIGN KEY (feature_property_id) - REFERENCES feature_property(feature_property_id); - - -- add indexes for foreign keys - CREATE INDEX search_taxonomy_idx1 ON search_taxonomy(submission_feature_id); - - CREATE INDEX search_taxonomy_idx2 ON search_taxonomy(feature_property_id); - - CREATE INDEX search_taxonomy_idx3 ON search_taxonomy(value); - ---------------------------------------------------------------------------------------- -- Create audit and journal triggers ---------------------------------------------------------------------------------------- @@ -264,9 +218,6 @@ export async function up(knex: Knex): Promise { create trigger audit_search_spatial before insert or update or delete on search_spatial for each row execute procedure tr_audit_trigger(); create trigger journal_search_spatial after insert or update or delete on search_spatial for each row execute procedure tr_journal_trigger(); - - create trigger audit_search_taxonomy before insert or update or delete on search_taxonomy for each row execute procedure tr_audit_trigger(); - create trigger journal_search_taxonomy after insert or update or delete on search_taxonomy for each row execute procedure tr_journal_trigger(); `); } diff --git a/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts b/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts index b00afd514..8ea893cf5 100644 --- a/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts +++ b/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts @@ -19,24 +19,23 @@ export async function up(knex: Knex): Promise { set search_path=biohub,public; -- populate feature_property_type table - insert into feature_property_type (name, description, record_effective_date) values ('string', 'A text type', now()); + insert into feature_property_type (name, description, record_effective_date) values ('string', 'A string type', now()); insert into feature_property_type (name, description, record_effective_date) values ('number', 'A number type', now()); + insert into feature_property_type (name, description, record_effective_date) values ('datetime', 'A datetime type (ISO 8601)', now()); + insert into feature_property_type (name, description, record_effective_date) values ('spatial', 'A spatial type', now()); insert into feature_property_type (name, description, record_effective_date) values ('boolean', 'A boolean type', now()); insert into feature_property_type (name, description, record_effective_date) values ('object', 'An object type', now()); insert into feature_property_type (name, description, record_effective_date) values ('array', 'An array type', now()); - insert into feature_property_type (name, description, record_effective_date) values ('datetime', 'A datetime type (ISO 8601)', now()); - insert into feature_property_type (name, description, record_effective_date) values ('spatial', 'A spatial type', now()); - insert into feature_property_type (name, description, record_effective_date) values ('taxonomy', 'A taxonomic type', now()); -- populate feature_property table - insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('name', 'Name', 'The name of the record', (select feature_property_type_id from feature_property_type where name = 'string'), null, now()); - insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('description', 'Description', 'The description of the record', (select feature_property_type_id from feature_property_type where name = 'string'), null, now()); - insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('taxonomy', 'Taxonomy', 'The taxonomy associated to the record', (select feature_property_type_id from feature_property_type where name = 'taxonomy'), null, now()); - insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('date_range', 'Date Range', 'A date range', (select feature_property_type_id from feature_property_type where name = 'object'), null, now()); - insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('start_date', 'Start Date', 'The start date of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), (select feature_property_id from feature_property where name = 'date_range'), now()); - insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('end_date', 'End Date', 'The end date of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), (select feature_property_id from feature_property where name = 'date_range'), now()); - insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('geometry', 'Geometry', 'The location of the record', (select feature_property_type_id from feature_property_type where name = 'spatial'), null, now()); - insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('count', 'Count', 'The count of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('name', 'Name', 'The name of the record', (select feature_property_type_id from feature_property_type where name = 'string'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('description', 'Description', 'The description of the record', (select feature_property_type_id from feature_property_type where name = 'string'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('taxonomy', 'Taxonomy Id', 'The taxonomy Id associated to the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('date_range', 'Date Range', 'A date range', (select feature_property_type_id from feature_property_type where name = 'object'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('start_date', 'Start Date', 'The start date of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), (select feature_property_id from feature_property where name = 'date_range'), now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('end_date', 'End Date', 'The end date of the record', (select feature_property_type_id from feature_property_type where name = 'datetime'), (select feature_property_id from feature_property where name = 'date_range'), now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('geometry', 'Geometry', 'The location of the record', (select feature_property_type_id from feature_property_type where name = 'spatial'), null, now()); + insert into feature_property (name, display_name, description, feature_property_type_id, parent_feature_property_id, record_effective_date) values ('count', 'Count', 'The count of the record', (select feature_property_type_id from feature_property_type where name = 'number'), null, now()); -- populate feature_type table insert into feature_type (name, display_name, description, record_effective_date) values ('dataset', 'Dataset', 'A related collection of data (ie: survey)', now()); diff --git a/database/src/seeds/04_mock_test_data.ts b/database/src/seeds/04_mock_test_data.ts index 14280d405..21842c6e0 100644 --- a/database/src/seeds/04_mock_test_data.ts +++ b/database/src/seeds/04_mock_test_data.ts @@ -299,7 +299,7 @@ const insertSearchTaxonomy = (options: { submission_feature_id: number }) => ` values ( ${options.submission_feature_id}, - (select feature_property_id from feature_property where name = 'taxonomy'), + (select feature_property_id from feature_property where name = 'number'), $$${faker.number.int({ min: 10000, max: 99999 })}$$ ); `; From 66d18c371c6019762ed13ec5ec9f19770fc02759 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Wed, 22 Nov 2023 15:44:27 -0800 Subject: [PATCH 07/20] Add env var to control seeding --- database/src/seeds/04_mock_test_data.ts | 11 +++++------ env_config/env.docker | 8 ++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/database/src/seeds/04_mock_test_data.ts b/database/src/seeds/04_mock_test_data.ts index 21842c6e0..a46c22210 100644 --- a/database/src/seeds/04_mock_test_data.ts +++ b/database/src/seeds/04_mock_test_data.ts @@ -3,8 +3,9 @@ import { faker } from '@faker-js/faker'; import random from 'geojson-random'; import { Knex } from 'knex'; -// Disable mock data seeding by default, unless manually enabled -const ENABLED = false; +// Disable mock data seeding by default. Set `ENABLE_MOCK_FEATURE_DATA=true` to enable. +const ENABLE_MOCK_FEATURE_SEEDING = Boolean(process.env.ENABLE_MOCK_FEATURE_SEEDING === 'true' || false); +const NUM_MOCK_FEATURE_SUBMISSIONS = Number(process.env.NUM_MOCK_FEATURE_SUBMISSIONS || 0); /** * Sample query for performance testing. @@ -40,7 +41,7 @@ const ENABLED = false; * @return {*} {Promise} */ export async function seed(knex: Knex): Promise { - if (!ENABLED) { + if (!ENABLE_MOCK_FEATURE_SEEDING) { return knex.raw(`SELECT null;`); // dummy query to appease knex } @@ -49,9 +50,7 @@ export async function seed(knex: Knex): Promise { SET SEARCH_PATH = 'biohub','public'; `); - const numSubmissions = 10000; // Number of submissions (ie: surveys) - - for (let i = 0; i < numSubmissions; i++) { + for (let i = 0; i < NUM_MOCK_FEATURE_SUBMISSIONS; i++) { await insertRecord(knex); } } diff --git a/env_config/env.docker b/env_config/env.docker index bcaecad14..7b42778fc 100644 --- a/env_config/env.docker +++ b/env_config/env.docker @@ -151,3 +151,11 @@ GEOSERVER_ADMIN_PASSWORD=geoserver ROOT_WEBAPP_REDIRECT=true TOMCAT_EXTRAS=false + +# ------------------------------------------------------------------------------ +# Seeding - Development +# ------------------------------------------------------------------------------ +# Enable or disable the mock data seeding in '04_mock_test_data.ts' +ENABLE_MOCK_FEATURE_SEEDING=false +# Configure how many feature submission records to seed +NUM_MOCK_FEATURE_SUBMISSIONS=0 From 6703f4327a7d51853de51d85291d4ed22dc7297e Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Wed, 22 Nov 2023 16:52:15 -0800 Subject: [PATCH 08/20] Add security join table --- .../20231709000001_security_tables.ts | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/database/src/migrations/20231709000001_security_tables.ts b/database/src/migrations/20231709000001_security_tables.ts index dd43485d2..e926ce9cf 100644 --- a/database/src/migrations/20231709000001_security_tables.ts +++ b/database/src/migrations/20231709000001_security_tables.ts @@ -47,6 +47,34 @@ export async function up(knex: Knex): Promise { ---------------------------------------------------------------------------------------- + CREATE TABLE feature_submission_security( + feature_submission_security_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + feature_submission_id integer NOT NULL, + security_rule_id integer NOT NULL, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT feature_submission_security_pk PRIMARY KEY (feature_submission_security_id) + ); + + COMMENT ON COLUMN feature_submission_security.feature_submission_security_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN feature_submission_security.feature_submission_id IS 'Foreign key to the feature_submission table.'; + COMMENT ON COLUMN feature_submission_security.security_rule_id IS 'Foreign key to the security_rule table.'; + COMMENT ON COLUMN feature_submission_security.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN feature_submission_security.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN feature_submission_security.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN feature_submission_security.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN feature_submission_security.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN feature_submission_security.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN feature_submission_security.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE feature_submission_security IS 'A join table between feature_submission and security_rule. Defines which security rules are applied to the a feature submission.'; + + ---------------------------------------------------------------------------------------- + CREATE TABLE security_string( security_string_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), security_rule_id integer NOT NULL, @@ -188,6 +216,27 @@ export async function up(knex: Knex): Promise { -- Add unique end-date key constraint (don't allow 2 records with the same name and a NULL record_end_date) CREATE UNIQUE INDEX security_rule_nuk1 ON security_rule(name, (record_end_date is NULL)) where record_end_date is null; + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: feature_submission_security + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same feature_submission_id, security_rule_id, and a NULL record_end_date) + CREATE UNIQUE INDEX feature_submission_security_nuk1 ON feature_submission_security(feature_submission_id, security_rule_id, (record_end_date is NULL)) where record_end_date is null; + + -- Add foreign key constraint + ALTER TABLE feature_submission_security ADD CONSTRAINT feature_submission_security_fk1 + FOREIGN KEY (feature_submission_id) + REFERENCES feature_submission(feature_submission_id); + + ALTER TABLE feature_submission_security ADD CONSTRAINT feature_submission_security_fk2 + FOREIGN KEY (security_rule_id) + REFERENCES security_rule(security_rule_id); + + -- add indexes for foreign keys + CREATE INDEX feature_submission_security_idx1 ON feature_submission_security(feature_submission_id); + + CREATE INDEX feature_submission_security_idx2 ON feature_submission_security(security_rule_id); + ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: security_string ---------------------------------------------------------------------------------------- From 2560f1e4547fdfc88d2acf1d678ddd483f06b181 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Wed, 22 Nov 2023 17:11:05 -0800 Subject: [PATCH 09/20] Port pipeline fixes from SIMS. Update seed comment with latest sample query. ignore-skip --- .github/workflows/cleanClosedPR.yml | 4 ++-- .github/workflows/deploy.yml | 8 +++++-- database/src/seeds/04_mock_test_data.ts | 32 +++++++++++-------------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/workflows/cleanClosedPR.yml b/.github/workflows/cleanClosedPR.yml index dc3597214..3b48ab00d 100644 --- a/.github/workflows/cleanClosedPR.yml +++ b/.github/workflows/cleanClosedPR.yml @@ -86,5 +86,5 @@ jobs: env: POD_SELECTOR: biohub-platform run: | - oc --namespace a0ec71-dev get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found" $1}' | bash - oc --namespace a0ec71-tools get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found" $1}' | bash + oc --namespace a0ec71-dev get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found " $1}' | bash + oc --namespace a0ec71-tools get all,pvc,secret,pods,ReplicationController,DeploymentConfig,HorizontalPodAutoscaler,imagestreamtag -o name | grep $POD_SELECTOR | grep $PR_NUMBER | awk '{print "oc delete --ignore-not-found " $1}' | bash diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c1b8da4a6..26c05479b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -5,6 +5,8 @@ name: PR-Based Deploy on OpenShift on: pull_request: types: [opened, reopened, synchronize, ready_for_review] + branches-ignore: + - prod concurrency: group: ${{ github.workflow }}-${{ github.event.number }} @@ -41,9 +43,11 @@ jobs: - checkEnv outputs: paths_result: ${{ steps.skip_check.outputs.paths_result }} - # Set to `true` if the latest commit message contains `ignore-skip` anywhere in the message. + # Set to `true` if the latest commit message contains `ignore-skip` anywhere in the message OR the base branch + # is dev, test, or prod. # Used to disable duplicate action skipping, if needed. - ignore_skip: ${{ contains(steps.head_commit_message.outputs.commit_message, 'ignore-skip') }} + ignore_skip: ${{ contains(steps.head_commit_message.outputs.commit_message, 'ignore-skip') || + github.head_ref == 'dev' || github.head_ref == 'test' || github.head_ref == 'prod' }} steps: - id: skip_check uses: fkirc/skip-duplicate-actions@v5 diff --git a/database/src/seeds/04_mock_test_data.ts b/database/src/seeds/04_mock_test_data.ts index a46c22210..962f2c83f 100644 --- a/database/src/seeds/04_mock_test_data.ts +++ b/database/src/seeds/04_mock_test_data.ts @@ -8,27 +8,23 @@ const ENABLE_MOCK_FEATURE_SEEDING = Boolean(process.env.ENABLE_MOCK_FEATURE_SEED const NUM_MOCK_FEATURE_SUBMISSIONS = Number(process.env.NUM_MOCK_FEATURE_SUBMISSIONS || 0); /** - * Sample query for performance testing. + * Search query for performance testing. * * -- Select feature_submissions on multiple conditions (AND) - * select * from submission_feature - * where feature_type_id = 1 and submission_feature_id in ( - * select distinct t1.submission_feature_id - * FROM search_string t1 + * SELECT * FROM submission_feature WHERE submission_feature_id IN ( + * SELECT DISTINCT t1.submission_feature_id FROM submission_feature t1 * WHERE EXISTS ( - * select 1 from search_string t3 where t3.submission_feature_id = t1.submission_feature_id and t3.value like '%cor%' - * ) and exists ( - * select 1 from search_string t4 where t4.submission_feature_id = t1.submission_feature_id and t4.value like '%arx%' - * ) and exists ( - * select 1 from search_number t5 where t5.submission_feature_id = t1.submission_feature_id and t5.value > 40 and t5.value < 50 - * ) and exists ( - * select 1 from search_taxonomy t6 where t6.submission_feature_id = t1.submission_feature_id and t6.value > 50000 and t6.value < 50100 - * ) and exists ( - * select 1 from search_datetime t7 where t7.submission_feature_id = t1.submission_feature_id and t7.value > '2023-08-01' and t7.value < '2024-04-01' and t7.feature_property_id = (select feature_property_id from feature_property where name = 'start_date') - * ) and exists ( - * select 1 from search_datetime t7 where t7.submission_feature_id = t1.submission_feature_id and t7.value > '2023-08-01' and t7.value < '2024-04-01' and t7.feature_property_id = (select feature_property_id from feature_property where name = 'end_date') - * ) and exists ( - * select 1 from search_spatial t8 where t8.submission_feature_id = t1.submission_feature_id and public.ST_INTERSECTS(t8.value, public.ST_GeomFromGeoJSON('{"coordinates":[[[-128.12596524778567,50.90095573861839],[-128.6951954392062,50.75063500834236],[-127.71373499792975,49.63640480052965],[-125.38308025753057,48.53083459202276],[-123.3647465830768,48.15806226354249],[-122.94623399379441,48.36504151433127],[-123.37439502763095,49.13209156231335],[-124.66835857611437,49.81654191782255],[-126.6572708981094,50.607171392416745],[-127.89342678974776,50.9888374217299],[-128.12596524778567,50.90095573861839]]],"type":"Polygon"}')) + * SELECT 1 FROM search_string t3 WHERE t3.submission_feature_id = t1.submission_feature_id AND t3.value LIKE '%cor%' + * ) AND EXISTS ( + * SELECT 1 FROM search_string t4 WHERE t4.submission_feature_id = t1.submission_feature_id AND t4.value LIKE '%arx%' + * ) AND EXISTS ( + * SELECT 1 FROM search_number t5 WHERE t5.submission_feature_id = t1.submission_feature_id AND t5.feature_property_id = (SELECT feature_property_id FROM feature_property fp WHERE fp.name = 'count') AND t5.value > 40 AND t5.value < 50 + * ) AND EXISTS ( + * SELECT 1 FROM search_datetime t7 WHERE t7.submission_feature_id = t1.submission_feature_id AND t7.value > '2023-08-01' AND t7.value < '2024-04-01' AND t7.feature_property_id = (SELECT feature_property_id FROM feature_property WHERE name = 'start_date') + * ) AND EXISTS ( + * SELECT 1 FROM search_datetime t8 WHERE t8.submission_feature_id = t1.submission_feature_id AND t8.value > '2023-08-01' AND t8.value < '2024-04-01' AND t8.feature_property_id = (SELECT feature_property_id FROM feature_property WHERE name = 'end_date') + * ) AND EXISTS ( + * SELECT 1 FROM search_spatial t9 WHERE t9.submission_feature_id = t1.submission_feature_id AND public.ST_INTERSECTS(t9.value, public.ST_GeomFromGeoJSON('{"coordinates":[[[-128.12596524778567,50.90095573861839],[-128.6951954392062,50.75063500834236],[-127.71373499792975,49.63640480052965],[-125.38308025753057,48.53083459202276],[-123.3647465830768,48.15806226354249],[-122.94623399379441,48.36504151433127],[-123.37439502763095,49.13209156231335],[-124.66835857611437,49.81654191782255],[-126.6572708981094,50.607171392416745],[-127.89342678974776,50.9888374217299],[-128.12596524778567,50.90095573861839]]],"type":"Polygon"}')) * ) * ); */ From 0a529f246cea6ba34aa363e8b8942341ca638639 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Wed, 22 Nov 2023 17:30:56 -0800 Subject: [PATCH 10/20] Fix typo --- .../20231709000001_security_tables.ts | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/database/src/migrations/20231709000001_security_tables.ts b/database/src/migrations/20231709000001_security_tables.ts index e926ce9cf..88d06c7b7 100644 --- a/database/src/migrations/20231709000001_security_tables.ts +++ b/database/src/migrations/20231709000001_security_tables.ts @@ -47,9 +47,9 @@ export async function up(knex: Knex): Promise { ---------------------------------------------------------------------------------------- - CREATE TABLE feature_submission_security( - feature_submission_security_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - feature_submission_id integer NOT NULL, + CREATE TABLE submission_feature_security( + submission_feature_security_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + submission_feature_id integer NOT NULL, security_rule_id integer NOT NULL, record_effective_date date NOT NULL, record_end_date date, @@ -58,20 +58,20 @@ export async function up(knex: Knex): Promise { update_date timestamptz(6), update_user integer, revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT feature_submission_security_pk PRIMARY KEY (feature_submission_security_id) + CONSTRAINT submission_feature_security_pk PRIMARY KEY (submission_feature_security_id) ); - COMMENT ON COLUMN feature_submission_security.feature_submission_security_id IS 'System generated surrogate primary key identifier.'; - COMMENT ON COLUMN feature_submission_security.feature_submission_id IS 'Foreign key to the feature_submission table.'; - COMMENT ON COLUMN feature_submission_security.security_rule_id IS 'Foreign key to the security_rule table.'; - COMMENT ON COLUMN feature_submission_security.record_effective_date IS 'Record level effective date.'; - COMMENT ON COLUMN feature_submission_security.record_end_date IS 'Record level end date.'; - COMMENT ON COLUMN feature_submission_security.create_date IS 'The datetime the record was created.'; - COMMENT ON COLUMN feature_submission_security.create_user IS 'The id of the user who created the record as identified in the system user table.'; - COMMENT ON COLUMN feature_submission_security.update_date IS 'The datetime the record was updated.'; - COMMENT ON COLUMN feature_submission_security.update_user IS 'The id of the user who updated the record as identified in the system user table.'; - COMMENT ON COLUMN feature_submission_security.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE feature_submission_security IS 'A join table between feature_submission and security_rule. Defines which security rules are applied to the a feature submission.'; + COMMENT ON COLUMN submission_feature_security.submission_feature_security_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN submission_feature_security.submission_feature_id IS 'Foreign key to the submission_feature table.'; + COMMENT ON COLUMN submission_feature_security.security_rule_id IS 'Foreign key to the security_rule table.'; + COMMENT ON COLUMN submission_feature_security.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN submission_feature_security.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN submission_feature_security.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN submission_feature_security.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN submission_feature_security.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN submission_feature_security.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN submission_feature_security.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE submission_feature_security IS 'A join table between submission_feature and security_rule. Defines which security rules are applied to the a feature submission.'; ---------------------------------------------------------------------------------------- @@ -217,25 +217,25 @@ export async function up(knex: Knex): Promise { CREATE UNIQUE INDEX security_rule_nuk1 ON security_rule(name, (record_end_date is NULL)) where record_end_date is null; ---------------------------------------------------------------------------------------- - -- Create Indexes and Constraints for table: feature_submission_security + -- Create Indexes and Constraints for table: submission_feature_security ---------------------------------------------------------------------------------------- - -- Add unique end-date key constraint (don't allow 2 records with the same feature_submission_id, security_rule_id, and a NULL record_end_date) - CREATE UNIQUE INDEX feature_submission_security_nuk1 ON feature_submission_security(feature_submission_id, security_rule_id, (record_end_date is NULL)) where record_end_date is null; + -- Add unique end-date key constraint (don't allow 2 records with the same submission_feature_id, security_rule_id, and a NULL record_end_date) + CREATE UNIQUE INDEX submission_feature_security_nuk1 ON submission_feature_security(submission_feature_id, security_rule_id, (record_end_date is NULL)) where record_end_date is null; -- Add foreign key constraint - ALTER TABLE feature_submission_security ADD CONSTRAINT feature_submission_security_fk1 - FOREIGN KEY (feature_submission_id) - REFERENCES feature_submission(feature_submission_id); + ALTER TABLE submission_feature_security ADD CONSTRAINT submission_feature_security_fk1 + FOREIGN KEY (submission_feature_id) + REFERENCES submission_feature(submission_feature_id); - ALTER TABLE feature_submission_security ADD CONSTRAINT feature_submission_security_fk2 + ALTER TABLE submission_feature_security ADD CONSTRAINT submission_feature_security_fk2 FOREIGN KEY (security_rule_id) REFERENCES security_rule(security_rule_id); -- add indexes for foreign keys - CREATE INDEX feature_submission_security_idx1 ON feature_submission_security(feature_submission_id); + CREATE INDEX submission_feature_security_idx1 ON submission_feature_security(submission_feature_id); - CREATE INDEX feature_submission_security_idx2 ON feature_submission_security(security_rule_id); + CREATE INDEX submission_feature_security_idx2 ON submission_feature_security(security_rule_id); ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: security_string From 2dbaecc2bf71db707513fb491f42add29c0e7c36 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 23 Nov 2023 14:57:21 -0800 Subject: [PATCH 11/20] Remove old biohub tables, add submission message table, rename migrations, port user seed updates from sims --- Makefile | 12 +- .../openapi/schemas/biohub-data-submission.ts | 9 +- .../src/migrations/20220225205948_release.ts | 30 +- ...2015_add_record_sims_eml_transformation.ts | 275 --- ...patial_transform_eml_project_boundaries.ts | 214 -- ...transform_eml_project_boundary_centroid.ts | 164 -- ...0_update_record_sims_eml_transformation.ts | 282 --- ...5_remove_unique_id_artifact_persecution.ts | 20 - .../20231109000002_search_tables.ts | 8 +- ...s.ts => 20231117000001_security_tables.ts} | 8 +- ...s => 20231117000002_security_functions.ts} | 0 .../20231122000001_submission_status.ts | 119 ++ .../api_get_context_system_user_role_id.sql | 27 - .../src/migrations/release.0.8.0/biohub.sql | 1880 ++--------------- .../populate_persecution_or_harm.sql | 49 - .../populate_persecution_or_harm_type.sql | 9 - .../populate_proprietary_type.sql | 7 - .../populate_submission_message_class.sql | 3 - .../populate_submission_message_type.sql | 8 - .../populate_submission_status_type.sql | 25 - .../tr_generated_audit_triggers.sql | 40 +- .../tr_generated_journal_triggers.sql | 40 +- .../release.0.8.0/vw_generated_dapi_views.sql | 31 - database/src/seeds/01_db_system_users.ts | 113 +- .../02_populate_feature_lookup_tables.ts} | 6 +- ..._security_transform_eml_dwc_occurrences.ts | 106 - .../src/seeds/03_dwc_spatial_transform.ts | 185 -- 27 files changed, 365 insertions(+), 3305 deletions(-) delete mode 100644 database/src/migrations/20220602152015_add_record_sims_eml_transformation.ts delete mode 100644 database/src/migrations/20220602152016_spatial_transform_eml_project_boundaries.ts delete mode 100644 database/src/migrations/20220602152016_spatial_transform_eml_project_boundary_centroid.ts delete mode 100644 database/src/migrations/20230518125700_update_record_sims_eml_transformation.ts delete mode 100644 database/src/migrations/20230531012345_remove_unique_id_artifact_persecution.ts rename database/src/migrations/{20231709000001_security_tables.ts => 20231117000001_security_tables.ts} (99%) rename database/src/migrations/{20231709000002_security_functions.ts => 20231117000002_security_functions.ts} (100%) create mode 100644 database/src/migrations/20231122000001_submission_status.ts delete mode 100644 database/src/migrations/release.0.8.0/api_get_context_system_user_role_id.sql delete mode 100644 database/src/migrations/release.0.8.0/populate_persecution_or_harm.sql delete mode 100644 database/src/migrations/release.0.8.0/populate_persecution_or_harm_type.sql delete mode 100644 database/src/migrations/release.0.8.0/populate_proprietary_type.sql delete mode 100644 database/src/migrations/release.0.8.0/populate_submission_message_class.sql delete mode 100644 database/src/migrations/release.0.8.0/populate_submission_message_type.sql delete mode 100644 database/src/migrations/release.0.8.0/populate_submission_status_type.sql delete mode 100644 database/src/migrations/release.0.8.0/vw_generated_dapi_views.sql rename database/src/{migrations/20231109000003_populate_feature_lookup_tables.ts => seeds/02_populate_feature_lookup_tables.ts} (98%) delete mode 100644 database/src/seeds/02_security_transform_eml_dwc_occurrences.ts delete mode 100644 database/src/seeds/03_dwc_spatial_transform.ts diff --git a/Makefile b/Makefile index 8cc2ba40c..127e689e9 100644 --- a/Makefile +++ b/Makefile @@ -90,13 +90,15 @@ build-backend: ## Builds all backend containers @echo "===============================================" @echo "Make: build-backend - building backend images" @echo "===============================================" - @docker-compose -f docker-compose.yml build db db_setup api queue + @docker-compose -f docker-compose.yml build db db_setup api + # @docker-compose -f docker-compose.yml build db db_setup api queue run-backend: ## Runs all backend containers @echo "===============================================" @echo "Make: run-backend - running backend images" @echo "===============================================" - @docker-compose -f docker-compose.yml up -d db db_setup api queue + @docker-compose -f docker-compose.yml up -d db db_setup api + # @docker-compose -f docker-compose.yml up -d db db_setup api queue ## ------------------------------------------------------------------------------ ## Build/Run Backend+Web Commands (backend + web frontend) @@ -107,13 +109,15 @@ build-web: ## Builds all backend+web containers @echo "===============================================" @echo "Make: build-web - building web images" @echo "===============================================" - @docker-compose -f docker-compose.yml build db db_setup api queue app + @docker-compose -f docker-compose.yml build db db_setup api app + # @docker-compose -f docker-compose.yml build db db_setup api queue app run-web: ## Runs all backend+web containers @echo "===============================================" @echo "Make: run-web - running web images" @echo "===============================================" - @docker-compose -f docker-compose.yml up -d db db_setup api queue app + @docker-compose -f docker-compose.yml up -d db db_setup api app + # @docker-compose -f docker-compose.yml up -d db db_setup api queue app ## ------------------------------------------------------------------------------ ## Commands to shell into the target container diff --git a/api/src/openapi/schemas/biohub-data-submission.ts b/api/src/openapi/schemas/biohub-data-submission.ts index 8e48b422c..1d06a5cba 100644 --- a/api/src/openapi/schemas/biohub-data-submission.ts +++ b/api/src/openapi/schemas/biohub-data-submission.ts @@ -1,4 +1,4 @@ -export const BioHubFeatureCollection = { +export const BioHubDataSubmission = { title: 'BioHub Data Submission', type: 'object', required: ['id', 'type', 'features'], @@ -9,7 +9,12 @@ export const BioHubFeatureCollection = { }, type: { type: 'string', - enum: ['Submission'] + enum: ['dataset'] + }, + properties: { + title: 'Feature properties', + type: 'object', + properties: {} }, features: { type: 'array', diff --git a/database/src/migrations/20220225205948_release.ts b/database/src/migrations/20220225205948_release.ts index c53e0f07b..591c942d8 100644 --- a/database/src/migrations/20220225205948_release.ts +++ b/database/src/migrations/20220225205948_release.ts @@ -28,9 +28,6 @@ export async function up(knex: Knex): Promise { path.join(__dirname, DB_RELEASE, 'tr_generated_audit_triggers.sql') ); const api_get_context_user_id = fs.readFileSync(path.join(__dirname, DB_RELEASE, 'api_get_context_user_id.sql')); - const api_get_context_system_user_role_id = fs.readFileSync( - path.join(__dirname, DB_RELEASE, 'api_get_context_system_user_role_id.sql') - ); const tr_journal_trigger = fs.readFileSync(path.join(__dirname, DB_RELEASE, 'tr_journal_trigger.sql')); const tr_generated_journal_triggers = fs.readFileSync( path.join(__dirname, DB_RELEASE, 'tr_generated_journal_triggers.sql') @@ -46,22 +43,6 @@ export async function up(knex: Knex): Promise { const populate_system_metadata_constant = fs.readFileSync( path.join(__dirname, DB_RELEASE, 'populate_system_metadata_constant.sql') ); - const populate_submission_status_type = fs.readFileSync( - path.join(__dirname, DB_RELEASE, 'populate_submission_status_type.sql') - ); - const populate_submission_message_class = fs.readFileSync( - path.join(__dirname, DB_RELEASE, 'populate_submission_message_class.sql') - ); - const populate_submission_message_type = fs.readFileSync( - path.join(__dirname, DB_RELEASE, 'populate_submission_message_type.sql') - ); - const populate_proprietary_type = fs.readFileSync(path.join(__dirname, DB_RELEASE, 'populate_proprietary_type.sql')); - const populate_persecution_or_harm_type = fs.readFileSync( - path.join(__dirname, DB_RELEASE, 'populate_persecution_or_harm_type.sql') - ); - const populate_persecution_or_harm = fs.readFileSync( - path.join(__dirname, DB_RELEASE, 'populate_persecution_or_harm.sql') - ); await knex.raw(` -- set up spatial extensions @@ -79,6 +60,7 @@ export async function up(knex: Knex): Promise { GRANT USAGE ON SCHEMA biohub TO ${DB_USER_API}; alter role ${DB_USER_API} set search_path to "$user", biohub, public; + -- alter default privileges for the biohub_api user so that it is granted access to all future tables/functions/etc ALTER DEFAULT PRIVILEGES IN SCHEMA biohub, public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO biohub_api; @@ -91,13 +73,13 @@ export async function up(knex: Knex): Promise { ALTER DEFAULT PRIVILEGES IN SCHEMA biohub, public GRANT USAGE, SELECT ON SEQUENCES TO biohub_api; + -- create tables/triggers/functions/etc ${biohub_ddl} ${populate_user_identity_source} ${api_set_context} ${tr_audit_trigger} ${tr_generated_audit_triggers} ${api_get_context_user_id} - ${api_get_context_system_user_role_id} ${tr_journal_trigger} ${tr_generated_journal_triggers} ${api_get_system_constant} @@ -105,15 +87,9 @@ export async function up(knex: Knex): Promise { ${create_sequences} -- populate look up tables - ${populate_system_constants} ${populate_system_role} + ${populate_system_constants} ${populate_system_metadata_constant} - ${populate_submission_status_type} - ${populate_submission_message_class} - ${populate_submission_message_type} - ${populate_proprietary_type} - ${populate_persecution_or_harm_type} - ${populate_persecution_or_harm} set role postgres; set search_path = biohub, public; diff --git a/database/src/migrations/20220602152015_add_record_sims_eml_transformation.ts b/database/src/migrations/20220602152015_add_record_sims_eml_transformation.ts deleted file mode 100644 index 7205960bd..000000000 --- a/database/src/migrations/20220602152015_add_record_sims_eml_transformation.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { Knex } from 'knex'; - -/** - * EML source Transform. - * - * Generates a JSON object containing key information from an EML file. - * - * Note: The intention of this transform is to only parse data that is relevant to supporting search. Fields that are - * not used in searching should not be included. - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function up(knex: Knex): Promise { - const transformSQL = ` - WITH submissionmetadata AS ( - SELECT - * - FROM - submission_metadata - WHERE - submission_id = ? - AND record_end_timestamp IS NULL - ), - eml AS ( - SELECT - Jsonb_path_query(eml_json_source, '$."eml:eml"') eml - FROM - submissionmetadata - ), - datasets AS ( - SELECT - Jsonb_path_query(eml, '$.**.dataset') dataset - FROM - eml - ), - projects AS ( - SELECT - p.proj_n, - p.project - FROM - datasets, - Jsonb_path_query_first(dataset, '$.**.project') WITH ordinality p(project, proj_n) - ), - related_projects AS ( - SELECT - p.proj_n, - p.project - FROM - datasets, - jsonb_path_query_first(dataset, '$.**.relatedProject[*]') WITH ordinality p(project, proj_n) - ), - projects_funding AS ( - SELECT - p.proj_n, - pf.fund_n, - pf.funding - FROM - projects p, - jsonb_path_query_first(project, '$.funding.section') WITH ordinality pf(funding, fund_n) - ), - related_projects_funding AS ( - SELECT - p.proj_n, - pf.fund_n, - pf.funding - FROM - related_projects p, - jsonb_path_query_first(project, '$.funding.section') WITH ordinality pf(funding, fund_n) - ), - projects_fundings AS ( - SELECT - pf.proj_n, - pf.fund_n, - pfs.funds_n, - jsonb_build_object( - 'agencyName', - pfs.projects_fundings -> 'para', - 'fundingStartDate', - jsonb_path_query_first( - pfs.projects_fundings, - '$.section[*] \\? (@.title == "Funding Start Date").para' - ), - 'fundingEndDate', - jsonb_path_query_first( - pfs.projects_fundings, - '$.section[*] \\? (@.title == "Funding End Date").para' - ) - ) funding_object - FROM - projects_funding pf, - jsonb_array_elements(funding) WITH ordinality pfs(projects_fundings, funds_n) - ), - related_projects_fundings AS ( - SELECT - pf.proj_n, - pf.fund_n, - pfs.funds_n, - jsonb_build_object( - 'agencyName', - pfs.related_projects_fundings -> 'para', - 'fundingStartDate', - jsonb_path_query_first( - pfs.related_projects_fundings, - '$.section[*] \\? (@.title == "Funding Start Date").para' - ), - 'fundingEndDate', - jsonb_path_query_first( - pfs.related_projects_fundings, - '$.section[*] \\? (@.title == "Funding End Date").para' - ) - ) funding_object - FROM - related_projects_funding pf, - jsonb_array_elements(funding) WITH ordinality pfs(related_projects_fundings, funds_n) - ), - projects_funding_arrs AS ( - SELECT - proj_n, - jsonb_agg(funding_object) funding_arr - FROM - projects_fundings - GROUP BY - proj_n - ), - related_projects_funding_arrs AS ( - SELECT - proj_n, - jsonb_agg(funding_object) funding_arr - FROM - related_projects_fundings - GROUP BY - proj_n - ), - project_objects AS ( - SELECT - jsonb_build_object( - 'projectId', - aps.project -> '@_id', - 'projectTitle', - aps.project -> 'title', - 'projectOrganizationName', - aps.project -> 'personnel' -> 'organizationName', - 'projectAbstract', - jsonb_path_query_first(aps.project, '$.abstract.section'), - 'taxonomicCoverage', - jsonb_path_query_first( - aps.project, - '$.studyAreaDescription.coverage.taxonomicCoverage' - ), - 'fundingSource', - pfas.funding_arr - ) project_object - FROM - projects aps - LEFT JOIN projects_funding_arrs pfas ON pfas.proj_n = aps.proj_n - ), - related_project_objects AS ( - SELECT - jsonb_build_object( - 'projectId', - aps.project -> '@_id', - 'projectTitle', - aps.project -> 'title', - 'projectOrganizationName', - aps.project -> 'personnel' -> 'organizationName', - 'projectAbstract', - jsonb_path_query_first(aps.project, '$.abstract.section'), - 'taxonomicCoverage', - jsonb_path_query_first( - aps.project, - '$.studyAreaDescription.coverage.taxonomicCoverage' - ), - 'fundingSource', - pfas.funding_arr - ) project_object - FROM - related_projects aps - LEFT JOIN related_projects_funding_arrs pfas ON pfas.proj_n = aps.proj_n - ), - project_objects_arr AS ( - SELECT - jsonb_agg(project_object) project_objects_arr - FROM - project_objects - ), - related_project_objects_arr AS ( - SELECT - jsonb_agg(project_object) project_objects_arr - FROM - related_project_objects - ) - SELECT - jsonb_strip_nulls( - jsonb_build_object( - 'datasetTitle', - d.dataset -> 'title', - 'datasetId', - d.dataset -> '@_id', - 'sourceSystem', - d.dataset -> '@_system', - 'publishDate', - d.dataset -> 'pubDate', - 'project', - poa.project_objects_arr, - 'relatedProject', - rpoa.project_objects_arr, - 'additionalMetadata', - jsonb_build_object( - 'projectIUCNConservationActions', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.IUCNConservationActions.IUCNConservationAction' - ), - 'projectStakeholderPartnerships', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.stakeholderPartnerships.stakeholderPartnership' - ), - 'projectActivities', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.projectActivities.projectActivity' - ), - 'projectFirstNations', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.firstNationPartnerships.firstNationPartnership' - ), - 'projectSurveyProprietors', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.projectSurveyProprietors.projectSurveyProprietor' - ) - ) - ) - ) result_data - FROM - eml e, - datasets d, - project_objects_arr poa, - related_project_objects_arr rpoa; - `; - - await knex.raw(` - SET SCHEMA 'biohub'; - SET SEARCH_PATH = 'biohub'; - - insert into source_transform ( - system_user_id, - version, - metadata_index, - metadata_transform - ) values ( - (select system_user_id from system_user where user_identifier = 'service-account-SIMS-SVC-4464'), - '1.0', - 'biohub_metadata', - $transform$ - ${transformSQL} - $transform$ - ); - `); -} - -/** - * Not used. - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function down(knex: Knex): Promise { - await knex.raw(``); -} diff --git a/database/src/migrations/20220602152016_spatial_transform_eml_project_boundaries.ts b/database/src/migrations/20220602152016_spatial_transform_eml_project_boundaries.ts deleted file mode 100644 index 23d4421e7..000000000 --- a/database/src/migrations/20220602152016_spatial_transform_eml_project_boundaries.ts +++ /dev/null @@ -1,214 +0,0 @@ -import { Knex } from 'knex'; - -/** - * EML Spatial Transform. - * - * Generates a GeoJSON FeatureCollection for the boundary specified in the geographicCoverage section of an EML file. - * - * Note: This parses all instances of `geographicCoverage` in the EML file (ie: in `relatedProjects` sections) - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function up(knex: Knex): Promise { - const transformSQL = ` - with submissionmetadata as ( - select - eml_json_source - from - submission_metadata - where - submission_id = ? - and record_end_timestamp is null - ), - project_coverage as ( - select - c.cov_n, - 'project' project_type, - c.coverage - from - submissionmetadata, - jsonb_path_query( - eml_json_source, - '$.**.project.studyAreaDescription.**.geographicCoverage' - ) with ordinality c(coverage, cov_n) - ), - related_project_coverages as ( - select - c.cov_n, - 'relatedProject' project_type, - c.coverage - from - submissionmetadata, - jsonb_path_query( - eml_json_source, - '$.**.relatedProject.studyAreaDescription.**.geographicCoverage' - ) with ordinality c(coverage, cov_n) - ), - descriptions as ( - select - cov_n, - project_type, - coverage -> 'geographicDescription' description - from - project_coverage - union - select - cov_n, - project_type, - coverage -> 'geographicDescription' description - from - related_project_coverages - ), - coverages as ( - select - * - from - project_coverage - union - select - * - from - related_project_coverages - ), - polys as ( - select - c.cov_n, - c.project_type, - p.poly_n, - p.points points - from - coverages c, - jsonb_path_query( - coverage, - '$.**.datasetGPolygon[*].datasetGPolygonOuterGRing.gRingPoint' - ) with ordinality p(points, poly_n) - ), - latlongs as ( - select - p.cov_n, - p.project_type, - p.poly_n, - arr.point_n, - arr.point ->> 'gRingLatitude' lat, - arr.point ->> 'gRingLongitude' long - from - polys p, - jsonb_array_elements(points) with ordinality arr(point, point_n) - ), - points as ( - select - ll.cov_n, - ll.project_type, - ll.poly_n, - ll.point_n, - json_build_array(ll.long :: float, ll.lat :: float) point - from - latlongs ll - ), - polys2 as ( - select - cov_n, - project_type, - poly_n, - jsonb_agg( - point - order by - point_n - ) poly - from - points - group by - cov_n, - project_type, - poly_n - ), - multipoly as ( - select - cov_n, - project_type, - jsonb_agg(poly) mpoly - from - polys2 - group by - project_type, - cov_n - ), - features as ( - select - json_build_object( - 'type', - 'Feature', - 'geometry', - json_build_object( - 'type', - 'Polygon', - 'coordinates', - f.mpoly - ), - 'properties', - json_build_object( - 'type', - 'Boundary', - 'description', - f.description, - 'project type', - f.project_type - ) - ) feature - from - ( - select - d.description, - m.mpoly, - m.project_type - from - multipoly m, - descriptions d - where - d.cov_n = m.cov_n - and d.project_type = m.project_type - ) f - ) - select - json_build_object( - 'type', - 'FeatureCollection', - 'features', - jsonb_agg(feature) - ) result_data - from - features - `; - - await knex.raw(` - SET SCHEMA 'biohub'; - SET SEARCH_PATH = 'biohub'; - - insert into spatial_transform ( - name, - description, - record_effective_date, - transform - ) values ( - 'EML Dataset Boundaries', - 'Extracts EML dataset geographic coverage boundaries and properties.', - now(), - $transform$ - ${transformSQL} - $transform$ - ); - `); -} - -/** - * Not used. - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function down(knex: Knex): Promise { - await knex.raw(``); -} diff --git a/database/src/migrations/20220602152016_spatial_transform_eml_project_boundary_centroid.ts b/database/src/migrations/20220602152016_spatial_transform_eml_project_boundary_centroid.ts deleted file mode 100644 index 402ce5842..000000000 --- a/database/src/migrations/20220602152016_spatial_transform_eml_project_boundary_centroid.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { Knex } from 'knex'; - -/** - * EML Spatial Transform. - * - * Generates a GeoJSON FeatureCollection for the centroid for the boundary specified in the geographicCoverage section - * of an EML file. - * - * Note: This parses all instances of `geographicCoverage` in the EML file (ie: in `relatedProjects` sections) - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function up(knex: Knex): Promise { - const transformSQL = ` - with submissionmetadata as ( - select - * - from - submission_metadata - where - submission_id = ? - and record_end_timestamp is null - ), - submissionuuid as ( - select - uuid - from - submission, - submissionmetadata - where - submission.submission_id = submissionmetadata.submission_id - ), - coverages as ( - select - c.cov_n, - c.coverage - from - submissionmetadata, - jsonb_path_query(eml_json_source, '$.**.geographicCoverage') with ordinality c(coverage, cov_n) - ), - polys as ( - select - c.cov_n, - p.poly_n, - p.points points - from - coverages c, - jsonb_path_query( - coverage, - '$.**.datasetGPolygon[*].datasetGPolygonOuterGRing.gRingPoint' - ) with ordinality p(points, poly_n) - ), - latlongs as ( - select - p.cov_n, - p.poly_n, - arr.point_n, - arr.point ->> 'gRingLatitude' lat, - arr.point ->> 'gRingLongitude' long - from - polys p, - jsonb_array_elements(points) with ordinality arr(point, point_n) - ), - points as ( - select - ll.cov_n, - ll.poly_n, - ll.point_n, - ll.long :: float || ' ' || ll.lat :: float point - from - latlongs ll - ), - polys2 as ( - select - cov_n, - poly_n, - array_agg( - point - order by - point_n - ) poly - from - points - group by - cov_n, - poly_n - ), - string_polys as ( - select - '(' || array_to_string(poly, ',') || ')' strp - from - polys2 - ), - geojson_centroid as ( - select - st_asgeojson( - ST_PointOnSurface( - ST_Multi('POLYGON(' || string_agg(strp, ',') || ')') - ) - ) centroid - from - string_polys - ) - select - jsonb_build_object( - 'type', - 'FeatureCollection', - 'features', - jsonb_build_array( - jsonb_build_object( - 'type', - 'Feature', - 'geometry', - gc.centroid :: json, - 'properties', - jsonb_build_object( - 'type', - 'Boundary Centroid', - 'datasetID', - su.uuid, - 'datasetTitle', - jsonb_path_query(sm.eml_json_source, '$.**.dataset.title') - ) - ) - ) - ) result_data - from - submissionmetadata sm, - submissionuuid su, - geojson_centroid gc - `; - - await knex.raw(` - SET SCHEMA 'biohub'; - SET SEARCH_PATH = 'biohub'; - - insert into spatial_transform ( - name, - description, - record_effective_date, - transform - ) values ( - 'EML Dataset Boundaries Centroid', - 'Extracts centroid from EML geographic coverage boundaries.', - now(), - $transform$ - ${transformSQL} - $transform$ - ); - `); -} - -/** - * Not used. - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function down(knex: Knex): Promise { - await knex.raw(``); -} diff --git a/database/src/migrations/20230518125700_update_record_sims_eml_transformation.ts b/database/src/migrations/20230518125700_update_record_sims_eml_transformation.ts deleted file mode 100644 index f5ae282b3..000000000 --- a/database/src/migrations/20230518125700_update_record_sims_eml_transformation.ts +++ /dev/null @@ -1,282 +0,0 @@ -import { Knex } from 'knex'; - -/** - * EML source Transform. - * - * Generates a JSON object containing key information from an EML file. - * - * Note: The intention of this transform is to only parse data that is relevant to supporting search. Fields that are - * not used in searching should not be included. - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function up(knex: Knex): Promise { - const transformSQL = ` - WITH submissionmetadata AS ( - SELECT - * - FROM - submission_metadata - WHERE - submission_id = ? - AND record_end_timestamp IS NULL - ), - eml AS ( - SELECT - Jsonb_path_query(eml_json_source, '$."eml:eml"') eml - FROM - submissionmetadata - ), - datasets AS ( - SELECT - Jsonb_path_query(eml, '$.**.dataset') dataset - FROM - eml - ), - projects AS ( - SELECT - p.proj_n, - p.project - FROM - datasets, - Jsonb_path_query_first(dataset, '$.**.project') WITH ordinality p(project, proj_n) - ), - related_projects AS ( - SELECT - p.proj_n, - p.project - FROM - datasets, - jsonb_path_query_first(dataset, '$.**.relatedProject[*]') WITH ordinality p(project, proj_n) - ), - projects_funding AS ( - SELECT - p.proj_n, - pf.fund_n, - pf.funding - FROM - projects p, - jsonb_path_query_first(project, '$.funding.section') WITH ordinality pf(funding, fund_n) - ), - related_projects_funding AS ( - SELECT - p.proj_n, - pf.fund_n, - pf.funding - FROM - related_projects p, - jsonb_path_query_first(project, '$.funding.section') WITH ordinality pf(funding, fund_n) - ), - projects_fundings AS ( - SELECT - pf.proj_n, - pf.fund_n, - pfs.funds_n, - jsonb_build_object( - 'agencyName', - pfs.projects_fundings -> 'para', - 'fundingStartDate', - jsonb_path_query_first( - pfs.projects_fundings, - '$.section[*] \\? (@.title == "Funding Start Date").para' - ), - 'fundingEndDate', - jsonb_path_query_first( - pfs.projects_fundings, - '$.section[*] \\? (@.title == "Funding End Date").para' - ) - ) funding_object - FROM - projects_funding pf, - jsonb_array_elements(funding) WITH ordinality pfs(projects_fundings, funds_n) - ), - related_projects_fundings AS ( - SELECT - pf.proj_n, - pf.fund_n, - pfs.funds_n, - jsonb_build_object( - 'agencyName', - pfs.related_projects_fundings -> 'para', - 'fundingStartDate', - jsonb_path_query_first( - pfs.related_projects_fundings, - '$.section[*] \\? (@.title == "Funding Start Date").para' - ), - 'fundingEndDate', - jsonb_path_query_first( - pfs.related_projects_fundings, - '$.section[*] \\? (@.title == "Funding End Date").para' - ) - ) funding_object - FROM - related_projects_funding pf, - jsonb_array_elements(funding) WITH ordinality pfs(related_projects_fundings, funds_n) - ), - projects_funding_arrs AS ( - SELECT - proj_n, - jsonb_agg(funding_object) funding_arr - FROM - projects_fundings - GROUP BY - proj_n - ), - related_projects_funding_arrs AS ( - SELECT - proj_n, - jsonb_agg(funding_object) funding_arr - FROM - related_projects_fundings - GROUP BY - proj_n - ), - project_objects AS ( - SELECT - jsonb_build_object( - 'projectId', - aps.project -> '@_id', - 'projectTitle', - aps.project -> 'title', - 'projectOrganizationName', - aps.project -> 'personnel' -> 'organizationName', - 'projectAbstract', - jsonb_path_query_first(aps.project, '$.abstract.section'), - 'taxonomicCoverage', - jsonb_path_query_first( - aps.project, - '$.studyAreaDescription.coverage.taxonomicCoverage' - ), - 'fundingSource', - pfas.funding_arr - ) project_object - FROM - projects aps - LEFT JOIN projects_funding_arrs pfas ON pfas.proj_n = aps.proj_n - ), - related_project_objects AS ( - SELECT - jsonb_build_object( - 'projectId', - aps.project -> '@_id', - 'projectTitle', - aps.project -> 'title', - 'projectOrganizationName', - aps.project -> 'personnel' -> 'organizationName', - 'projectAbstract', - jsonb_path_query_first(aps.project, '$.abstract.section'), - 'taxonomicCoverage', - jsonb_path_query_first( - aps.project, - '$.studyAreaDescription.coverage.taxonomicCoverage' - ), - 'fundingSource', - pfas.funding_arr - ) project_object - FROM - related_projects aps - LEFT JOIN related_projects_funding_arrs pfas ON pfas.proj_n = aps.proj_n - ), - project_objects_arr AS ( - SELECT - jsonb_agg(project_object) project_objects_arr - FROM - project_objects - ), - related_project_objects_arr AS ( - SELECT - jsonb_agg(project_object) project_objects_arr - FROM - related_project_objects - ), - project_type AS ( - SELECT - jsonb_path_query_array(jsonb_agg(data_array -> 'metadata' -> 'types' ->> 'type'), '$[*] \\? (@ != null)') as project_types - FROM - submission s, - submission_metadata sm, - json_array_elements(sm.eml_json_source::json->'eml:eml'->'additionalMetadata') as data_array, - submissionmetadata smd - WHERE smd.submission_id = s.submission_id - AND s.submission_id = sm.submission_id - AND sm.record_end_timestamp is null - AND (data_array ->> 'describes') = s.uuid::text - ) - SELECT - jsonb_strip_nulls( - jsonb_build_object( - 'datasetTitle', - d.dataset -> 'title', - 'datasetId', - d.dataset -> '@_id', - 'sourceSystem', - d.dataset -> '@_system', - 'publishDate', - d.dataset -> 'pubDate', - 'project', - poa.project_objects_arr, - 'relatedProject', - rpoa.project_objects_arr, - 'submitterSystem', - 'sims', - 'primaryKeywords', - pt.project_types, - 'additionalMetadata', - jsonb_build_object( - 'projectIUCNConservationActions', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.IUCNConservationActions.IUCNConservationAction' - ), - 'projectStakeholderPartnerships', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.stakeholderPartnerships.stakeholderPartnership' - ), - 'projectActivities', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.projectActivities.projectActivity' - ), - 'projectFirstNations', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.firstNationPartnerships.firstNationPartnership' - ), - 'projectSurveyProprietors', - jsonb_path_query_first( - e.eml, - '$.additionalMetadata[*].metadata.projectSurveyProprietors.projectSurveyProprietor' - ) - ) - ) - ) result_data - FROM - eml e, - datasets d, - project_objects_arr poa, - related_project_objects_arr rpoa, - project_type pt; - `; - await knex.raw(` - SET SCHEMA 'biohub'; - SET SEARCH_PATH = 'biohub'; - - update source_transform - set metadata_transform=$transform$${transformSQL}$transform$ - where system_user_id = (select system_user_id from system_user where user_identifier = 'service-account-SIMS-SVC-4464'); - `); -} - -/** - * Not used. - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function down(knex: Knex): Promise { - await knex.raw(``); -} diff --git a/database/src/migrations/20230531012345_remove_unique_id_artifact_persecution.ts b/database/src/migrations/20230531012345_remove_unique_id_artifact_persecution.ts deleted file mode 100644 index 86d39ede2..000000000 --- a/database/src/migrations/20230531012345_remove_unique_id_artifact_persecution.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Knex } from 'knex'; - -/** - * Remove unique id from artifact_persecution table - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function up(knex: Knex): Promise { - await knex.raw(` - set search_path=biohub; - DROP INDEX if exists artifact_persecution_uk1; - CREATE UNIQUE INDEX artifact_persecution_uk1 ON artifact_persecution(artifact_id, persecution_or_harm_id); - `); -} - -export async function down(knex: Knex): Promise { - await knex.raw(``); -} diff --git a/database/src/migrations/20231109000002_search_tables.ts b/database/src/migrations/20231109000002_search_tables.ts index 8800d5b44..fc2c64654 100644 --- a/database/src/migrations/20231109000002_search_tables.ts +++ b/database/src/migrations/20231109000002_search_tables.ts @@ -42,7 +42,7 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN search_string.update_date IS 'The datetime the record was updated.'; COMMENT ON COLUMN search_string.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN search_string.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE search_string IS 'String search values'; + COMMENT ON TABLE search_string IS 'String search values.'; ---------------------------------------------------------------------------------------- @@ -68,7 +68,7 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN search_number.update_date IS 'The datetime the record was updated.'; COMMENT ON COLUMN search_number.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN search_number.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE search_number IS 'Number search values'; + COMMENT ON TABLE search_number IS 'Number search values.'; ---------------------------------------------------------------------------------------- @@ -94,7 +94,7 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN search_datetime.update_date IS 'The datetime the record was updated.'; COMMENT ON COLUMN search_datetime.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN search_datetime.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE search_datetime IS 'Datetime search values'; + COMMENT ON TABLE search_datetime IS 'Datetime search values.'; ---------------------------------------------------------------------------------------- @@ -120,7 +120,7 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN search_spatial.update_date IS 'The spatial the record was updated.'; COMMENT ON COLUMN search_spatial.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN search_spatial.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE search_spatial IS 'Spatial search values'; + COMMENT ON TABLE search_spatial IS 'Spatial search values.'; ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: search_string diff --git a/database/src/migrations/20231709000001_security_tables.ts b/database/src/migrations/20231117000001_security_tables.ts similarity index 99% rename from database/src/migrations/20231709000001_security_tables.ts rename to database/src/migrations/20231117000001_security_tables.ts index 88d06c7b7..f920a5eff 100644 --- a/database/src/migrations/20231709000001_security_tables.ts +++ b/database/src/migrations/20231117000001_security_tables.ts @@ -105,7 +105,7 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN security_string.update_date IS 'The datetime the record was updated.'; COMMENT ON COLUMN security_string.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN security_string.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE security_string IS 'String search values'; + COMMENT ON TABLE security_string IS 'String security condition.'; ---------------------------------------------------------------------------------------- @@ -139,7 +139,7 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN security_number.update_date IS 'The datetime the record was updated.'; COMMENT ON COLUMN security_number.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN security_number.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE security_number IS 'Numeric search values'; + COMMENT ON TABLE security_number IS 'Number security condition.'; ---------------------------------------------------------------------------------------- @@ -173,7 +173,7 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN security_datetime.update_date IS 'The datetime the record was updated.'; COMMENT ON COLUMN security_datetime.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN security_datetime.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE security_datetime IS 'Numeric search values'; + COMMENT ON TABLE security_datetime IS 'Datetime security condition.'; ---------------------------------------------------------------------------------------- @@ -207,7 +207,7 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN security_spatial.update_date IS 'The spatial the record was updated.'; COMMENT ON COLUMN security_spatial.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN security_spatial.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE security_spatial IS 'Numeric search values'; + COMMENT ON TABLE security_spatial IS 'Spatial security condition.'; ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: security_rule diff --git a/database/src/migrations/20231709000002_security_functions.ts b/database/src/migrations/20231117000002_security_functions.ts similarity index 100% rename from database/src/migrations/20231709000002_security_functions.ts rename to database/src/migrations/20231117000002_security_functions.ts diff --git a/database/src/migrations/20231122000001_submission_status.ts b/database/src/migrations/20231122000001_submission_status.ts new file mode 100644 index 000000000..a5d63996a --- /dev/null +++ b/database/src/migrations/20231122000001_submission_status.ts @@ -0,0 +1,119 @@ +import { Knex } from 'knex'; + +/** + * Add tables: + * - search_string + * - search_string + * - search_number + * - search_datetime + * - search_spatial + * - search_taxonomy + * + * @export + * @param {Knex} knex + * @return {*} {Promise} + */ +export async function up(knex: Knex): Promise { + await knex.raw(` + ---------------------------------------------------------------------------------------- + -- Create tables + ---------------------------------------------------------------------------------------- + set search_path=biohub,public; + + CREATE TABLE submission_message( + submission_message_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + submission_message_type_id integer NOT NULL, + submission_feature_id integer NOT NULL, + label varchar(250) NOT NULL, + message varchar(500) NOT NULL, + data jsonb, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT submission_message_pk PRIMARY KEY (submission_message_id) + ); + + COMMENT ON COLUMN submission_message.submission_message_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN submission_message.submission_message_type_id IS 'Foreign key to the submission_message_type_id table.'; + COMMENT ON COLUMN submission_message.submission_feature_id IS 'Foreign key to the submission_feature table.'; + COMMENT ON COLUMN submission_message.label IS 'The message label.'; + COMMENT ON COLUMN submission_message.message IS 'The message text.'; + COMMENT ON COLUMN submission_message.data IS 'The message data.'; + COMMENT ON COLUMN submission_message.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN submission_message.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN submission_message.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN submission_message.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN submission_message.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE submission_message IS 'Messages about the submission.'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE submission_message_type( + submission_message_type_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + name varchar(50) NOT NULL, + description varchar(50) NOT NULL, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT submission_message_type_pk PRIMARY KEY (submission_message_type_id) + ); + + COMMENT ON COLUMN submission_message_type.submission_message_type_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN submission_message_type.name IS 'The name of the submission_message_type record.'; + COMMENT ON COLUMN submission_message_type.description IS 'The description of the submission_message_type record.'; + COMMENT ON COLUMN submission_message_type.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN submission_message_type.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN submission_message_type.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN submission_message_type.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN submission_message_type.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE submission_message_type IS 'Submission message types.'; + + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: submission_message + ---------------------------------------------------------------------------------------- + + -- Add foreign key constraint + ALTER TABLE submission_message ADD CONSTRAINT submission_message_fk1 + FOREIGN KEY (submission_feature_id) + REFERENCES submission_feature(submission_feature_id); + + -- Add foreign key constraint + ALTER TABLE submission_message ADD CONSTRAINT submission_message_fk2 + FOREIGN KEY (submission_message_type_id) + REFERENCES submission_message_type(submission_message_type_id); + + -- add indexes for foreign keys + CREATE INDEX submission_message_idx1 ON submission_message(submission_feature_id); + + -- add indexes for foreign keys + CREATE INDEX submission_message_idx2 ON submission_message(submission_message_type_id); + + ---------------------------------------------------------------------------------------- + -- Create audit and journal triggers + ---------------------------------------------------------------------------------------- + + create trigger audit_submission_message before insert or update or delete on submission_message for each row execute procedure tr_audit_trigger(); + create trigger journal_submission_message after insert or update or delete on submission_message for each row execute procedure tr_journal_trigger(); + + create trigger audit_submission_message_type before insert or update or delete on submission_message_type for each row execute procedure tr_audit_trigger(); + create trigger journal_submission_message_type after insert or update or delete on submission_message_type for each row execute procedure tr_journal_trigger(); + + ---------------------------------------------------------------------------------------- + -- Populate lookup tables + ---------------------------------------------------------------------------------------- + + -- populate submission_message_type table + insert into submission_message_type (name, description) values ('info', 'An informational message.'); + insert into submission_message_type (name, description) values ('warn', 'A warning message.'); + insert into submission_message_type (name, description) values ('error', 'An error message.'); + insert into submission_message_type (name, description) values ('debug', 'A debug message.'); + `); +} + +export async function down(knex: Knex): Promise { + await knex.raw(``); +} diff --git a/database/src/migrations/release.0.8.0/api_get_context_system_user_role_id.sql b/database/src/migrations/release.0.8.0/api_get_context_system_user_role_id.sql deleted file mode 100644 index 7d074995e..000000000 --- a/database/src/migrations/release.0.8.0/api_get_context_system_user_role_id.sql +++ /dev/null @@ -1,27 +0,0 @@ --- api_get_context_system_user_role_id.sql - -CREATE OR REPLACE FUNCTION api_get_context_system_user_role_id() - RETURNS integer - LANGUAGE 'plpgsql' - COST 100 - stable -AS $$ --- ******************************************************************* --- Procedure: api_get_context_system_user_role_id --- Purpose: returns the context system user role id from the invokers --- temp table --- --- MODIFICATION HISTORY --- Person Date Comments --- ---------------- ----------- -------------------------------------- --- roland.stens@gov.bc.ca --- 2021-06-03 initial release --- ******************************************************************* -declare - _system_role_id system_user_role.system_role_id%type; -begin - select value::integer into _system_role_id from biohub_context_temp where tag = 'system_user_role_id'; - - return _system_role_id; -end; -$$; diff --git a/database/src/migrations/release.0.8.0/biohub.sql b/database/src/migrations/release.0.8.0/biohub.sql index 5f957eebf..277b770ee 100644 --- a/database/src/migrations/release.0.8.0/biohub.sql +++ b/database/src/migrations/release.0.8.0/biohub.sql @@ -1,11 +1,3 @@ --- --- ER/Studio Data Architect SQL Code Generation --- Project : BioHub.DM1 --- --- Date Created : Monday, May 29, 2023 20:18:39 --- Target DBMS : PostgreSQL 10.x-12.x --- - -- -- TABLE: artifact -- @@ -31,188 +23,28 @@ CREATE TABLE artifact( update_user integer, revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT artifact_pk PRIMARY KEY (artifact_id) -) -; - - - -COMMENT ON COLUMN artifact.artifact_id IS 'Surrogate primary key identifier. This value should be selected from the appropriate sequence and populated manually.' -; -COMMENT ON COLUMN artifact.submission_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN artifact.uuid IS 'The universally unique identifier for the record.' -; -COMMENT ON COLUMN artifact.file_name IS 'The name of the artifact.' -; -COMMENT ON COLUMN artifact.file_type IS 'The artifact type. Artifact type examples include video, audio and field data.' -; -COMMENT ON COLUMN artifact.title IS 'The title of the artifact.' -; -COMMENT ON COLUMN artifact.description IS 'The description of the record.' -; -COMMENT ON COLUMN artifact.file_size IS 'The size of the artifact in bytes.' -; -COMMENT ON COLUMN artifact.key IS 'The identifying key to the file in the storage system.' -; -COMMENT ON COLUMN artifact.security_review_timestamp IS 'The timestamp that the security review of the submission artifact was completed.' -; -COMMENT ON COLUMN artifact.foi_reason IS 'A boolean flag indicating whether the data is secured due to Freedom of Information data being present.' -; -COMMENT ON COLUMN artifact.security_reason_name IS 'The name of the custom security reason.' -; -COMMENT ON COLUMN artifact.security_reason_description IS 'A reason description that is secures this data and is specific to this artifact or dataset.' -; -COMMENT ON COLUMN artifact.security_reason_end_date IS 'Custom security reason end date.' -; -COMMENT ON COLUMN artifact.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN artifact.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN artifact.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN artifact.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN artifact.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE artifact IS 'A listing of historical data submission artifacts. The record with the most recent security review timestamp is the currently published data set for each artifact identified by UUID. -' -; - --- --- TABLE: artifact_government_interest --- - -CREATE TABLE artifact_government_interest( - artifact_government_interest_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - artifact_id integer NOT NULL, - wldtaxonomic_units_id integer NOT NULL, - data_type varchar(300), - description varchar(3000), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT artifact_government_interest_pk PRIMARY KEY (artifact_government_interest_id) -) -; - - - -COMMENT ON COLUMN artifact_government_interest.artifact_government_interest_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN artifact_government_interest.artifact_id IS 'Surrogate primary key identifier. This value should be selected from the appropriate sequence and populated manually.' -; -COMMENT ON COLUMN artifact_government_interest.wldtaxonomic_units_id IS 'A foreign reference to the taxonomic unit id.' -; -COMMENT ON COLUMN artifact_government_interest.data_type IS 'A description of the type of data that is secured.' -; -COMMENT ON COLUMN artifact_government_interest.description IS 'The description of the record.' -; -COMMENT ON COLUMN artifact_government_interest.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN artifact_government_interest.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN artifact_government_interest.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN artifact_government_interest.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN artifact_government_interest.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE artifact_government_interest IS 'An intersection table relating submission artifacts to government interests.' -; - --- --- TABLE: artifact_persecution --- - -CREATE TABLE artifact_persecution( - artifact_persecution_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - persecution_or_harm_id integer NOT NULL, - artifact_id integer NOT NULL, - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT artifact_persecution_pk PRIMARY KEY (artifact_persecution_id) -) -; - - - -COMMENT ON COLUMN artifact_persecution.artifact_persecution_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN artifact_persecution.persecution_or_harm_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN artifact_persecution.artifact_id IS 'Surrogate primary key identifier. This value should be selected from the appropriate sequence and populated manually.' -; -COMMENT ON COLUMN artifact_persecution.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN artifact_persecution.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN artifact_persecution.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN artifact_persecution.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN artifact_persecution.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE artifact_persecution IS 'An intersection table defining associations between artifacts and persecution or harm security labels. -' -; - --- --- TABLE: artifact_proprietary --- - -CREATE TABLE artifact_proprietary( - artifact_proprietary_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - artifact_id integer NOT NULL, - proprietary_type_id integer NOT NULL, - first_nations_id integer, - proprietor varchar(30), - description varchar(3000), - start_date timestamptz(6), - end_date timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT artifact_proprietary_pk PRIMARY KEY (artifact_proprietary_id) -) -; - - - -COMMENT ON COLUMN artifact_proprietary.artifact_proprietary_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN artifact_proprietary.artifact_id IS 'Surrogate primary key identifier. This value should be selected from the appropriate sequence and populated manually.' -; -COMMENT ON COLUMN artifact_proprietary.proprietary_type_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN artifact_proprietary.first_nations_id IS 'A foreign reference to the first nations id.' -; -COMMENT ON COLUMN artifact_proprietary.proprietor IS 'The proprietor or owner of the artifact.' -; -COMMENT ON COLUMN artifact_proprietary.description IS 'The description of the record.' -; -COMMENT ON COLUMN artifact_proprietary.start_date IS 'The record start date.' -; -COMMENT ON COLUMN artifact_proprietary.end_date IS 'The record end date.' -; -COMMENT ON COLUMN artifact_proprietary.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN artifact_proprietary.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN artifact_proprietary.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN artifact_proprietary.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN artifact_proprietary.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE artifact_proprietary IS 'An intersection table defining associations between artifacts and proprietary security labels.' -; +); + +COMMENT ON COLUMN artifact.artifact_id IS 'Surrogate primary key identifier. This value should be selected from the appropriate sequence and populated manually.'; +COMMENT ON COLUMN artifact.submission_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN artifact.uuid IS 'The universally unique identifier for the record.'; +COMMENT ON COLUMN artifact.file_name IS 'The name of the artifact.'; +COMMENT ON COLUMN artifact.file_type IS 'The artifact type. Artifact type examples include video, audio and field data.'; +COMMENT ON COLUMN artifact.title IS 'The title of the artifact.'; +COMMENT ON COLUMN artifact.description IS 'The description of the record.'; +COMMENT ON COLUMN artifact.file_size IS 'The size of the artifact in bytes.'; +COMMENT ON COLUMN artifact.key IS 'The identifying key to the file in the storage system.'; +COMMENT ON COLUMN artifact.security_review_timestamp IS 'The timestamp that the security review of the submission artifact was completed.'; +COMMENT ON COLUMN artifact.foi_reason IS 'A boolean flag indicating whether the data is secured due to Freedom of Information data being present.'; +COMMENT ON COLUMN artifact.security_reason_name IS 'The name of the custom security reason.'; +COMMENT ON COLUMN artifact.security_reason_description IS 'A reason description that is secures this data and is specific to this artifact or dataset.'; +COMMENT ON COLUMN artifact.security_reason_end_date IS 'Custom security reason end date.'; +COMMENT ON COLUMN artifact.create_date IS 'The datetime the record was created.'; +COMMENT ON COLUMN artifact.create_user IS 'The id of the user who created the record as identified in the system user table.'; +COMMENT ON COLUMN artifact.update_date IS 'The datetime the record was updated.'; +COMMENT ON COLUMN artifact.update_user IS 'The id of the user who updated the record as identified in the system user table.'; +COMMENT ON COLUMN artifact.revision_count IS 'Revision count used for concurrency control.'; +COMMENT ON TABLE artifact IS 'A listing of historical data submission artifacts. The record with the most recent security review timestamp is the currently published data set for each artifact identified by UUID.'; -- -- TABLE: audit_log @@ -227,388 +59,16 @@ CREATE TABLE audit_log( before_value json, after_value json, CONSTRAINT audit_log_pk PRIMARY KEY (audit_log_id) -) -; - - - -COMMENT ON COLUMN audit_log.audit_log_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN audit_log.system_user_id IS 'The system user id affecting the data change.' -; -COMMENT ON COLUMN audit_log.create_date IS 'The date and time of record creation.' -; -COMMENT ON COLUMN audit_log.table_name IS 'The table name of the data record.' -; -COMMENT ON COLUMN audit_log.operation IS 'The operation that affected the data change (ie. INSERT, UPDATE, DELETE, TRUNCATE).' -; -COMMENT ON COLUMN audit_log.before_value IS 'The JSON representation of the before value of the record.' -; -COMMENT ON COLUMN audit_log.after_value IS 'The JSON representation of the after value of the record.' -; -COMMENT ON TABLE audit_log IS 'Holds record level audit log data for the entire database.' -; - --- --- TABLE: persecution_or_harm --- - -CREATE TABLE persecution_or_harm( - persecution_or_harm_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - persecution_or_harm_type_id integer NOT NULL, - wldtaxonomic_units_id integer, - name varchar(300) NOT NULL, - description varchar(3000), - start_date timestamptz(6), - end_date timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT persecution_or_harm_pk PRIMARY KEY (persecution_or_harm_id) -) -; - - - -COMMENT ON COLUMN persecution_or_harm.persecution_or_harm_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN persecution_or_harm.persecution_or_harm_type_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN persecution_or_harm.wldtaxonomic_units_id IS 'A foreign reference to the taxonomic unit id.' -; -COMMENT ON COLUMN persecution_or_harm.name IS 'The name of the record.' -; -COMMENT ON COLUMN persecution_or_harm.description IS 'The description of the record.' -; -COMMENT ON COLUMN persecution_or_harm.start_date IS 'The record start date.' -; -COMMENT ON COLUMN persecution_or_harm.end_date IS 'The record end date.' -; -COMMENT ON COLUMN persecution_or_harm.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN persecution_or_harm.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN persecution_or_harm.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN persecution_or_harm.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN persecution_or_harm.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE persecution_or_harm IS 'Describes persecution or harm security reasons.' -; - --- --- TABLE: persecution_or_harm_type --- - -CREATE TABLE persecution_or_harm_type( - persecution_or_harm_type_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - name varchar(100) NOT NULL, - description varchar(3000), - record_effective_date timestamptz(6) DEFAULT now() NOT NULL, - record_end_date timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT persecution_or_harm_type_pk PRIMARY KEY (persecution_or_harm_type_id) -) -; - - - -COMMENT ON COLUMN persecution_or_harm_type.persecution_or_harm_type_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN persecution_or_harm_type.name IS 'The name of the record.' -; -COMMENT ON COLUMN persecution_or_harm_type.description IS 'The description of the record.' -; -COMMENT ON COLUMN persecution_or_harm_type.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN persecution_or_harm_type.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN persecution_or_harm_type.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN persecution_or_harm_type.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN persecution_or_harm_type.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN persecution_or_harm_type.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN persecution_or_harm_type.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE persecution_or_harm_type IS 'Describes persecution or harm security rule types.' -; - --- --- TABLE: proprietary_type --- - -CREATE TABLE proprietary_type( - proprietary_type_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - name varchar(100) NOT NULL, - description varchar(3000), - record_effective_date timestamptz(6) DEFAULT now() NOT NULL, - record_end_date timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT proprietary_type_pk PRIMARY KEY (proprietary_type_id) -) -; - - - -COMMENT ON COLUMN proprietary_type.proprietary_type_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN proprietary_type.name IS 'The name of the record.' -; -COMMENT ON COLUMN proprietary_type.description IS 'The description of the record.' -; -COMMENT ON COLUMN proprietary_type.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN proprietary_type.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN proprietary_type.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN proprietary_type.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN proprietary_type.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN proprietary_type.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN proprietary_type.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE proprietary_type IS 'Describes proprietary security rule types.' -; - --- --- TABLE: security_transform --- - -CREATE TABLE security_transform( - security_transform_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - persecution_or_harm_id integer NOT NULL, - name varchar(100) NOT NULL, - description varchar(3000), - notes varchar(3000), - transform text NOT NULL, - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT security_transform_pk PRIMARY KEY (security_transform_id) -) -; - - - -COMMENT ON COLUMN security_transform.security_transform_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN security_transform.persecution_or_harm_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN security_transform.name IS 'The name of the record.' -; -COMMENT ON COLUMN security_transform.description IS 'The description of the record.' -; -COMMENT ON COLUMN security_transform.notes IS 'Notes associated with the record.' -; -COMMENT ON COLUMN security_transform.transform IS 'A SQL statement or fragment suitable for the identification of spatial components from submission spatial components and subsequent population of a machine readable dataset that describes the secured map viewable attributes of that component.' -; -COMMENT ON COLUMN security_transform.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN security_transform.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN security_transform.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN security_transform.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN security_transform.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE security_transform IS 'Security transforms are SQL statements or fragments that dynamically operate on submission spatial components to provide a secure view of the component for map display.' -; - --- --- TABLE: security_transform_submission --- - -CREATE TABLE security_transform_submission( - security_transform_submission_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - submission_spatial_component_id integer NOT NULL, - security_transform_id integer NOT NULL, - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT security_transform_submission_pk PRIMARY KEY (security_transform_submission_id) -) -; - - - -COMMENT ON COLUMN security_transform_submission.security_transform_submission_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN security_transform_submission.submission_spatial_component_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN security_transform_submission.security_transform_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN security_transform_submission.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN security_transform_submission.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN security_transform_submission.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN security_transform_submission.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN security_transform_submission.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE security_transform_submission IS 'A associative entity that joins security transforms with submission spatial components.' -; - --- --- TABLE: source_transform --- - -CREATE TABLE source_transform( - source_transform_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - system_user_id integer NOT NULL, - version varchar(20) NOT NULL, - metadata_transform text NOT NULL, - metadata_index varchar(100) NOT NULL, - record_effective_date timestamptz(6) DEFAULT now() NOT NULL, - record_end_date timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT source_transform_pk PRIMARY KEY (source_transform_id) -) -; - - - -COMMENT ON COLUMN source_transform.source_transform_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN source_transform.system_user_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN source_transform.version IS 'The version number of the transformation data set for a specific source system. Examples include "0.1" and "2.0.1".' -; -COMMENT ON COLUMN source_transform.metadata_transform IS 'Describes a SQL statement that transforms the JSON representation of the source submissions EML data for indexing in Elastic Search.' -; -COMMENT ON COLUMN source_transform.metadata_index IS 'The search engine layer index that the metadata transform conforms to. This attribute provides the index name that is the target for the metadata produced by the associated "metadata transform" template.' -; -COMMENT ON COLUMN source_transform.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN source_transform.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN source_transform.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN source_transform.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN source_transform.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN source_transform.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN source_transform.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE source_transform IS 'Stores data transform information for data sources. This information is used by data ingest logic to lookup version information and transformations for processing data submissions. Note that foreign keys to system users should be restricted to users with a user identity source of "SYSTEM".' -; - --- --- TABLE: spatial_transform --- - -CREATE TABLE spatial_transform( - spatial_transform_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - name varchar(100) NOT NULL, - description varchar(3000), - notes varchar(3000), - transform text NOT NULL, - record_effective_date timestamptz(6) DEFAULT now() NOT NULL, - record_end_date timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT spatial_transform_pk PRIMARY KEY (spatial_transform_id) -) -; - - - -COMMENT ON COLUMN spatial_transform.spatial_transform_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN spatial_transform.name IS 'The name of the record.' -; -COMMENT ON COLUMN spatial_transform.description IS 'The description of the record.' -; -COMMENT ON COLUMN spatial_transform.notes IS 'Notes associated with the record.' -; -COMMENT ON COLUMN spatial_transform.transform IS 'A SQL statement or fragment suitable for the identification of spatial components from submission source and subsequent population of a machine readable dataset that describes the map viewable attributes of that component.' -; -COMMENT ON COLUMN spatial_transform.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN spatial_transform.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN spatial_transform.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN spatial_transform.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN spatial_transform.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN spatial_transform.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN spatial_transform.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE spatial_transform IS 'Spatial transforms are SQL statements that dynamically operate on submission source to extract spatial components of interest for map display.' -; - --- --- TABLE: spatial_transform_submission --- - -CREATE TABLE spatial_transform_submission( - spatial_transform_submission_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - spatial_transform_id integer NOT NULL, - submission_spatial_component_id integer NOT NULL, - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT spatial_transform_submission_pk PRIMARY KEY (spatial_transform_submission_id) -) -; - - +); -COMMENT ON COLUMN spatial_transform_submission.spatial_transform_submission_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN spatial_transform_submission.spatial_transform_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN spatial_transform_submission.submission_spatial_component_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN spatial_transform_submission.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN spatial_transform_submission.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN spatial_transform_submission.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN spatial_transform_submission.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN spatial_transform_submission.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE spatial_transform_submission IS 'A associative entity that joins spatial transforms with submission spatial components.' -; +COMMENT ON COLUMN audit_log.audit_log_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN audit_log.system_user_id IS 'The system user id affecting the data change.'; +COMMENT ON COLUMN audit_log.create_date IS 'The date and time of record creation.'; +COMMENT ON COLUMN audit_log.table_name IS 'The table name of the data record.'; +COMMENT ON COLUMN audit_log.operation IS 'The operation that affected the data change (ie. INSERT, UPDATE, DELETE, TRUNCATE).'; +COMMENT ON COLUMN audit_log.before_value IS 'The JSON representation of the before value of the record.'; +COMMENT ON COLUMN audit_log.after_value IS 'The JSON representation of the after value of the record.'; +COMMENT ON TABLE audit_log IS 'Holds record level audit log data for the entire database.'; -- -- TABLE: submission @@ -616,7 +76,6 @@ COMMENT ON TABLE spatial_transform_submission IS 'A associative entity that join CREATE TABLE submission( submission_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - source_transform_id integer NOT NULL, uuid uuid DEFAULT public.gen_random_uuid() NOT NULL, create_date timestamptz(6) DEFAULT now() NOT NULL, create_user integer NOT NULL, @@ -624,73 +83,16 @@ CREATE TABLE submission( update_user integer, revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT submission_pk PRIMARY KEY (submission_id) -) -; +); - - -COMMENT ON COLUMN submission.submission_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission.source_transform_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission.uuid IS 'The universally unique identifier for the submission as supplied by the source system.' -; -COMMENT ON COLUMN submission.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission IS 'Provides a listing of data submissions.' -; - --- --- TABLE: submission_government_interest --- - -CREATE TABLE submission_government_interest( - submission_government_interest_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - submission_id integer NOT NULL, - wldtaxonomic_units_id integer NOT NULL, - data_type varchar(300) NOT NULL, - description varchar(3000), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT submission_government_interest_pk PRIMARY KEY (submission_government_interest_id) -) -; - - - -COMMENT ON COLUMN submission_government_interest.submission_government_interest_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_government_interest.submission_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_government_interest.wldtaxonomic_units_id IS 'A foreign reference to the taxonomic unit id.' -; -COMMENT ON COLUMN submission_government_interest.data_type IS 'A description of the type of data that is secured.' -; -COMMENT ON COLUMN submission_government_interest.description IS 'The description of the record.' -; -COMMENT ON COLUMN submission_government_interest.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_government_interest.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_government_interest.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_government_interest.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_government_interest.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_government_interest IS 'An intersection table relating submissions to government interests.' -; +COMMENT ON COLUMN submission.submission_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN submission.uuid IS 'The universally unique identifier for the submission as supplied by the source system.'; +COMMENT ON COLUMN submission.create_date IS 'The datetime the record was created.'; +COMMENT ON COLUMN submission.create_user IS 'The id of the user who created the record as identified in the system user table.'; +COMMENT ON COLUMN submission.update_date IS 'The datetime the record was updated.'; +COMMENT ON COLUMN submission.update_user IS 'The id of the user who updated the record as identified in the system user table.'; +COMMENT ON COLUMN submission.revision_count IS 'Revision count used for concurrency control.'; +COMMENT ON TABLE submission IS 'Provides a listing of data submissions.'; -- -- TABLE: submission_job_queue @@ -710,423 +112,21 @@ CREATE TABLE submission_job_queue( update_user integer, revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT submission_job_queue_pk PRIMARY KEY (submission_job_queue_id) -) -; - - - -COMMENT ON COLUMN submission_job_queue.submission_job_queue_id IS 'Surrogate primary key identifier. This value should be selected from the appropriate sequence and populated manually. -' -; -COMMENT ON COLUMN submission_job_queue.submission_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_job_queue.job_start_timestamp IS 'The timestamp of the job process instantiation.' -; -COMMENT ON COLUMN submission_job_queue.job_end_timestamp IS 'The timestamp of the job process completion.' -; -COMMENT ON COLUMN submission_job_queue.security_request IS 'A document supplied by the submitter outlining a security request for submission observations.' -; -COMMENT ON COLUMN submission_job_queue.key IS 'The identifying key to the file in the storage system.' -; -COMMENT ON COLUMN submission_job_queue.attempt_count IS 'The number of times this job queue record has been attempted.' -; -COMMENT ON COLUMN submission_job_queue.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_job_queue.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_job_queue.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_job_queue.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_job_queue.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_job_queue IS 'A listing of data submission job processes and their details including start and end times.' -; - --- --- TABLE: submission_message --- - -CREATE TABLE submission_message( - submission_message_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - submission_message_type_id integer NOT NULL, - submission_status_id integer NOT NULL, - event_timestamp timestamptz(6) NOT NULL, - message varchar(3000), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT submission_message_pk PRIMARY KEY (submission_message_id) -) -; - - - -COMMENT ON COLUMN submission_message.submission_message_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_message.submission_message_type_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_message.submission_status_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_message.event_timestamp IS 'The timestamp of the associated event.' -; -COMMENT ON COLUMN submission_message.message IS 'The description of the record.' -; -COMMENT ON COLUMN submission_message.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_message.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_message.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_message.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_message.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_message IS 'Intersection table to track submission messages.' -; - --- --- TABLE: submission_message_class --- - -CREATE TABLE submission_message_class( - submission_message_class_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - name varchar(50) NOT NULL, - description varchar(250), - record_effective_date timestamptz(6) DEFAULT now() NOT NULL, - record_end_date timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT submission_message_class_pk PRIMARY KEY (submission_message_class_id) -) -; - - - -COMMENT ON COLUMN submission_message_class.submission_message_class_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_message_class.name IS 'The name of the record.' -; -COMMENT ON COLUMN submission_message_class.description IS 'The description of the record.' -; -COMMENT ON COLUMN submission_message_class.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN submission_message_class.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN submission_message_class.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_message_class.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_message_class.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_message_class.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_message_class.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_message_class IS 'The classification of submission message types available to report.' -; - --- --- TABLE: submission_message_type --- - -CREATE TABLE submission_message_type( - submission_message_type_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - submission_message_class_id integer NOT NULL, - name varchar(50) NOT NULL, - record_end_date timestamptz(6), - record_effective_date timestamptz(6) DEFAULT now() NOT NULL, - description varchar(250), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT submission_message_type_pk PRIMARY KEY (submission_message_type_id) -) -; - - - -COMMENT ON COLUMN submission_message_type.submission_message_type_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_message_type.submission_message_class_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_message_type.name IS 'The name of the record.' -; -COMMENT ON COLUMN submission_message_type.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN submission_message_type.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN submission_message_type.description IS 'The description of the record.' -; -COMMENT ON COLUMN submission_message_type.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_message_type.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_message_type.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_message_type.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_message_type.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_message_type IS 'The types of submission messages available to report. These messages may include metrics and validation concerns.' -; - --- --- TABLE: submission_metadata --- - -CREATE TABLE submission_metadata( - submission_metadata_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - submission_id integer NOT NULL, - eml_source text NOT NULL, - eml_json_source jsonb, - dataset_search_criteria jsonb, - record_effective_timestamp timestamptz(6), - record_end_timestamp timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT submission_metadata_pk PRIMARY KEY (submission_metadata_id) -) -; - - - -COMMENT ON COLUMN submission_metadata.submission_metadata_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_metadata.submission_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_metadata.eml_source IS 'The Ecological Metadata Language source as extracted from the submission.' -; -COMMENT ON COLUMN submission_metadata.eml_json_source IS 'The JSON representation of the Ecological Metadata Language source for the submission.' -; -COMMENT ON COLUMN submission_metadata.dataset_search_criteria IS 'Describes the object the system sends to elastic search' -; -COMMENT ON COLUMN submission_metadata.record_effective_timestamp IS 'Record level effective timestamp.' -; -COMMENT ON COLUMN submission_metadata.record_end_timestamp IS 'Record level end timestamp.' -; -COMMENT ON COLUMN submission_metadata.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_metadata.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_metadata.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_metadata.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_metadata.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_metadata IS 'Provides a historical listing of data submission metadata.' -; - --- --- TABLE: submission_observation --- - -CREATE TABLE submission_observation( - submission_observation_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - submission_id integer NOT NULL, - darwin_core_source jsonb NOT NULL, - submission_security_request jsonb, - security_review_timestamp timestamptz(6), - foi_reason boolean, - security_reason_name varchar(300), - security_reason_description varchar(3000), - security_reason_end_date timestamptz(6), - record_effective_timestamp timestamptz(6), - record_end_timestamp timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT submission_observation_pk PRIMARY KEY (submission_observation_id) -) -; - - - -COMMENT ON COLUMN submission_observation.submission_observation_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_observation.submission_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_observation.darwin_core_source IS 'The denormalized Darwin Core source as extracted from the submission.' -; -COMMENT ON COLUMN submission_observation.submission_security_request IS 'A JSON document describing a submitters desire the secure submission data.' -; -COMMENT ON COLUMN submission_observation.security_review_timestamp IS 'The timestamp of the associated event.' -; -COMMENT ON COLUMN submission_observation.foi_reason IS 'A boolean flag indicating whether the data is secured due to Freedom of Information data being present.' -; -COMMENT ON COLUMN submission_observation.security_reason_name IS 'The name of the custom security reason.' -; -COMMENT ON COLUMN submission_observation.security_reason_description IS 'A reason description that is secures this data and is specific to this artifact or dataset.' -; -COMMENT ON COLUMN submission_observation.security_reason_end_date IS 'Custom security reason end date.' -; -COMMENT ON COLUMN submission_observation.record_effective_timestamp IS 'Record level effective timestamp.' -; -COMMENT ON COLUMN submission_observation.record_end_timestamp IS 'Record level end timestamp.' -; -COMMENT ON COLUMN submission_observation.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_observation.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_observation.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_observation.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_observation.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_observation IS 'A listing of historical data submission observation data. The record with the most recent security review timestamp is the currently published data set for each submission.' -; - --- --- TABLE: submission_spatial_component --- - -CREATE TABLE submission_spatial_component( - submission_spatial_component_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - submission_observation_id integer NOT NULL, - spatial_component jsonb NOT NULL, - geometry geometry(geometry, 3005), - geography geography(geometry), - secured_spatial_component jsonb, - secured_geometry geometry(geometry, 3005), - secured_geography geography(geometry), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT submission_spatial_component_pk PRIMARY KEY (submission_spatial_component_id) -) -; - - - -COMMENT ON COLUMN submission_spatial_component.submission_spatial_component_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_spatial_component.submission_observation_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_spatial_component.spatial_component IS 'A spatial component is a JSON attribute representation of a viewable map object.' -; -COMMENT ON COLUMN submission_spatial_component.geometry IS 'The containing geometry of the spatial component attribute.' -; -COMMENT ON COLUMN submission_spatial_component.geography IS 'The containing geography of the spatial component attribute.' -; -COMMENT ON COLUMN submission_spatial_component.secured_spatial_component IS 'A secure spatial component is a JSON attribute representation of a viewable map object that has been adjusted by a security rule that targets that representation.' -; -COMMENT ON COLUMN submission_spatial_component.secured_geometry IS 'The containing geometry of the secured spatial component attribute.' -; -COMMENT ON COLUMN submission_spatial_component.secured_geography IS 'The containing geography of the secured spatial component attribute.' -; -COMMENT ON COLUMN submission_spatial_component.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_spatial_component.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_spatial_component.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_spatial_component.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_spatial_component.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_spatial_component IS 'Submission spatial components are spatial features and their desired map representations as extracted from submission source data.' -; - --- --- TABLE: submission_status --- - -CREATE TABLE submission_status( - submission_status_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - submission_id integer NOT NULL, - submission_status_type_id integer NOT NULL, - event_timestamp timestamptz(6) NOT NULL, - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT submission_status_pk PRIMARY KEY (submission_status_id) -) -; - - - -COMMENT ON COLUMN submission_status.submission_status_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_status.submission_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_status.submission_status_type_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_status.event_timestamp IS 'The timestamp of the associated event.' -; -COMMENT ON COLUMN submission_status.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_status.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_status.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_status.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_status.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_status IS 'Provides a history of submission statuses.' -; - --- --- TABLE: submission_status_type --- - -CREATE TABLE submission_status_type( - submission_status_type_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - name varchar(50) NOT NULL, - description varchar(250), - record_effective_date timestamptz(6) DEFAULT now() NOT NULL, - record_end_date timestamptz(6), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT submission_status_type_pk PRIMARY KEY (submission_status_type_id) -) -; - - - -COMMENT ON COLUMN submission_status_type.submission_status_type_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN submission_status_type.name IS 'The name of the record.' -; -COMMENT ON COLUMN submission_status_type.description IS 'The description of the record.' -; -COMMENT ON COLUMN submission_status_type.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN submission_status_type.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN submission_status_type.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN submission_status_type.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_status_type.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN submission_status_type.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN submission_status_type.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE submission_status_type IS 'The status types of submissions. Typical status types are those that represent submissions being submitted or rejected.' -; +); + +COMMENT ON COLUMN submission_job_queue.submission_job_queue_id IS 'Surrogate primary key identifier. This value should be selected from the appropriate sequence and populated manually.'; +COMMENT ON COLUMN submission_job_queue.submission_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN submission_job_queue.job_start_timestamp IS 'The timestamp of the job process instantiation.'; +COMMENT ON COLUMN submission_job_queue.job_end_timestamp IS 'The timestamp of the job process completion.'; +COMMENT ON COLUMN submission_job_queue.security_request IS 'A document supplied by the submitter outlining a security request for submission observations.'; +COMMENT ON COLUMN submission_job_queue.key IS 'The identifying key to the file in the storage system.'; +COMMENT ON COLUMN submission_job_queue.attempt_count IS 'The number of times this job queue record has been attempted.'; +COMMENT ON COLUMN submission_job_queue.create_date IS 'The datetime the record was created.'; +COMMENT ON COLUMN submission_job_queue.create_user IS 'The id of the user who created the record as identified in the system user table.'; +COMMENT ON COLUMN submission_job_queue.update_date IS 'The datetime the record was updated.'; +COMMENT ON COLUMN submission_job_queue.update_user IS 'The id of the user who updated the record as identified in the system user table.'; +COMMENT ON COLUMN submission_job_queue.revision_count IS 'Revision count used for concurrency control.'; +COMMENT ON TABLE submission_job_queue IS 'A listing of data submission job processes and their details including start and end times.'; -- -- TABLE: system_constant @@ -1144,33 +144,19 @@ CREATE TABLE system_constant( update_user integer, revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT system_constant_pk PRIMARY KEY (system_constant_id) -) -; - - +); -COMMENT ON COLUMN system_constant.system_constant_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_constant.constant_name IS 'The lookup name of the constant.' -; -COMMENT ON COLUMN system_constant.character_value IS 'The string value of the constant.' -; -COMMENT ON COLUMN system_constant.numeric_value IS 'The numeric value of the constant.' -; -COMMENT ON COLUMN system_constant.description IS 'The description of the record.' -; -COMMENT ON COLUMN system_constant.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN system_constant.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN system_constant.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN system_constant.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN system_constant.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE system_constant IS 'A list of system constants necessary for system functionality. Such constants are not editable by system administrators as they are used by internal logic.' -; +COMMENT ON COLUMN system_constant.system_constant_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN system_constant.constant_name IS 'The lookup name of the constant.'; +COMMENT ON COLUMN system_constant.character_value IS 'The string value of the constant.'; +COMMENT ON COLUMN system_constant.numeric_value IS 'The numeric value of the constant.'; +COMMENT ON COLUMN system_constant.description IS 'The description of the record.'; +COMMENT ON COLUMN system_constant.create_date IS 'The datetime the record was created.'; +COMMENT ON COLUMN system_constant.create_user IS 'The id of the user who created the record as identified in the system user table.'; +COMMENT ON COLUMN system_constant.update_date IS 'The datetime the record was updated.'; +COMMENT ON COLUMN system_constant.update_user IS 'The id of the user who updated the record as identified in the system user table.'; +COMMENT ON COLUMN system_constant.revision_count IS 'Revision count used for concurrency control.'; +COMMENT ON TABLE system_constant IS 'A list of system constants necessary for system functionality. Such constants are not editable by system administrators as they are used by internal logic.'; -- -- TABLE: system_metadata_constant @@ -1188,33 +174,19 @@ CREATE TABLE system_metadata_constant( update_user integer, revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT system_metadata_constant_pk PRIMARY KEY (system_metadata_constant_id) -) -; +); - - -COMMENT ON COLUMN system_metadata_constant.system_metadata_constant_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_metadata_constant.constant_name IS 'The lookup name of the constant.' -; -COMMENT ON COLUMN system_metadata_constant.character_value IS 'The string value of the constant.' -; -COMMENT ON COLUMN system_metadata_constant.numeric_value IS 'The numeric value of the constant.' -; -COMMENT ON COLUMN system_metadata_constant.description IS 'The description of the record.' -; -COMMENT ON COLUMN system_metadata_constant.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN system_metadata_constant.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN system_metadata_constant.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN system_metadata_constant.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN system_metadata_constant.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE system_metadata_constant IS 'A list of system metadata constants associated with the business. Such constants are editable by system administrators and are used when publishing data.' -; +COMMENT ON COLUMN system_metadata_constant.system_metadata_constant_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN system_metadata_constant.constant_name IS 'The lookup name of the constant.'; +COMMENT ON COLUMN system_metadata_constant.character_value IS 'The string value of the constant.'; +COMMENT ON COLUMN system_metadata_constant.numeric_value IS 'The numeric value of the constant.'; +COMMENT ON COLUMN system_metadata_constant.description IS 'The description of the record.'; +COMMENT ON COLUMN system_metadata_constant.create_date IS 'The datetime the record was created.'; +COMMENT ON COLUMN system_metadata_constant.create_user IS 'The id of the user who created the record as identified in the system user table.'; +COMMENT ON COLUMN system_metadata_constant.update_date IS 'The datetime the record was updated.'; +COMMENT ON COLUMN system_metadata_constant.update_user IS 'The id of the user who updated the record as identified in the system user table.'; +COMMENT ON COLUMN system_metadata_constant.revision_count IS 'Revision count used for concurrency control.'; +COMMENT ON TABLE system_metadata_constant IS 'A list of system metadata constants associated with the business. Such constants are editable by system administrators and are used when publishing data.'; -- -- TABLE: system_role @@ -1233,35 +205,20 @@ CREATE TABLE system_role( update_user integer, revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT system_role_pk PRIMARY KEY (system_role_id) -) -; - - - -COMMENT ON COLUMN system_role.system_role_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_role.name IS 'The name of the record.' -; -COMMENT ON COLUMN system_role.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN system_role.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN system_role.description IS 'The description of the record.' -; -COMMENT ON COLUMN system_role.notes IS 'Notes associated with the record.' -; -COMMENT ON COLUMN system_role.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN system_role.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN system_role.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN system_role.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN system_role.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE system_role IS 'Agency or Ministry funding the project.' -; +); + +COMMENT ON COLUMN system_role.system_role_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN system_role.name IS 'The name of the record.'; +COMMENT ON COLUMN system_role.record_effective_date IS 'Record level effective date.'; +COMMENT ON COLUMN system_role.record_end_date IS 'Record level end date.'; +COMMENT ON COLUMN system_role.description IS 'The description of the record.'; +COMMENT ON COLUMN system_role.notes IS 'Notes associated with the record.'; +COMMENT ON COLUMN system_role.create_date IS 'The datetime the record was created.'; +COMMENT ON COLUMN system_role.create_user IS 'The id of the user who created the record as identified in the system user table.'; +COMMENT ON COLUMN system_role.update_date IS 'The datetime the record was updated.'; +COMMENT ON COLUMN system_role.update_user IS 'The id of the user who updated the record as identified in the system user table.'; +COMMENT ON COLUMN system_role.revision_count IS 'Revision count used for concurrency control.'; +COMMENT ON TABLE system_role IS 'Agency or Ministry funding the project.'; -- -- TABLE: system_user @@ -1280,33 +237,19 @@ CREATE TABLE system_user( update_user integer, revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT system_user_pk PRIMARY KEY (system_user_id) -) -; - - +); -COMMENT ON COLUMN system_user.system_user_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_user.user_identity_source_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_user.user_identifier IS 'The identifier of the user.' -; -COMMENT ON COLUMN system_user.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN system_user.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN system_user.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN system_user.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN system_user.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN system_user.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN system_user.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE system_user IS 'Agency or Ministry funding the project.' -; +COMMENT ON COLUMN system_user.system_user_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN system_user.user_identity_source_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN system_user.user_identifier IS 'The identifier of the user.'; +COMMENT ON COLUMN system_user.record_effective_date IS 'Record level effective date.'; +COMMENT ON COLUMN system_user.record_end_date IS 'Record level end date.'; +COMMENT ON COLUMN system_user.create_date IS 'The datetime the record was created.'; +COMMENT ON COLUMN system_user.create_user IS 'The id of the user who created the record as identified in the system user table.'; +COMMENT ON COLUMN system_user.update_date IS 'The datetime the record was updated.'; +COMMENT ON COLUMN system_user.update_user IS 'The id of the user who updated the record as identified in the system user table.'; +COMMENT ON COLUMN system_user.revision_count IS 'Revision count used for concurrency control.'; +COMMENT ON TABLE system_user IS 'Agency or Ministry funding the project.'; -- -- TABLE: system_user_role @@ -1322,76 +265,17 @@ CREATE TABLE system_user_role( update_user integer, revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT system_user_role_pk PRIMARY KEY (system_user_role_id) -) -; +); - - -COMMENT ON COLUMN system_user_role.system_user_role_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_user_role.system_user_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_user_role.system_role_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_user_role.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN system_user_role.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN system_user_role.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN system_user_role.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN system_user_role.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE system_user_role IS 'A associative entity that joins system users and system role types.' -; - --- --- TABLE: system_user_security_exception --- - -CREATE TABLE system_user_security_exception( - system_user_security_exception_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - system_user_id integer NOT NULL, - persecution_or_harm_id integer NOT NULL, - start_date timestamptz(6), - end_date timestamptz(6), - notes varchar(3000), - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT system_user_security_exception_pk PRIMARY KEY (system_user_security_exception_id) -) -; - - - -COMMENT ON COLUMN system_user_security_exception.system_user_security_exception_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_user_security_exception.system_user_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_user_security_exception.persecution_or_harm_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN system_user_security_exception.start_date IS 'The record start date.' -; -COMMENT ON COLUMN system_user_security_exception.end_date IS 'The record end date.' -; -COMMENT ON COLUMN system_user_security_exception.notes IS 'Notes associated with the record.' -; -COMMENT ON COLUMN system_user_security_exception.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN system_user_security_exception.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN system_user_security_exception.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN system_user_security_exception.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN system_user_security_exception.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE system_user_security_exception IS 'Identifies persecution or harm security reasons for which particular system users are exempt, thus allowing those users to view unsecured map representations of those spatial components.' -; +COMMENT ON COLUMN system_user_role.system_user_role_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN system_user_role.system_user_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN system_user_role.system_role_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN system_user_role.create_date IS 'The datetime the record was created.'; +COMMENT ON COLUMN system_user_role.create_user IS 'The id of the user who created the record as identified in the system user table.'; +COMMENT ON COLUMN system_user_role.update_date IS 'The datetime the record was updated.'; +COMMENT ON COLUMN system_user_role.update_user IS 'The id of the user who updated the record as identified in the system user table.'; +COMMENT ON COLUMN system_user_role.revision_count IS 'Revision count used for concurrency control.'; +COMMENT ON TABLE system_user_role IS 'A associative entity that joins system users and system role types.'; -- -- TABLE: user_identity_source @@ -1410,471 +294,92 @@ CREATE TABLE user_identity_source( update_user integer, revision_count integer DEFAULT 0 NOT NULL, CONSTRAINT user_identity_source_pk PRIMARY KEY (user_identity_source_id) -) -; - - - -COMMENT ON COLUMN user_identity_source.user_identity_source_id IS 'System generated surrogate primary key identifier.' -; -COMMENT ON COLUMN user_identity_source.name IS 'The name of the record.' -; -COMMENT ON COLUMN user_identity_source.record_effective_date IS 'Record level effective date.' -; -COMMENT ON COLUMN user_identity_source.record_end_date IS 'Record level end date.' -; -COMMENT ON COLUMN user_identity_source.description IS 'The description of the record.' -; -COMMENT ON COLUMN user_identity_source.notes IS 'Notes associated with the record.' -; -COMMENT ON COLUMN user_identity_source.create_date IS 'The datetime the record was created.' -; -COMMENT ON COLUMN user_identity_source.create_user IS 'The id of the user who created the record as identified in the system user table.' -; -COMMENT ON COLUMN user_identity_source.update_date IS 'The datetime the record was updated.' -; -COMMENT ON COLUMN user_identity_source.update_user IS 'The id of the user who updated the record as identified in the system user table.' -; -COMMENT ON COLUMN user_identity_source.revision_count IS 'Revision count used for concurrency control.' -; -COMMENT ON TABLE user_identity_source IS 'The source of the user identifier. This source is traditionally the system that authenticates the user. Example sources could include IDIR, BCEID and DATABASE.' -; +); + +COMMENT ON COLUMN user_identity_source.user_identity_source_id IS 'System generated surrogate primary key identifier.'; +COMMENT ON COLUMN user_identity_source.name IS 'The name of the record.'; +COMMENT ON COLUMN user_identity_source.record_effective_date IS 'Record level effective date.'; +COMMENT ON COLUMN user_identity_source.record_end_date IS 'Record level end date.'; +COMMENT ON COLUMN user_identity_source.description IS 'The description of the record.'; +COMMENT ON COLUMN user_identity_source.notes IS 'Notes associated with the record.'; +COMMENT ON COLUMN user_identity_source.create_date IS 'The datetime the record was created.'; +COMMENT ON COLUMN user_identity_source.create_user IS 'The id of the user who created the record as identified in the system user table.'; +COMMENT ON COLUMN user_identity_source.update_date IS 'The datetime the record was updated.'; +COMMENT ON COLUMN user_identity_source.update_user IS 'The id of the user who updated the record as identified in the system user table.'; +COMMENT ON COLUMN user_identity_source.revision_count IS 'Revision count used for concurrency control.'; +COMMENT ON TABLE user_identity_source IS 'The source of the user identifier. This source is traditionally the system that authenticates the user. Example sources could include IDIR, BCEID and DATABASE.'; -- -- INDEX: "Ref165191" -- -CREATE INDEX "Ref165191" ON artifact(submission_id) -; --- --- INDEX: "Ref228199" --- - -CREATE INDEX "Ref228199" ON artifact_government_interest(artifact_id) -; --- --- INDEX: artifact_persecution_uk1 --- - -CREATE UNIQUE INDEX artifact_persecution_uk1 ON artifact_persecution(persecution_or_harm_id) -; --- --- INDEX: "Ref228195" --- - -CREATE INDEX "Ref228195" ON artifact_persecution(artifact_id) -; --- --- INDEX: "Ref254202" --- - -CREATE INDEX "Ref254202" ON artifact_persecution(persecution_or_harm_id) -; --- --- INDEX: "Ref228194" --- - -CREATE INDEX "Ref228194" ON artifact_proprietary(artifact_id) -; --- --- INDEX: "Ref233196" --- - -CREATE INDEX "Ref233196" ON artifact_proprietary(proprietary_type_id) -; --- --- INDEX: persecution_or_harm_uk1 --- - -CREATE UNIQUE INDEX persecution_or_harm_uk1 ON persecution_or_harm(persecution_or_harm_type_id, name, wldtaxonomic_units_id) -; --- --- INDEX: "Ref252201" --- - -CREATE INDEX "Ref252201" ON persecution_or_harm(persecution_or_harm_type_id) -; --- --- INDEX: persecution_or_harm_type_nuk1 --- - -CREATE UNIQUE INDEX persecution_or_harm_type_nuk1 ON persecution_or_harm_type(name, record_end_date) -; --- --- INDEX: proprietary_type_nuk1 --- - -CREATE UNIQUE INDEX proprietary_type_nuk1 ON proprietary_type(name, record_end_date) -; --- --- INDEX: security_transform_uk1 --- +CREATE INDEX "Ref165191" ON artifact(submission_id); -CREATE UNIQUE INDEX security_transform_uk1 ON security_transform(name, persecution_or_harm_id) -; --- --- INDEX: "Ref254203" --- - -CREATE INDEX "Ref254203" ON security_transform(persecution_or_harm_id) -; --- --- INDEX: security_transform_submission_uk1 --- - -CREATE UNIQUE INDEX security_transform_submission_uk1 ON security_transform_submission(submission_spatial_component_id, security_transform_id) -; --- --- INDEX: "Ref169186" --- - -CREATE INDEX "Ref169186" ON security_transform_submission(submission_spatial_component_id) -; --- --- INDEX: "Ref218187" --- - -CREATE INDEX "Ref218187" ON security_transform_submission(security_transform_id) -; --- --- INDEX: source_transform_nuk1 --- - -CREATE UNIQUE INDEX source_transform_nuk1 ON source_transform(system_user_id, version, record_end_date) -; --- --- INDEX: "Ref191183" --- - -CREATE INDEX "Ref191183" ON source_transform(system_user_id) -; --- --- INDEX: spatial_transform_nuk1 --- - -CREATE UNIQUE INDEX spatial_transform_nuk1 ON spatial_transform(name, record_end_date) -; --- --- INDEX: spatial_transform_submission_uk1 --- - -CREATE UNIQUE INDEX spatial_transform_submission_uk1 ON spatial_transform_submission(spatial_transform_id, submission_spatial_component_id) -; --- --- INDEX: "Ref207184" --- - -CREATE INDEX "Ref207184" ON spatial_transform_submission(spatial_transform_id) -; --- --- INDEX: "Ref169185" --- - -CREATE INDEX "Ref169185" ON spatial_transform_submission(submission_spatial_component_id) -; -- -- INDEX: submission_uk1 -- -CREATE UNIQUE INDEX submission_uk1 ON submission(uuid) -; --- --- INDEX: "Ref199182" --- - -CREATE INDEX "Ref199182" ON submission(source_transform_id) -; --- --- INDEX: submission_government_interest_uk1 --- - -CREATE UNIQUE INDEX submission_government_interest_uk1 ON submission_government_interest(submission_id) -; --- --- INDEX: "Ref165198" --- +CREATE UNIQUE INDEX submission_uk1 ON submission(uuid); -CREATE INDEX "Ref165198" ON submission_government_interest(submission_id) -; -- -- INDEX: "Ref165208" -- -CREATE INDEX "Ref165208" ON submission_job_queue(submission_id) -; --- --- INDEX: "Ref184166" --- - -CREATE INDEX "Ref184166" ON submission_message(submission_status_id) -; --- --- INDEX: "Ref182167" --- +CREATE INDEX "Ref165208" ON submission_job_queue(submission_id); -CREATE INDEX "Ref182167" ON submission_message(submission_message_type_id) -; --- --- INDEX: submission_message_class_nuk1 --- - -CREATE UNIQUE INDEX submission_message_class_nuk1 ON submission_message_class(name, record_end_date) -; --- --- INDEX: submission_message_type_nuk1 --- - -CREATE UNIQUE INDEX submission_message_type_nuk1 ON submission_message_type(name, record_end_date) -; --- --- INDEX: "Ref189177" --- - -CREATE INDEX "Ref189177" ON submission_message_type(submission_message_class_id) -; --- --- INDEX: submission_metadata_nuk1 --- - -CREATE UNIQUE INDEX submission_metadata_nuk1 ON submission_metadata(submission_id, record_end_timestamp) -; --- --- INDEX: "Ref165207" --- - -CREATE INDEX "Ref165207" ON submission_metadata(submission_id) -; --- --- INDEX: "Ref165205" --- - -CREATE INDEX "Ref165205" ON submission_observation(submission_id) -; --- --- INDEX: "Ref255206" --- - -CREATE INDEX "Ref255206" ON submission_spatial_component(submission_observation_id) -; --- --- INDEX: "Ref165163" --- - -CREATE INDEX "Ref165163" ON submission_status(submission_id) -; --- --- INDEX: "Ref183164" --- - -CREATE INDEX "Ref183164" ON submission_status(submission_status_type_id) -; --- --- INDEX: submission_status_type_nuk1 --- - -CREATE UNIQUE INDEX submission_status_type_nuk1 ON submission_status_type(name, record_end_date) -; -- -- INDEX: system_constant_uk1 -- -CREATE UNIQUE INDEX system_constant_uk1 ON system_constant(constant_name) -; +CREATE UNIQUE INDEX system_constant_uk1 ON system_constant(constant_name); -- -- INDEX: system_metadata_constant_uk1 -- -CREATE UNIQUE INDEX system_metadata_constant_uk1 ON system_metadata_constant(constant_name) -; +CREATE UNIQUE INDEX system_metadata_constant_uk1 ON system_metadata_constant(constant_name); -- -- INDEX: system_role_nuk1 -- -CREATE UNIQUE INDEX system_role_nuk1 ON system_role(name, record_end_date) -; +CREATE UNIQUE INDEX system_role_nuk1 ON system_role(name, record_end_date); -- -- INDEX: system_user_nuk1 -- -CREATE UNIQUE INDEX system_user_nuk1 ON system_user(user_identifier, user_identity_source_id, record_end_date) -; +CREATE UNIQUE INDEX system_user_nuk1 ON system_user(user_identifier, user_identity_source_id, record_end_date); -- -- INDEX: "Ref190178" -- -CREATE INDEX "Ref190178" ON system_user(user_identity_source_id) -; +CREATE INDEX "Ref190178" ON system_user(user_identity_source_id); -- -- INDEX: system_user_role_uk1 -- -CREATE UNIQUE INDEX system_user_role_uk1 ON system_user_role(system_user_id, system_role_id) -; +CREATE UNIQUE INDEX system_user_role_uk1 ON system_user_role(system_user_id, system_role_id); -- -- INDEX: "Ref191179" -- -CREATE INDEX "Ref191179" ON system_user_role(system_user_id) -; +CREATE INDEX "Ref191179" ON system_user_role(system_user_id); -- -- INDEX: "Ref192180" -- -CREATE INDEX "Ref192180" ON system_user_role(system_role_id) -; --- --- INDEX: system_user_security_exception_uk1 --- - -CREATE UNIQUE INDEX system_user_security_exception_uk1 ON system_user_security_exception(system_user_id, persecution_or_harm_id) -; --- --- INDEX: "Ref191189" --- +CREATE INDEX "Ref192180" ON system_user_role(system_role_id); -CREATE INDEX "Ref191189" ON system_user_security_exception(system_user_id) -; --- --- INDEX: "Ref254204" --- - -CREATE INDEX "Ref254204" ON system_user_security_exception(persecution_or_harm_id) -; -- -- INDEX: user_identity_source_nuk1 -- -CREATE UNIQUE INDEX user_identity_source_nuk1 ON user_identity_source(name, record_end_date) -; +CREATE UNIQUE INDEX user_identity_source_nuk1 ON user_identity_source(name, record_end_date); -- -- TABLE: artifact -- ALTER TABLE artifact ADD CONSTRAINT "Refsubmission191" FOREIGN KEY (submission_id) - REFERENCES submission(submission_id) -; - - --- --- TABLE: artifact_government_interest --- - -ALTER TABLE artifact_government_interest ADD CONSTRAINT "Refartifact199" - FOREIGN KEY (artifact_id) - REFERENCES artifact(artifact_id) -; - - --- --- TABLE: artifact_persecution --- - -ALTER TABLE artifact_persecution ADD CONSTRAINT "Refartifact195" - FOREIGN KEY (artifact_id) - REFERENCES artifact(artifact_id) -; - -ALTER TABLE artifact_persecution ADD CONSTRAINT "Refpersecution_or_harm202" - FOREIGN KEY (persecution_or_harm_id) - REFERENCES persecution_or_harm(persecution_or_harm_id) -; - - --- --- TABLE: artifact_proprietary --- - -ALTER TABLE artifact_proprietary ADD CONSTRAINT "Refartifact194" - FOREIGN KEY (artifact_id) - REFERENCES artifact(artifact_id) -; - -ALTER TABLE artifact_proprietary ADD CONSTRAINT "Refproprietary_type196" - FOREIGN KEY (proprietary_type_id) - REFERENCES proprietary_type(proprietary_type_id) -; - - --- --- TABLE: persecution_or_harm --- - -ALTER TABLE persecution_or_harm ADD CONSTRAINT "Refpersecution_or_harm_type201" - FOREIGN KEY (persecution_or_harm_type_id) - REFERENCES persecution_or_harm_type(persecution_or_harm_type_id) -; - - --- --- TABLE: security_transform --- - -ALTER TABLE security_transform ADD CONSTRAINT "Refpersecution_or_harm203" - FOREIGN KEY (persecution_or_harm_id) - REFERENCES persecution_or_harm(persecution_or_harm_id) -; - - --- --- TABLE: security_transform_submission --- - -ALTER TABLE security_transform_submission ADD CONSTRAINT "Refsubmission_spatial_component186" - FOREIGN KEY (submission_spatial_component_id) - REFERENCES submission_spatial_component(submission_spatial_component_id) -; - -ALTER TABLE security_transform_submission ADD CONSTRAINT "Refsecurity_transform187" - FOREIGN KEY (security_transform_id) - REFERENCES security_transform(security_transform_id) -; - - --- --- TABLE: source_transform --- - -ALTER TABLE source_transform ADD CONSTRAINT "Refsystem_user183" - FOREIGN KEY (system_user_id) - REFERENCES system_user(system_user_id) -; - - --- --- TABLE: spatial_transform_submission --- - -ALTER TABLE spatial_transform_submission ADD CONSTRAINT "Refspatial_transform184" - FOREIGN KEY (spatial_transform_id) - REFERENCES spatial_transform(spatial_transform_id) -; - -ALTER TABLE spatial_transform_submission ADD CONSTRAINT "Refsubmission_spatial_component185" - FOREIGN KEY (submission_spatial_component_id) - REFERENCES submission_spatial_component(submission_spatial_component_id) -; - - --- --- TABLE: submission --- - -ALTER TABLE submission ADD CONSTRAINT "Refsource_transform182" - FOREIGN KEY (source_transform_id) - REFERENCES source_transform(source_transform_id) -; - - --- --- TABLE: submission_government_interest --- - -ALTER TABLE submission_government_interest ADD CONSTRAINT "Refsubmission198" - FOREIGN KEY (submission_id) - REFERENCES submission(submission_id) -; - + REFERENCES submission(submission_id); -- -- TABLE: submission_job_queue @@ -1882,79 +387,7 @@ ALTER TABLE submission_government_interest ADD CONSTRAINT "Refsubmission198" ALTER TABLE submission_job_queue ADD CONSTRAINT "Refsubmission208" FOREIGN KEY (submission_id) - REFERENCES submission(submission_id) -; - - --- --- TABLE: submission_message --- - -ALTER TABLE submission_message ADD CONSTRAINT "Refsubmission_status166" - FOREIGN KEY (submission_status_id) - REFERENCES submission_status(submission_status_id) -; - -ALTER TABLE submission_message ADD CONSTRAINT "Refsubmission_message_type167" - FOREIGN KEY (submission_message_type_id) - REFERENCES submission_message_type(submission_message_type_id) -; - - --- --- TABLE: submission_message_type --- - -ALTER TABLE submission_message_type ADD CONSTRAINT "Refsubmission_message_class177" - FOREIGN KEY (submission_message_class_id) - REFERENCES submission_message_class(submission_message_class_id) -; - - --- --- TABLE: submission_metadata --- - -ALTER TABLE submission_metadata ADD CONSTRAINT "Refsubmission207" - FOREIGN KEY (submission_id) - REFERENCES submission(submission_id) -; - - --- --- TABLE: submission_observation --- - -ALTER TABLE submission_observation ADD CONSTRAINT "Refsubmission205" - FOREIGN KEY (submission_id) - REFERENCES submission(submission_id) -; - - --- --- TABLE: submission_spatial_component --- - -ALTER TABLE submission_spatial_component ADD CONSTRAINT "Refsubmission_observation206" - FOREIGN KEY (submission_observation_id) - REFERENCES submission_observation(submission_observation_id) -; - - --- --- TABLE: submission_status --- - -ALTER TABLE submission_status ADD CONSTRAINT "Refsubmission163" - FOREIGN KEY (submission_id) - REFERENCES submission(submission_id) -; - -ALTER TABLE submission_status ADD CONSTRAINT "Refsubmission_status_type164" - FOREIGN KEY (submission_status_type_id) - REFERENCES submission_status_type(submission_status_type_id) -; - + REFERENCES submission(submission_id); -- -- TABLE: system_user @@ -1962,9 +395,7 @@ ALTER TABLE submission_status ADD CONSTRAINT "Refsubmission_status_type164" ALTER TABLE system_user ADD CONSTRAINT "Refuser_identity_source178" FOREIGN KEY (user_identity_source_id) - REFERENCES user_identity_source(user_identity_source_id) -; - + REFERENCES user_identity_source(user_identity_source_id); -- -- TABLE: system_user_role @@ -1972,27 +403,8 @@ ALTER TABLE system_user ADD CONSTRAINT "Refuser_identity_source178" ALTER TABLE system_user_role ADD CONSTRAINT "Refsystem_user179" FOREIGN KEY (system_user_id) - REFERENCES system_user(system_user_id) -; + REFERENCES system_user(system_user_id); ALTER TABLE system_user_role ADD CONSTRAINT "Refsystem_role180" FOREIGN KEY (system_role_id) - REFERENCES system_role(system_role_id) -; - - --- --- TABLE: system_user_security_exception --- - -ALTER TABLE system_user_security_exception ADD CONSTRAINT "Refsystem_user189" - FOREIGN KEY (system_user_id) - REFERENCES system_user(system_user_id) -; - -ALTER TABLE system_user_security_exception ADD CONSTRAINT "Refpersecution_or_harm204" - FOREIGN KEY (persecution_or_harm_id) - REFERENCES persecution_or_harm(persecution_or_harm_id) -; - - + REFERENCES system_role(system_role_id); diff --git a/database/src/migrations/release.0.8.0/populate_persecution_or_harm.sql b/database/src/migrations/release.0.8.0/populate_persecution_or_harm.sql deleted file mode 100644 index f3c3a06d2..000000000 --- a/database/src/migrations/release.0.8.0/populate_persecution_or_harm.sql +++ /dev/null @@ -1,49 +0,0 @@ --- populate_persecution_or_harm.sql - -do $$ -declare - _persecution_or_harm_type_id persecution_or_harm_type.persecution_or_harm_type_id%type; -begin - select persecution_or_harm_type_id from persecution_or_harm_type into _persecution_or_harm_type_id where name = 'Hibernation and Maternity'; - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1942, 'Big Brown Bat winter hibernation sites and maternity roosts', 'Data or information about the location of Big Brown Bat winter hibernation sites and maternity roosts.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1947, 'Californian myotis winter hibernation sites and maternity roosts', 'Californian myotis winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1953, 'Fringed Myotis winter hibernation sites and maternity roosts', 'Fringed Myotis winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1950, 'All Keens Myotis winter hibernation sites and maternity roosts', 'All Keens Myotis winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1951, 'Little Brown Myotis winter hibernation sites and maternity roosts', 'Little Brown Myotis winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1949, 'Long-eared Myotis winter hibernation sites and maternity roosts', 'Long-eared Myotis winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1954, 'Long-legged Myotis winter hibernation sites and maternity roosts', 'Long-legged Myotis winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1948, 'Western Small-footed myotis winter hibernation sites and maternity roosts', 'Western Small-footed myotis winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1955, 'Yuma Myotis winter hibernation sites and maternity roosts', 'Yuma Myotis winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1952, 'Northern Myotis winter hibernation sites and maternity roosts', 'Northern Myotis winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1941, 'Pallid bat winter hibernation sites and maternity roosts', 'Data or information about the location of Pallid bat winter hibernation sites and maternity roosts.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1944, 'Silver-haired bat winter hibernation sites and maternity roosts', 'Silver-haired bat winter hibernation sites and maternity roosts should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1956, 'Townsends Big-eared bat winter hibernation sites and maternity roosts', 'Data or information about the location of Townsends Big-eared bat winter hibernation sites and maternity roosts.'); - - select persecution_or_harm_type_id from persecution_or_harm_type into _persecution_or_harm_type_id where name = 'Taxon'; - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 2063, 'Bighorn sheep data', 'Bighorn sheep occurrence data, survey and telemetry observations, harvest data should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 2062, 'Mountain goat data', 'Mountain goat occurrence data, survey and telemetry observations, harvest data should be secured to limit poaching risk.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1716, 'Spotted Owl data', 'Data or information about the location of survey observations (SOs) or telemetry observations (TOs) or incidental observations (IOs) or feature observations (FOs) of Spotted Owl.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 23921, 'Dall''s sheep data', 'All Dall''s sheep occurrence data, survey and telemetry observations, harvest data should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 23922, 'Stone''s sheep occurrence data', 'All Stone''s sheep occurrence data, survey and telemetry observations, harvest data should be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 30287, 'Slender Yoke-moss data', 'All Slender Yoke-moss observation data should be secured.'); - - select persecution_or_harm_type_id from persecution_or_harm_type into _persecution_or_harm_type_id where name = 'Spawning Areas'; - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1377, 'Bull Trout spawning areas', 'Bull Trout spawning areas are at risk of poaching and should be automatically secured.'); - - select persecution_or_harm_type_id from persecution_or_harm_type into _persecution_or_harm_type_id where name = 'Hibernacula'; - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 27102, 'Gopher Snake hibernacula', 'Gopher Snake hibernacula are susceptible to persecution and should be automatically secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 27100, 'North American Racer hibernacula', 'North American Racer hibernacula are susceptible to persecution and should be automatically secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 30965, 'Western Rattlesnake hibernacula', 'Western Rattlesnake hibernacula are susceptible to persecution and should be automatically secured.'); - - select persecution_or_harm_type_id from persecution_or_harm_type into _persecution_or_harm_type_id where name = 'Nests'; - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1582, 'Gyrfalcon nest locations', 'Gyrfalcon nest locations should all be secured.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1581, 'Peregrine Falcon nest locations', 'Data or information about the location of nests of Peregrine Falcon.'); - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1583, 'Prairie Falcon nest location', 'Data or information about the location of nests of Prairie Falcon.'); - - select persecution_or_harm_type_id from persecution_or_harm_type into _persecution_or_harm_type_id where name = 'Nests'; - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 1594, 'Sharp-tailed Grouse nests', 'Data or information about the location of leks of Sharp-tailed Grouse.'); - - select persecution_or_harm_type_id from persecution_or_harm_type into _persecution_or_harm_type_id where name = 'Mineral Lick'; - insert into persecution_or_harm (persecution_or_harm_type_id, wldtaxonomic_units_id, name, description) values (_persecution_or_harm_type_id, 917, 'Ungulate mineral lick locations', 'All ungulate mineral lick locations should be secured.'); -end -$$; \ No newline at end of file diff --git a/database/src/migrations/release.0.8.0/populate_persecution_or_harm_type.sql b/database/src/migrations/release.0.8.0/populate_persecution_or_harm_type.sql deleted file mode 100644 index ce5028dc7..000000000 --- a/database/src/migrations/release.0.8.0/populate_persecution_or_harm_type.sql +++ /dev/null @@ -1,9 +0,0 @@ --- populate_persecution_or_harm_type.sql - -insert into persecution_or_harm_type (name, description) values ('Hibernation and Maternity',''); -insert into persecution_or_harm_type (name, description) values ('Taxon',''); -insert into persecution_or_harm_type (name, description) values ('Spawning Areas',''); -insert into persecution_or_harm_type (name, description) values ('Hibernacula',''); -insert into persecution_or_harm_type (name, description) values ('Nests',''); -insert into persecution_or_harm_type (name, description) values ('Leks',''); -insert into persecution_or_harm_type (name, description) values ('Mineral Lick',''); \ No newline at end of file diff --git a/database/src/migrations/release.0.8.0/populate_proprietary_type.sql b/database/src/migrations/release.0.8.0/populate_proprietary_type.sql deleted file mode 100644 index 128f02c40..000000000 --- a/database/src/migrations/release.0.8.0/populate_proprietary_type.sql +++ /dev/null @@ -1,7 +0,0 @@ --- populate_proprietary_type.sql - -insert into proprietary_type (name, description) values ('Ownership',''); -insert into proprietary_type (name, description) values ('Private Land',''); -insert into proprietary_type (name, description) values ('Awaiting Publication',''); -insert into proprietary_type (name, description) values ('Environmental Assessment Office Permit',''); -insert into proprietary_type (name, description) values ('First Nations',''); \ No newline at end of file diff --git a/database/src/migrations/release.0.8.0/populate_submission_message_class.sql b/database/src/migrations/release.0.8.0/populate_submission_message_class.sql deleted file mode 100644 index 85b87f22a..000000000 --- a/database/src/migrations/release.0.8.0/populate_submission_message_class.sql +++ /dev/null @@ -1,3 +0,0 @@ --- populate_submission_message_class.sql - -insert into submission_message_class (name, record_effective_date, description) values ('Class', now(), 'temp class to be removed'); diff --git a/database/src/migrations/release.0.8.0/populate_submission_message_type.sql b/database/src/migrations/release.0.8.0/populate_submission_message_type.sql deleted file mode 100644 index a58013c19..000000000 --- a/database/src/migrations/release.0.8.0/populate_submission_message_type.sql +++ /dev/null @@ -1,8 +0,0 @@ --- populate_submission_message_type.sql - --- term related -insert into submission_message_type (name, record_effective_date, description, submission_message_class_id) values ('Notice', now(), 'Notice', (select submission_message_class_id from submission_message_class where name = 'Class')); -insert into submission_message_type (name, record_effective_date, description, submission_message_class_id) values ('Error', now(), 'An error has occurred', (select submission_message_class_id from submission_message_class where name = 'Class')); -insert into submission_message_type (name, record_effective_date, description, submission_message_class_id) values ('Warning', now(), 'A warning has occurred', (select submission_message_class_id from submission_message_class where name = 'Class')); -insert into submission_message_type (name, record_effective_date, description, submission_message_class_id) values ('Debug', now(), 'A debug information', (select submission_message_class_id from submission_message_class where name = 'Class')); - diff --git a/database/src/migrations/release.0.8.0/populate_submission_status_type.sql b/database/src/migrations/release.0.8.0/populate_submission_status_type.sql deleted file mode 100644 index 6265204a9..000000000 --- a/database/src/migrations/release.0.8.0/populate_submission_status_type.sql +++ /dev/null @@ -1,25 +0,0 @@ --- populate_submission_status_type.sql - -insert into submission_status_type (name, record_effective_date, description) values ('Published', now(), 'The submission has been publised.'); -insert into submission_status_type (name, record_effective_date, description) values ('Rejected', now(), 'The submission has been rejected.'); -insert into submission_status_type (name, record_effective_date, description) values ('Out Dated Record', now(), 'Submission record has been updated to new record'); -insert into submission_status_type (name, record_effective_date, description) values ('Ingested', now(), 'Submission data ingested'); -insert into submission_status_type (name, record_effective_date, description) values ('Uploaded', now(), 'Submission data uploaded to S3'); -insert into submission_status_type (name, record_effective_date, description) values ('Validated', now(), 'Submission data validated'); -insert into submission_status_type (name, record_effective_date, description) values ('Secured', now(), 'Submission data secured'); -insert into submission_status_type (name, record_effective_date, description) values ('EML Ingested', now(), 'Submission EML ingested'); -insert into submission_status_type (name, record_effective_date, description) values ('EML To JSON', now(), 'EML transformed to JSON'); -insert into submission_status_type (name, record_effective_date, description) values ('Metadata To ES', now(), 'Metadata uploaded to ES'); -insert into submission_status_type (name, record_effective_date, description) values ('Normalized', now(), 'Submission data normalized'); -insert into submission_status_type (name, record_effective_date, description) values ('Spatial Transform Unsecure', now(), 'Transformed unsecure spatial data'); -insert into submission_status_type (name, record_effective_date, description) values ('Spatial Transform Secure', now(), 'Transformed secure spatial data'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed Ingestion', now(), 'Submission data failed to ingest'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed Upload', now(), 'Submission data failed to upload to S3'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed Validation', now(), 'Submission data failed to validate'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed Security', now(), 'Submission data failed secure'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed EML Ingestion', now(), 'Submission EML failed to ingest'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed EML To JSON', now(), 'EML transform to JSON failed'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed Metadata To ES', now(), 'Metadata failed to upload'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed Normalization', now(), 'Submission data failed to normalize'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed Spatial Transform Unsecure', now(), 'Transform failed on unsecure spatial data'); -insert into submission_status_type (name, record_effective_date, description) values ('Failed Spatial Transform Secure', now(), 'Transform failed on secure spatial data'); diff --git a/database/src/migrations/release.0.8.0/tr_generated_audit_triggers.sql b/database/src/migrations/release.0.8.0/tr_generated_audit_triggers.sql index 7f3623864..ec7319f55 100644 --- a/database/src/migrations/release.0.8.0/tr_generated_audit_triggers.sql +++ b/database/src/migrations/release.0.8.0/tr_generated_audit_triggers.sql @@ -1,31 +1,9 @@ - create trigger audit_artifact before insert or update or delete on biohub.artifact for each row execute procedure tr_audit_trigger(); - create trigger audit_artifact_government_interest before insert or update or delete on biohub.artifact_government_interest for each row execute procedure tr_audit_trigger(); - create trigger audit_artifact_persecution before insert or update or delete on biohub.artifact_persecution for each row execute procedure tr_audit_trigger(); - create trigger audit_artifact_proprietary before insert or update or delete on biohub.artifact_proprietary for each row execute procedure tr_audit_trigger(); - create trigger audit_persecution_or_harm_type before insert or update or delete on biohub.persecution_or_harm_type for each row execute procedure tr_audit_trigger(); - create trigger audit_security_transform_submission before insert or update or delete on biohub.security_transform_submission for each row execute procedure tr_audit_trigger(); - create trigger audit_spatial_transform before insert or update or delete on biohub.spatial_transform for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_government_interest before insert or update or delete on biohub.submission_government_interest for each row execute procedure tr_audit_trigger(); - create trigger audit_submission before insert or update or delete on biohub.submission for each row execute procedure tr_audit_trigger(); - create trigger audit_proprietary_type before insert or update or delete on biohub.proprietary_type for each row execute procedure tr_audit_trigger(); - create trigger audit_security_transform before insert or update or delete on biohub.security_transform for each row execute procedure tr_audit_trigger(); - create trigger audit_source_transform before insert or update or delete on biohub.source_transform for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_message_class before insert or update or delete on biohub.submission_message_class for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_message before insert or update or delete on biohub.submission_message for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_status_type before insert or update or delete on biohub.submission_status_type for each row execute procedure tr_audit_trigger(); - create trigger audit_system_user before insert or update or delete on biohub.system_user for each row execute procedure tr_audit_trigger(); - create trigger audit_system_constant before insert or update or delete on biohub.system_constant for each row execute procedure tr_audit_trigger(); - create trigger audit_system_metadata_constant before insert or update or delete on biohub.system_metadata_constant for each row execute procedure tr_audit_trigger(); - create trigger audit_system_user_security_exception before insert or update or delete on biohub.system_user_security_exception for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_status before insert or update or delete on biohub.submission_status for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_observation before insert or update or delete on biohub.submission_observation for each row execute procedure tr_audit_trigger(); - create trigger audit_system_user_role before insert or update or delete on biohub.system_user_role for each row execute procedure tr_audit_trigger(); - create trigger audit_system_role before insert or update or delete on biohub.system_role for each row execute procedure tr_audit_trigger(); - create trigger audit_persecution_or_harm before insert or update or delete on biohub.persecution_or_harm for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_spatial_component before insert or update or delete on biohub.submission_spatial_component for each row execute procedure tr_audit_trigger(); - create trigger audit_spatial_transform_submission before insert or update or delete on biohub.spatial_transform_submission for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_job_queue before insert or update or delete on biohub.submission_job_queue for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_message_type before insert or update or delete on biohub.submission_message_type for each row execute procedure tr_audit_trigger(); - create trigger audit_submission_metadata before insert or update or delete on biohub.submission_metadata for each row execute procedure tr_audit_trigger(); - create trigger audit_user_identity_source before insert or update or delete on biohub.user_identity_source for each row execute procedure tr_audit_trigger(); - +create trigger audit_artifact before insert or update or delete on biohub.artifact for each row execute procedure tr_audit_trigger(); +create trigger audit_submission before insert or update or delete on biohub.submission for each row execute procedure tr_audit_trigger(); +create trigger audit_system_user before insert or update or delete on biohub.system_user for each row execute procedure tr_audit_trigger(); +create trigger audit_system_constant before insert or update or delete on biohub.system_constant for each row execute procedure tr_audit_trigger(); +create trigger audit_system_metadata_constant before insert or update or delete on biohub.system_metadata_constant for each row execute procedure tr_audit_trigger(); +create trigger audit_system_user_role before insert or update or delete on biohub.system_user_role for each row execute procedure tr_audit_trigger(); +create trigger audit_system_role before insert or update or delete on biohub.system_role for each row execute procedure tr_audit_trigger(); +create trigger audit_submission_job_queue before insert or update or delete on biohub.submission_job_queue for each row execute procedure tr_audit_trigger(); +create trigger audit_user_identity_source before insert or update or delete on biohub.user_identity_source for each row execute procedure tr_audit_trigger(); diff --git a/database/src/migrations/release.0.8.0/tr_generated_journal_triggers.sql b/database/src/migrations/release.0.8.0/tr_generated_journal_triggers.sql index b933bcc18..fe7fefbee 100644 --- a/database/src/migrations/release.0.8.0/tr_generated_journal_triggers.sql +++ b/database/src/migrations/release.0.8.0/tr_generated_journal_triggers.sql @@ -1,31 +1,9 @@ - create trigger journal_artifact after insert or update or delete on biohub.artifact for each row execute procedure tr_journal_trigger(); - create trigger journal_artifact_government_interest after insert or update or delete on biohub.artifact_government_interest for each row execute procedure tr_journal_trigger(); - create trigger journal_artifact_persecution after insert or update or delete on biohub.artifact_persecution for each row execute procedure tr_journal_trigger(); - create trigger journal_artifact_proprietary after insert or update or delete on biohub.artifact_proprietary for each row execute procedure tr_journal_trigger(); - create trigger journal_persecution_or_harm_type after insert or update or delete on biohub.persecution_or_harm_type for each row execute procedure tr_journal_trigger(); - create trigger journal_security_transform_submission after insert or update or delete on biohub.security_transform_submission for each row execute procedure tr_journal_trigger(); - create trigger journal_spatial_transform after insert or update or delete on biohub.spatial_transform for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_government_interest after insert or update or delete on biohub.submission_government_interest for each row execute procedure tr_journal_trigger(); - create trigger journal_submission after insert or update or delete on biohub.submission for each row execute procedure tr_journal_trigger(); - create trigger journal_proprietary_type after insert or update or delete on biohub.proprietary_type for each row execute procedure tr_journal_trigger(); - create trigger journal_security_transform after insert or update or delete on biohub.security_transform for each row execute procedure tr_journal_trigger(); - create trigger journal_source_transform after insert or update or delete on biohub.source_transform for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_message_class after insert or update or delete on biohub.submission_message_class for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_message after insert or update or delete on biohub.submission_message for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_status_type after insert or update or delete on biohub.submission_status_type for each row execute procedure tr_journal_trigger(); - create trigger journal_system_user after insert or update or delete on biohub.system_user for each row execute procedure tr_journal_trigger(); - create trigger journal_system_constant after insert or update or delete on biohub.system_constant for each row execute procedure tr_journal_trigger(); - create trigger journal_system_metadata_constant after insert or update or delete on biohub.system_metadata_constant for each row execute procedure tr_journal_trigger(); - create trigger journal_system_user_security_exception after insert or update or delete on biohub.system_user_security_exception for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_status after insert or update or delete on biohub.submission_status for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_observation after insert or update or delete on biohub.submission_observation for each row execute procedure tr_journal_trigger(); - create trigger journal_system_user_role after insert or update or delete on biohub.system_user_role for each row execute procedure tr_journal_trigger(); - create trigger journal_system_role after insert or update or delete on biohub.system_role for each row execute procedure tr_journal_trigger(); - create trigger journal_persecution_or_harm after insert or update or delete on biohub.persecution_or_harm for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_spatial_component after insert or update or delete on biohub.submission_spatial_component for each row execute procedure tr_journal_trigger(); - create trigger journal_spatial_transform_submission after insert or update or delete on biohub.spatial_transform_submission for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_job_queue after insert or update or delete on biohub.submission_job_queue for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_message_type after insert or update or delete on biohub.submission_message_type for each row execute procedure tr_journal_trigger(); - create trigger journal_submission_metadata after insert or update or delete on biohub.submission_metadata for each row execute procedure tr_journal_trigger(); - create trigger journal_user_identity_source after insert or update or delete on biohub.user_identity_source for each row execute procedure tr_journal_trigger(); - +create trigger journal_artifact after insert or update or delete on biohub.artifact for each row execute procedure tr_journal_trigger(); +create trigger journal_submission after insert or update or delete on biohub.submission for each row execute procedure tr_journal_trigger(); +create trigger journal_system_user after insert or update or delete on biohub.system_user for each row execute procedure tr_journal_trigger(); +create trigger journal_system_constant after insert or update or delete on biohub.system_constant for each row execute procedure tr_journal_trigger(); +create trigger journal_system_metadata_constant after insert or update or delete on biohub.system_metadata_constant for each row execute procedure tr_journal_trigger(); +create trigger journal_system_user_role after insert or update or delete on biohub.system_user_role for each row execute procedure tr_journal_trigger(); +create trigger journal_system_role after insert or update or delete on biohub.system_role for each row execute procedure tr_journal_trigger(); +create trigger journal_submission_job_queue after insert or update or delete on biohub.submission_job_queue for each row execute procedure tr_journal_trigger(); +create trigger journal_user_identity_source after insert or update or delete on biohub.user_identity_source for each row execute procedure tr_journal_trigger(); diff --git a/database/src/migrations/release.0.8.0/vw_generated_dapi_views.sql b/database/src/migrations/release.0.8.0/vw_generated_dapi_views.sql deleted file mode 100644 index 1486168f8..000000000 --- a/database/src/migrations/release.0.8.0/vw_generated_dapi_views.sql +++ /dev/null @@ -1,31 +0,0 @@ - create or replace view artifact as select * from biohub.artifact; - create or replace view artifact_government_interest as select * from biohub.artifact_government_interest; - create or replace view artifact_persecution as select * from biohub.artifact_persecution; - create or replace view artifact_proprietary as select * from biohub.artifact_proprietary; - create or replace view persecution_or_harm as select * from biohub.persecution_or_harm; - create or replace view persecution_or_harm_type as select * from biohub.persecution_or_harm_type; - create or replace view proprietary_type as select * from biohub.proprietary_type; - create or replace view security_transform as select * from biohub.security_transform; - create or replace view security_transform_submission as select * from biohub.security_transform_submission; - create or replace view source_transform as select * from biohub.source_transform; - create or replace view spatial_transform as select * from biohub.spatial_transform; - create or replace view spatial_transform_submission as select * from biohub.spatial_transform_submission; - create or replace view submission as select * from biohub.submission; - create or replace view submission_government_interest as select * from biohub.submission_government_interest; - create or replace view submission_job_queue as select * from biohub.submission_job_queue; - create or replace view submission_message as select * from biohub.submission_message; - create or replace view submission_message_class as select * from biohub.submission_message_class; - create or replace view submission_message_type as select * from biohub.submission_message_type; - create or replace view submission_metadata as select * from biohub.submission_metadata; - create or replace view submission_observation as select * from biohub.submission_observation; - create or replace view submission_spatial_component as select * from biohub.submission_spatial_component; - create or replace view submission_status as select * from biohub.submission_status; - create or replace view submission_status_type as select * from biohub.submission_status_type; - create or replace view system_constant as select * from biohub.system_constant; - create or replace view system_metadata_constant as select * from biohub.system_metadata_constant; - create or replace view system_role as select * from biohub.system_role; - create or replace view system_user as select * from biohub.system_user; - create or replace view system_user_role as select * from biohub.system_user_role; - create or replace view system_user_security_exception as select * from biohub.system_user_security_exception; - create or replace view user_identity_source as select * from biohub.user_identity_source; - diff --git a/database/src/seeds/01_db_system_users.ts b/database/src/seeds/01_db_system_users.ts index 34cf5e6a7..7c0080092 100644 --- a/database/src/seeds/01_db_system_users.ts +++ b/database/src/seeds/01_db_system_users.ts @@ -1,80 +1,75 @@ import { Knex } from 'knex'; +const DB_SCHEMA = process.env.DB_SCHEMA; const DB_ADMIN = process.env.DB_ADMIN; -export enum SYSTEM_IDENTITY_SOURCE { +enum SYSTEM_IDENTITY_SOURCE { DATABASE = 'DATABASE', IDIR = 'IDIR', BCEID_BASIC = 'BCEIDBASIC', - BCEID_BUSINESS = 'BCEIDBUSINESS', - SYSTEM = 'SYSTEM' + BCEID_BUSINESS = 'BCEIDBUSINESS' } -export enum SYSTEM_USER_ROLE_ID { - SYSTEM_ADMINISTRATOR = 1, - DATA_ADMINISTRATOR = 2 +enum SYSTEM_USER_ROLE_NAME { + SYSTEM_ADMINISTRATOR = 'System Administrator', + DATA_ADMINISTRATOR = 'Data Administrator' } -const systemUsers = [ +interface SystemUserSeed { + identifier: string; + type: SYSTEM_IDENTITY_SOURCE; + role_name: SYSTEM_USER_ROLE_NAME; + user_guid: string; +} + +const systemUsers: SystemUserSeed[] = [ { identifier: 'arosenth', type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, + role_name: SYSTEM_USER_ROLE_NAME.SYSTEM_ADMINISTRATOR, user_guid: 'DFE2CC5E345E4B1E813EC1DC10852064' }, - { - identifier: 'aagahche', - type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, - user_guid: 'DF86C48FAD244498B0881AF8DBB7645F' - }, - { - identifier: 'cgarrett', - type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, - user_guid: '067361FFF3514B5E80AAAAD795E6741D' - }, { identifier: 'cupshall', type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, + role_name: SYSTEM_USER_ROLE_NAME.SYSTEM_ADMINISTRATOR, user_guid: 'C42DFA74A976490A819BC85FF5E254E4' }, { identifier: 'jxdunsdo', type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, + role_name: SYSTEM_USER_ROLE_NAME.SYSTEM_ADMINISTRATOR, user_guid: '82E8D3B4BAD045E8AD3980D426EA781C' }, { identifier: 'keinarss', type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, + role_name: SYSTEM_USER_ROLE_NAME.SYSTEM_ADMINISTRATOR, user_guid: 'F4663727DE89489C8B7CFA81E4FA99B3' }, { identifier: 'nphura', type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, + role_name: SYSTEM_USER_ROLE_NAME.SYSTEM_ADMINISTRATOR, user_guid: '813B096BC1BC4AAAB2E39DDE58F432E2' }, { - identifier: 'robmunro', + identifier: 'achirico', type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, - user_guid: '72013F74B95A4FBEB53BDB4B494E5550' + role_name: SYSTEM_USER_ROLE_NAME.SYSTEM_ADMINISTRATOR, + user_guid: 'E3A279530D164485BF43C6FE7A49E175' }, { - identifier: 'zochampi', + identifier: 'mdeluca', type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, - user_guid: '349F20767A834FB582A18E8D378973E7' + role_name: SYSTEM_USER_ROLE_NAME.SYSTEM_ADMINISTRATOR, + user_guid: '0054CF4823A744309BE399C34B6B0F43' }, { - identifier: 'achirico', + identifier: 'mauberti', type: SYSTEM_IDENTITY_SOURCE.IDIR, - roleId: SYSTEM_USER_ROLE_ID.SYSTEM_ADMINISTRATOR, - user_guid: 'E3A279530D164485BF43C6FE7A49E175' + role_name: SYSTEM_USER_ROLE_NAME.SYSTEM_ADMINISTRATOR, + user_guid: '62EC624E50844486A046DC9709854F8D' } ]; @@ -85,23 +80,27 @@ const systemUsers = [ */ export async function seed(knex: Knex): Promise { await knex.raw(` - set schema 'biohub'; - set search_path = 'biohub'; + set schema '${DB_SCHEMA}'; + set search_path = ${DB_SCHEMA}; `); for (const systemUser of systemUsers) { // check if user is already in the system users table - const response = await knex.raw(getSystemUserSQL(systemUser.identifier)); + const response = await knex.raw(` + ${getSystemUserSQL(systemUser)} + `); // if the fetch returns no rows, then the user is not in the system users table and should be added if (!response?.rows?.[0]) { // Add system user - await knex.raw(insertSystemUserSQL(systemUser.identifier, systemUser.type, systemUser.user_guid.toLowerCase())); + await knex.raw(` + ${insertSystemUserSQL(systemUser)} + `); - if (systemUser.roleId) { - // Add system role - await knex.raw(insertSystemUserRoleSQL(systemUser.identifier, systemUser.roleId)); - } + // Add system administrator role + await knex.raw(` + ${insertSystemUserRoleSQL(systemUser)} + `); } } } @@ -109,43 +108,42 @@ export async function seed(knex: Knex): Promise { /** * SQL to fetch an existing system user row. * - * @param {string} userIdentifier + * @param {SystemUserSeed} systemUser */ -const getSystemUserSQL = (userIdentifier: string) => ` +const getSystemUserSQL = (systemUser: SystemUserSeed) => ` SELECT user_identifier FROM system_user WHERE - user_identifier = '${userIdentifier}'; + LOWER(user_identifier) = LOWER('${systemUser.identifier}'); `; /** * SQL to insert a system user row. * - * @param {string} userIdentifier - * @param {string} userType + * @param {SystemUserSeed} systemUser */ -const insertSystemUserSQL = (userIdentifier: string, userType: string, userGuid?: string) => ` +const insertSystemUserSQL = (systemUser: SystemUserSeed) => ` INSERT INTO system_user ( user_identity_source_id, user_identifier, + user_guid, record_effective_date, create_date, - create_user, - user_guid + create_user ) SELECT user_identity_source_id, - '${userIdentifier}', + '${systemUser.identifier}', + LOWER('${systemUser.user_guid}'), now(), now(), - (SELECT system_user_id from system_user where user_identifier = '${DB_ADMIN}'), - '${userGuid}' + (SELECT system_user_id from system_user where LOWER(user_identifier) = LOWER('${DB_ADMIN}')) FROM user_identity_source WHERE - name = '${userType}' + LOWER(name) = LOWER('${systemUser.type}') AND record_end_date is null; `; @@ -153,15 +151,14 @@ const insertSystemUserSQL = (userIdentifier: string, userType: string, userGuid? /** * SQL to insert a system user role row. * - * @param {string} userIdentifier - * @param {number} roleId + * @param {SystemUserSeed} systemUser */ -const insertSystemUserRoleSQL = (userIdentifier: string, roleId: number) => ` +const insertSystemUserRoleSQL = (systemUser: SystemUserSeed) => ` INSERT INTO system_user_role ( system_user_id, system_role_id ) VALUES ( - (SELECT system_user_id from system_user where user_identifier = '${userIdentifier}'), - ${roleId} + (SELECT system_user_id from system_user where LOWER(user_identifier) = LOWER('${systemUser.identifier}')), + (SELECT system_role_id from system_role where LOWER(name) = LOWER('${systemUser.role_name}')) ); `; diff --git a/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts b/database/src/seeds/02_populate_feature_lookup_tables.ts similarity index 98% rename from database/src/migrations/20231109000003_populate_feature_lookup_tables.ts rename to database/src/seeds/02_populate_feature_lookup_tables.ts index 8ea893cf5..54443eecd 100644 --- a/database/src/migrations/20231109000003_populate_feature_lookup_tables.ts +++ b/database/src/seeds/02_populate_feature_lookup_tables.ts @@ -11,7 +11,7 @@ import { Knex } from 'knex'; * @param {Knex} knex * @return {*} {Promise} */ -export async function up(knex: Knex): Promise { +export async function seed(knex: Knex): Promise { await knex.raw(` ---------------------------------------------------------------------------------------- -- Create tables @@ -87,7 +87,3 @@ export async function up(knex: Knex): Promise { insert into feature_type_property (feature_type_id, feature_property_id, record_effective_date) values ((select feature_type_id from feature_type where name = 'telemetry'), (select feature_property_id from feature_property where name = 'end_date'), now()); `); } - -export async function down(knex: Knex): Promise { - await knex.raw(``); -} diff --git a/database/src/seeds/02_security_transform_eml_dwc_occurrences.ts b/database/src/seeds/02_security_transform_eml_dwc_occurrences.ts deleted file mode 100644 index 1b94dfce5..000000000 --- a/database/src/seeds/02_security_transform_eml_dwc_occurrences.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { Knex } from 'knex'; - -/** - * Add spatial transform - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function seed(knex: Knex): Promise { - await knex.raw(` - SET SCHEMA 'biohub'; - SET SEARCH_PATH = 'biohub'; - `); - - const response = await knex.raw(checkTransformExists()); - - if (!response?.rows?.[0]) { - await knex.raw(insertSecurityTransform()); - } else { - await knex.raw(updateSecurityTransform()); - } -} - -const checkTransformExists = () => ` - SELECT - security_transform_id - FROM - security_transform - WHERE - name = 'DwC Occurrences'; -`; - -const insertSecurityTransform = () => ` - INSERT into security_transform - (persecution_or_harm_id, name, description, transform) - VALUES ( - 1, 'DwC Occurrences', 'Assigns Persecution and Harm Rules', ${transformString} - ); -`; - -const updateSecurityTransform = () => ` - UPDATE - security_transform - SET - transform = ${transformString} - WHERE - name = 'DwC Occurrences'; -`; - -const transformString = ` - $transform$ - WITH w_submission_observation AS ( - select - submission_observation_id - from - submission_observation - where - submission_id = ? - and record_end_timestamp is null - ), - w_spatial_component AS ( - SELECT - spatial_component, - submission_spatial_component_id - FROM - submission_spatial_component ssc, - w_submission_observation - WHERE - ssc.submission_observation_id = w_submission_observation.submission_observation_id - AND jsonb_path_exists( - spatial_component, - '$.features[*] \\? (@.properties.type == "Occurrence")' - ) - ) - SELECT - jsonb_build_object( - 'submission_spatial_component_id', - wsc.submission_spatial_component_id, - 'spatial_data', - CASE - WHEN json_build_array( - Lower( - wsc.spatial_component -> 'features' -> 0 -> 'properties' -> 'dwc' ->> 'taxonID' - ) - ) :: jsonb <@ json_build_array( - 'mountain goat', - 'bighorn sheep', - 'thinhorn sheep', - 'spotted owl', - 'm-oram', - 'm-ovca', - 'm-ovca-ca', - 'm-ovda', - 'm-ovda-da', - 'm-ovda-st', - 'b-spow', - 'b-spow-ca' - ) :: jsonb THEN json_build_object() - ELSE NULL - END - ) spatial_component - FROM - w_spatial_component wsc; - $transform$ -`; diff --git a/database/src/seeds/03_dwc_spatial_transform.ts b/database/src/seeds/03_dwc_spatial_transform.ts deleted file mode 100644 index fe20ee917..000000000 --- a/database/src/seeds/03_dwc_spatial_transform.ts +++ /dev/null @@ -1,185 +0,0 @@ -import { Knex } from 'knex'; - -/** - * Add spatial transform - * - * @export - * @param {Knex} knex - * @return {*} {Promise} - */ -export async function seed(knex: Knex): Promise { - await knex.raw(` - SET SCHEMA 'biohub'; - SET SEARCH_PATH = 'biohub'; - `); - - const response = await knex.raw(checkTransformExists()); - - if (!response?.rows?.[0]) { - await knex.raw(insertSpatialTransform()); - } else { - await knex.raw(updateSpatialTransform()); - } -} - -const checkTransformExists = () => ` - SELECT - spatial_transform_id - FROM - spatial_transform - WHERE - name = 'DwC Occurrences'; -`; - -const insertSpatialTransform = () => ` - INSERT into spatial_transform - (name, description, record_effective_date, transform) - VALUES ( - 'DwC Occurrences', 'Extracts occurrences and properties from DwC JSON source.', now(),${transformString} - ); -`; - -const updateSpatialTransform = () => ` - UPDATE - spatial_transform SET transform = ${transformString} - WHERE - name = 'DwC Occurrences'; -`; - -const transformString = ` - $transform$ - with w_submission_observation as ( - select - * - from - submission_observation - where - submission_id = ? - and record_end_timestamp is null - ), - w_occurrences as ( - select - submission_observation_id, - occs - from - w_submission_observation, - jsonb_path_query(darwin_core_source, '$.occurrence') occs - ), - w_occurrence as ( - select - submission_observation_id, - jsonb_array_elements(occs) occ - from - w_occurrences - ), - w_events as ( - select - evns - from - w_submission_observation, - jsonb_path_query(darwin_core_source, '$.event') evns - ), - w_event as ( - select - jsonb_array_elements(evns) evn - from - w_events - ), - w_locations as ( - select - locs - from - w_submission_observation, - jsonb_path_query(darwin_core_source, '$.location') locs - ), - w_location as ( - select - jsonb_array_elements(locs) loc - from - w_locations - ), - w_location_coord as ( - select - coalesce(st_x(pt), 0) x, - coalesce(st_y(pt), 0) y, - loc - from - w_location, - ST_SetSRID( - ST_MakePoint( - (nullif(loc ->> 'decimalLongitude', '')) :: float, - (nullif(loc ->> 'decimalLatitude', '')) :: float - ), - 4326 - ) pt - ), - w_normal as ( - select - distinct o.submission_observation_id, - o.occ, - ec.*, - e.evn - from - w_occurrence o - left outer join w_location_coord ec on (ec.loc -> 'eventID' = o.occ -> 'eventID') - left outer join w_event e on (e.evn -> 'eventID' = o.occ -> 'eventID') - ) - select - jsonb_build_object( - 'type', - 'FeatureCollection', - 'features', - jsonb_build_array( - jsonb_build_object( - 'type', - 'Feature', - 'geometry', - jsonb_build_object( - 'type', - 'Point', - 'coordinates', - json_build_array(n.x, n.y) - ), - 'properties', - jsonb_build_object( - 'type', - 'Occurrence', - 'dwc', - jsonb_build_object( - 'type', - 'PhysicalObject', - 'basisOfRecord', - 'Occurrence', - 'datasetID', - n.submission_observation_id, - 'occurrenceID', - n.occ -> 'occurrenceID', - 'sex', - n.occ -> 'sex', - 'lifeStage', - n.occ -> 'lifeStage', - 'taxonID', - n.occ -> 'taxonID', - 'vernacularName', - n.occ -> 'vernacularName', - 'scientificName', - n.occ -> 'scientificName', - 'occurrenceRemarks', - n.occ -> 'occurrenceRemarks', - 'individualCount', - n.occ -> 'individualCount', - 'eventDate', - n.evn -> 'eventDate', - 'verbatimSRS', - n.loc -> 'verbatimSRS', - 'verbatimCoordinates', - n.loc -> 'verbatimCoordinates' - ) - ) - ) - ) - ) result_data - from - w_normal n; - $transform$ -`; From d72994dcc1f62c80ffdf5f56eb7797f28327f08b Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 23 Nov 2023 15:06:36 -0800 Subject: [PATCH 12/20] Port pipeline resource enhancements from sims --- api/.pipeline/config.js | 19 +++++++++++-------- api/.pipeline/lib/api.deploy.js | 10 +++++++++- api/.pipeline/templates/api.dc.yaml | 3 +++ app/.pipeline/config.js | 8 ++++---- database/.pipeline/config.js | 10 +++++----- 5 files changed, 32 insertions(+), 18 deletions(-) diff --git a/api/.pipeline/config.js b/api/.pipeline/config.js index 02f4b02bf..122576a51 100644 --- a/api/.pipeline/config.js +++ b/api/.pipeline/config.js @@ -62,9 +62,9 @@ const phases = { env: 'build', tz: config.timezone.api, branch: branch, - cpuRequest: '100m', + cpuRequest: '50m', cpuLimit: '1250m', - memoryRequest: '512Mi', + memoryRequest: '100Mi', memoryLimit: '3Gi' }, dev: { @@ -87,9 +87,10 @@ const phases = { tz: config.timezone.api, sso: config.sso.dev, logLevel: 'debug', - cpuRequest: '100m', + nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes) + cpuRequest: '50m', cpuLimit: '500m', - memoryRequest: '512Mi', + memoryRequest: '100Mi', memoryLimit: '2Gi', replicas: '1', replicasMax: (isStaticDeployment && '2') || '1' @@ -114,9 +115,10 @@ const phases = { tz: config.timezone.api, sso: config.sso.test, logLevel: 'info', - cpuRequest: '200m', + nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes) + cpuRequest: '50m', cpuLimit: '1000m', - memoryRequest: '512Mi', + memoryRequest: '100Mi', memoryLimit: '2Gi', replicas: '2', replicasMax: '3' @@ -141,9 +143,10 @@ const phases = { tz: config.timezone.api, sso: config.sso.prod, logLevel: 'info', - cpuRequest: '200m', + nodeOptions: '--max_old_space_size=1500', // 75% of memoryLimit (bytes) + cpuRequest: '50m', cpuLimit: '1000m', - memoryRequest: '512Mi', + memoryRequest: '100Mi', memoryLimit: '2Gi', replicas: '2', replicasMax: '3' diff --git a/api/.pipeline/lib/api.deploy.js b/api/.pipeline/lib/api.deploy.js index 3f772d6e9..e60b994bb 100644 --- a/api/.pipeline/lib/api.deploy.js +++ b/api/.pipeline/lib/api.deploy.js @@ -31,22 +31,30 @@ const apiDeploy = async (settings) => { HOST: phases[phase].host, CHANGE_ID: phases.build.changeId || changeId, APP_HOST: phases[phase].appHost, + // Node NODE_ENV: phases[phase].env || 'dev', + NODE_OPTIONS: phases[phase].nodeOptions, + // Elastic Search ELASTICSEARCH_URL: phases[phase].elasticsearchURL, ELASTICSEARCH_EML_INDEX: phases[phase].elasticsearchEmlIndex, ELASTICSEARCH_TAXONOMY_INDEX: phases[phase].elasticsearchTaxonomyIndex, + // S3 (Object Store) S3_KEY_PREFIX: phases[phase].s3KeyPrefix, + // Database TZ: phases[phase].tz, + DB_SERVICE_NAME: `${phases[phase].dbName}-postgresql${phases[phase].suffix}`, + // Keycloak KEYCLOAK_ADMIN_USERNAME: phases[phase].sso.adminUserName, KEYCLOAK_SECRET: phases[phase].sso.keycloakSecret, KEYCLOAK_SECRET_ADMIN_PASSWORD: phases[phase].sso.keycloakSecretAdminPassword, - DB_SERVICE_NAME: `${phases[phase].dbName}-postgresql${phases[phase].suffix}`, KEYCLOAK_HOST: phases[phase].sso.url, KEYCLOAK_CLIENT_ID: phases[phase].sso.clientId, KEYCLOAK_REALM: phases[phase].sso.realm, KEYCLOAK_INTEGRATION_ID: phases[phase].sso.integrationId, KEYCLOAK_API_HOST: phases[phase].sso.apiHost, + // Log Level LOG_LEVEL: phases[phase].logLevel || 'info', + // OPenshift Resources CPU_REQUEST: phases[phase].cpuRequest, CPU_LIMIT: phases[phase].cpuLimit, MEMORY_REQUEST: phases[phase].memoryRequest, diff --git a/api/.pipeline/templates/api.dc.yaml b/api/.pipeline/templates/api.dc.yaml index cef961b2e..5993f1b49 100644 --- a/api/.pipeline/templates/api.dc.yaml +++ b/api/.pipeline/templates/api.dc.yaml @@ -35,6 +35,7 @@ parameters: description: Application Environment type variable required: true value: 'dev' + - name: NODE_OPTIONS - name: ELASTICSEARCH_URL description: Platform Elasticsearch URL required: true @@ -223,6 +224,8 @@ objects: value: ${CHANGE_ID} - name: NODE_ENV value: ${NODE_ENV} + - name: NODE_OPTIONS + value: ${NODE_OPTIONS} - name: ELASTICSEARCH_URL value: ${ELASTICSEARCH_URL} - name: ELASTICSEARCH_EML_INDEX diff --git a/app/.pipeline/config.js b/app/.pipeline/config.js index 40947b443..7172cc5e0 100644 --- a/app/.pipeline/config.js +++ b/app/.pipeline/config.js @@ -62,7 +62,7 @@ const phases = { tag: tag, env: 'build', branch: branch, - cpuRequest: '100m', + cpuRequest: '50m', cpuLimit: '1000m', memoryRequest: '512Mi', memoryLimit: '3Gi' @@ -86,7 +86,7 @@ const phases = { sso: config.sso.dev, cpuRequest: '50m', cpuLimit: (isStaticDeployment && '300m') || '200m', - memoryRequest: '50Mi', + memoryRequest: '100Mi', memoryLimit: (isStaticDeployment && '300Mi') || '200Mi', replicas: '1', replicasMax: (isStaticDeployment && '2') || '1' @@ -107,7 +107,7 @@ const phases = { maxUploadFileSize, env: 'test', sso: config.sso.test, - cpuRequest: '100m', + cpuRequest: '50m', cpuLimit: '400m', memoryRequest: '100Mi', memoryLimit: '400Mi', @@ -130,7 +130,7 @@ const phases = { maxUploadFileSize, env: 'prod', sso: config.sso.prod, - cpuRequest: '100m', + cpuRequest: '50m', cpuLimit: '400m', memoryRequest: '100Mi', memoryLimit: '400Mi', diff --git a/database/.pipeline/config.js b/database/.pipeline/config.js index c0969304b..8b640e3e4 100644 --- a/database/.pipeline/config.js +++ b/database/.pipeline/config.js @@ -80,7 +80,7 @@ const phases = { volumeCapacity: (isStaticDeployment && '3Gi') || '500Mi', cpuRequest: '50m', cpuLimit: '200m', - memoryRequest: '512Mi', + memoryRequest: '100Mi', memoryLimit: '2Gi', replicas: '1' }, @@ -97,9 +97,9 @@ const phases = { tz: config.timezone.db, dbSetupDockerfilePath: dbSetupDockerfilePath, volumeCapacity: '3Gi', - cpuRequest: '100m', + cpuRequest: '50m', cpuLimit: '500m', - memoryRequest: '512Mi', + memoryRequest: '100Mi', memoryLimit: '3Gi', replicas: '1' }, @@ -116,9 +116,9 @@ const phases = { tz: config.timezone.db, dbSetupDockerfilePath: dbSetupDockerfilePath, volumeCapacity: '5Gi', - cpuRequest: '100m', + cpuRequest: '50m', cpuLimit: '500m', - memoryRequest: '512Mi', + memoryRequest: '100Mi', memoryLimit: '3Gi', replicas: '1' } From f0ff4109d3c6a2a50c1d5484bd2e37cbe15f8581 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 23 Nov 2023 15:15:34 -0800 Subject: [PATCH 13/20] ignore-skip From c0a656c1357c3ab8d6c9e08a9193cd17aa82d360 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 23 Nov 2023 16:22:38 -0800 Subject: [PATCH 14/20] Rename migrations, update comments --- ...948_release.ts => 20220225205948_release_0.8.0.ts} | 0 .../migrations/20231122000001_submission_status.ts | 11 +++++------ ...lookup_tables.ts => 02_populate_feature_tables.ts} | 0 3 files changed, 5 insertions(+), 6 deletions(-) rename database/src/migrations/{20220225205948_release.ts => 20220225205948_release_0.8.0.ts} (100%) rename database/src/seeds/{02_populate_feature_lookup_tables.ts => 02_populate_feature_tables.ts} (100%) diff --git a/database/src/migrations/20220225205948_release.ts b/database/src/migrations/20220225205948_release_0.8.0.ts similarity index 100% rename from database/src/migrations/20220225205948_release.ts rename to database/src/migrations/20220225205948_release_0.8.0.ts diff --git a/database/src/migrations/20231122000001_submission_status.ts b/database/src/migrations/20231122000001_submission_status.ts index a5d63996a..a8cafdb3e 100644 --- a/database/src/migrations/20231122000001_submission_status.ts +++ b/database/src/migrations/20231122000001_submission_status.ts @@ -2,12 +2,11 @@ import { Knex } from 'knex'; /** * Add tables: - * - search_string - * - search_string - * - search_number - * - search_datetime - * - search_spatial - * - search_taxonomy + * - submission_message + * - submission_message_type + * + * Populate tables: + * - submission_message_type * * @export * @param {Knex} knex diff --git a/database/src/seeds/02_populate_feature_lookup_tables.ts b/database/src/seeds/02_populate_feature_tables.ts similarity index 100% rename from database/src/seeds/02_populate_feature_lookup_tables.ts rename to database/src/seeds/02_populate_feature_tables.ts From a2a1ff7592b57595f709381e99c12166d766fd5b Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Thu, 23 Nov 2023 16:23:02 -0800 Subject: [PATCH 15/20] ignore-skip From 2f7868b8af0bda81b68708fc9c3b052baad3bda9 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 24 Nov 2023 10:31:15 -0800 Subject: [PATCH 16/20] Add publish timestamp to submission table. Improve old generated index names --- .../src/migrations/release.0.8.0/biohub.sql | 44 +++++++++++-------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/database/src/migrations/release.0.8.0/biohub.sql b/database/src/migrations/release.0.8.0/biohub.sql index 277b770ee..11e1b7ade 100644 --- a/database/src/migrations/release.0.8.0/biohub.sql +++ b/database/src/migrations/release.0.8.0/biohub.sql @@ -77,6 +77,7 @@ COMMENT ON TABLE audit_log IS 'Holds record level audit log data for the entire CREATE TABLE submission( submission_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), uuid uuid DEFAULT public.gen_random_uuid() NOT NULL, + publish_timestamp timestamptz(6), create_date timestamptz(6) DEFAULT now() NOT NULL, create_user integer NOT NULL, update_date timestamptz(6), @@ -87,6 +88,7 @@ CREATE TABLE submission( COMMENT ON COLUMN submission.submission_id IS 'System generated surrogate primary key identifier.'; COMMENT ON COLUMN submission.uuid IS 'The universally unique identifier for the submission as supplied by the source system.'; +COMMENT ON COLUMN submission.publish_timestamp IS 'The timestamp of when the submission was published. Null indicates the submission is not published.'; COMMENT ON COLUMN submission.create_date IS 'The datetime the record was created.'; COMMENT ON COLUMN submission.create_user IS 'The id of the user who created the record as identified in the system user table.'; COMMENT ON COLUMN submission.update_date IS 'The datetime the record was updated.'; @@ -310,10 +312,10 @@ COMMENT ON COLUMN user_identity_source.revision_count IS 'Revision count used fo COMMENT ON TABLE user_identity_source IS 'The source of the user identifier. This source is traditionally the system that authenticates the user. Example sources could include IDIR, BCEID and DATABASE.'; -- --- INDEX: "Ref165191" +-- INDEX: "artifact_idx1" -- -CREATE INDEX "Ref165191" ON artifact(submission_id); +CREATE INDEX artifact_idx1 ON artifact(submission_id); -- -- INDEX: submission_uk1 @@ -322,62 +324,68 @@ CREATE INDEX "Ref165191" ON artifact(submission_id); CREATE UNIQUE INDEX submission_uk1 ON submission(uuid); -- --- INDEX: "Ref165208" +-- INDEX: "submission_job_queue_idx1" -- -CREATE INDEX "Ref165208" ON submission_job_queue(submission_id); +CREATE INDEX submission_job_queue_idx1 ON submission_job_queue(submission_id); -- -- INDEX: system_constant_uk1 -- CREATE UNIQUE INDEX system_constant_uk1 ON system_constant(constant_name); + -- -- INDEX: system_metadata_constant_uk1 -- CREATE UNIQUE INDEX system_metadata_constant_uk1 ON system_metadata_constant(constant_name); + -- -- INDEX: system_role_nuk1 -- -CREATE UNIQUE INDEX system_role_nuk1 ON system_role(name, record_end_date); +CREATE UNIQUE INDEX system_role_nuk1 ON system_role(name, (record_end_date is NULL)) where record_end_date is null; + -- -- INDEX: system_user_nuk1 -- -CREATE UNIQUE INDEX system_user_nuk1 ON system_user(user_identifier, user_identity_source_id, record_end_date); +CREATE UNIQUE INDEX system_user_nuk1 ON system_user(user_identifier, user_identity_source_id, (record_end_date is NULL)) where record_end_date is null; + -- --- INDEX: "Ref190178" +-- INDEX: "system_user_idx1" -- -CREATE INDEX "Ref190178" ON system_user(user_identity_source_id); +CREATE INDEX system_user_id_idx1 ON system_user(user_identity_source_id); -- -- INDEX: system_user_role_uk1 -- CREATE UNIQUE INDEX system_user_role_uk1 ON system_user_role(system_user_id, system_role_id); + -- --- INDEX: "Ref191179" +-- INDEX: "system_user_role_idx1" -- -CREATE INDEX "Ref191179" ON system_user_role(system_user_id); +CREATE INDEX system_user_role_idx1 ON system_user_role(system_user_id); -- --- INDEX: "Ref192180" +-- INDEX: "system_user_role_idx2" -- -CREATE INDEX "Ref192180" ON system_user_role(system_role_id); +CREATE INDEX system_user_role_idx2 ON system_user_role(system_role_id); -- -- INDEX: user_identity_source_nuk1 -- -CREATE UNIQUE INDEX user_identity_source_nuk1 ON user_identity_source(name, record_end_date); +CREATE UNIQUE INDEX user_identity_source_nuk1 ON user_identity_source(name, (record_end_date is NULL)) where record_end_date is null; + -- -- TABLE: artifact -- -ALTER TABLE artifact ADD CONSTRAINT "Refsubmission191" +ALTER TABLE artifact ADD CONSTRAINT artifact_fk1 FOREIGN KEY (submission_id) REFERENCES submission(submission_id); @@ -385,7 +393,7 @@ ALTER TABLE artifact ADD CONSTRAINT "Refsubmission191" -- TABLE: submission_job_queue -- -ALTER TABLE submission_job_queue ADD CONSTRAINT "Refsubmission208" +ALTER TABLE submission_job_queue ADD CONSTRAINT submission_job_queue_fk1 FOREIGN KEY (submission_id) REFERENCES submission(submission_id); @@ -393,7 +401,7 @@ ALTER TABLE submission_job_queue ADD CONSTRAINT "Refsubmission208" -- TABLE: system_user -- -ALTER TABLE system_user ADD CONSTRAINT "Refuser_identity_source178" +ALTER TABLE system_user ADD CONSTRAINT system_user_fk1 FOREIGN KEY (user_identity_source_id) REFERENCES user_identity_source(user_identity_source_id); @@ -401,10 +409,10 @@ ALTER TABLE system_user ADD CONSTRAINT "Refuser_identity_source178" -- TABLE: system_user_role -- -ALTER TABLE system_user_role ADD CONSTRAINT "Refsystem_user179" +ALTER TABLE system_user_role ADD CONSTRAINT system_user_role_fk1 FOREIGN KEY (system_user_id) REFERENCES system_user(system_user_id); -ALTER TABLE system_user_role ADD CONSTRAINT "Refsystem_role180" +ALTER TABLE system_user_role ADD CONSTRAINT system_user_role_fk2 FOREIGN KEY (system_role_id) REFERENCES system_role(system_role_id); From 786a90c5ced1f0985297f8696070492e5e9438dd Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 24 Nov 2023 15:09:51 -0800 Subject: [PATCH 17/20] Update artifact table to remove old security columns. Add artifact security join table. --- .../20231117000001_security_tables.ts | 49 +++++++++++++++++++ .../src/migrations/release.0.8.0/biohub.sql | 10 +--- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/database/src/migrations/20231117000001_security_tables.ts b/database/src/migrations/20231117000001_security_tables.ts index f920a5eff..2f2ccc9a9 100644 --- a/database/src/migrations/20231117000001_security_tables.ts +++ b/database/src/migrations/20231117000001_security_tables.ts @@ -72,6 +72,34 @@ export async function up(knex: Knex): Promise { COMMENT ON COLUMN submission_feature_security.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN submission_feature_security.revision_count IS 'Revision count used for concurrency control.'; COMMENT ON TABLE submission_feature_security IS 'A join table between submission_feature and security_rule. Defines which security rules are applied to the a feature submission.'; + + ---------------------------------------------------------------------------------------- + + CREATE TABLE artifact_security( + artifact_security_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + artifact_id integer NOT NULL, + security_rule_id integer NOT NULL, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT artifact_security_pk PRIMARY KEY (artifact_security_id) + ); + + COMMENT ON COLUMN artifact_security.artifact_security_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN artifact_security.artifact_id IS 'Foreign key to the artifact table.'; + COMMENT ON COLUMN artifact_security.security_rule_id IS 'Foreign key to the security_rule table.'; + COMMENT ON COLUMN artifact_security.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN artifact_security.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN artifact_security.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN artifact_security.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN artifact_security.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN artifact_security.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN artifact_security.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE artifact_security IS 'A join table between artifact and security_rule. Defines which security rules are applied to the an artifact.'; ---------------------------------------------------------------------------------------- @@ -237,6 +265,27 @@ export async function up(knex: Knex): Promise { CREATE INDEX submission_feature_security_idx2 ON submission_feature_security(security_rule_id); + ---------------------------------------------------------------------------------------- + -- Create Indexes and Constraints for table: artifact_security + ---------------------------------------------------------------------------------------- + + -- Add unique end-date key constraint (don't allow 2 records with the same artifact_id, security_rule_id, and a NULL record_end_date) + CREATE UNIQUE INDEX artifact_security_nuk1 ON artifact_security(artifact_id, security_rule_id, (record_end_date is NULL)) where record_end_date is null; + + -- Add foreign key constraint + ALTER TABLE artifact_security ADD CONSTRAINT artifact_security_fk1 + FOREIGN KEY (artifact_id) + REFERENCES artifact(artifact_id); + + ALTER TABLE artifact_security ADD CONSTRAINT artifact_security_fk2 + FOREIGN KEY (security_rule_id) + REFERENCES security_rule(security_rule_id); + + -- add indexes for foreign keys + CREATE INDEX artifact_security_idx1 ON artifact_security(artifact_id); + + CREATE INDEX artifact_security_idx2 ON artifact_security(security_rule_id); + ---------------------------------------------------------------------------------------- -- Create Indexes and Constraints for table: security_string ---------------------------------------------------------------------------------------- diff --git a/database/src/migrations/release.0.8.0/biohub.sql b/database/src/migrations/release.0.8.0/biohub.sql index 11e1b7ade..0ae040fa9 100644 --- a/database/src/migrations/release.0.8.0/biohub.sql +++ b/database/src/migrations/release.0.8.0/biohub.sql @@ -13,10 +13,6 @@ CREATE TABLE artifact( file_size integer, key varchar(1000), security_review_timestamp timestamptz(6), - foi_reason boolean, - security_reason_name varchar(300), - security_reason_description varchar(3000), - security_reason_end_date timestamptz(6), create_date timestamptz(6) DEFAULT now() NOT NULL, create_user integer NOT NULL, update_date timestamptz(6), @@ -35,16 +31,12 @@ COMMENT ON COLUMN artifact.description IS 'The description of the record.'; COMMENT ON COLUMN artifact.file_size IS 'The size of the artifact in bytes.'; COMMENT ON COLUMN artifact.key IS 'The identifying key to the file in the storage system.'; COMMENT ON COLUMN artifact.security_review_timestamp IS 'The timestamp that the security review of the submission artifact was completed.'; -COMMENT ON COLUMN artifact.foi_reason IS 'A boolean flag indicating whether the data is secured due to Freedom of Information data being present.'; -COMMENT ON COLUMN artifact.security_reason_name IS 'The name of the custom security reason.'; -COMMENT ON COLUMN artifact.security_reason_description IS 'A reason description that is secures this data and is specific to this artifact or dataset.'; -COMMENT ON COLUMN artifact.security_reason_end_date IS 'Custom security reason end date.'; COMMENT ON COLUMN artifact.create_date IS 'The datetime the record was created.'; COMMENT ON COLUMN artifact.create_user IS 'The id of the user who created the record as identified in the system user table.'; COMMENT ON COLUMN artifact.update_date IS 'The datetime the record was updated.'; COMMENT ON COLUMN artifact.update_user IS 'The id of the user who updated the record as identified in the system user table.'; COMMENT ON COLUMN artifact.revision_count IS 'Revision count used for concurrency control.'; -COMMENT ON TABLE artifact IS 'A listing of historical data submission artifacts. The record with the most recent security review timestamp is the currently published data set for each artifact identified by UUID.'; +COMMENT ON TABLE artifact IS 'A listing of historical data submission artifacts.'; -- -- TABLE: audit_log From beb3670026781007961ea39e59128fa69601df8e Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 24 Nov 2023 15:10:10 -0800 Subject: [PATCH 18/20] ignore-skip From ad7dc492afbe15199d88da95d3ffaf852b906ef2 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 24 Nov 2023 15:32:43 -0800 Subject: [PATCH 19/20] Rename migration --- .../20231117000001_security_tables.ts | 46 +++++++++---------- ...s => 20231122000001_submission_message.ts} | 0 2 files changed, 23 insertions(+), 23 deletions(-) rename database/src/migrations/{20231122000001_submission_status.ts => 20231122000001_submission_message.ts} (100%) diff --git a/database/src/migrations/20231117000001_security_tables.ts b/database/src/migrations/20231117000001_security_tables.ts index 2f2ccc9a9..d538c965c 100644 --- a/database/src/migrations/20231117000001_security_tables.ts +++ b/database/src/migrations/20231117000001_security_tables.ts @@ -76,30 +76,30 @@ export async function up(knex: Knex): Promise { ---------------------------------------------------------------------------------------- CREATE TABLE artifact_security( - artifact_security_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), - artifact_id integer NOT NULL, - security_rule_id integer NOT NULL, - record_effective_date date NOT NULL, - record_end_date date, - create_date timestamptz(6) DEFAULT now() NOT NULL, - create_user integer NOT NULL, - update_date timestamptz(6), - update_user integer, - revision_count integer DEFAULT 0 NOT NULL, - CONSTRAINT artifact_security_pk PRIMARY KEY (artifact_security_id) - ); + artifact_security_id integer GENERATED ALWAYS AS IDENTITY (START WITH 1 INCREMENT BY 1), + artifact_id integer NOT NULL, + security_rule_id integer NOT NULL, + record_effective_date date NOT NULL, + record_end_date date, + create_date timestamptz(6) DEFAULT now() NOT NULL, + create_user integer NOT NULL, + update_date timestamptz(6), + update_user integer, + revision_count integer DEFAULT 0 NOT NULL, + CONSTRAINT artifact_security_pk PRIMARY KEY (artifact_security_id) + ); - COMMENT ON COLUMN artifact_security.artifact_security_id IS 'System generated surrogate primary key identifier.'; - COMMENT ON COLUMN artifact_security.artifact_id IS 'Foreign key to the artifact table.'; - COMMENT ON COLUMN artifact_security.security_rule_id IS 'Foreign key to the security_rule table.'; - COMMENT ON COLUMN artifact_security.record_effective_date IS 'Record level effective date.'; - COMMENT ON COLUMN artifact_security.record_end_date IS 'Record level end date.'; - COMMENT ON COLUMN artifact_security.create_date IS 'The datetime the record was created.'; - COMMENT ON COLUMN artifact_security.create_user IS 'The id of the user who created the record as identified in the system user table.'; - COMMENT ON COLUMN artifact_security.update_date IS 'The datetime the record was updated.'; - COMMENT ON COLUMN artifact_security.update_user IS 'The id of the user who updated the record as identified in the system user table.'; - COMMENT ON COLUMN artifact_security.revision_count IS 'Revision count used for concurrency control.'; - COMMENT ON TABLE artifact_security IS 'A join table between artifact and security_rule. Defines which security rules are applied to the an artifact.'; + COMMENT ON COLUMN artifact_security.artifact_security_id IS 'System generated surrogate primary key identifier.'; + COMMENT ON COLUMN artifact_security.artifact_id IS 'Foreign key to the artifact table.'; + COMMENT ON COLUMN artifact_security.security_rule_id IS 'Foreign key to the security_rule table.'; + COMMENT ON COLUMN artifact_security.record_effective_date IS 'Record level effective date.'; + COMMENT ON COLUMN artifact_security.record_end_date IS 'Record level end date.'; + COMMENT ON COLUMN artifact_security.create_date IS 'The datetime the record was created.'; + COMMENT ON COLUMN artifact_security.create_user IS 'The id of the user who created the record as identified in the system user table.'; + COMMENT ON COLUMN artifact_security.update_date IS 'The datetime the record was updated.'; + COMMENT ON COLUMN artifact_security.update_user IS 'The id of the user who updated the record as identified in the system user table.'; + COMMENT ON COLUMN artifact_security.revision_count IS 'Revision count used for concurrency control.'; + COMMENT ON TABLE artifact_security IS 'A join table between artifact and security_rule. Defines which security rules are applied to the an artifact.'; ---------------------------------------------------------------------------------------- diff --git a/database/src/migrations/20231122000001_submission_status.ts b/database/src/migrations/20231122000001_submission_message.ts similarity index 100% rename from database/src/migrations/20231122000001_submission_status.ts rename to database/src/migrations/20231122000001_submission_message.ts From 010f04b3d8d74457613684d60b08fad8b8e30b37 Mon Sep 17 00:00:00 2001 From: Nick Phura Date: Fri, 24 Nov 2023 15:58:22 -0800 Subject: [PATCH 20/20] ignore-skip