From 2cb5ff5dbcfc56180d8769b8b42d82ec81ecc6a6 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Tue, 21 Feb 2023 20:21:27 +0200 Subject: [PATCH 01/71] update couchdb container --- couchdb/10-docker-default.ini | 2 +- couchdb/Dockerfile | 2 +- couchdb/docker-entrypoint.sh | 6 +++--- couchdb/set-up-cluster.sh | 16 +++++++++++----- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/couchdb/10-docker-default.ini b/couchdb/10-docker-default.ini index 4b47e9cd667..cdff5aebb1b 100644 --- a/couchdb/10-docker-default.ini +++ b/couchdb/10-docker-default.ini @@ -1,7 +1,7 @@ ; couchdb/local.d/package.ini [fabric] -request_timeout = infinity +request_timeout = 31536000 [query_server_config] os_process_limit = 1000 diff --git a/couchdb/Dockerfile b/couchdb/Dockerfile index 6be8b5c7f21..ee4a8014ad4 100644 --- a/couchdb/Dockerfile +++ b/couchdb/Dockerfile @@ -1,4 +1,4 @@ -FROM couchdb:2.3.1 as base_couchdb_build +FROM couchdb:3.3.1 as base_couchdb_build # Add configuration COPY --chown=couchdb:couchdb 10-docker-default.ini /opt/couchdb/etc/default.d/ diff --git a/couchdb/docker-entrypoint.sh b/couchdb/docker-entrypoint.sh index bde8d76a447..b86880377f5 100755 --- a/couchdb/docker-entrypoint.sh +++ b/couchdb/docker-entrypoint.sh @@ -35,7 +35,7 @@ setSecret() { COUCHDB_SECRET=$(cat /proc/sys/kernel/random/uuid) fi # Set secret only if not already present - if [ -z $(grep -Pzor "\[couch_httpd_auth\]\nsecret =" /opt/couchdb/etc/local.d/*.ini)]; then + if [ -z "$(grep -Pzor "\[couch_httpd_auth\]\nsecret =" /opt/couchdb/etc/local.d/*.ini)" ]; then printf "\n[couch_httpd_auth]\nsecret = %s\n" "$COUCHDB_SECRET" >> $CLUSTER_CREDENTIALS fi } @@ -45,7 +45,7 @@ setUuid() { COUCHDB_UUID=$(cat /proc/sys/kernel/random/uuid) fi # Set uuid only if not already present - if [ -z $(grep -Pzor "\[couchdb\]\nuuid =" /opt/couchdb/etc/local.d/*.ini)]; then + if [ -z "$(grep -Pzor "\[couchdb\]\nuuid =" /opt/couchdb/etc/local.d/*.ini)" ]; then printf "\n[couchdb]\nuuid = %s\n" "$COUCHDB_UUID" >> $CLUSTER_CREDENTIALS fi } @@ -94,7 +94,7 @@ if [ "$1" = '/opt/couchdb/bin/couchdb' ]; then fi if [ "$COUCHDB_LOG_LEVEL" ]; then - if ! grep -Pzoqr "\[log\]\nlevel =" /opt/couchdb/etc/local.d/*.ini; then + if ! grep -Pzor "\[log\]\nlevel =" /opt/couchdb/etc/local.d/*.ini; then printf "\n[log]\nlevel = %s\n" "$COUCHDB_LOG_LEVEL" >> $CLUSTER_CREDENTIALS fi fi diff --git a/couchdb/set-up-cluster.sh b/couchdb/set-up-cluster.sh index c3a40d55984..74731a377f5 100644 --- a/couchdb/set-up-cluster.sh +++ b/couchdb/set-up-cluster.sh @@ -91,20 +91,26 @@ add_peers_to_cluster() { check_cluster_membership } +create_system_databases() { + for db in _users _replicator _global_changes; do + if ! curl -sfX PUT "http://$COUCHDB_USER:$COUCHDB_PASSWORD@$SVC_NAME:5984/$db" > /dev/null; then + echo "Failed to create system database '$db'" + fi + done +} + main(){ + echo "MAIN" check_if_couchdb_is_ready http://$COUCHDB_USER:$COUCHDB_PASSWORD@$SVC_NAME:5984 # only attempt clustering if CLUSTER_PEER_IPS environment variable is present. if [ ! -z "$CLUSTER_PEER_IPS" ]; then enable_cluster add_peers_to_cluster + create_system_databases fi # only attempt to setup initial databases if single node is used if [ -z "$CLUSTER_PEER_IPS" ] && [ -z "$COUCHDB_SYNC_ADMINS_NODE" ]; then - for db in _users _replicator _global_changes; do - if ! curl -sfX PUT "http://$COUCHDB_USER:$COUCHDB_PASSWORD@$SVC_NAME:5984/$db" > /dev/null; then - echo "Failed to create system database '$db'" - fi - done + create_system_databases fi # end process exit 1 From 244ff4ab9f5b8372ebee2eefbdffb4122803c9ce Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 22 Feb 2023 13:35:40 +0200 Subject: [PATCH 02/71] update e2e tests --- couchdb/10-docker-default.ini | 8 ++--- .../api/controllers/db-doc.spec.js | 2 -- tests/integration/api/routing.spec.js | 4 +-- .../cht-conf/cht-conf-actions.spec.js | 2 +- .../integration/couchdb/couch_chttpd.spec.js | 36 ++++++++++++------- .../couchdb/couch_httpd_script/index.js | 4 +-- 6 files changed, 32 insertions(+), 24 deletions(-) diff --git a/couchdb/10-docker-default.ini b/couchdb/10-docker-default.ini index cdff5aebb1b..c397a502c4d 100644 --- a/couchdb/10-docker-default.ini +++ b/couchdb/10-docker-default.ini @@ -1,7 +1,7 @@ ; couchdb/local.d/package.ini [fabric] -request_timeout = 31536000 +request_timeout = 31536000 ; 1 year [query_server_config] os_process_limit = 1000 @@ -10,6 +10,8 @@ os_process_limit = 1000 os_process_timeout = 60000 max_dbs_open = 5000 attachment_stream_buffer_size = 16384 +users_db_security_editable = true +max_document_size = 134217728 ; 128 MiB [chttpd] port = 5984 @@ -19,14 +21,12 @@ socket_options = [{sndbuf, 262144}, {nodelay, true}] require_valid_user = true [httpd] -port = 5986 -bind_address = 0.0.0.0 secure_rewrites = false max_http_request_size = 134217728 ; 128 MiB WWW-Authenticate = Basic realm="Medic Mobile Web Services" [couch_httpd_auth] -# timeout is set to 1 year in seconds, 31536000 +; timeout is set to 1 year in seconds, 31536000 timeout = 31536000 allow_persistent_cookies = true require_valid_user = true diff --git a/tests/integration/api/controllers/db-doc.spec.js b/tests/integration/api/controllers/db-doc.spec.js index 2e4b77cd038..f3e05885474 100644 --- a/tests/integration/api/controllers/db-doc.spec.js +++ b/tests/integration/api/controllers/db-doc.spec.js @@ -666,10 +666,8 @@ describe('db-doc handler', () => { // can read reports about deleted patients results.forEach((result, idx) => { if (reportScenarios[idx].allowed) { - console.log(idx, reportScenarios[idx].doc); chai.expect(result).to.deep.include(reportScenarios[idx].doc, idx); } else { - console.log(idx); chai.expect(result).to.deep.nested.include({ statusCode: 403, 'responseBody.error': 'forbidden'}, idx); } }); diff --git a/tests/integration/api/routing.spec.js b/tests/integration/api/routing.spec.js index 47ce37db757..1d3d2920c05 100644 --- a/tests/integration/api/routing.spec.js +++ b/tests/integration/api/routing.spec.js @@ -898,12 +898,12 @@ describe('routing', () => { describe('admin access to Fauxton', () => { it('should allow access with a trailing slash', async () => { const response = await utils.request({ path: '/_utils/', json: false }); - expect(response).to.include('Fauxton Release'); + expect(response).to.include('Project Fauxton'); }); it('should allow access without a trailing slash', async () => { const response = await utils.request({ path: '/_utils', json: false }); - expect(response).to.include('Fauxton Release'); + expect(response).to.include('Project Fauxton'); }); }); }); diff --git a/tests/integration/cht-conf/cht-conf-actions.spec.js b/tests/integration/cht-conf/cht-conf-actions.spec.js index f2234ffc2b5..19d430ec388 100644 --- a/tests/integration/cht-conf/cht-conf-actions.spec.js +++ b/tests/integration/cht-conf/cht-conf-actions.spec.js @@ -51,7 +51,7 @@ describe('cht-conf actions tests', () => { it('should upload branding', async () => { const branding = await utils.getDoc('branding').catch(error => { if(error){ - console.log(error); + console.err(error); } }); expect(branding.title).to.equal('Medic'); diff --git a/tests/integration/couchdb/couch_chttpd.spec.js b/tests/integration/couchdb/couch_chttpd.spec.js index 47cf22013aa..de7dd12333a 100644 --- a/tests/integration/couchdb/couch_chttpd.spec.js +++ b/tests/integration/couchdb/couch_chttpd.spec.js @@ -34,11 +34,28 @@ const getLogs = async () => { } }; -describe('accessing couch_httpd', () => { - it('should not be available to the host', async () => { +const expectCorrectMetadata = (metadata) => { + expect(metadata._id).to.equal(constants.DB_NAME); + // 12 shards, evenly distributed across 3 nodes + expect(Object.keys(metadata.by_range).length).to.equal(12); + expect(metadata.by_node).to.have.keys([ + 'couchdb@couchdb-1.local', 'couchdb@couchdb-2.local', 'couchdb@couchdb-3.local' + ]); + expect(metadata.by_node['couchdb@couchdb-1.local'].length).to.equal(4); + expect(metadata.by_node['couchdb@couchdb-2.local'].length).to.equal(4); + expect(metadata.by_node['couchdb@couchdb-3.local'].length).to.equal(4); +}; + +describe('accessing couch clustering endpoint', () => { + it('should block unauthenticated access through the host network', async () => { await expect( - utils.request({ uri: `https://localhost:5986/_dbs/${constants.DB_NAME}` }) - ).to.be.rejectedWith('Error: connect ECONNREFUSED 127.0.0.1:5986'); + utils.request({ uri: `https://localhost/_node/_local/_dbs/${constants.DB_NAME}` }) + ).to.be.rejectedWith(Error, 'Authentication required'); + }); + + it('should allow authenticated access through host network', async () => { + const metadata = await utils.request({ path: `/_node/_local/_dbs/${constants.DB_NAME}` }); + expectCorrectMetadata(metadata); }); it('should block unauthenticated access through docker network', async () => { @@ -53,14 +70,7 @@ describe('accessing couch_httpd', () => { const logs = await getLogs(); expect(logs.length).to.equal(1); const metadata = logs[0]; - expect(metadata._id).to.equal(constants.DB_NAME); - // 12 shards, evenly distributed across 3 nodes - expect(Object.keys(metadata.by_range).length).to.equal(12); - expect(metadata.by_node).to.have.keys([ - 'couchdb@couchdb-1.local', 'couchdb@couchdb-2.local', 'couchdb@couchdb-3.local' - ]); - expect(metadata.by_node['couchdb@couchdb-1.local'].length).to.equal(4); - expect(metadata.by_node['couchdb@couchdb-2.local'].length).to.equal(4); - expect(metadata.by_node['couchdb@couchdb-3.local'].length).to.equal(4); + expectCorrectMetadata(metadata); }); }); + diff --git a/tests/integration/couchdb/couch_httpd_script/index.js b/tests/integration/couchdb/couch_httpd_script/index.js index 3183fccb0fe..6486ec76e45 100644 --- a/tests/integration/couchdb/couch_httpd_script/index.js +++ b/tests/integration/couchdb/couch_httpd_script/index.js @@ -4,8 +4,8 @@ const { COUCH_AUTH } = process.env; (() => { const options = { hostname: 'couchdb-1.local', - port: 5986, - path: '/_dbs/medic-test', + port: 5984, + path: '/_node/_local/_dbs/medic-test', method: 'GET', auth: COUCH_AUTH, headers: { From 64c49e723d45e19156cfe9d6ab2044f84db42b35 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 22 Feb 2023 13:47:59 +0200 Subject: [PATCH 03/71] update e2e tests --- api/src/db.js | 26 +++++++++ api/src/migrations/add-national_admin-role.js | 54 ++----------------- api/src/services/config-watcher.js | 10 ++++ 3 files changed, 39 insertions(+), 51 deletions(-) diff --git a/api/src/db.js b/api/src/db.js index b6b6dc7be39..e5996dcfdde 100644 --- a/api/src/db.js +++ b/api/src/db.js @@ -2,6 +2,7 @@ const PouchDB = require('pouchdb-core'); const logger = require('./logger'); const environment = require('./environment'); const rpn = require('request-promise-native'); +const request = require('request-promise-native'); PouchDB.plugin(require('pouchdb-adapter-http')); PouchDB.plugin(require('pouchdb-find')); PouchDB.plugin(require('pouchdb-mapreduce')); @@ -162,4 +163,29 @@ if (UNIT_TEST_ENV) { throw new Error(`Error while saving docs: ${errors.join(', ')}`); }; + + const DEFAULT_SECURITY_STRUCTURE = { + names: [], + roles: [], + }; + + module.exports.addRoleToSecurity = async (dbname, role, addAsAdmin) => { + const securityUrl = new URL(environment.serverUrl); + securityUrl.pathname = `${dbname}/_security`; + + const securityObject = await rpn.get({ url: securityUrl.toString(), json: true }); + const property = addAsAdmin ? 'admins' : 'members'; + if (!securityObject[property]) { + securityObject[property] = DEFAULT_SECURITY_STRUCTURE; + } + + if (securityObject[property].roles.includes(role)) { + return; + } + + logger.info(`Adding ${role} role to ${dbname} ${property}`); + securityObject[property].roles.push(role); + return await rpn.put({ url: securityUrl.toString(), json: true, body: securityObject }); + }; + } diff --git a/api/src/migrations/add-national_admin-role.js b/api/src/migrations/add-national_admin-role.js index 9112fc6c33e..a35ab4fbe1e 100644 --- a/api/src/migrations/add-national_admin-role.js +++ b/api/src/migrations/add-national_admin-role.js @@ -1,61 +1,13 @@ -const request = require('request-promise-native'); -const url = require('url'); const environment = require('../environment'); -const logger = require('../logger'); +const db = require('../db'); -const DEFAULT_STRUCTURE = { - names: [], - roles: [], -}; - -const addRole = (dbname, role) => { - return request.get({ - url: url.format({ - protocol: environment.protocol, - hostname: environment.host, - port: environment.port, - pathname: `${dbname}/_security`, - }), - auth: { - user: environment.username, - pass: environment.password - }, - json: true - }) - .then(body => { - // In CouchDB 1.x, if you have not written to the _security object before - // it is empty. - if (!body.admins) { - body.admins = DEFAULT_STRUCTURE; - } - - if (!body.admins.roles.includes(role)) { - logger.info(`Adding ${role} role to ${dbname} admins`); - body.admins.roles.push(role); - } - return request.put({ - url: url.format({ - protocol: environment.protocol, - hostname: environment.host, - port: environment.port, - pathname: `${dbname}/_security`, - }), - auth: { - user: environment.username, - pass: environment.password - }, - json: true, - body: body - }); - }); -}; module.exports = { name: 'add-national_admin-role', created: new Date(2017, 3, 30), run: () => { return Promise.resolve() - .then(() => addRole('_users', 'national_admin')) - .then(() => addRole(environment.db, 'national_admin')); + .then(() => db.addRoleToSecurity('_users', 'national_admin')) + .then(() => db.addRoleToSecurity(environment.db, 'national_admin')); } }; diff --git a/api/src/services/config-watcher.js b/api/src/services/config-watcher.js index 4ff6a645c34..a860ca1993c 100644 --- a/api/src/services/config-watcher.js +++ b/api/src/services/config-watcher.js @@ -9,6 +9,7 @@ const generateXform = require('./generate-xform'); const generateServiceWorker = require('../generate-service-worker'); const manifest = require('./manifest'); const config = require('../config'); +const environment = require('../environment'); const MEDIC_DDOC_ID = '_design/medic'; @@ -87,6 +88,7 @@ const handleSettingsChange = () => { logger.error('Failed to reload settings: %o', err); process.exit(1); }) + .then(() => addUserRolesToDb()) .then(() => initTransitionLib()) .then(() => logger.debug('Settings updated')); }; @@ -170,9 +172,17 @@ const listen = () => { }); }; +const addUserRolesToDb = async () => { + const roles = config.get('roles'); + for (const role of Object.keys(roles)) { + await db.addRoleToSecurity(environment.db, role, false); + } +}; + module.exports = { load, listen, updateServiceWorker, loadTranslations, + addUserRolesToDb, }; From 7d166b7545a1c6ac914e125629a3d2f3833faedc Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 22 Feb 2023 17:30:37 +0200 Subject: [PATCH 04/71] update unit tests --- api/src/db.js | 4 +-- api/src/services/config-watcher.js | 4 +++ .../mocha/services/config-watcher.spec.js | 35 +++++++++++++++++++ .../integration/couchdb/couch_chttpd.spec.js | 4 +-- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/api/src/db.js b/api/src/db.js index e5996dcfdde..f74ee0a4036 100644 --- a/api/src/db.js +++ b/api/src/db.js @@ -2,7 +2,6 @@ const PouchDB = require('pouchdb-core'); const logger = require('./logger'); const environment = require('./environment'); const rpn = require('request-promise-native'); -const request = require('request-promise-native'); PouchDB.plugin(require('pouchdb-adapter-http')); PouchDB.plugin(require('pouchdb-find')); PouchDB.plugin(require('pouchdb-mapreduce')); @@ -41,7 +40,8 @@ if (UNIT_TEST_ENV) { 'allDbs', 'activeTasks', 'saveDocs', - 'createVault' + 'createVault', + 'addRoleToSecurity', ]; const notStubbed = (first, second) => { diff --git a/api/src/services/config-watcher.js b/api/src/services/config-watcher.js index a860ca1993c..bea85789b27 100644 --- a/api/src/services/config-watcher.js +++ b/api/src/services/config-watcher.js @@ -174,6 +174,10 @@ const listen = () => { const addUserRolesToDb = async () => { const roles = config.get('roles'); + if (!roles || typeof roles !== 'object') { + return; + } + for (const role of Object.keys(roles)) { await db.addRoleToSecurity(environment.db, role, false); } diff --git a/api/tests/mocha/services/config-watcher.spec.js b/api/tests/mocha/services/config-watcher.spec.js index 4d705816f90..b1dace19610 100644 --- a/api/tests/mocha/services/config-watcher.spec.js +++ b/api/tests/mocha/services/config-watcher.spec.js @@ -11,6 +11,7 @@ const generateXform = require('../../../src/services/generate-xform'); const config = require('../../../src/config'); const bootstrap = require('../../../src/services/config-watcher'); const manifest = require('../../../src/services/manifest'); +const environment = require('../../../src/environment'); let on; const emitChange = (change) => { @@ -226,12 +227,46 @@ describe('Configuration', () => { it('reloads settings settings doc is updated', () => { settingsService.update.resolves(); settingsService.get.resolves({ settings: 'yes' }); + sinon.stub(db, 'addRoleToSecurity'); + sinon.stub(config, 'get').withArgs('roles').returns({ chw: {} }); + sinon.stub(environment, 'db').get(() => 'medicdb'); return emitChange({ id: 'settings' }).then(() => { chai.expect(settingsService.update.callCount).to.equal(1); chai.expect(settingsService.get.callCount).to.equal(1); chai.expect(config.set.callCount).to.equal(1); chai.expect(config.set.args[0]).to.deep.equal([{ settings: 'yes' }]); + chai.expect(config.get.withArgs('roles').callCount).to.equal(1); + chai.expect(db.addRoleToSecurity.args).to.deep.equal([['medicdb', 'chw', false]]); + }); + }); + + it('should add all configured user roles to the main database', () => { + settingsService.update.resolves(); + settingsService.get.resolves({ settings: 'yes' }); + sinon.stub(db, 'addRoleToSecurity'); + sinon.stub(config, 'get') + .withArgs('roles') + .returns({ + chw1: {}, + chw2: {}, + chw3: {}, + chw4: {}, + }); + sinon.stub(environment, 'db').get(() => 'medicdb'); + + return emitChange({ id: 'settings' }).then(() => { + chai.expect(settingsService.update.callCount).to.equal(1); + chai.expect(settingsService.get.callCount).to.equal(1); + chai.expect(config.set.callCount).to.equal(1); + chai.expect(config.set.args[0]).to.deep.equal([{ settings: 'yes' }]); + chai.expect(config.get.withArgs('roles').callCount).to.equal(1); + chai.expect(db.addRoleToSecurity.args).to.deep.equal([ + ['medicdb', 'chw1', false], + ['medicdb', 'chw2', false], + ['medicdb', 'chw3', false], + ['medicdb', 'chw4', false], + ]); }); }); diff --git a/tests/integration/couchdb/couch_chttpd.spec.js b/tests/integration/couchdb/couch_chttpd.spec.js index de7dd12333a..6f0daf636c0 100644 --- a/tests/integration/couchdb/couch_chttpd.spec.js +++ b/tests/integration/couchdb/couch_chttpd.spec.js @@ -49,8 +49,8 @@ const expectCorrectMetadata = (metadata) => { describe('accessing couch clustering endpoint', () => { it('should block unauthenticated access through the host network', async () => { await expect( - utils.request({ uri: `https://localhost/_node/_local/_dbs/${constants.DB_NAME}` }) - ).to.be.rejectedWith(Error, 'Authentication required'); + utils.request({ uri: `https://localhost/_node/_local/_dbs/${constants.DB_NAME}`, noAuth: true }) + ).to.be.rejectedWith(Error, 'unauthorized'); }); it('should allow authenticated access through host network', async () => { From 30edc75b228c32ec6abccb854746abe1aa6a035d Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 22 Feb 2023 22:18:02 +0200 Subject: [PATCH 05/71] add missed code. --- api/src/services/config-watcher.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/services/config-watcher.js b/api/src/services/config-watcher.js index bea85789b27..08200b57369 100644 --- a/api/src/services/config-watcher.js +++ b/api/src/services/config-watcher.js @@ -101,7 +101,7 @@ const handleTranslationsChange = () => { }; const handleFormChange = (change) => { - logger.info('Detected form change for', change.id); + logger.info('Detected form change for: %s', change.id); if (change.deleted) { return Promise.resolve(); } @@ -133,6 +133,7 @@ const load = () => { loadViewMaps(); return loadTranslations() .then(() => loadSettings()) + .then(() => addUserRolesToDb()) .then(() => initTransitionLib()) .then(() => db.createVault()); }; @@ -141,7 +142,6 @@ const listen = () => { db.medic .changes({ live: true, since: 'now', return_docs: false }) .on('change', change => { - if (tombstoneUtils.isTombstoneId(change.id)) { return Promise.resolve(); } From 5034915c128ff0cceb1a3eac1434744794b6bb18 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 22 Feb 2023 22:20:28 +0200 Subject: [PATCH 06/71] the story behind the messy commits is ... I had all the changes ready then mistakenly force checked out another branch (for a code review), and, naturally, failed to restore correctly from history repeatedly. --- api/src/db.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/db.js b/api/src/db.js index f74ee0a4036..95a402be9a2 100644 --- a/api/src/db.js +++ b/api/src/db.js @@ -183,7 +183,7 @@ if (UNIT_TEST_ENV) { return; } - logger.info(`Adding ${role} role to ${dbname} ${property}`); + logger.info(`Adding "${role}" role to ${dbname} ${property}`); securityObject[property].roles.push(role); return await rpn.put({ url: securityUrl.toString(), json: true, body: securityObject }); }; From 4df0da54ad40ea72391f193254523d488ef6c74a Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 23 Feb 2023 14:46:17 +0200 Subject: [PATCH 07/71] add e2e tests --- .../default/users/create-meta-db.wdio-spec.js | 17 +++++++++++------ .../couchdb/db-access-for-custom-roles.spec.js | 0 2 files changed, 11 insertions(+), 6 deletions(-) create mode 100644 tests/integration/couchdb/db-access-for-custom-roles.spec.js diff --git a/tests/e2e/default/users/create-meta-db.wdio-spec.js b/tests/e2e/default/users/create-meta-db.wdio-spec.js index 7c70921de72..06b3acc5e99 100644 --- a/tests/e2e/default/users/create-meta-db.wdio-spec.js +++ b/tests/e2e/default/users/create-meta-db.wdio-spec.js @@ -3,6 +3,7 @@ const utils = require('../../../utils'); const usersPage = require('../../../page-objects/default/users/user.wdio.page'); const commonElements = require('../../../page-objects/default/common/common.wdio.page'); const loginPage = require('../../../page-objects/default/login/login.wdio.page'); +const uuid = require('uuid').v4; const username = 'fulltester'; const fullName = 'Roger Milla'; @@ -28,16 +29,20 @@ describe('Create user meta db : ', () => { await loginPage.login({ username, password }); await commonElements.waitForPageLoaded(); - const doc = { _id: 'this is a random uuid' }; + const doc = { _id: uuid() }; await utils.requestOnTestMetaDb(_.defaults({ method: 'POST', body: doc }, options)); const response = await utils.requestOnTestMetaDb(_.defaults({ path: '/_changes' }, options)); const changes = response.results; - expect(changes.length).to.equal(2); - const ids = changes.map(change => change.id).sort(); - expect(ids[0]).to.equal('_design/medic-user'); - expect(ids[1]).to.equal(doc._id); + const ids = changes.map(change => change.id); + expect(ids).to.include.members(['_design/medic-user', doc._id]); + + // admins can also read and write from the users meta db + const adminDoc = { _id: uuid() }; + await utils.requestOnTestMetaDb({ method: 'POST', body: doc, username: options.userName }); + const adminChanges = await utils.requestOnTestMetaDb({ path: '/_changes', username: options.userName }); + const adminChangeIds = adminChanges.results.map(change => change.id); + expect(adminChangeIds).to.include.members(['_design/medic-user', doc._id, adminDoc._id]); }); - }); diff --git a/tests/integration/couchdb/db-access-for-custom-roles.spec.js b/tests/integration/couchdb/db-access-for-custom-roles.spec.js new file mode 100644 index 00000000000..e69de29bb2d From 5b2c0456a280ac2f5bd453642ea3ec57174ddafe Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 23 Feb 2023 23:46:34 +0200 Subject: [PATCH 08/71] add e2e tests --- .../create-user-for-contacts.wdio-spec.js | 110 +++++------------- .../db/db-access-for-custom-roles.spec.js | 76 ++++++++++++ tests/e2e/default/db/db-sync.wdio-spec.js | 59 ++-------- .../default/users/create-meta-db.wdio-spec.js | 4 +- tests/integration/.mocharc.js | 6 +- tests/utils/browser.js | 81 ++++++++++++- 6 files changed, 203 insertions(+), 133 deletions(-) create mode 100644 tests/e2e/default/db/db-access-for-custom-roles.spec.js diff --git a/tests/e2e/default/client-side-transitions/create-user-for-contacts.wdio-spec.js b/tests/e2e/default/client-side-transitions/create-user-for-contacts.wdio-spec.js index 2d5627437b7..8e23372d35a 100644 --- a/tests/e2e/default/client-side-transitions/create-user-for-contacts.wdio-spec.js +++ b/tests/e2e/default/client-side-transitions/create-user-for-contacts.wdio-spec.js @@ -1,7 +1,7 @@ -/* global window */ const { expect } = require('chai'); const fs = require('fs'); const utils = require('../../../utils'); +const browserDbUtils = require('../../../utils/browser'); const loginPage = require('../../../page-objects/default/login/login.wdio.page'); const commonPage = require('../../../page-objects/default/common/common.wdio.page'); const reportsPage = require('../../../page-objects/default/reports/reports.wdio.page'); @@ -106,54 +106,6 @@ const populateReplaceUserForm = async (formTitle) => { await genericForm.nextPage(); }; -const saveLocalDocFromBrowser = async (doc) => { - const { err, result } = await browser.executeAsync((doc, callback) => { - const db = window.CHTCore.DB.get(); - return db - .put(doc) - .then(result => callback({ result })) - .catch(err => callback({ err })); - }, doc); - - if (err) { - throw err; - } - - return result; -}; - -const getLocalDocFromBrowser = async (docId) => { - const { err, result } = await browser.executeAsync((docId, callback) => { - const db = window.CHTCore.DB.get(); - return db - .get(docId, { conflicts: true }) - .then(result => callback({ result })) - .catch(err => callback({ err })); - }, docId); - - if (err) { - throw err; - } - - return result; -}; - -const getManyLocalDocsFromBrowser = async (docIds) => { - const { err, result } = await browser.executeAsync((docIds, callback) => { - const db = window.CHTCore.DB.get(); - return db - .allDocs({ keys: docIds, include_docs: true }) - .then(response => callback({ result: response.rows.map(row => row.doc) })) - .catch(err => callback({ err })); - }, docIds); - - if (err) { - throw err; - } - - return result; -}; - const sync = async () => { await commonPage.openHamburgerMenu(); await (await commonPage.syncButton()).click(); @@ -302,22 +254,22 @@ describe('Create user for contacts', () => { const basicReportId1 = await submitBasicForm(); // Replace user report created - const replaceUserReport = await getLocalDocFromBrowser(reportId); + const replaceUserReport = await browserDbUtils.getDoc(reportId); assertReplaceUserReport(replaceUserReport, originalContactId); const { replacement_contact_id: replacementContactId } = replaceUserReport.fields; // Basic form reports re-parented - const basicReports = await getManyLocalDocsFromBrowser([basicReportId0, basicReportId1]); + const basicReports = await browserDbUtils.getDocs([basicReportId0, basicReportId1]); basicReports.forEach((report) => expect(report.contact._id).to.equal(replacementContactId)); // Existing report not re-parented - const existingBasicReport = await getLocalDocFromBrowser(existingBasicReportId); + const existingBasicReport = await browserDbUtils.getDoc(existingBasicReportId); expect(existingBasicReport.contact._id).to.equal(originalContactId); // Original contact updated to PENDING - const originalContact = await getLocalDocFromBrowser(originalContactId); + const originalContact = await browserDbUtils.getDoc(originalContactId); assertOriginalContactUpdated(originalContact, ORIGINAL_USER.username, replacementContactId, 'PENDING'); - const newContact = await getLocalDocFromBrowser(replacementContactId); + const newContact = await browserDbUtils.getDoc(replacementContactId); assertNewContact(newContact, ORIGINAL_USER, originalContact); // Set as primary contact - const district = await getLocalDocFromBrowser(DISTRICT._id); + const district = await browserDbUtils.getDoc(DISTRICT._id); expect(district.contact._id).to.equal(replacementContactId); await browser.throttle('online'); @@ -481,19 +433,19 @@ describe('Create user for contacts', () => { const basicReportId1 = await submitBasicForm(); // Replace user report created - const replaceUserReport = await getLocalDocFromBrowser(reportId); + const replaceUserReport = await browserDbUtils.getDoc(reportId); assertReplaceUserReport(replaceUserReport, originalContactId); const { replacement_contact_id: replacementContactId0 } = replaceUserReport.fields; // Basic form reports re-parented - const basicReports = await getManyLocalDocsFromBrowser([basicReportId0, basicReportId1]); + const basicReports = await browserDbUtils.getDocs([basicReportId0, basicReportId1]); basicReports.forEach((report) => expect(report.contact._id).to.equal(replacementContactId0)); // Original contact updated to PENDING - let originalContact = await getLocalDocFromBrowser(originalContactId); + let originalContact = await browserDbUtils.getDoc(originalContactId); assertOriginalContactUpdated(originalContact, ORIGINAL_USER.username, replacementContactId0, 'PENDING'); - const newContact = await getLocalDocFromBrowser(replacementContactId0); + const newContact = await browserDbUtils.getDoc(replacementContactId0); assertNewContact(newContact, ORIGINAL_USER, originalContact); // Set as primary contact - let district = await getLocalDocFromBrowser(DISTRICT._id); + let district = await browserDbUtils.getDoc(DISTRICT._id); expect(district.contact._id).to.equal(replacementContactId0); // Submit another replace user form @@ -508,19 +460,19 @@ describe('Create user for contacts', () => { const basicReportId2 = await submitBasicForm(); const basicReportId3 = await submitBasicForm(); - const replaceUserReport1 = await getLocalDocFromBrowser(reportId1); + const replaceUserReport1 = await browserDbUtils.getDoc(reportId1); const { replacement_contact_id: replacementContactId1 } = replaceUserReport1.fields; assertReplaceUserReport(replaceUserReport, originalContactId); // Basic form reports re-parented - const basicReports1 = await getManyLocalDocsFromBrowser([basicReportId2, basicReportId3]); + const basicReports1 = await browserDbUtils.getDocs([basicReportId2, basicReportId3]); basicReports1.forEach((report) => expect(report.contact._id).to.equal(replacementContactId1)); // Original contact updated to have new replacement contact id - originalContact = await getLocalDocFromBrowser(originalContactId); + originalContact = await browserDbUtils.getDoc(originalContactId); assertOriginalContactUpdated(originalContact, ORIGINAL_USER.username, replacementContactId1, 'PENDING'); - const newContact1 = await getLocalDocFromBrowser(replacementContactId1); + const newContact1 = await browserDbUtils.getDoc(replacementContactId1); assertNewContact(newContact1, ORIGINAL_USER, originalContact); // Set as primary contact - district = await getLocalDocFromBrowser(DISTRICT._id); + district = await browserDbUtils.getDoc(DISTRICT._id); expect(district.contact._id).to.equal(replacementContactId1); await browser.throttle('online'); @@ -650,23 +602,23 @@ describe('Create user for contacts', () => { const reportId = await reportsPage.getLastSubmittedReportId(); // Replace user report created - const replaceUserReport = await getLocalDocFromBrowser(reportId); + const replaceUserReport = await browserDbUtils.getDoc(reportId); assertReplaceUserReport(replaceUserReport, originalContactId); const replacementContactId = replaceUserReport.fields.replacement_contact_id; // Original contact updated to PENDING - let originalContact = await getLocalDocFromBrowser(originalContactId); + let originalContact = await browserDbUtils.getDoc(originalContactId); assertOriginalContactUpdated(originalContact, ORIGINAL_USER.username, replacementContactId, 'PENDING'); - const newContact = await getLocalDocFromBrowser(replacementContactId); + const newContact = await browserDbUtils.getDoc(replacementContactId); assertNewContact(newContact, ORIGINAL_USER, originalContact); // Set as primary contact - const district = await getLocalDocFromBrowser(DISTRICT._id); + const district = await browserDbUtils.getDoc(DISTRICT._id); expect(district.contact._id).to.equal(replacementContactId); // Submit several forms to be re-parented const basicReportId0 = await submitBasicForm(); const basicReportId1 = await submitBasicForm(); // Basic form reports re-parented - const basicReports = await getManyLocalDocsFromBrowser([basicReportId0, basicReportId1]); + const basicReports = await browserDbUtils.getDocs([basicReportId0, basicReportId1]); basicReports.forEach((report) => expect(report.contact._id).to.equal(replacementContactId)); // Logout before syncing @@ -729,7 +681,7 @@ describe('Create user for contacts', () => { await commonPage.goToPeople(originalContactId); // Local version of contact should be updated and have conflict - const localOriginalContact = await waitForConflicts(() => getLocalDocFromBrowser(originalContactId)); + const localOriginalContact = await waitForConflicts(() => browserDbUtils.getDoc(originalContactId)); originalContact = await waitForConflicts(() => utils.getDoc(originalContactId, null, '?conflicts=true')); expect(localOriginalContact).to.deep.equal(originalContact); // Other user replace data exists on the contact @@ -801,23 +753,23 @@ describe('Create user for contacts', () => { const basicReportId1 = await submitBasicForm(); // Replace user report created - const replaceUserReport = await getLocalDocFromBrowser(reportId); + const replaceUserReport = await browserDbUtils.getDoc(reportId); assertReplaceUserReport(replaceUserReport, originalContactId); const { replacement_contact_id: replacementContactId } = replaceUserReport.fields; // Basic form reports re-parented - const basicReports = await getManyLocalDocsFromBrowser([basicReportId0, basicReportId1]); + const basicReports = await browserDbUtils.getDocs([basicReportId0, basicReportId1]); basicReports.forEach((report) => expect(report.contact._id).to.equal(replacementContactId)); // Original contact updated to PENDING - const originalContact = await getLocalDocFromBrowser(originalContactId); + const originalContact = await browserDbUtils.getDoc(originalContactId); assertOriginalContactUpdated(originalContact, ORIGINAL_USER.username, replacementContactId, 'PENDING'); - const newContact = await getLocalDocFromBrowser(replacementContactId); + const newContact = await browserDbUtils.getDoc(replacementContactId); assertNewContact(newContact, ORIGINAL_USER, originalContact); // Set as primary contact - const district = await getLocalDocFromBrowser(DISTRICT._id); + const district = await browserDbUtils.getDoc(DISTRICT._id); expect(district.contact._id).to.equal(replacementContactId); // Remove phone number from contact to force the transition to fail - await saveLocalDocFromBrowser({ + await browserDbUtils.createDoc({ ...newContact, phone: undefined, }); @@ -854,12 +806,12 @@ describe('Create user for contacts', () => { await loginPage.login(ORIGINAL_USER); await commonPage.waitForPageLoaded(); await browser.throttle('offline'); - const finalOriginalContactLocal = await getLocalDocFromBrowser(originalContactId); + const finalOriginalContactLocal = await browserDbUtils.getDoc(originalContactId); assertOriginalContactUpdated(finalOriginalContactLocal, ORIGINAL_USER.username, replacementContactId, 'ERROR'); await commonPage.goToReports(); const basicReportId2 = await submitBasicForm(); const basicReportId3 = await submitBasicForm(); - const subsequentBasicReports = await getManyLocalDocsFromBrowser([basicReportId2, basicReportId3]); + const subsequentBasicReports = await browserDbUtils.getDocs([basicReportId2, basicReportId3]); subsequentBasicReports.forEach((report) => expect(report.contact._id).to.equal(originalContactId)); }); }); diff --git a/tests/e2e/default/db/db-access-for-custom-roles.spec.js b/tests/e2e/default/db/db-access-for-custom-roles.spec.js new file mode 100644 index 00000000000..76f83a4cc06 --- /dev/null +++ b/tests/e2e/default/db/db-access-for-custom-roles.spec.js @@ -0,0 +1,76 @@ +const commonElements = require('../../../page-objects/default/common/common.wdio.page'); +const utils = require('../../../utils'); +const browserDbUtils = require('../../../utils/browser'); +const loginPage = require('../../../page-objects/default/login/login.wdio.page'); +const uuid = require('uuid').v4; +const personFactory = require('../../../factories/cht/contacts/person'); +const place = require('../../../factories/cht/contacts/place'); +const places = place.generateHierarchy(); +const clinic = places.get('clinic'); + +const contact = personFactory.build( + { + parent: { + _id: clinic._id, + parent: clinic.parent + }, + phone: '+254712345670' + }); + +const docs = [...places.values(), contact]; +const newRole = 'new_chw'; + +const addRole = async (role = newRole) => { + const settings = await utils.getSettings(); + settings.roles[role] = { ...settings.roles.chw }; + await utils.updateSettings(settings, true); +}; + +const username = uuid(); +const user = { + _id: `org.couchdb.user:${username}`, + type: 'user', + password: uuid(), + facility_id: clinic._id, + contact_id: contact._id, + roles: [ newRole ] +}; + +describe('Database access for new roles', () => { + before(async () => { + await utils.saveDocs(docs); + await addRole(newRole); + await utils.createUsers([user]); + }); + + it('user with custom role should be able to log in', async () => { + await loginPage.login({ username: username, password: user.password }); + }); + + it('should be able to sync documents up', async () => { + const report = { + _id: uuid(), + type: 'data_record', + form: 'something', + contact: { _id: contact._id }, + fields: { patient_id: contact._id, }, + }; + await browserDbUtils.createDoc(report); + await commonElements.sync(); + + await utils.get(report._id); + }); + + it('should be able to sync documents down', async () => { + const report = { + _id: uuid(), + type: 'data_record', + form: 'something', + contact: { _id: contact._id }, + fields: { patient_id: contact._id, }, + }; + await utils.saveDoc(report); + await commonElements.sync(); + await browserDbUtils.getDoc(report._id); + }); +}); diff --git a/tests/e2e/default/db/db-sync.wdio-spec.js b/tests/e2e/default/db/db-sync.wdio-spec.js index 4a134561791..767606885e9 100644 --- a/tests/e2e/default/db/db-sync.wdio-spec.js +++ b/tests/e2e/default/db/db-sync.wdio-spec.js @@ -5,6 +5,7 @@ const loginPage = require('../../../page-objects/default/login/login.wdio.page') const reportsPage = require('../../../page-objects/default/reports/reports.wdio.page'); const chai = require('chai'); const uuid = require('uuid').v4; +const browserDbUtils = require('../../../utils/browser'); /* global window */ @@ -147,46 +148,6 @@ describe('db-sync', () => { }, ]; - const updateDoc = async (docId, changes, overwrite = false) => { - const { err, result } = await browser.executeAsync((docId, changes, overwrite, callback) => { - const db = window.CHTCore.DB.get(); - return db - .get(docId) - .then(doc => { - if (overwrite) { - doc = { _rev: doc._rev, _id: doc._id }; - } - - Object.assign(doc, changes); - return db.put(doc); - }) - .then(result => callback({ result })) - .catch(err => callback({ err })); - }, docId, changes, overwrite); - - if (err) { - throw err; - } - - return result; - }; - - const createDoc = async (doc) => { - const { err, result } = await browser.executeAsync((doc, callback) => { - const db = window.CHTCore.DB.get(); - return db - .put(doc) - .then(result => callback({ result })) - .catch(err => callback({ err })); - }, doc); - - if (err) { - throw err; - } - - return result; - }; - const getServerRevs = async (docIds) => { const result = await utils.requestOnMedicDb({ path: '/_all_docs', qs: { keys: JSON.stringify(docIds) } }); return result.rows; @@ -205,11 +166,11 @@ describe('db-sync', () => { }); it('should not filter allowed docs', async () => { - await updateDoc(report1, { extra: '1' }); - await updateDoc(report2, { extra: '2' }); - await updateDoc(patientId, { extra: '3' }); + await browserDbUtils.updateDoc(report1, { extra: '1' }); + await browserDbUtils.updateDoc(report2, { extra: '2' }); + await browserDbUtils.updateDoc(patientId, { extra: '3' }); const newReport = { ...initialReports[0], _id: uuid(), extra: '4' }; - const { rev } = await createDoc(newReport); + const { rev } = await browserDbUtils.createDoc(newReport); newReport._rev = rev; await commonElements.sync(); @@ -229,7 +190,7 @@ describe('db-sync', () => { }); it('should not filter deletes', async () => { - await updateDoc(report1, { _deleted: true }); + await browserDbUtils.updateDoc(report1, { _deleted: true }); await commonElements.sync(); @@ -251,7 +212,7 @@ describe('db-sync', () => { ]; const serverRevs = await getServerRevs(docIds); for (const docId of docIds) { - await updateDoc(docId, { something: 'random' }); + await browserDbUtils.updateDoc(docId, { something: 'random' }); } await commonElements.sync(); @@ -261,7 +222,7 @@ describe('db-sync', () => { }); it('should filter locally purged docs', async () => { - const { rev:localRev } = await updateDoc(report3, { _deleted: true, purged: true }, true); + const { rev:localRev } = await browserDbUtils.updateDoc(report3, { _deleted: true, purged: true }, true); await commonElements.sync(); @@ -273,9 +234,9 @@ describe('db-sync', () => { it('should filter ddocs', async () => { const newDdoc = { _id: '_design/test' }; - await createDoc(newDdoc); + await browserDbUtils.createDoc(newDdoc); const serverRevs = await getServerRevs(['_design/medic-client']); - await updateDoc('_design/medic-client', { something: 'random' }); + await browserDbUtils.updateDoc('_design/medic-client', { something: 'random' }); // updating the ddoc will throw the "upgrade" popup, ignore it! await commonElements.closeReloadModal(); await commonElements.sync(); diff --git a/tests/e2e/default/users/create-meta-db.wdio-spec.js b/tests/e2e/default/users/create-meta-db.wdio-spec.js index 06b3acc5e99..13f6437e9ac 100644 --- a/tests/e2e/default/users/create-meta-db.wdio-spec.js +++ b/tests/e2e/default/users/create-meta-db.wdio-spec.js @@ -40,8 +40,8 @@ describe('Create user meta db : ', () => { // admins can also read and write from the users meta db const adminDoc = { _id: uuid() }; - await utils.requestOnTestMetaDb({ method: 'POST', body: doc, username: options.userName }); - const adminChanges = await utils.requestOnTestMetaDb({ path: '/_changes', username: options.userName }); + await utils.requestOnTestMetaDb({ method: 'POST', body: adminDoc, userName: options.userName }); + const adminChanges = await utils.requestOnTestMetaDb({ path: '/_changes', userName: options.userName }); const adminChangeIds = adminChanges.results.map(change => change.id); expect(adminChangeIds).to.include.members(['_design/medic-user', doc._id, adminDoc._id]); }); diff --git a/tests/integration/.mocharc.js b/tests/integration/.mocharc.js index a8a40378cf2..fe1db137af6 100644 --- a/tests/integration/.mocharc.js +++ b/tests/integration/.mocharc.js @@ -4,6 +4,8 @@ const chai = require('chai'); chai.use(chaiExclude); chai.use(chaiAsPromised); +global.expect = chai.expect; + module.exports = { allowUncaught: false, color: true, @@ -11,8 +13,8 @@ module.exports = { fullTrace: true, asyncOnly: false, spec: [ - 'tests/integration/!(cht-conf)/**/*.spec.js', - 'tests/integration/cht-conf/**/*.spec.js', // Executing last to not side-effect sentinel tests. + 'tests/integration/!(cht-conf)/**/db-access*.spec.js', + // 'tests/integration/cht-conf/**/*.spec.js', // Executing last to not side-effect sentinel tests. ], timeout: 200 * 1000, //API takes a little long to start up reporter: 'spec', diff --git a/tests/utils/browser.js b/tests/utils/browser.js index 9ddc84586df..3fe4952563a 100644 --- a/tests/utils/browser.js +++ b/tests/utils/browser.js @@ -1,4 +1,7 @@ const fs = require('fs'); + +/* global window */ + const feedBackDocs = async (testName = 'allLogs', existingDocIds = []) => { const feedBackDocs = await browser.executeAsync(feedBackDocsScript); const flattened = feedBackDocs.flat(); @@ -32,7 +35,83 @@ const getCookies = (...cookieNameList) => { return browser.getCookies(cookieNameList); }; +const createDoc = async (doc) => { + const { err, result } = await browser.executeAsync((doc, callback) => { + const db = window.CHTCore.DB.get(); + return db + .put(doc) + .then(result => callback({ result })) + .catch(err => callback({ err })); + }, doc); + + if (err) { + throw err; + } + + return result; +}; + +const updateDoc = async (docId, changes, overwrite = false) => { + const { err, result } = await browser.executeAsync((docId, changes, overwrite, callback) => { + const db = window.CHTCore.DB.get(); + return db + .get(docId) + .then(doc => { + if (overwrite) { + doc = { _rev: doc._rev, _id: doc._id }; + } + + Object.assign(doc, changes); + return db.put(doc); + }) + .then(result => callback({ result })) + .catch(err => callback({ err })); + }, docId, changes, overwrite); + + if (err) { + throw err; + } + + return result; +}; + +const getDoc = async (docId) => { + const { err, result } = await browser.executeAsync((docId, callback) => { + const db = window.CHTCore.DB.get(); + return db + .get(docId, { conflicts: true }) + .then(result => callback({ result })) + .catch(err => callback({ err })); + }, docId); + + if (err) { + throw err; + } + + return result; +}; + +const getDocs = async (docIds) => { + const { err, result } = await browser.executeAsync((docIds, callback) => { + const db = window.CHTCore.DB.get(); + return db + .allDocs({ keys: docIds, include_docs: true }) + .then(response => callback({ result: response.rows.map(row => row.doc) })) + .catch(err => callback({ err })); + }, docIds); + + if (err) { + throw err; + } + + return result; +}; + module.exports = { feedBackDocs, - getCookies + getCookies, + createDoc, + updateDoc, + getDoc, + getDocs, }; From 9c5c5a63884fa43721f304182a496120ee6439b9 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Fri, 24 Feb 2023 00:06:33 +0200 Subject: [PATCH 09/71] remove old file --- tests/integration/couchdb/db-access-for-custom-roles.spec.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/integration/couchdb/db-access-for-custom-roles.spec.js diff --git a/tests/integration/couchdb/db-access-for-custom-roles.spec.js b/tests/integration/couchdb/db-access-for-custom-roles.spec.js deleted file mode 100644 index e69de29bb2d..00000000000 From 7ed10b5df3ea90baade2386b87aba11df3c1ae8e Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Fri, 24 Feb 2023 00:31:40 +0200 Subject: [PATCH 10/71] adds unit test --- api/src/db.js | 10 ++- api/tests/mocha/db.spec.js | 131 +++++++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) diff --git a/api/src/db.js b/api/src/db.js index 95a402be9a2..9acc42f42cb 100644 --- a/api/src/db.js +++ b/api/src/db.js @@ -170,6 +170,10 @@ if (UNIT_TEST_ENV) { }; module.exports.addRoleToSecurity = async (dbname, role, addAsAdmin) => { + if (!dbname || !role) { + return; + } + const securityUrl = new URL(environment.serverUrl); securityUrl.pathname = `${dbname}/_security`; @@ -179,13 +183,17 @@ if (UNIT_TEST_ENV) { securityObject[property] = DEFAULT_SECURITY_STRUCTURE; } + if (!securityObject[property].roles) { + throw new Error('Invalid database security %o', securityObject); + } + if (securityObject[property].roles.includes(role)) { return; } logger.info(`Adding "${role}" role to ${dbname} ${property}`); securityObject[property].roles.push(role); - return await rpn.put({ url: securityUrl.toString(), json: true, body: securityObject }); + await rpn.put({ url: securityUrl.toString(), json: true, body: securityObject }); }; } diff --git a/api/tests/mocha/db.spec.js b/api/tests/mocha/db.spec.js index 236bc363838..17293d7d971 100644 --- a/api/tests/mocha/db.spec.js +++ b/api/tests/mocha/db.spec.js @@ -202,4 +202,135 @@ describe('db', () => { expect(db.close.args).to.deep.equal([[dbObject]]); }); }); + + describe('addRoleToSecurity', () => { + it('should add role as member to db security', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); + sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] }, members: { roles: ['role2'] } }); + sinon.stub(rpn, 'put').resolves(); + + await db.addRoleToSecurity('dbname', 'rolename'); + + expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); + expect(rpn.put.args).to.deep.equal([[ + { + url: 'http://admin:pass@couchdb:5984/dbname/_security', + json: true, + body: { + admins: { roles: ['role1'] }, + members: { roles: ['role2', 'rolename'] }, + } + } + ]]); + }); + + it('should add role as admin to db security', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); + sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] }, members: { roles: ['role2'] } }); + sinon.stub(rpn, 'put').resolves(); + + await db.addRoleToSecurity('dbname', 'rolename', true); + + expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); + expect(rpn.put.args).to.deep.equal([[ + { + url: 'http://admin:pass@couchdb:5984/dbname/_security', + json: true, + body: { + admins: { roles: ['role1', 'rolename'], }, + members: { roles: ['role2'] }, + } + } + ]]); + }); + + it('should skip member roles that already exist', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); + sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] }, members: { roles: ['role2'] } }); + sinon.stub(rpn, 'put').resolves(); + + await db.addRoleToSecurity('dbname', 'role2'); + + expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); + expect(rpn.put.called).to.equal(false); + }); + + it('should skip admin roles that already exist', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); + sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] }, members: { roles: ['role2'] } }); + sinon.stub(rpn, 'put').resolves(); + + await db.addRoleToSecurity('dbname', 'role1', true); + + expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); + expect(rpn.put.called).to.equal(false); + }); + + it('should set members security property if not existing', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); + sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] } }); + sinon.stub(rpn, 'put').resolves(); + + await db.addRoleToSecurity('dbname', 'rolename'); + + expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); + expect(rpn.put.args).to.deep.equal([[ + { + url: 'http://admin:pass@couchdb:5984/dbname/_security', + json: true, + body: { + admins: { roles: ['role1'] }, + members: { roles: ['rolename'], names: [], }, + } + } + ]]); + }); + + it('should set admins security property if not existing', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pwd@host:6984'); + sinon.stub(rpn, 'get').resolves({ members: { roles: ['role2'] } }); + sinon.stub(rpn, 'put').resolves(); + + await db.addRoleToSecurity('name', 'arole', true); + + expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pwd@host:6984/name/_security', json: true } ]]); + expect(rpn.put.args).to.deep.equal([[ + { + url: 'http://admin:pwd@host:6984/name/_security', + json: true, + body: { + admins: { roles: ['arole'], names: [] }, + members: { roles: ['role2'] }, + } + } + ]]); + }); + + it('should throw get security errors', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); + sinon.stub(rpn, 'get').rejects(new Error('not_found')); + + await expect(db.addRoleToSecurity('data', 'attr')).to.be.rejectedWith(Error, 'not_found'); + }); + + it('should throw put security errors', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); + sinon.stub(rpn, 'get').resolves({ members: { roles: ['role2'] } }); + sinon.stub(rpn, 'put').rejects(new Error('forbidden or something')); + + await expect(db.addRoleToSecurity('data', 'attr')).to.be.rejectedWith(Error, 'forbidden or something'); + }); + + it('should throw error when security is invalid', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); + sinon.stub(rpn, 'get').resolves({ members: 'this is actually a string' }); + + await expect(db.addRoleToSecurity('data', 'attr')).to.be.rejectedWith(Error, 'Invalid database security'); + }); + + it('should skip when missing dbname or role', async () => { + await db.addRoleToSecurity('', 'arole'); + await db.addRoleToSecurity('dbanme', ); + }); + }); }); From f7ea09d174228c75eadfff3ec2b8562c8ba731e9 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Fri, 24 Feb 2023 06:18:39 +0200 Subject: [PATCH 11/71] remove test only --- tests/integration/.mocharc.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/.mocharc.js b/tests/integration/.mocharc.js index fe1db137af6..e0e83f2fe05 100644 --- a/tests/integration/.mocharc.js +++ b/tests/integration/.mocharc.js @@ -13,8 +13,8 @@ module.exports = { fullTrace: true, asyncOnly: false, spec: [ - 'tests/integration/!(cht-conf)/**/db-access*.spec.js', - // 'tests/integration/cht-conf/**/*.spec.js', // Executing last to not side-effect sentinel tests. + 'tests/integration/!(cht-conf)/**/*.spec.js', + 'tests/integration/cht-conf/**/*.spec.js', // Executing last to not side-effect sentinel tests. ], timeout: 200 * 1000, //API takes a little long to start up reporter: 'spec', From b1a600c840cdffcadac13eb5e0764ed3638fc2a3 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Mon, 27 Feb 2023 10:30:46 +0200 Subject: [PATCH 12/71] remove test echo --- couchdb/set-up-cluster.sh | 1 - tests/conf.js | 4 ++-- tests/e2e/protractor/sms/rapidpro.spec.js | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/couchdb/set-up-cluster.sh b/couchdb/set-up-cluster.sh index 74731a377f5..884234d95d2 100644 --- a/couchdb/set-up-cluster.sh +++ b/couchdb/set-up-cluster.sh @@ -100,7 +100,6 @@ create_system_databases() { } main(){ - echo "MAIN" check_if_couchdb_is_ready http://$COUCHDB_USER:$COUCHDB_PASSWORD@$SVC_NAME:5984 # only attempt clustering if CLUSTER_PEER_IPS environment variable is present. if [ ! -z "$CLUSTER_PEER_IPS" ]; then diff --git a/tests/conf.js b/tests/conf.js index e073af89b74..f6e633d97fa 100644 --- a/tests/conf.js +++ b/tests/conf.js @@ -7,7 +7,7 @@ chai.use(require('chai-shallow-deep-equal')); chai.config.truncateThreshold = 0; chai.use(require('chai-exclude')); -const NUMBER_OF_RETRIES = 5; +// const NUMBER_OF_RETRIES = 5; const baseConfig = { params:{ @@ -67,7 +67,7 @@ const baseConfig = { }, afterLaunch: async () => { await utils.endSession(); - return retry.afterLaunch(NUMBER_OF_RETRIES); + //return retry.afterLaunch(NUMBER_OF_RETRIES); }, onPrepare: async () => { retry.onPrepare(); diff --git a/tests/e2e/protractor/sms/rapidpro.spec.js b/tests/e2e/protractor/sms/rapidpro.spec.js index b2f9d56786c..a7396d626d6 100644 --- a/tests/e2e/protractor/sms/rapidpro.spec.js +++ b/tests/e2e/protractor/sms/rapidpro.spec.js @@ -612,7 +612,8 @@ describe('RapidPro SMS Gateway', () => { await utils.saveDocs(docs); const iterations = docsCount / 25; // batch size is 25 - await browser.wait(() => messagesEndpointRequests.length === docsCount, (iterations + 2) * 1000 ); + // increased number of wait time after Couch3 upgrade + await browser.wait(() => messagesEndpointRequests.length === docsCount, (iterations + 10) * 1000 ); const queriedBroadcasts = messagesEndpointRequests.map(request => request[0].broadcast).sort(); const expectedBroadcasts = docs.map(doc => doc.tasks[0].gateway_ref).sort(); From afda1a683f07b70d3362709e2918528f5d362b2b Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Mon, 27 Feb 2023 12:40:55 +0200 Subject: [PATCH 13/71] reload page before loading analytics --- tests/e2e/protractor/targets/target-aggregates.spec.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/e2e/protractor/targets/target-aggregates.spec.js b/tests/e2e/protractor/targets/target-aggregates.spec.js index 12c8d755118..4dab76f59c9 100644 --- a/tests/e2e/protractor/targets/target-aggregates.spec.js +++ b/tests/e2e/protractor/targets/target-aggregates.spec.js @@ -348,6 +348,7 @@ describe('Target aggregates', () => { await utils.saveDocs(targetDocs); await updateSettings(targetsConfig, user); + await commonElements.goToReports(true); await commonElements.goToAnalytics(); await analytics.goToTargetAggregates(true); await helper.takeScreenshot('targets.png'); @@ -450,6 +451,7 @@ describe('Target aggregates', () => { await utils.saveDocs(targetDocs); await updateSettings(targetsConfig, user, contactSummaryScript); + await commonElements.goToReports(true); await commonElements.goToAnalytics(); await analytics.goToTargetAggregates(true); From 57fe903ef62930ec3b820d4ecafa07e9972a1615 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 22 Mar 2023 16:36:23 +0200 Subject: [PATCH 14/71] use lua-load-per-thread --- haproxy/default_frontend.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/haproxy/default_frontend.cfg b/haproxy/default_frontend.cfg index 0d49d3a50af..f450105b7cb 100644 --- a/haproxy/default_frontend.cfg +++ b/haproxy/default_frontend.cfg @@ -6,9 +6,9 @@ global maxconn 150000 spread-checks 5 - lua-load /usr/local/etc/haproxy/parse_basic.lua - lua-load /usr/local/etc/haproxy/parse_cookie.lua - lua-load /usr/local/etc/haproxy/replace_password.lua + lua-load-per-thread /usr/local/etc/haproxy/parse_basic.lua + lua-load-per-thread /usr/local/etc/haproxy/parse_cookie.lua + lua-load-per-thread /usr/local/etc/haproxy/replace_password.lua log stdout len 65535 local2 debug tune.bufsize 1638400 tune.http.maxhdr 1010 From 0e77a02b3e05b4d3154075528b65d45ee60546d4 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 5 Apr 2023 10:13:53 +0300 Subject: [PATCH 15/71] upgrade to haproxy 2.7 --- haproxy/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haproxy/Dockerfile b/haproxy/Dockerfile index eb10ccee6da..ef9bdf6a5cd 100644 --- a/haproxy/Dockerfile +++ b/haproxy/Dockerfile @@ -1,4 +1,4 @@ -FROM haproxy:2.6 +FROM haproxy:2.7 USER root RUN apt-get update && apt-get install rsyslog luarocks gettext jq curl -y From bfd5cb4f323b1ee773dcef7cecc5fb986fba9ba6 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 5 Apr 2023 14:23:11 +0300 Subject: [PATCH 16/71] add option abortonclose --- haproxy/default_frontend.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/haproxy/default_frontend.cfg b/haproxy/default_frontend.cfg index f450105b7cb..0a6647912ca 100644 --- a/haproxy/default_frontend.cfg +++ b/haproxy/default_frontend.cfg @@ -5,6 +5,7 @@ # log 127.0.0.1 local0 warning global maxconn 150000 + nbproc 4 spread-checks 5 lua-load-per-thread /usr/local/etc/haproxy/parse_basic.lua lua-load-per-thread /usr/local/etc/haproxy/parse_cookie.lua @@ -31,6 +32,7 @@ defaults option forwardfor option redispatch option http-server-close + option abortonclose timeout client 15000000 timeout server 360000000 timeout connect 1500000 From 10df1b19e0d546ec8d5bbae03c775e201a5ae29f Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 6 Apr 2023 13:51:32 +0300 Subject: [PATCH 17/71] remove tune options from haproxy config --- haproxy/default_frontend.cfg | 4 ---- 1 file changed, 4 deletions(-) diff --git a/haproxy/default_frontend.cfg b/haproxy/default_frontend.cfg index 0a6647912ca..a0dbea744b3 100644 --- a/haproxy/default_frontend.cfg +++ b/haproxy/default_frontend.cfg @@ -5,14 +5,11 @@ # log 127.0.0.1 local0 warning global maxconn 150000 - nbproc 4 spread-checks 5 lua-load-per-thread /usr/local/etc/haproxy/parse_basic.lua lua-load-per-thread /usr/local/etc/haproxy/parse_cookie.lua lua-load-per-thread /usr/local/etc/haproxy/replace_password.lua log stdout len 65535 local2 debug - tune.bufsize 1638400 - tune.http.maxhdr 1010 # https://www.haproxy.com/documentation/hapee/latest/onepage/#3.2-tune.bufsize # At least the global maxconn @@ -32,7 +29,6 @@ defaults option forwardfor option redispatch option http-server-close - option abortonclose timeout client 15000000 timeout server 360000000 timeout connect 1500000 From f7f158c9ed980b7d5008b4c3cd2e2cc53078d9cb Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 6 Apr 2023 14:57:23 +0300 Subject: [PATCH 18/71] remove old comment about tune.bufsize --- haproxy/default_frontend.cfg | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/haproxy/default_frontend.cfg b/haproxy/default_frontend.cfg index a0dbea744b3..b4b485b915b 100644 --- a/haproxy/default_frontend.cfg +++ b/haproxy/default_frontend.cfg @@ -11,17 +11,6 @@ global lua-load-per-thread /usr/local/etc/haproxy/replace_password.lua log stdout len 65535 local2 debug -# https://www.haproxy.com/documentation/hapee/latest/onepage/#3.2-tune.bufsize -# At least the global maxconn -# parameter should be decreased by the same factor as this one is increased. If an -# HTTP request is larger than (tune.bufsize - tune.maxrewrite), HAProxy will -# return HTTP 400 (Bad Request) error. Similarly if an HTTP response is larger -# than this size, HAProxy will return HTTP 502 (Bad Gateway). - -# https://www.haproxy.com/documentation/hapee/latest/onepage/#3.2-tune.http.maxhdr -# Similarly, too large responses -# are blocked with "502 Bad Gateway". - defaults mode http option http-ignore-probes From ade5bb16393748026f8585ab44cfc13b2423eb2d Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Mon, 17 Apr 2023 16:15:19 +0300 Subject: [PATCH 19/71] disable repeatedly failing test --- tests/e2e/protractor/targets/target-aggregates.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/protractor/targets/target-aggregates.spec.js b/tests/e2e/protractor/targets/target-aggregates.spec.js index 4dab76f59c9..5e8f780eb07 100644 --- a/tests/e2e/protractor/targets/target-aggregates.spec.js +++ b/tests/e2e/protractor/targets/target-aggregates.spec.js @@ -279,7 +279,7 @@ describe('Target aggregates', () => { } }); - it('should display correct data', async () => { + xit('should display correct data', async () => { const targetsConfig = [ { id: 'count_no_goal', type: 'count', title: genTitle('count no goal'), aggregate: true }, { id: 'count_with_goal', type: 'count', title: genTitle('count with goal'), goal: 20, aggregate: true }, From 0120c35b6f1bd5ae2cc3fd908a3deabab9f3a090 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Fri, 28 Apr 2023 17:51:26 +0300 Subject: [PATCH 20/71] "patch" node-fetch in api and sentinel. --- api/Dockerfile | 1 + api/package-lock.json | 102 +++++++++++++++-------- api/package.json | 1 + sentinel/Dockerfile | 1 + sentinel/package-lock.json | 54 +++++++++--- sentinel/package.json | 1 + tests/integration/.mocharc.js | 1 + tests/integration/api/routing.spec.js | 1 - tests/integration/api/server.spec.js | 25 +++++- tests/integration/sentinel/queue.spec.js | 24 +++++- tests/utils.js | 5 ++ 11 files changed, 165 insertions(+), 51 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index cb33930a102..69da74e3ca4 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -15,5 +15,6 @@ WORKDIR /api COPY api/ /api COPY shared-libs /shared-libs ENV NODE_PATH=/api/node_modules +RUN rm -rf /api/node_modules/pouchdb-fetch/node_modules/node-fetch ENTRYPOINT ["/bin/bash", "/api/docker-entrypoint.sh", "main"] diff --git a/api/package-lock.json b/api/package-lock.json index 9d4c0824867..f182f995ead 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -45,6 +45,7 @@ "morgan": "^1.10.0", "mustache": "^4.2.0", "node-cache": "^5.1.2", + "node-fetch": "^2.6.9", "node-html-parser": "^3.0.4", "object-path": "^0.11.8", "openrosa-formlist": "github:medic/openrosa-formlist#sax", @@ -111,7 +112,7 @@ "dependencies": { "@medic/phone-number": "file:../phone-number", "bikram-sambat-bootstrap": "^1.5.0", - "google-libphonenumber": "^3.2.30", + "google-libphonenumber": "^3.2.31", "gsm": "^0.1.4", "lodash": "^4.17.21", "moment": "^2.29.1", @@ -124,7 +125,7 @@ "version": "1.0.0", "license": "Apache-2.0", "dependencies": { - "google-libphonenumber": "^3.2.30" + "google-libphonenumber": "^3.2.31" } }, "../shared-libs/purging-utils": { @@ -196,7 +197,7 @@ "async": "^3.2.3", "bikram-sambat": "^1.7.0", "bikram-sambat-bootstrap": "^1.5.0", - "google-libphonenumber": "^3.2.30", + "google-libphonenumber": "^3.2.31", "gsm": "^0.1.4", "lodash": "4.17.19", "moment": "^2.29.1", @@ -2307,9 +2308,9 @@ } }, "node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", "dependencies": { "whatwg-url": "^5.0.0" }, @@ -2723,6 +2724,25 @@ "node-fetch": "2.6.7" } }, + "node_modules/pouchdb-fetch/node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/pouchdb-find": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/pouchdb-find/-/pouchdb-find-7.3.1.tgz", @@ -2848,9 +2868,9 @@ "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, "node_modules/punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", "engines": { "node": ">=6" } @@ -2940,9 +2960,9 @@ } }, "node_modules/readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -3821,9 +3841,9 @@ } }, "node_modules/winston-transport/node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -3845,9 +3865,9 @@ } }, "node_modules/winston/node_modules/readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -3956,7 +3976,7 @@ "requires": { "@medic/phone-number": "file:../phone-number", "bikram-sambat-bootstrap": "^1.5.0", - "google-libphonenumber": "^3.2.30", + "google-libphonenumber": "^3.2.31", "gsm": "^0.1.4", "lodash": "^4.17.21", "moment": "^2.29.1", @@ -3967,7 +3987,7 @@ "@medic/phone-number": { "version": "file:../shared-libs/phone-number", "requires": { - "google-libphonenumber": "^3.2.30" + "google-libphonenumber": "^3.2.31" } }, "@medic/purging-utils": { @@ -4023,7 +4043,7 @@ "async": "^3.2.3", "bikram-sambat": "^1.7.0", "bikram-sambat-bootstrap": "^1.5.0", - "google-libphonenumber": "^3.2.30", + "google-libphonenumber": "^3.2.31", "gsm": "^0.1.4", "jsverify": "^0.8.4", "lodash": "4.17.19", @@ -5576,9 +5596,9 @@ } }, "node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", "requires": { "whatwg-url": "^5.0.0" } @@ -5904,6 +5924,16 @@ "abort-controller": "3.0.0", "fetch-cookie": "0.11.0", "node-fetch": "2.6.7" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } + } } }, "pouchdb-find": { @@ -6019,9 +6049,9 @@ "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, "punycode": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==" }, "qs": { "version": "6.11.0", @@ -6083,9 +6113,9 @@ } }, "readable-stream": { - "version": "2.3.7", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", - "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -6782,9 +6812,9 @@ "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" }, "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -6804,9 +6834,9 @@ }, "dependencies": { "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", "requires": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", diff --git a/api/package.json b/api/package.json index 43931134271..a03d52d6050 100644 --- a/api/package.json +++ b/api/package.json @@ -52,6 +52,7 @@ "morgan": "^1.10.0", "mustache": "^4.2.0", "node-cache": "^5.1.2", + "node-fetch": "^2.6.9", "node-html-parser": "^3.0.4", "object-path": "^0.11.8", "openrosa-formlist": "github:medic/openrosa-formlist#sax", diff --git a/sentinel/Dockerfile b/sentinel/Dockerfile index 7ca40993e13..747d4a6cc5c 100644 --- a/sentinel/Dockerfile +++ b/sentinel/Dockerfile @@ -14,5 +14,6 @@ WORKDIR /sentinel COPY sentinel/ /sentinel COPY shared-libs /shared-libs ENV NODE_PATH=/sentinel/node_modules +RUN rm -rf /sentinel/node_modules/pouchdb-fetch/node_modules/node-fetch ENTRYPOINT ["/bin/bash", "/sentinel/docker-entrypoint.sh", "main"] diff --git a/sentinel/package-lock.json b/sentinel/package-lock.json index 5c27194743f..d2d698cc082 100644 --- a/sentinel/package-lock.json +++ b/sentinel/package-lock.json @@ -33,6 +33,7 @@ "lodash": "4.17.19", "moment": "^2.29.1", "mustache": "^4.2.0", + "node-fetch": "^2.6.9", "object-path": "^0.11.8", "pouchdb-adapter-http": "^7.3.1", "pouchdb-core": "^7.3.1", @@ -75,7 +76,7 @@ "dependencies": { "@medic/phone-number": "file:../phone-number", "bikram-sambat-bootstrap": "^1.5.0", - "google-libphonenumber": "^3.2.30", + "google-libphonenumber": "^3.2.31", "gsm": "^0.1.4", "lodash": "^4.17.21", "moment": "^2.29.1", @@ -100,7 +101,7 @@ "version": "1.0.0", "license": "Apache-2.0", "dependencies": { - "google-libphonenumber": "^3.2.30" + "google-libphonenumber": "^3.2.31" } }, "../shared-libs/purging-utils": { @@ -163,7 +164,7 @@ "async": "^3.2.3", "bikram-sambat": "^1.7.0", "bikram-sambat-bootstrap": "^1.5.0", - "google-libphonenumber": "^3.2.30", + "google-libphonenumber": "^3.2.31", "gsm": "^0.1.4", "lodash": "4.17.19", "moment": "^2.29.1", @@ -753,9 +754,9 @@ } }, "node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", "dependencies": { "whatwg-url": "^5.0.0" }, @@ -897,6 +898,25 @@ "node-fetch": "2.6.7" } }, + "node_modules/pouchdb-fetch/node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/pouchdb-generate-replication-id": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/pouchdb-generate-replication-id/-/pouchdb-generate-replication-id-7.3.1.tgz", @@ -1340,7 +1360,7 @@ "requires": { "@medic/phone-number": "file:../phone-number", "bikram-sambat-bootstrap": "^1.5.0", - "google-libphonenumber": "^3.2.30", + "google-libphonenumber": "^3.2.31", "gsm": "^0.1.4", "lodash": "^4.17.21", "moment": "^2.29.1", @@ -1361,7 +1381,7 @@ "@medic/phone-number": { "version": "file:../shared-libs/phone-number", "requires": { - "google-libphonenumber": "^3.2.30" + "google-libphonenumber": "^3.2.31" } }, "@medic/purging-utils": { @@ -1410,7 +1430,7 @@ "async": "^3.2.3", "bikram-sambat": "^1.7.0", "bikram-sambat-bootstrap": "^1.5.0", - "google-libphonenumber": "^3.2.30", + "google-libphonenumber": "^3.2.31", "gsm": "^0.1.4", "jsverify": "^0.8.4", "lodash": "4.17.19", @@ -1837,9 +1857,9 @@ "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==" }, "node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.9.tgz", + "integrity": "sha512-DJm/CJkZkRjKKj4Zi4BsKVZh3ValV5IR5s7LVZnW+6YMh0W1BfNA8XSs6DLMGYlId5F3KnA70uu2qepcR08Qqg==", "requires": { "whatwg-url": "^5.0.0" } @@ -1962,6 +1982,16 @@ "abort-controller": "3.0.0", "fetch-cookie": "0.11.0", "node-fetch": "2.6.7" + }, + "dependencies": { + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } + } } }, "pouchdb-generate-replication-id": { diff --git a/sentinel/package.json b/sentinel/package.json index 737026533e6..05aaf4a7690 100644 --- a/sentinel/package.json +++ b/sentinel/package.json @@ -33,6 +33,7 @@ "lodash": "4.17.19", "moment": "^2.29.1", "mustache": "^4.2.0", + "node-fetch": "^2.6.9", "object-path": "^0.11.8", "pouchdb-adapter-http": "^7.3.1", "pouchdb-core": "^7.3.1", diff --git a/tests/integration/.mocharc.js b/tests/integration/.mocharc.js index a8a40378cf2..c36d17301eb 100644 --- a/tests/integration/.mocharc.js +++ b/tests/integration/.mocharc.js @@ -3,6 +3,7 @@ const chaiAsPromised = require('chai-as-promised'); const chai = require('chai'); chai.use(chaiExclude); chai.use(chaiAsPromised); +global.expect = chai.expect; module.exports = { allowUncaught: false, diff --git a/tests/integration/api/routing.spec.js b/tests/integration/api/routing.spec.js index b38121ada16..a48215d7458 100644 --- a/tests/integration/api/routing.spec.js +++ b/tests/integration/api/routing.spec.js @@ -2,7 +2,6 @@ const _ = require('lodash'); const utils = require('../../utils'); const constants = require('../../constants'); const moment = require('moment'); -const { expect } = require('chai'); const password = 'passwordSUP3RS3CR37!'; diff --git a/tests/integration/api/server.spec.js b/tests/integration/api/server.spec.js index bf74518c67b..e8260a3fa4b 100644 --- a/tests/integration/api/server.spec.js +++ b/tests/integration/api/server.spec.js @@ -2,7 +2,6 @@ const utils = require('../../utils'); const request = require('request'); const constants = require('../../constants'); const _ = require('lodash'); -const {expect} = require('chai'); describe('server', () => { describe('JSON-only endpoints', () => { @@ -181,4 +180,28 @@ describe('server', () => { expect(attachmentBody).to.equal(png); }); }); + + describe('API changes feed', () => { + it('should respond to changes even after services are restarted', async () => { + await utils.stopHaproxy(); // this will also crash API + await utils.startHaproxy(); + // the nginx restart is required because of https://github.com/medic/cht-core/issues/8205 + await utils.stopNginx(); + await utils.startNginx(); + await utils.listenForApi(); + + const forms = await utils.db.allDocs({ + start_key: 'form:', + end_key: 'form:\ufff0', + include_docs: true, + limit: 1, + }); + const formDoc = forms.rows[0].doc; + delete formDoc._attachments['form.html']; + delete formDoc._attachments['model.xml']; + await utils.saveDoc(formDoc); + const updatedFormDoc = await utils.getDoc(formDoc._id); + expect(updatedFormDoc._attachments).to.have.keys(['xml', 'form.html', 'model.xml']); + }); + }); }); diff --git a/tests/integration/sentinel/queue.spec.js b/tests/integration/sentinel/queue.spec.js index 19b87b12151..549c06b2861 100644 --- a/tests/integration/sentinel/queue.spec.js +++ b/tests/integration/sentinel/queue.spec.js @@ -1,7 +1,6 @@ const utils = require('../../utils'); const sentinelUtils = require('../../utils/sentinel'); const uuid = require('uuid').v4; -const { expect } = require('chai'); const NBR_DOCS = 300; @@ -119,4 +118,27 @@ describe('Sentinel queue drain', () => { }); }); }).timeout(300 * 1000); + + it('queue should work after restarting haproxy', async () => { + await utils.stopHaproxy(); // this will also crash Sentinel and API + await utils.startHaproxy(); + // the nginx restart is required because of https://github.com/medic/cht-core/issues/8205 + await utils.stopNginx(); + await utils.startNginx(); + await utils.listenForApi(); + const settings = { transitions: { update_clinics: true } }; + await utils.updateSettings(settings, 'api'); + + const doc = { + _id: uuid(), + type: 'data_record', + from: 'phone1', + fields: { patient_id: 'patient' }, + reported_date: new Date().getTime(), + }; + await utils.saveDoc(doc); + await sentinelUtils.waitForSentinel(); + const [info] = await sentinelUtils.getInfoDocs(doc._id); + expect(info.transitions.update_clinics.ok).to.be.true; + }); }); diff --git a/tests/utils.js b/tests/utils.js index 3edf0f07d73..be796c4fc9e 100644 --- a/tests/utils.js +++ b/tests/utils.js @@ -1245,6 +1245,11 @@ module.exports = { await startService('api'); await listenForApi(); }, + stopHaproxy: () => stopService('haproxy'), + startHaproxy: () => startService('haproxy'), + + stopNginx: () => stopService('nginx'), + startNginx: () => startService('nginx'), saveCredentials: (key, password) => { const options = { From a66ae9d2c6357c7675e620b7270756729faa3bea Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Fri, 28 Apr 2023 22:01:13 +0300 Subject: [PATCH 21/71] rm in grunt only. --- Gruntfile.js | 3 ++- api/Dockerfile | 1 - haproxy/entrypoint.sh | 2 -- sentinel/Dockerfile | 1 - tests/integration/sentinel/queue.spec.js | 2 ++ 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index 1cbd5f46047..829c3bea3cb 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -300,6 +300,7 @@ module.exports = function(grunt) { `cd ${service}`, `npm ci --production`, `npm dedupe`, + `rm -rf ./node_modules/pouchdb-fetch/node_modules/node-fetch`, `cd ../`, `docker build -f ./${service}/Dockerfile --tag ${buildVersions.getImageTag(service)} .`, ].join(' && ') @@ -385,7 +386,7 @@ module.exports = function(grunt) { }, 'npm-ci-modules': { cmd: ['webapp', 'api', 'sentinel', 'admin'] - .map(dir => `echo "[${dir}]" && cd ${dir} && npm ci --legacy-peer-deps && cd ..`) + .map(dir => `echo "[${dir}]" && cd ${dir} && npm ci --legacy-peer-deps && rm -rf ./node_modules/pouchdb-fetch/node_modules/node-fetch && cd ..`) .join(' && '), }, 'start-webdriver': { diff --git a/api/Dockerfile b/api/Dockerfile index 69da74e3ca4..cb33930a102 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -15,6 +15,5 @@ WORKDIR /api COPY api/ /api COPY shared-libs /shared-libs ENV NODE_PATH=/api/node_modules -RUN rm -rf /api/node_modules/pouchdb-fetch/node_modules/node-fetch ENTRYPOINT ["/bin/bash", "/api/docker-entrypoint.sh", "main"] diff --git a/haproxy/entrypoint.sh b/haproxy/entrypoint.sh index 27e76483d4b..07cacb85954 100644 --- a/haproxy/entrypoint.sh +++ b/haproxy/entrypoint.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e -# Make sure service is running -service rsyslog start DEFAULT="/usr/local/etc/haproxy/default_frontend.cfg" BACKEND="/usr/local/etc/haproxy/backend.cfg" diff --git a/sentinel/Dockerfile b/sentinel/Dockerfile index 747d4a6cc5c..7ca40993e13 100644 --- a/sentinel/Dockerfile +++ b/sentinel/Dockerfile @@ -14,6 +14,5 @@ WORKDIR /sentinel COPY sentinel/ /sentinel COPY shared-libs /shared-libs ENV NODE_PATH=/sentinel/node_modules -RUN rm -rf /sentinel/node_modules/pouchdb-fetch/node_modules/node-fetch ENTRYPOINT ["/bin/bash", "/sentinel/docker-entrypoint.sh", "main"] diff --git a/tests/integration/sentinel/queue.spec.js b/tests/integration/sentinel/queue.spec.js index 549c06b2861..178923635fb 100644 --- a/tests/integration/sentinel/queue.spec.js +++ b/tests/integration/sentinel/queue.spec.js @@ -126,6 +126,7 @@ describe('Sentinel queue drain', () => { await utils.stopNginx(); await utils.startNginx(); await utils.listenForApi(); + const settings = { transitions: { update_clinics: true } }; await utils.updateSettings(settings, 'api'); @@ -137,6 +138,7 @@ describe('Sentinel queue drain', () => { reported_date: new Date().getTime(), }; await utils.saveDoc(doc); + console.log(doc); await sentinelUtils.waitForSentinel(); const [info] = await sentinelUtils.getInfoDocs(doc._id); expect(info.transitions.update_clinics.ok).to.be.true; From 709e11c6ef4e7dcd9e2d87d3f1a9ad714cba29df Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Mon, 1 May 2023 07:40:27 +0300 Subject: [PATCH 22/71] revert unwanted change --- haproxy/entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/haproxy/entrypoint.sh b/haproxy/entrypoint.sh index 07cacb85954..27e76483d4b 100644 --- a/haproxy/entrypoint.sh +++ b/haproxy/entrypoint.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +# Make sure service is running +service rsyslog start DEFAULT="/usr/local/etc/haproxy/default_frontend.cfg" BACKEND="/usr/local/etc/haproxy/backend.cfg" From e2773db4d2ee1051bc4e196c79b0fec2b99588f6 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Mon, 1 May 2023 07:43:37 +0300 Subject: [PATCH 23/71] add comment. --- Gruntfile.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Gruntfile.js b/Gruntfile.js index 829c3bea3cb..3a56ee7f911 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -386,6 +386,8 @@ module.exports = function(grunt) { }, 'npm-ci-modules': { cmd: ['webapp', 'api', 'sentinel', 'admin'] + // removing pouchdb-fetch/node-fetch forces PouchDb to use a newer version node-fetch + // https://github.com/medic/cht-core/issues/8173 .map(dir => `echo "[${dir}]" && cd ${dir} && npm ci --legacy-peer-deps && rm -rf ./node_modules/pouchdb-fetch/node_modules/node-fetch && cd ..`) .join(' && '), }, From a6ed30baf06bbef41f62d740b4393404f9aea928 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 3 May 2023 14:16:30 +0300 Subject: [PATCH 24/71] remove code that is no longer necessary --- tests/integration/sentinel/queue.spec.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/integration/sentinel/queue.spec.js b/tests/integration/sentinel/queue.spec.js index 178923635fb..d5b46177fbb 100644 --- a/tests/integration/sentinel/queue.spec.js +++ b/tests/integration/sentinel/queue.spec.js @@ -122,9 +122,6 @@ describe('Sentinel queue drain', () => { it('queue should work after restarting haproxy', async () => { await utils.stopHaproxy(); // this will also crash Sentinel and API await utils.startHaproxy(); - // the nginx restart is required because of https://github.com/medic/cht-core/issues/8205 - await utils.stopNginx(); - await utils.startNginx(); await utils.listenForApi(); const settings = { transitions: { update_clinics: true } }; From fce25bd5d10c19667877e2d4325790d1cb8d4371 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 4 May 2023 11:21:13 +0300 Subject: [PATCH 25/71] partial cherrypick of haproxy edits Because haproxy fails to start with this error: rsyslogd: pidfile '/run/rsyslogd.pid' and pid 14 already exist. If you want to run multiple instances of rsyslog, you need to specify different pid files for them (-i option). rsyslogd: run failed with error -3000 (see rsyslog.h or try https://www.rsyslog.com/e/3000 to learn what that number means) --- haproxy/Dockerfile | 3 +-- haproxy/entrypoint.sh | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/haproxy/Dockerfile b/haproxy/Dockerfile index eb10ccee6da..2434cd41236 100644 --- a/haproxy/Dockerfile +++ b/haproxy/Dockerfile @@ -1,13 +1,12 @@ FROM haproxy:2.6 USER root -RUN apt-get update && apt-get install rsyslog luarocks gettext jq curl -y +RUN apt-get update && apt-get install luarocks gettext jq curl -y COPY entrypoint.sh / RUN chmod +x /entrypoint.sh ADD default_frontend.cfg /usr/local/etc/haproxy ADD backend.cfg.template /usr/local/etc/haproxy -ADD rsyslog.conf /etc/rsyslog.conf COPY scripts /usr/local/etc/haproxy/ ENTRYPOINT ["/entrypoint.sh"] diff --git a/haproxy/entrypoint.sh b/haproxy/entrypoint.sh index 27e76483d4b..07cacb85954 100644 --- a/haproxy/entrypoint.sh +++ b/haproxy/entrypoint.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e -# Make sure service is running -service rsyslog start DEFAULT="/usr/local/etc/haproxy/default_frontend.cfg" BACKEND="/usr/local/etc/haproxy/backend.cfg" From c6d1527f1766f1652a1b61942e9b6cd3892d1531 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 4 May 2023 12:48:39 +0300 Subject: [PATCH 26/71] don't wait for API when haproxy is down, it'll never start. --- tests/integration/api/server.spec.js | 4 +++- tests/utils.js | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/integration/api/server.spec.js b/tests/integration/api/server.spec.js index 9d059f60fcc..3569a0fe991 100644 --- a/tests/integration/api/server.spec.js +++ b/tests/integration/api/server.spec.js @@ -213,10 +213,12 @@ describe('server', () => { await utils.stopHaproxy(); await utils.stopApi(); - await utils.startApi(); + await utils.startApi(false); await utils.startHaproxy(); await utils.request('/'); + + await utils.listenForApi(); }); }); }); diff --git a/tests/utils.js b/tests/utils.js index be796c4fc9e..1988579430c 100644 --- a/tests/utils.js +++ b/tests/utils.js @@ -1241,9 +1241,9 @@ module.exports = { startSentinel: () => startService('sentinel'), stopApi: () => stopService('api'), - startApi: async () => { + startApi: async (listen = true) => { await startService('api'); - await listenForApi(); + listen && await listenForApi(); }, stopHaproxy: () => stopService('haproxy'), startHaproxy: () => startService('haproxy'), From bf8f239678f0ceb038c95f5f1c77c15487bd8809 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 4 May 2023 16:40:57 +0300 Subject: [PATCH 27/71] don't wait for API when haproxy is down, it'll never start. --- tests/integration/api/server.spec.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/api/server.spec.js b/tests/integration/api/server.spec.js index 3569a0fe991..78b701ce087 100644 --- a/tests/integration/api/server.spec.js +++ b/tests/integration/api/server.spec.js @@ -216,8 +216,6 @@ describe('server', () => { await utils.startApi(false); await utils.startHaproxy(); - await utils.request('/'); - await utils.listenForApi(); }); }); From 144c273f27e2d58bfdd773ceab1a6cd44dbf7464 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Fri, 5 May 2023 06:18:50 +0300 Subject: [PATCH 28/71] always retry writing the infodoc. --- sentinel/src/schedule/outbound.js | 35 ++++++----------- sentinel/tests/unit/schedule/outbound.js | 10 +++-- shared-libs/infodoc/src/infodoc.js | 44 +++++++++++++-------- shared-libs/infodoc/test/infodoc.js | 49 ++++++++++++++++++++++-- 4 files changed, 90 insertions(+), 48 deletions(-) diff --git a/sentinel/src/schedule/outbound.js b/sentinel/src/schedule/outbound.js index a2608c4d2cd..b5f4b44d780 100644 --- a/sentinel/src/schedule/outbound.js +++ b/sentinel/src/schedule/outbound.js @@ -3,6 +3,8 @@ const db = require('../db'); const logger = require('../lib/logger'); const lineage = require('@medic/lineage')(Promise, db.medic); const outbound = require('@medic/outbound')(logger); +const infodocLib = require('@medic/infodoc'); +infodocLib.initLib(db.medic, db.sentinel); const CONFIGURED_PUSHES = 'outbound'; const BATCH_SIZE = 1000; @@ -102,7 +104,8 @@ const removeConfigKeyFromTask = (taskDoc, key) => { * Push a payload, cleanup afterwards if successful * */ -const singlePush = (taskDoc, medicDoc, infoDoc, config, key) => Promise.resolve() +const singlePush = (taskDoc, medicDoc, infoDoc, config, key) => Promise + .resolve() .then(() => { if (!config) { // The outbound config entry has been deleted / renamed / something! @@ -112,28 +115,14 @@ const singlePush = (taskDoc, medicDoc, infoDoc, config, key) => Promise.resolve( return removeConfigKeyFromTask(taskDoc, key); } - return outbound.send(config, key, medicDoc, infoDoc) - .then(() => { - // Worked, remove entry from queue and store infodoc that outbound service has updated - return removeConfigKeyFromTask(taskDoc, key) - .then(() => db.sentinel.put(infoDoc)) - .catch(err => { - if (err.status !== 409) { - logger.error('Failed to save infodoc'); - logger.error(err); - throw err; - } - - // While we held the infodoc open something else wrote to it. Presume it wasn't - // anything task related, load a fresh version and overwrite - return db.sentinel.get(infoDoc._id) - .then(freshInfoDoc => { - freshInfoDoc.completed_tasks = infoDoc.completed_tasks; - return db.sentinel.put(freshInfoDoc); - }); - }); - }); - }).catch(() => { + return outbound + .send(config, key, medicDoc, infoDoc) + // Worked, remove entry from queue and store infodoc that outbound service has updated + .then(() => removeConfigKeyFromTask(taskDoc, key)) + .then(() => infodocLib.saveCompletedTasks(medicDoc._id, infoDoc)); + }) + .catch((err) => { + logger.warn(`Unable to push ${medicDoc._id} for ${key}: %eo`, err); // Failed! // Don't remove the entry from the task's queue so it will be tried again next time }); diff --git a/sentinel/tests/unit/schedule/outbound.js b/sentinel/tests/unit/schedule/outbound.js index 90c4321f291..5ceee5e7d8e 100644 --- a/sentinel/tests/unit/schedule/outbound.js +++ b/sentinel/tests/unit/schedule/outbound.js @@ -321,6 +321,7 @@ describe('outbound schedule', () => { let sentinelPut; let sentinelGet; let send; + let infodocSaveCompletedTasks; const restores = []; @@ -334,8 +335,10 @@ describe('outbound schedule', () => { restores.push(outbound.__set__('mapDocumentToPayload', mapDocumentToPayload)); send = sinon.stub(); - restores.push(outbound.__set__('outbound', { send: send })); + infodocSaveCompletedTasks = sinon.stub(); + restores.push(outbound.__set__('outbound', { send: send })); + restores.push(outbound.__set__('infodocLib', { saveCompletedTasks: infodocSaveCompletedTasks })); sentinelPut = sinon.stub(db.sentinel, 'put'); sentinelGet = sinon.stub(db.sentinel, 'get'); }); @@ -372,7 +375,7 @@ describe('outbound schedule', () => { assert.equal(task._deleted, true); assert.equal(task._rev, '1-abc'); assert.equal(send.callCount, 1); - assert.equal(sentinelPut.callCount, 2); + assert.equal(infodocSaveCompletedTasks.callCount, 1); }); }); @@ -397,8 +400,7 @@ describe('outbound schedule', () => { return outbound.__get__('singlePush')(task, doc, config, 'test-config-1') .then(() => { assert.equal(task.queue.length, 1); - assert.equal(sentinelGet.callCount, 0); - assert.equal(sentinelPut.callCount, 0); + assert.equal(infodocSaveCompletedTasks.callCount, 0); }); }); diff --git a/shared-libs/infodoc/src/infodoc.js b/shared-libs/infodoc/src/infodoc.js index efb981e418f..212f48488c0 100644 --- a/shared-libs/infodoc/src/infodoc.js +++ b/shared-libs/infodoc/src/infodoc.js @@ -152,27 +152,36 @@ const updateTransition = (change, transition, ok) => { }; const saveTransitions = change => { - return db.sentinel.get(getInfoDocId(change.id)) - .catch(err => { - if (err.status !== 404) { - throw err; - } + return saveProperty(change.id, change.info, 'transitions', {}); +}; - return change.info; - }) - .then(doc => { - doc.transitions = change.info.transitions || {}; - return db.sentinel.put(doc); - }) - .catch(err => { - if (err.status !== 409) { - throw err; - } +const saveCompletedTasks = (id, infodoc) => { + return saveProperty(id, infodoc, 'completed_tasks', {}); +}; - return saveTransitions(change); - }); +const saveProperty = async (id, infodoc, property, defaultValue) => { + let updatedInfoDoc; + try { + updatedInfoDoc = await db.sentinel.get(getInfoDocId(id)); + updatedInfoDoc[property] = (infodoc && infodoc[property]) || defaultValue; + } catch (err) { + if (err.status !== 404) { + throw err; + } + updatedInfoDoc = infodoc; + } + + try { + return await db.sentinel.put(updatedInfoDoc); + } catch (err) { + if (err.status !== 409) { + throw err; + } + return saveProperty(id, infodoc, property, defaultValue); + } }; + const bulkUpdate = infoDocs => { if (!infoDocs || !infoDocs.length) { return Promise.resolve(); @@ -276,6 +285,7 @@ module.exports = { bulkGet: changes => resolveInfoDocs(changes, false), bulkUpdate: bulkUpdate, saveTransitions: saveTransitions, + saveCompletedTasks: saveCompletedTasks, // Used to update infodoc metadata that occurs at write time. A delete does not count as a write // in this instance, as deletes resolve as infodoc cleanups once sentinel's background-cleanup diff --git a/shared-libs/infodoc/test/infodoc.js b/shared-libs/infodoc/test/infodoc.js index 283dbd407f4..cf64d2461ee 100644 --- a/shared-libs/infodoc/test/infodoc.js +++ b/shared-libs/infodoc/test/infodoc.js @@ -471,12 +471,53 @@ describe('infodoc', () => { }; sinon.stub(db.sentinel, 'get').resolves(info); const sentinelPut = sinon.stub(db.sentinel, 'put'); - sentinelPut.onFirstCall().rejects({status: 409}); - sentinelPut.onSecondCall().resolves(); + sentinelPut.rejects({ status: 409 }); + sentinelPut.onCall(20).resolves(); return lib.saveTransitions(change).then(() => { - assert.equal(db.sentinel.get.callCount, 2); - assert.equal(db.sentinel.put.callCount, 2); + assert.equal(db.sentinel.get.callCount, 21); + assert.equal(db.sentinel.put.callCount, 21); + assert.deepEqual(db.sentinel.put.args[20], [{ ...info, transitions: change.info.transitions }]); + }); + }); + }); + + describe('saveCompletedTasks', () => { + it('saveCompletedTasks should update infodoc', () => { + const serverInfo = { _id: 'some-info', _rev: 2, doc_id: 'some' }; + sinon.stub(db.sentinel, 'get').resolves(serverInfo); + sinon.stub(db.sentinel, 'put').resolves(); + const providedInfo = { + _id: 'some-info', + _rev: 1, + completed_tasks: { completed: 'tasks' }, + }; + + return lib.saveCompletedTasks('some', providedInfo).then(() => { + assert.equal(db.sentinel.get.callCount, 1); + assert.deepEqual(db.sentinel.get.args[0], ['some-info']); + assert.equal(db.sentinel.put.callCount, 1); + assert.deepEqual(db.sentinel.put.args[0], [{ ...serverInfo, completed_tasks: providedInfo.completed_tasks }]); + }); + }); + + it('should handle conflicts correctly', () => { + const serverInfo = { _id: 'some-info', _rev: 2, doc_id: 'some' }; + sinon.stub(db.sentinel, 'get').resolves(serverInfo); + const sentinelPut = sinon.stub(db.sentinel, 'put'); + sentinelPut.rejects({ status: 409 }); + sentinelPut.onCall(45).resolves(); + + const providedInfo = { + _id: 'some-info', + _rev: 1, + completed_tasks: { success: 'tasks', failure: 'othertasks' }, + }; + + return lib.saveCompletedTasks('some', providedInfo).then(() => { + assert.equal(db.sentinel.get.callCount, 46); + assert.equal(db.sentinel.put.callCount, 46); + assert.deepEqual(db.sentinel.put.args[45], [{ ...serverInfo, completed_tasks: providedInfo.completed_tasks }]); }); }); }); From 98e42e64b937719072712573f921ae28a5d5ec07 Mon Sep 17 00:00:00 2001 From: Diana Date: Fri, 5 May 2023 18:21:59 +0300 Subject: [PATCH 29/71] fix eslint --- sentinel/tests/unit/schedule/outbound.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/sentinel/tests/unit/schedule/outbound.js b/sentinel/tests/unit/schedule/outbound.js index 5ceee5e7d8e..b0fde702c92 100644 --- a/sentinel/tests/unit/schedule/outbound.js +++ b/sentinel/tests/unit/schedule/outbound.js @@ -319,7 +319,6 @@ describe('outbound schedule', () => { describe('single push', () => { let mapDocumentToPayload; let sentinelPut; - let sentinelGet; let send; let infodocSaveCompletedTasks; @@ -340,7 +339,6 @@ describe('outbound schedule', () => { restores.push(outbound.__set__('outbound', { send: send })); restores.push(outbound.__set__('infodocLib', { saveCompletedTasks: infodocSaveCompletedTasks })); sentinelPut = sinon.stub(db.sentinel, 'put'); - sentinelGet = sinon.stub(db.sentinel, 'get'); }); afterEach(() => restores.forEach(restore => restore())); From f4c797152f30f7a816d39eb5756e612c3beafbf8 Mon Sep 17 00:00:00 2001 From: Diana Date: Tue, 9 May 2023 16:48:00 +0300 Subject: [PATCH 30/71] add e2e test --- .../sentinel/schedules/outbound.spec.js | 66 ++++++++++++++----- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/tests/integration/sentinel/schedules/outbound.spec.js b/tests/integration/sentinel/schedules/outbound.spec.js index 760246a6b25..f701fb614a2 100644 --- a/tests/integration/sentinel/schedules/outbound.spec.js +++ b/tests/integration/sentinel/schedules/outbound.spec.js @@ -13,6 +13,16 @@ const outboundConfig = (port) => ({ }, relevant_to: 'false' }, + also_working: { + destination: { + base_url: utils.hostURL(port), + path: '/test-working' + }, + mapping: { + id: 'doc._id' + }, + relevant_to: 'false' + }, broken: { destination: { base_url: utils.hostURL(port), @@ -34,12 +44,12 @@ const tasks = [{ _id: `task:outbound:test-aaa`, type: 'task:outbound', doc_id: 'test-aaa', - queue: ['working', 'broken'], + queue: ['working', 'also_working', 'broken'], }, { _id: `task:outbound:test-zzz`, type: 'task:outbound', doc_id: 'test-zzz', - queue: ['working'], + queue: ['working', 'also_working'], }]; const express = require('express'); @@ -51,13 +61,11 @@ destinationApp.use(jsonParser); destinationApp.post('/test-working', (req, res) => inboxes.working.push(req.body) && res.send('true')); destinationApp.post('/test-broken', (req, res) => inboxes.broken.push(req.body) && res.status(500).end()); let server; +let port; -const waitForPushes = () => { +const waitForPushes = (expectedTasks = 1) => { return getTasks().then(result => { - // waiting for 1 task left should imply that the first task, which should stay because it points - // to a broken endpoint, has executed, since the second task has executed successfully and been - // deleted - if (result.rows.length === 1) { + if (result.rows.length === expectedTasks) { return; } return utils.delayPromise(waitForPushes, 100); @@ -75,7 +83,11 @@ const wipeTasks = () => { describe('Outbound', () => { before(() => { + // get a random port assigned. we will reuse this port when starting the server again. + // the known port is necessary for the outbound config server = destinationApp.listen(); + port = server.address().port; + server.close(); }); after(() => { @@ -85,28 +97,44 @@ describe('Outbound', () => { afterEach(() => utils.revertDb([], true).then(() => wipeTasks())); it('should find existing outbound tasks and execute them, leaving them if the send was unsuccessful', () => { + const settings = { + outbound: outboundConfig(port), + transitions: { + mark_for_outbound: true, + } + }; return utils - .updateSettings({ outbound: outboundConfig(server.address().port) }, true) - .then(() => utils.stopSentinel()) + .updateSettings(settings, 'sentinel') + .then(() => console.log('settings')) .then(() => utils.saveDocs(docs)) - .then(() => utils.sentinelDb.bulkDocs(tasks)) + // pushes will fail if destination server is not up, so tasks will get created + .then(() => waitForPushes(2)) + .then(() => console.log('wait')) + .then(() => utils.stopSentinel()) .then(() => utils.startSentinel()) + .then(() => server = destinationApp.listen(port)) // and they will generate tasks .then(() => console.log('Waiting for schedules')) - .then(() => waitForPushes()) + // waiting for 1 task left should imply that the first task, which should stay because it points + // to a broken endpoint, has executed, since the second task has executed successfully and been + // deleted + .then(() => waitForPushes(1)) .then(() => { - chai.expect(inboxes.working).to.have.lengthOf(2); + console.log(inboxes.working); + chai.expect(inboxes.working).to.have.lengthOf(4); chai.expect(inboxes.broken).to.have.lengthOf(1); chai.expect(inboxes.working).to.have.deep.members([ - {id: 'test-aaa'}, - {id: 'test-zzz'} + { id: 'test-aaa' }, + { id: 'test-aaa' }, + { id: 'test-zzz' }, + { id: 'test-zzz' }, ]); chai.expect(inboxes.broken).to.have.deep.members([ {id: 'test-aaa'} ]); }) - .then(() => utils.sentinelDb.allDocs({ keys: tasks.map(task => task._id), include_docs: true })) + .then(() => utils.sentinelDb.allDocs({ keys: docs.map(doc => `task:outbound:${doc._id}`), include_docs: true })) .then(tasksResult => { chai.expect(tasksResult.rows).to.have.lengthOf(2); chai.expect(tasksResult.rows[0].doc).to.deep.equal({ @@ -130,14 +158,18 @@ describe('Outbound', () => { type: 'info', doc_id: 'test-aaa', 'completed_tasks[0].type': 'outbound', - 'completed_tasks[0].name': 'working' + 'completed_tasks[0].name': 'working', + 'completed_tasks[1].type': 'outbound', + 'completed_tasks[1].name': 'also_working', }); chai.expect(infoDocs[1]).to.nested.include({ _id: 'test-zzz-info', type: 'info', doc_id: 'test-zzz', 'completed_tasks[0].type': 'outbound', - 'completed_tasks[0].name': 'working' + 'completed_tasks[0].name': 'working', + 'completed_tasks[1].type': 'outbound', + 'completed_tasks[1].name': 'also_working', }); }).catch(err => { // We don't really have a reliable way to know when these writes happen, because of how From 9fb80651f26661e3f5427d70bfa60a69d5c0e286 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Tue, 9 May 2023 20:14:57 +0300 Subject: [PATCH 31/71] update test --- .../sentinel/schedules/outbound.spec.js | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/integration/sentinel/schedules/outbound.spec.js b/tests/integration/sentinel/schedules/outbound.spec.js index f701fb614a2..f3cf3d44daf 100644 --- a/tests/integration/sentinel/schedules/outbound.spec.js +++ b/tests/integration/sentinel/schedules/outbound.spec.js @@ -11,7 +11,7 @@ const outboundConfig = (port) => ({ mapping: { id: 'doc._id' }, - relevant_to: 'false' + relevant_to: 'doc._id.startsWith("test")' }, also_working: { destination: { @@ -21,7 +21,7 @@ const outboundConfig = (port) => ({ mapping: { id: 'doc._id' }, - relevant_to: 'false' + relevant_to: 'doc._id.startsWith("test")' }, broken: { destination: { @@ -31,13 +31,13 @@ const outboundConfig = (port) => ({ mapping: { id: 'doc._id' }, - relevant_to: 'false' + relevant_to: 'doc._id.startsWith("test-aaa")' } }); const docs = [ - {_id: 'test-aaa'}, - {_id: 'test-zzz'} + { _id: 'test-aaa' }, + { _id: 'test-zzz' } ]; const tasks = [{ @@ -68,7 +68,7 @@ const waitForPushes = (expectedTasks = 1) => { if (result.rows.length === expectedTasks) { return; } - return utils.delayPromise(waitForPushes, 100); + return utils.delayPromise(() => waitForPushes(expectedTasks), 100); }); }; @@ -105,21 +105,17 @@ describe('Outbound', () => { }; return utils .updateSettings(settings, 'sentinel') - .then(() => console.log('settings')) .then(() => utils.saveDocs(docs)) // pushes will fail if destination server is not up, so tasks will get created .then(() => waitForPushes(2)) - .then(() => console.log('wait')) .then(() => utils.stopSentinel()) .then(() => utils.startSentinel()) .then(() => server = destinationApp.listen(port)) // and they will generate tasks - .then(() => console.log('Waiting for schedules')) // waiting for 1 task left should imply that the first task, which should stay because it points // to a broken endpoint, has executed, since the second task has executed successfully and been // deleted .then(() => waitForPushes(1)) .then(() => { - console.log(inboxes.working); chai.expect(inboxes.working).to.have.lengthOf(4); chai.expect(inboxes.broken).to.have.lengthOf(1); @@ -137,7 +133,7 @@ describe('Outbound', () => { .then(() => utils.sentinelDb.allDocs({ keys: docs.map(doc => `task:outbound:${doc._id}`), include_docs: true })) .then(tasksResult => { chai.expect(tasksResult.rows).to.have.lengthOf(2); - chai.expect(tasksResult.rows[0].doc).to.deep.equal({ + chai.expect(tasksResult.rows[0].doc).excluding('created').to.deep.equal({ _id: `task:outbound:test-aaa`, _rev: tasksResult.rows[0].doc._rev, type: 'task:outbound', From cddd945930c63fdceaea7a595d604e3117fc83e9 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Tue, 9 May 2023 20:27:19 +0300 Subject: [PATCH 32/71] fix eslint and start server before restarting sentinel. --- .../integration/sentinel/schedules/outbound.spec.js | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/integration/sentinel/schedules/outbound.spec.js b/tests/integration/sentinel/schedules/outbound.spec.js index f3cf3d44daf..df03b68f8d8 100644 --- a/tests/integration/sentinel/schedules/outbound.spec.js +++ b/tests/integration/sentinel/schedules/outbound.spec.js @@ -40,17 +40,6 @@ const docs = [ { _id: 'test-zzz' } ]; -const tasks = [{ - _id: `task:outbound:test-aaa`, - type: 'task:outbound', - doc_id: 'test-aaa', - queue: ['working', 'also_working', 'broken'], -}, { - _id: `task:outbound:test-zzz`, - type: 'task:outbound', - doc_id: 'test-zzz', - queue: ['working', 'also_working'], -}]; const express = require('express'); const bodyParser = require('body-parser'); @@ -109,8 +98,8 @@ describe('Outbound', () => { // pushes will fail if destination server is not up, so tasks will get created .then(() => waitForPushes(2)) .then(() => utils.stopSentinel()) - .then(() => utils.startSentinel()) .then(() => server = destinationApp.listen(port)) // and they will generate tasks + .then(() => utils.startSentinel()) // waiting for 1 task left should imply that the first task, which should stay because it points // to a broken endpoint, has executed, since the second task has executed successfully and been // deleted From ca1a6549d8ec4a009ba80f20ad3ac4dca6815342 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 10 May 2023 07:56:55 +0300 Subject: [PATCH 33/71] merge things. --- tests/e2e/default/db/db-sync.wdio-spec.js | 1 - tests/integration/infodocs/infodocs.spec.js | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/e2e/default/db/db-sync.wdio-spec.js b/tests/e2e/default/db/db-sync.wdio-spec.js index 417a5a3f14c..12037822b22 100644 --- a/tests/e2e/default/db/db-sync.wdio-spec.js +++ b/tests/e2e/default/db/db-sync.wdio-spec.js @@ -6,7 +6,6 @@ const loginPage = require('../../../page-objects/default/login/login.wdio.page') const reportsPage = require('../../../page-objects/default/reports/reports.wdio.page'); const chai = require('chai'); const uuid = require('uuid').v4; -const browserDbUtils = require('../../../utils/browser'); /* global window */ diff --git a/tests/integration/infodocs/infodocs.spec.js b/tests/integration/infodocs/infodocs.spec.js index c0970e3906f..1ca0069006a 100644 --- a/tests/integration/infodocs/infodocs.spec.js +++ b/tests/integration/infodocs/infodocs.spec.js @@ -10,7 +10,7 @@ const sentinelUtils = require('../../utils/sentinel'); // process. // /* eslint-disable no-console */ -const delayedInfoDocsOf = ids => sentinelUtils.waitForSentinel(ids).then(() => sentinelUtils.getInfoDocs(ids)); +const delayedInfoDocsOf = ids => sentinelUtils.waitForSentinel().then(() => sentinelUtils.getInfoDocs(ids)); describe('infodocs', () => { afterEach(() => utils.revertDb([], true)); From aa15d4afc3d1fd8ee019ebeda0790837c84ae408 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 10 May 2023 10:59:05 +0300 Subject: [PATCH 34/71] wait longer to sync passwords in cluster. --- tests/integration/api/controllers/users.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api/controllers/users.spec.js b/tests/integration/api/controllers/users.spec.js index 309b79ce3dd..d65b3837d4e 100644 --- a/tests/integration/api/controllers/users.spec.js +++ b/tests/integration/api/controllers/users.spec.js @@ -1157,7 +1157,7 @@ describe('Users API', () => { chai.expect(responseUser.token_login).to.have.keys('expiration_date'); }); - await utils.delayPromise(1000); + await utils.delayPromise(4000); for (const user of users) { let [userInDb, userSettings] = await Promise.all([getUser(user), getUserSettings(user)]); From 2f9f918ce50b0947a799a7c4ef00d641573566b8 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 10 May 2023 15:07:49 +0300 Subject: [PATCH 35/71] reorder refresh in protractor target test --- tests/e2e/protractor/targets/target-aggregates.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/protractor/targets/target-aggregates.spec.js b/tests/e2e/protractor/targets/target-aggregates.spec.js index 5e8f780eb07..7ca55168345 100644 --- a/tests/e2e/protractor/targets/target-aggregates.spec.js +++ b/tests/e2e/protractor/targets/target-aggregates.spec.js @@ -242,7 +242,7 @@ describe('Target aggregates', () => { '^target~' ]; - afterEach(() => utils.revertDb(DOCS_TO_KEEP)); + afterEach(() => utils.revertDb(DOCS_TO_KEEP, true).then(() => utils.refreshToGetNewSettings())); it('should display no data when no targets are uploaded', async () => { const targetsConfig = [ @@ -279,7 +279,7 @@ describe('Target aggregates', () => { } }); - xit('should display correct data', async () => { + it('should display correct data', async () => { const targetsConfig = [ { id: 'count_no_goal', type: 'count', title: genTitle('count no goal'), aggregate: true }, { id: 'count_with_goal', type: 'count', title: genTitle('count with goal'), goal: 20, aggregate: true }, From 8ed0fde5c26825f51e62c7d41fee5a132c712c9c Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Tue, 16 May 2023 13:40:35 +0300 Subject: [PATCH 36/71] reload session before asserting languages. --- tests/e2e/default/translations/enabled-languages.wdio-spec.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e/default/translations/enabled-languages.wdio-spec.js b/tests/e2e/default/translations/enabled-languages.wdio-spec.js index fae74e611fc..b81706dee65 100644 --- a/tests/e2e/default/translations/enabled-languages.wdio-spec.js +++ b/tests/e2e/default/translations/enabled-languages.wdio-spec.js @@ -21,7 +21,8 @@ describe('Enabling/disabling languages', () => { ], }; await utils.updateSettings(settings, true); - await browser.refresh(); + await browser.reloadSession(); + await browser.url('/'); }); after(async () => { From 0256ed62252fc5eadd8ad44f9e28c1789fc80a12 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Tue, 16 May 2023 21:50:24 +0300 Subject: [PATCH 37/71] call _sync_shards to sync password updates across the cluster --- tests/integration/api/controllers/users.spec.js | 10 +++++----- tests/utils.js | 5 +++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/integration/api/controllers/users.spec.js b/tests/integration/api/controllers/users.spec.js index d65b3837d4e..759cd3f3762 100644 --- a/tests/integration/api/controllers/users.spec.js +++ b/tests/integration/api/controllers/users.spec.js @@ -1045,7 +1045,7 @@ describe('Users API', () => { contact: { id: user.contact._id }, }))); - await utils.delayPromise(1000); + await utils.syncShards(); for (const user of users) { let [userInDb, userSettings] = await Promise.all([getUser(user), getUserSettings(user)]); @@ -1157,7 +1157,7 @@ describe('Users API', () => { chai.expect(responseUser.token_login).to.have.keys('expiration_date'); }); - await utils.delayPromise(4000); + await utils.syncShards(); for (const user of users) { let [userInDb, userSettings] = await Promise.all([getUser(user), getUserSettings(user)]); @@ -1335,7 +1335,7 @@ describe('Users API', () => { return expectSendableSms(loginTokenDoc); }) - .then(() => utils.delayPromise(1000)) + .then(() => utils.syncShards()) .then(() => expectPasswordLoginToFail(user)) .then(() => expectTokenLoginToSucceed(tokenUrl)) .then(() => Promise.all([ getUser(user), getUserSettings(user) ])) @@ -1414,7 +1414,7 @@ describe('Users API', () => { return expectSendableSms(loginTokenDoc); }) - .then(() => utils.delayPromise(1000)) + .then(() => utils.syncShards()) .then(() => expectPasswordLoginToFail(user)) .then(() => expectTokenLoginToSucceed(tokenUrl)) .then(() => Promise.all([ getUser(user), getUserSettings(user) ])) @@ -1450,7 +1450,7 @@ describe('Users API', () => { }) .then(response => { chai.expect(response.token_login).to.be.undefined; - return utils.delayPromise(1000); + return utils.syncShards(); }) .then(() => expectPasswordLoginToFail(user)) .then(() => Promise.all([ getUser(user), getUserSettings(user) ])) diff --git a/tests/utils.js b/tests/utils.js index 450477c1218..8048ef68761 100644 --- a/tests/utils.js +++ b/tests/utils.js @@ -882,6 +882,10 @@ const formDocProcessing = async (docs) => { }; }; +const syncShards = async () => { + await request({ path: '/medic/_sync_shards', method: 'POST' }); +}; + module.exports = { hostURL, parseCookieResponse, @@ -1400,6 +1404,7 @@ module.exports = { makeTempDir, SW_SUCCESSFUL_REGEX: /Service worker generated successfully/, updatePermissions, + syncShards, ONE_YEAR_IN_S, }; From b7c7b6dcee5a697cf3f581ecee33850f3975bf19 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 17 May 2023 06:11:00 +0300 Subject: [PATCH 38/71] wait after starting haproxy --- tests/integration/api/server.spec.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/api/server.spec.js b/tests/integration/api/server.spec.js index 78b701ce087..7fab7213275 100644 --- a/tests/integration/api/server.spec.js +++ b/tests/integration/api/server.spec.js @@ -185,6 +185,7 @@ describe('server', () => { it('should respond to changes even after services are restarted', async () => { await utils.stopHaproxy(); // this will also crash API await utils.startHaproxy(); + await utils.delayPromise(1000); await utils.listenForApi(); const forms = await utils.db.allDocs({ @@ -207,6 +208,7 @@ describe('server', () => { await utils.stopHaproxy(); await utils.stopApi(); await utils.startHaproxy(); + await utils.delayPromise(1000); await utils.startApi(); await utils.request('/'); @@ -215,6 +217,7 @@ describe('server', () => { await utils.stopApi(); await utils.startApi(false); await utils.startHaproxy(); + await utils.delayPromise(1000); await utils.listenForApi(); }); From f19f32f3c01d2a9464e1bf212d2af084531d8c4c Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 17 May 2023 06:40:48 +0300 Subject: [PATCH 39/71] wait for 10s for logs (in case many forms are pushed) --- tests/utils.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils.js b/tests/utils.js index 8048ef68761..7ba51d230bd 100644 --- a/tests/utils.js +++ b/tests/utils.js @@ -569,7 +569,7 @@ const waitForDockerLogs = (container, ...regex) => { // It takes a while until the process actually starts tailing logs, and initiating next test steps immediately // after watching results in a race condition, where the log is created before watching started. // As a fix, watch the logs with tail=1, so we always receive one log line immediately, then proceed with next - // steps of testing afterwards. + // steps of testing afterward. const params = `logs ${container} -f --tail=1`; const proc = spawn('docker', params.split(' '), { stdio: ['ignore', 'pipe', 'pipe'] }); let receivedFirstLine; @@ -580,7 +580,7 @@ const waitForDockerLogs = (container, ...regex) => { console.log('Found logs', logs, 'watched for', ...regex); reject(new Error('Timed out looking for details in logs.')); killSpawnedProcess(proc); - }, 6000); + }, 10000); const checkOutput = (data) => { if (!firstLine) { From db20407fb509d9e0406d8aa02c080638ae1cab10 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 17 May 2023 07:28:23 +0300 Subject: [PATCH 40/71] wait for 10s for logs (in case many forms are pushed) --- tests/integration/api/server.spec.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/api/server.spec.js b/tests/integration/api/server.spec.js index 7fab7213275..ed0e4e8ce26 100644 --- a/tests/integration/api/server.spec.js +++ b/tests/integration/api/server.spec.js @@ -185,8 +185,8 @@ describe('server', () => { it('should respond to changes even after services are restarted', async () => { await utils.stopHaproxy(); // this will also crash API await utils.startHaproxy(); - await utils.delayPromise(1000); await utils.listenForApi(); + await utils.delayPromise(1000); const forms = await utils.db.allDocs({ start_key: 'form:', @@ -208,9 +208,10 @@ describe('server', () => { await utils.stopHaproxy(); await utils.stopApi(); await utils.startHaproxy(); - await utils.delayPromise(1000); await utils.startApi(); + await utils.delayPromise(1000); + await utils.request('/'); await utils.stopHaproxy(); From 4dc5f0f5d133b03dd0be6f68232013b3c7ef2ae1 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 17 May 2023 08:01:23 +0300 Subject: [PATCH 41/71] wait for 10s instead of 5. --- tests/integration/transitions/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/transitions/utils.js b/tests/integration/transitions/utils.js index c6dd59f46bf..4d679f0ffcc 100644 --- a/tests/integration/transitions/utils.js +++ b/tests/integration/transitions/utils.js @@ -14,7 +14,7 @@ const getApiSmsChanges = (messages) => { const timeout = setTimeout(() => { listener.cancel(); reject('timer expired'); - }, 5000); + }, 10000); listener.on('change', change => { if (change.doc.sms_message) { if (ids.includes(change.id)) { From 27894840e2fecd33dd289348642932fe0b981cb2 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 5 Jul 2023 09:59:38 +0300 Subject: [PATCH 42/71] more queue time --- tests/integration/sentinel/queue.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/sentinel/queue.spec.js b/tests/integration/sentinel/queue.spec.js index cc008897d95..d43afd16b80 100644 --- a/tests/integration/sentinel/queue.spec.js +++ b/tests/integration/sentinel/queue.spec.js @@ -117,7 +117,7 @@ describe('Sentinel queue drain', () => { expect(tombstone.tombstone).to.have.property('type', 'data_record'); }); }); - }).timeout(300 * 1000); + }).timeout(400 * 1000); it('queue should work after restarting haproxy', async () => { await utils.stopHaproxy(); // this will also crash Sentinel and API From c953ee306b1d99296d9b9ddf6b8c8be7558b350f Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 6 Jul 2023 07:41:03 +0300 Subject: [PATCH 43/71] fix bad merge --- tests/utils/index.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/utils/index.js b/tests/utils/index.js index 9744a4d85e0..016cf059619 100644 --- a/tests/utils/index.js +++ b/tests/utils/index.js @@ -1268,6 +1268,4 @@ module.exports = { apiLogTestEnd, updateContainerNames, updatePermissions, - - ONE_YEAR_IN_S, }; From 5c05230f593abac893533c66e6cab80911c021bb Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 6 Jul 2023 16:40:36 +0300 Subject: [PATCH 44/71] add delay before pushing a new doc --- tests/e2e/default/users/create-meta-db.wdio-spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/e2e/default/users/create-meta-db.wdio-spec.js b/tests/e2e/default/users/create-meta-db.wdio-spec.js index 57cf22e0f5e..26b8e268b56 100644 --- a/tests/e2e/default/users/create-meta-db.wdio-spec.js +++ b/tests/e2e/default/users/create-meta-db.wdio-spec.js @@ -28,6 +28,7 @@ describe('Create user meta db : ', () => { await commonElements.logout(); await loginPage.login({ username, password }); await commonElements.waitForPageLoaded(); + await commonElements.goToReports(); const doc = { _id: uuid() }; await utils.requestOnTestMetaDb(_.defaults({ method: 'POST', body: doc }, options)); From 5a1c00b79f51acd3aed577a5435a77a7ddaa03ed Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sun, 9 Jul 2023 11:16:43 +0300 Subject: [PATCH 45/71] revert unwanted changes --- api/src/services/config-watcher.js | 1 + haproxy/Dockerfile | 2 +- shared-libs/infodoc/src/infodoc.js | 1 - tests/integration/cht-conf/cht-conf-actions.spec.js | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/services/config-watcher.js b/api/src/services/config-watcher.js index 78bbc2ec78a..d99d5cf03f8 100644 --- a/api/src/services/config-watcher.js +++ b/api/src/services/config-watcher.js @@ -147,6 +147,7 @@ const listen = () => { db.medic .changes({ live: true, since: 'now', return_docs: false }) .on('change', change => { + if (tombstoneUtils.isTombstoneId(change.id)) { return Promise.resolve(); } diff --git a/haproxy/Dockerfile b/haproxy/Dockerfile index 5c0cd31e73e..3f7e71f9571 100644 --- a/haproxy/Dockerfile +++ b/haproxy/Dockerfile @@ -1,4 +1,4 @@ -FROM haproxy:2.7 +FROM haproxy:2.6 USER root RUN apt-get update && apt-get install luarocks gettext jq curl -y diff --git a/shared-libs/infodoc/src/infodoc.js b/shared-libs/infodoc/src/infodoc.js index 7405f76f8d3..1c2844842b4 100644 --- a/shared-libs/infodoc/src/infodoc.js +++ b/shared-libs/infodoc/src/infodoc.js @@ -181,7 +181,6 @@ const saveProperty = async (id, infodoc, property, defaultValue = {}) => { } }; - const bulkUpdate = infoDocs => { if (!infoDocs || !infoDocs.length) { return Promise.resolve(); diff --git a/tests/integration/cht-conf/cht-conf-actions.spec.js b/tests/integration/cht-conf/cht-conf-actions.spec.js index f92524b9a99..2b743811e01 100644 --- a/tests/integration/cht-conf/cht-conf-actions.spec.js +++ b/tests/integration/cht-conf/cht-conf-actions.spec.js @@ -50,7 +50,7 @@ describe('cht-conf actions tests', () => { it('should upload branding', async () => { const branding = await utils.getDoc('branding').catch(error => { - if(error){ + if (error) { console.err(error); } }); From 13e4eaf6d60a81fb692b709afdcfb53385ce0e10 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sun, 9 Jul 2023 11:18:37 +0300 Subject: [PATCH 46/71] revert unwanted changes --- .../client-side-transitions/create-user-for-contacts.wdio-spec.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/e2e/default/client-side-transitions/create-user-for-contacts.wdio-spec.js diff --git a/tests/e2e/default/client-side-transitions/create-user-for-contacts.wdio-spec.js b/tests/e2e/default/client-side-transitions/create-user-for-contacts.wdio-spec.js deleted file mode 100644 index e69de29bb2d..00000000000 From 16dcc3d392c65cd47d4ae5a101c8724b28e30660 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sun, 9 Jul 2023 11:32:50 +0300 Subject: [PATCH 47/71] revert unwanted changes --- tests/integration/.mocharc.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/.mocharc.js b/tests/integration/.mocharc.js index a85c408fb5b..c5a5f243aaf 100644 --- a/tests/integration/.mocharc.js +++ b/tests/integration/.mocharc.js @@ -6,8 +6,6 @@ chai.use(chaiExclude); chai.use(chaiAsPromised); global.expect = chai.expect; -global.expect = chai.expect; - module.exports = { allowUncaught: false, color: true, From 2cbf56bc53af2ee56cecd2df69e750701f524a67 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sun, 9 Jul 2023 12:32:48 +0300 Subject: [PATCH 48/71] pretty paths in new e2e test --- .../db/db-access-for-custom-roles.spec.js | 20 ++++++++++--------- .../default/users/create-meta-db.wdio-spec.js | 10 +++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/e2e/default/db/db-access-for-custom-roles.spec.js b/tests/e2e/default/db/db-access-for-custom-roles.spec.js index 76f83a4cc06..1eeed0b0001 100644 --- a/tests/e2e/default/db/db-access-for-custom-roles.spec.js +++ b/tests/e2e/default/db/db-access-for-custom-roles.spec.js @@ -1,11 +1,13 @@ -const commonElements = require('../../../page-objects/default/common/common.wdio.page'); -const utils = require('../../../utils'); -const browserDbUtils = require('../../../utils/browser'); -const loginPage = require('../../../page-objects/default/login/login.wdio.page'); const uuid = require('uuid').v4; -const personFactory = require('../../../factories/cht/contacts/person'); -const place = require('../../../factories/cht/contacts/place'); -const places = place.generateHierarchy(); + +const utils = require('@utils'); +const commonPage = require('@page-objects/default/common/common.wdio.page'); +const browserDbUtils = require('@utils/cht-db'); +const loginPage = require('@page-objects/default/login/login.wdio.page'); +const personFactory = require('@factories/cht/contacts/person'); +const placeFactory = require('@factories/cht/contacts/place'); + +const places = placeFactory.generateHierarchy(); const clinic = places.get('clinic'); const contact = personFactory.build( @@ -56,7 +58,7 @@ describe('Database access for new roles', () => { fields: { patient_id: contact._id, }, }; await browserDbUtils.createDoc(report); - await commonElements.sync(); + await commonPage.sync(); await utils.get(report._id); }); @@ -70,7 +72,7 @@ describe('Database access for new roles', () => { fields: { patient_id: contact._id, }, }; await utils.saveDoc(report); - await commonElements.sync(); + await commonPage.sync(); await browserDbUtils.getDoc(report._id); }); }); diff --git a/tests/e2e/default/users/create-meta-db.wdio-spec.js b/tests/e2e/default/users/create-meta-db.wdio-spec.js index 26b8e268b56..3fc07fcedc0 100644 --- a/tests/e2e/default/users/create-meta-db.wdio-spec.js +++ b/tests/e2e/default/users/create-meta-db.wdio-spec.js @@ -1,7 +1,7 @@ const _ = require('lodash'); const utils = require('@utils'); const usersPage = require('@page-objects/default/users/user.wdio.page'); -const commonElements = require('@page-objects/default/common/common.wdio.page'); +const commonPage = require('@page-objects/default/common/common.wdio.page'); const loginPage = require('@page-objects/default/login/login.wdio.page'); const uuid = require('uuid').v4; @@ -24,11 +24,11 @@ describe('Create user meta db : ', () => { await usersPage.openAddUserDialog(); await usersPage.inputAddUserFields(username, fullName, 'program_officer', '', '', password); await usersPage.saveUser(); - await commonElements.goToMessages(); - await commonElements.logout(); + await commonPage.goToMessages(); + await commonPage.logout(); await loginPage.login({ username, password }); - await commonElements.waitForPageLoaded(); - await commonElements.goToReports(); + await commonPage.waitForPageLoaded(); + await commonPage.goToReports(); const doc = { _id: uuid() }; await utils.requestOnTestMetaDb(_.defaults({ method: 'POST', body: doc }, options)); From 3e02a4bc53c65ebdc9efd694d0d628d099f0f801 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sun, 9 Jul 2023 18:02:44 +0300 Subject: [PATCH 49/71] we rolled back time, telemetry date is supposed to be yesterday --- tests/e2e/default/telemetry/telemetry.wdio-spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/default/telemetry/telemetry.wdio-spec.js b/tests/e2e/default/telemetry/telemetry.wdio-spec.js index 9513650a835..785b180cac9 100644 --- a/tests/e2e/default/telemetry/telemetry.wdio-spec.js +++ b/tests/e2e/default/telemetry/telemetry.wdio-spec.js @@ -70,7 +70,7 @@ describe('Telemetry', () => { expect(telemetryEntry.doc).to.deep.nested.include({ 'metadata.year': moment().year(), 'metadata.month': moment().month() + 1, - 'metadata.day': moment().day() + 1, + 'metadata.day': moment().date() - 1, 'metadata.user': user.username, 'metadata.versions.app': clientDdoc.build_info.version, }); From ad51a7bbbe589e5d43993d1b4ebfbffdee97c417 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Tue, 11 Jul 2023 16:39:39 +0300 Subject: [PATCH 50/71] couchdb is supposed to be faster ffs --- tests/integration/infodocs/infodocs.spec.js | 2 +- tests/integration/sentinel/queue.spec.js | 2 +- tests/integration/transitions/utils.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/infodocs/infodocs.spec.js b/tests/integration/infodocs/infodocs.spec.js index 2b51d92c5f5..cf290db83a3 100644 --- a/tests/integration/infodocs/infodocs.spec.js +++ b/tests/integration/infodocs/infodocs.spec.js @@ -10,7 +10,7 @@ const sentinelUtils = require('@utils/sentinel'); // process. // /* eslint-disable no-console */ -const delayedInfoDocsOf = ids => sentinelUtils.waitForSentinel().then(() => sentinelUtils.getInfoDocs(ids)); +const delayedInfoDocsOf = ids => sentinelUtils.waitForSentinel(ids).then(() => sentinelUtils.getInfoDocs(ids)); describe('infodocs', () => { afterEach(() => utils.revertDb([], true)); diff --git a/tests/integration/sentinel/queue.spec.js b/tests/integration/sentinel/queue.spec.js index d43afd16b80..cc008897d95 100644 --- a/tests/integration/sentinel/queue.spec.js +++ b/tests/integration/sentinel/queue.spec.js @@ -117,7 +117,7 @@ describe('Sentinel queue drain', () => { expect(tombstone.tombstone).to.have.property('type', 'data_record'); }); }); - }).timeout(400 * 1000); + }).timeout(300 * 1000); it('queue should work after restarting haproxy', async () => { await utils.stopHaproxy(); // this will also crash Sentinel and API diff --git a/tests/integration/transitions/utils.js b/tests/integration/transitions/utils.js index f7bf7c33143..5f54d3f0ef4 100644 --- a/tests/integration/transitions/utils.js +++ b/tests/integration/transitions/utils.js @@ -14,7 +14,7 @@ const getApiSmsChanges = (messages) => { const timeout = setTimeout(() => { listener.cancel(); reject('timer expired'); - }, 10000); + }, 5000); listener.on('change', change => { if (change.doc.sms_message) { if (ids.includes(change.id)) { From 76afdf339174d87bc130c8d73409b2064abda5e0 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 12 Jul 2023 23:27:39 +0300 Subject: [PATCH 51/71] it's actually slower --- tests/integration/transitions/utils.js | 2 +- tests/utils/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/transitions/utils.js b/tests/integration/transitions/utils.js index 5f54d3f0ef4..f7bf7c33143 100644 --- a/tests/integration/transitions/utils.js +++ b/tests/integration/transitions/utils.js @@ -14,7 +14,7 @@ const getApiSmsChanges = (messages) => { const timeout = setTimeout(() => { listener.cancel(); reject('timer expired'); - }, 5000); + }, 10000); listener.on('change', change => { if (change.doc.sms_message) { if (ids.includes(change.id)) { diff --git a/tests/utils/index.js b/tests/utils/index.js index 4a89f314c09..4d1bbb66dc3 100644 --- a/tests/utils/index.js +++ b/tests/utils/index.js @@ -1072,7 +1072,7 @@ const waitForDockerLogs = (container, ...regex) => { console.log('Found logs', logs, 'watched for', ...regex); reject(new Error('Timed out looking for details in logs.')); killSpawnedProcess(proc); - }, 6000); + }, 10000); const checkOutput = (data) => { if (!firstLine) { From 6dfbb37ea0462ad98db9749cb76a2556f7c5197e Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Thu, 13 Jul 2023 07:43:02 +0300 Subject: [PATCH 52/71] it's actually slower --- tests/integration/sentinel/queue.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/sentinel/queue.spec.js b/tests/integration/sentinel/queue.spec.js index cc008897d95..d43afd16b80 100644 --- a/tests/integration/sentinel/queue.spec.js +++ b/tests/integration/sentinel/queue.spec.js @@ -117,7 +117,7 @@ describe('Sentinel queue drain', () => { expect(tombstone.tombstone).to.have.property('type', 'data_record'); }); }); - }).timeout(300 * 1000); + }).timeout(400 * 1000); it('queue should work after restarting haproxy', async () => { await utils.stopHaproxy(); // this will also crash Sentinel and API From 85047600ae45b3b3ba0905bfff05889d1013c19e Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Tue, 18 Jul 2023 18:39:00 +0300 Subject: [PATCH 53/71] review feedback --- api/src/db.js | 18 +++++---- api/tests/mocha/db.spec.js | 69 ++++++++++++++++++++++++++++------- couchdb/10-docker-default.ini | 2 +- couchdb/Dockerfile | 2 +- 4 files changed, 68 insertions(+), 23 deletions(-) diff --git a/api/src/db.js b/api/src/db.js index 88f5798f810..f730fc5f29a 100644 --- a/api/src/db.js +++ b/api/src/db.js @@ -165,14 +165,14 @@ if (UNIT_TEST_ENV) { throw new Error(`Error while saving docs: ${errors.join(', ')}`); }; - const DEFAULT_SECURITY_STRUCTURE = { + const getDefaultSecurityStructure = () => ({ names: [], roles: [], - }; + }); - module.exports.addRoleToSecurity = async (dbname, role, addAsAdmin) => { + const addRoleToSecurity = async (dbname, role, addAsAdmin) => { if (!dbname || !role) { - return; + throw new Error(`Cannot add security: invalid db name ${dbname} or role ${role}`); } const securityUrl = new URL(environment.serverUrl); @@ -180,12 +180,14 @@ if (UNIT_TEST_ENV) { const securityObject = await rpn.get({ url: securityUrl.toString(), json: true }); const property = addAsAdmin ? 'admins' : 'members'; + if (!securityObject[property]) { - securityObject[property] = DEFAULT_SECURITY_STRUCTURE; + securityObject[property] = getDefaultSecurityStructure(); } + console.log(securityObject); - if (!securityObject[property].roles) { - throw new Error('Invalid database security %o', securityObject); + if (!securityObject[property].roles || !Array.isArray(securityObject[property].roles)) { + securityObject[property].roles = []; } if (securityObject[property].roles.includes(role)) { @@ -197,4 +199,6 @@ if (UNIT_TEST_ENV) { await rpn.put({ url: securityUrl.toString(), json: true, body: securityObject }); }; + module.exports.addRoleAsAdmin = (dbname, role) => addRoleToSecurity(dbname, role, true); + module.exports.addRoleAsMember = (dbname, role) => addRoleToSecurity(dbname, role, false); } diff --git a/api/tests/mocha/db.spec.js b/api/tests/mocha/db.spec.js index 2a5a8cc1eee..ab49f34e08b 100644 --- a/api/tests/mocha/db.spec.js +++ b/api/tests/mocha/db.spec.js @@ -3,6 +3,7 @@ require('chai').use(require('chai-as-promised')); const { expect } = require('chai'); const rewire = require('rewire'); const rpn = require('request-promise-native'); +process.env.UNIT_TEST_ENV = 1; let db; let unitTestEnv; @@ -211,7 +212,7 @@ describe('db', () => { sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] }, members: { roles: ['role2'] } }); sinon.stub(rpn, 'put').resolves(); - await db.addRoleToSecurity('dbname', 'rolename'); + await db.addRoleAsMember('dbname', 'rolename'); expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); expect(rpn.put.args).to.deep.equal([[ @@ -231,7 +232,7 @@ describe('db', () => { sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] }, members: { roles: ['role2'] } }); sinon.stub(rpn, 'put').resolves(); - await db.addRoleToSecurity('dbname', 'rolename', true); + await db.addRoleAsAdmin('dbname', 'rolename'); expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); expect(rpn.put.args).to.deep.equal([[ @@ -251,7 +252,7 @@ describe('db', () => { sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] }, members: { roles: ['role2'] } }); sinon.stub(rpn, 'put').resolves(); - await db.addRoleToSecurity('dbname', 'role2'); + await db.addRoleAsMember('dbname', 'role2'); expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); expect(rpn.put.called).to.equal(false); @@ -262,7 +263,7 @@ describe('db', () => { sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] }, members: { roles: ['role2'] } }); sinon.stub(rpn, 'put').resolves(); - await db.addRoleToSecurity('dbname', 'role1', true); + await db.addRoleAsAdmin('dbname', 'role1'); expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); expect(rpn.put.called).to.equal(false); @@ -273,7 +274,7 @@ describe('db', () => { sinon.stub(rpn, 'get').resolves({ admins: { roles: ['role1'] } }); sinon.stub(rpn, 'put').resolves(); - await db.addRoleToSecurity('dbname', 'rolename'); + await db.addRoleAsMember('dbname', 'rolename'); expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pass@couchdb:5984/dbname/_security', json: true } ]]); expect(rpn.put.args).to.deep.equal([[ @@ -288,12 +289,39 @@ describe('db', () => { ]]); }); + it('should not mutate default roles', async () => { + sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); + sinon.stub(rpn, 'get').callsFake(() => ({ members: {} })); + sinon.stub(rpn, 'put').resolves(); + + await db.addRoleAsMember('dbname1', 'rolename1'); + await db.addRoleAsMember('dbname2', 'rolename2'); + + expect(rpn.put.args).to.deep.equal([[ + { + url: 'http://admin:pass@couchdb:5984/dbname1/_security', + json: true, + body: { + members: { roles: ['rolename1'] }, + } + } + ], [ + { + url: 'http://admin:pass@couchdb:5984/dbname2/_security', + json: true, + body: { + members: { roles: ['rolename2'] }, + } + } + ]]); + }); + it('should set admins security property if not existing', async () => { sinon.stub(env, 'serverUrl').get(() => 'http://admin:pwd@host:6984'); sinon.stub(rpn, 'get').resolves({ members: { roles: ['role2'] } }); sinon.stub(rpn, 'put').resolves(); - await db.addRoleToSecurity('name', 'arole', true); + await db.addRoleAsAdmin('name', 'arole'); expect(rpn.get.args).to.deep.equal([[ { url: 'http://admin:pwd@host:6984/name/_security', json: true } ]]); expect(rpn.put.args).to.deep.equal([[ @@ -312,7 +340,7 @@ describe('db', () => { sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); sinon.stub(rpn, 'get').rejects(new Error('not_found')); - await expect(db.addRoleToSecurity('data', 'attr')).to.be.rejectedWith(Error, 'not_found'); + await expect(db.addRoleAsMember('data', 'attr')).to.be.rejectedWith(Error, 'not_found'); }); it('should throw put security errors', async () => { @@ -320,19 +348,32 @@ describe('db', () => { sinon.stub(rpn, 'get').resolves({ members: { roles: ['role2'] } }); sinon.stub(rpn, 'put').rejects(new Error('forbidden or something')); - await expect(db.addRoleToSecurity('data', 'attr')).to.be.rejectedWith(Error, 'forbidden or something'); + await expect(db.addRoleAsMember('data', 'attr')).to.be.rejectedWith(Error, 'forbidden or something'); }); - it('should throw error when security is invalid', async () => { + it('should set default security when security is invalid', async () => { sinon.stub(env, 'serverUrl').get(() => 'http://admin:pass@couchdb:5984'); - sinon.stub(rpn, 'get').resolves({ members: 'this is actually a string' }); + sinon.stub(rpn, 'get').resolves({ members: false }); + sinon.stub(rpn, 'put').resolves(); + + await db.addRoleAsMember('data', 'attr'); + expect(rpn.put.args).to.deep.equal([[ + { + url: 'http://admin:pass@couchdb:5984/data/_security', + json: true, + body: { + members: { roles: ['attr'], names: [], }, + } + } + ]]); - await expect(db.addRoleToSecurity('data', 'attr')).to.be.rejectedWith(Error, 'Invalid database security'); }); - it('should skip when missing dbname or role', async () => { - await db.addRoleToSecurity('', 'arole'); - await db.addRoleToSecurity('dbanme', ); + it('should throw when missing dbname or role', async () => { + await expect(db.addRoleAsMember('', 'arole')) + .to.be.rejectedWith(Error, `Cannot add security: invalid db name or role arole`); + await expect(db.addRoleAsMember('dbanme', '')) + .to.be.rejectedWith(Error, `Cannot add security: invalid db name dbanme or role`); }); }); }); diff --git a/couchdb/10-docker-default.ini b/couchdb/10-docker-default.ini index c397a502c4d..81bda3f518b 100644 --- a/couchdb/10-docker-default.ini +++ b/couchdb/10-docker-default.ini @@ -1,7 +1,7 @@ ; couchdb/local.d/package.ini [fabric] -request_timeout = 31536000 ; 1 year +request_timeout = 3600 ; 1 hour in seconds [query_server_config] os_process_limit = 1000 diff --git a/couchdb/Dockerfile b/couchdb/Dockerfile index ee4a8014ad4..018a2c13c80 100644 --- a/couchdb/Dockerfile +++ b/couchdb/Dockerfile @@ -1,4 +1,4 @@ -FROM couchdb:3.3.1 as base_couchdb_build +FROM couchdb:3.3.2 as base_couchdb_build # Add configuration COPY --chown=couchdb:couchdb 10-docker-default.ini /opt/couchdb/etc/default.d/ From bf8ab8ed43f4d42577bb3073777da46e9dac47d2 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Tue, 18 Jul 2023 20:11:42 +0300 Subject: [PATCH 54/71] fix eslint --- api/src/db.js | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/db.js b/api/src/db.js index f730fc5f29a..67aac9ebde0 100644 --- a/api/src/db.js +++ b/api/src/db.js @@ -184,7 +184,6 @@ if (UNIT_TEST_ENV) { if (!securityObject[property]) { securityObject[property] = getDefaultSecurityStructure(); } - console.log(securityObject); if (!securityObject[property].roles || !Array.isArray(securityObject[property].roles)) { securityObject[property].roles = []; From f78fb3e781ea5f14fa67a267adc419358f56269f Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 19 Jul 2023 08:41:21 +0300 Subject: [PATCH 55/71] fix bad function reference --- api/src/db.js | 3 ++- api/src/migrations/add-national_admin-role.js | 4 ++-- api/src/services/config-watcher.js | 2 +- api/tests/mocha/services/config-watcher.spec.js | 16 ++++++++-------- couchdb/10-docker-default.ini | 2 +- package.json | 1 + 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/api/src/db.js b/api/src/db.js index 67aac9ebde0..89dc303444d 100644 --- a/api/src/db.js +++ b/api/src/db.js @@ -42,7 +42,8 @@ if (UNIT_TEST_ENV) { 'activeTasks', 'saveDocs', 'createVault', - 'addRoleToSecurity', + 'addRoleAsAdmin', + 'addRoleAsMember', ]; const notStubbed = (first, second) => { diff --git a/api/src/migrations/add-national_admin-role.js b/api/src/migrations/add-national_admin-role.js index a35ab4fbe1e..7eb43a9c247 100644 --- a/api/src/migrations/add-national_admin-role.js +++ b/api/src/migrations/add-national_admin-role.js @@ -7,7 +7,7 @@ module.exports = { created: new Date(2017, 3, 30), run: () => { return Promise.resolve() - .then(() => db.addRoleToSecurity('_users', 'national_admin')) - .then(() => db.addRoleToSecurity(environment.db, 'national_admin')); + .then(() => db.addRoleAsMember('_users', 'national_admin')) + .then(() => db.addRoleAsMember(environment.db, 'national_admin')); } }; diff --git a/api/src/services/config-watcher.js b/api/src/services/config-watcher.js index d99d5cf03f8..2eda3f5ec86 100644 --- a/api/src/services/config-watcher.js +++ b/api/src/services/config-watcher.js @@ -189,7 +189,7 @@ const addUserRolesToDb = async () => { } for (const role of Object.keys(roles)) { - await db.addRoleToSecurity(environment.db, role, false); + await db.addRoleAsMember(environment.db, role); } }; diff --git a/api/tests/mocha/services/config-watcher.spec.js b/api/tests/mocha/services/config-watcher.spec.js index 24ac930ef6c..f7eff615b8a 100644 --- a/api/tests/mocha/services/config-watcher.spec.js +++ b/api/tests/mocha/services/config-watcher.spec.js @@ -228,7 +228,7 @@ describe('Configuration', () => { it('reloads settings settings doc is updated', () => { settingsService.update.resolves(); settingsService.get.resolves({ settings: 'yes' }); - sinon.stub(db, 'addRoleToSecurity'); + sinon.stub(db, 'addRoleAsMember'); sinon.stub(config, 'get').withArgs('roles').returns({ chw: {} }); sinon.stub(environment, 'db').get(() => 'medicdb'); @@ -238,14 +238,14 @@ describe('Configuration', () => { chai.expect(config.set.callCount).to.equal(1); chai.expect(config.set.args[0]).to.deep.equal([{ settings: 'yes' }]); chai.expect(config.get.withArgs('roles').callCount).to.equal(1); - chai.expect(db.addRoleToSecurity.args).to.deep.equal([['medicdb', 'chw', false]]); + chai.expect(db.addRoleAsMember.args).to.deep.equal([['medicdb', 'chw']]); }); }); it('should add all configured user roles to the main database', () => { settingsService.update.resolves(); settingsService.get.resolves({ settings: 'yes' }); - sinon.stub(db, 'addRoleToSecurity'); + sinon.stub(db, 'addRoleAsMember'); sinon.stub(config, 'get') .withArgs('roles') .returns({ @@ -262,11 +262,11 @@ describe('Configuration', () => { chai.expect(config.set.callCount).to.equal(1); chai.expect(config.set.args[0]).to.deep.equal([{ settings: 'yes' }]); chai.expect(config.get.withArgs('roles').callCount).to.equal(1); - chai.expect(db.addRoleToSecurity.args).to.deep.equal([ - ['medicdb', 'chw1', false], - ['medicdb', 'chw2', false], - ['medicdb', 'chw3', false], - ['medicdb', 'chw4', false], + chai.expect(db.addRoleAsMember.args).to.deep.equal([ + ['medicdb', 'chw1'], + ['medicdb', 'chw2'], + ['medicdb', 'chw3'], + ['medicdb', 'chw4'], ]); }); }); diff --git a/couchdb/10-docker-default.ini b/couchdb/10-docker-default.ini index 81bda3f518b..f61e6758b32 100644 --- a/couchdb/10-docker-default.ini +++ b/couchdb/10-docker-default.ini @@ -1,7 +1,7 @@ ; couchdb/local.d/package.ini [fabric] -request_timeout = 3600 ; 1 hour in seconds +request_timeout = 31536000 ; 1 year in seconds [query_server_config] os_process_limit = 1000 diff --git a/package.json b/package.json index bee318bd3e8..16b39ff6b0c 100755 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "standard-wdio-local": "export VERSION=$(node ./scripts/build/get-version.js) && grunt e2e-env-setup && wdio run ./tests/e2e/standard/wdio.conf.js", "default-wdio-mobile-local": "export VERSION=$(node ./scripts/build/get-version.js) && grunt e2e-env-setup && wdio run ./tests/e2e/default-mobile/wdio.conf.js", "e2e-integration": "mocha --config tests/integration/.mocharc.js ", + "e2e-integration-local": "export VERSION=$(node ./scripts/build/get-version.js) && grunt e2e-env-setup && mocha --config tests/integration/.mocharc.js ", "wdio": "wdio run ./tests/e2e/default/wdio.conf.js", "upgrade-wdio": "wdio run ./tests/e2e/upgrade/wdio.conf.js", "standard-wdio": "wdio run ./tests/e2e/standard/wdio.conf.js", From f4f719d799fd97e6f30664c91821cfebf7e304cb Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 26 Jul 2023 13:35:02 +0300 Subject: [PATCH 56/71] fix e2e test --- ...=> db-access-for-custom-roles.wdio-spec.js} | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) rename tests/e2e/default/db/{db-access-for-custom-roles.spec.js => db-access-for-custom-roles.wdio-spec.js} (86%) diff --git a/tests/e2e/default/db/db-access-for-custom-roles.spec.js b/tests/e2e/default/db/db-access-for-custom-roles.wdio-spec.js similarity index 86% rename from tests/e2e/default/db/db-access-for-custom-roles.spec.js rename to tests/e2e/default/db/db-access-for-custom-roles.wdio-spec.js index 1eeed0b0001..f87e2fb9142 100644 --- a/tests/e2e/default/db/db-access-for-custom-roles.spec.js +++ b/tests/e2e/default/db/db-access-for-custom-roles.wdio-spec.js @@ -24,18 +24,22 @@ const newRole = 'new_chw'; const addRole = async (role = newRole) => { const settings = await utils.getSettings(); - settings.roles[role] = { ...settings.roles.chw }; + settings.roles[role] = { name: newRole, offline: true }; + Object.values(settings.permissions).forEach(roles => { + if (roles.includes('chw')) { + roles.push(newRole); + } + }); await utils.updateSettings(settings, true); }; const username = uuid(); const user = { - _id: `org.couchdb.user:${username}`, - type: 'user', + username: username, password: uuid(), - facility_id: clinic._id, - contact_id: contact._id, - roles: [ newRole ] + place: clinic._id, + contact: contact._id, + roles: [newRole] }; describe('Database access for new roles', () => { @@ -60,7 +64,7 @@ describe('Database access for new roles', () => { await browserDbUtils.createDoc(report); await commonPage.sync(); - await utils.get(report._id); + await utils.getDoc(report._id); }); it('should be able to sync documents down', async () => { From 3a1967543cc46cee336427fbd675652b3026334d Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Wed, 26 Jul 2023 17:06:05 +0300 Subject: [PATCH 57/71] add compressible types config --- couchdb/10-docker-default.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/couchdb/10-docker-default.ini b/couchdb/10-docker-default.ini index f61e6758b32..6f9c5702ef4 100644 --- a/couchdb/10-docker-default.ini +++ b/couchdb/10-docker-default.ini @@ -38,3 +38,7 @@ ssl_certificate_max_depth = 4 [cluster] q=12 n=1 + +[attachments] +compressible_types = text/*, application/javascript, application/json, application/xml +compression_level = 8 From efee2eb9375d09679812934d71e31e40e83d23c3 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Fri, 8 Sep 2023 09:30:35 +0300 Subject: [PATCH 58/71] remove national-admin Signed-off-by: Diana Barsan --- admin/src/js/services/session.js | 3 +- api/src/migrations/add-national_admin-role.js | 61 ------------------- config/default/app_settings.json | 3 + config/demo/app_settings.json | 1 - shared-libs/cht-script-api/src/auth.js | 3 +- shared-libs/user-management/src/roles.js | 1 - webapp/src/js/bootstrapper/index.js | 4 +- webapp/src/ts/services/session.service.ts | 3 +- 8 files changed, 7 insertions(+), 72 deletions(-) delete mode 100644 api/src/migrations/add-national_admin-role.js diff --git a/admin/src/js/services/session.js b/admin/src/js/services/session.js index 721e14f7a1c..70487669dee 100644 --- a/admin/src/js/services/session.js +++ b/admin/src/js/services/session.js @@ -92,8 +92,7 @@ const _ = require('lodash/core'); const isAdmin = function(userCtx) { userCtx = userCtx || getUserCtx(); - return hasRole(userCtx, '_admin') || - hasRole(userCtx, 'national_admin'); // deprecated: kept for backwards compatibility: #4525 + return hasRole(userCtx, '_admin'); }; const isDbAdmin = function(userCtx) { diff --git a/api/src/migrations/add-national_admin-role.js b/api/src/migrations/add-national_admin-role.js deleted file mode 100644 index 9112fc6c33e..00000000000 --- a/api/src/migrations/add-national_admin-role.js +++ /dev/null @@ -1,61 +0,0 @@ -const request = require('request-promise-native'); -const url = require('url'); -const environment = require('../environment'); -const logger = require('../logger'); - -const DEFAULT_STRUCTURE = { - names: [], - roles: [], -}; - -const addRole = (dbname, role) => { - return request.get({ - url: url.format({ - protocol: environment.protocol, - hostname: environment.host, - port: environment.port, - pathname: `${dbname}/_security`, - }), - auth: { - user: environment.username, - pass: environment.password - }, - json: true - }) - .then(body => { - // In CouchDB 1.x, if you have not written to the _security object before - // it is empty. - if (!body.admins) { - body.admins = DEFAULT_STRUCTURE; - } - - if (!body.admins.roles.includes(role)) { - logger.info(`Adding ${role} role to ${dbname} admins`); - body.admins.roles.push(role); - } - return request.put({ - url: url.format({ - protocol: environment.protocol, - hostname: environment.host, - port: environment.port, - pathname: `${dbname}/_security`, - }), - auth: { - user: environment.username, - pass: environment.password - }, - json: true, - body: body - }); - }); -}; - -module.exports = { - name: 'add-national_admin-role', - created: new Date(2017, 3, 30), - run: () => { - return Promise.resolve() - .then(() => addRole('_users', 'national_admin')) - .then(() => addRole(environment.db, 'national_admin')); - } -}; diff --git a/config/default/app_settings.json b/config/default/app_settings.json index 9b00522d708..bb6ab450460 100644 --- a/config/default/app_settings.json +++ b/config/default/app_settings.json @@ -62,6 +62,9 @@ "synthetic_date": "", "contact_display_short": "clinic.name", "roles": { + "national_admin": { + "name": "usertype.national_admin" + }, "data_entry": { "name": "usertype.data-entry" }, diff --git a/config/demo/app_settings.json b/config/demo/app_settings.json index cc4f0f3b0fd..67941d78315 100644 --- a/config/demo/app_settings.json +++ b/config/demo/app_settings.json @@ -186,7 +186,6 @@ "program_officer" ], "can_export_dhis": [ - "national_admin", "crfo" ], "can_verify_reports": [ diff --git a/shared-libs/cht-script-api/src/auth.js b/shared-libs/cht-script-api/src/auth.js index 80abe6f4d55..4822f1dd588 100644 --- a/shared-libs/cht-script-api/src/auth.js +++ b/shared-libs/cht-script-api/src/auth.js @@ -4,7 +4,6 @@ */ const ADMIN_ROLE = '_admin'; -const NATIONAL_ADMIN_ROLE = 'national_admin'; // Deprecated: kept for backwards compatibility: #4525 const DISALLOWED_PERMISSION_PREFIX = '!'; const isAdmin = (userRoles) => { @@ -12,7 +11,7 @@ const isAdmin = (userRoles) => { return false; } - return [ADMIN_ROLE, NATIONAL_ADMIN_ROLE].some(role => userRoles.includes(role)); + return [ADMIN_ROLE].some(role => userRoles.includes(role)); }; const groupPermissions = (permissions) => { diff --git a/shared-libs/user-management/src/roles.js b/shared-libs/user-management/src/roles.js index 4fd917dc3ea..5486f84d103 100644 --- a/shared-libs/user-management/src/roles.js +++ b/shared-libs/user-management/src/roles.js @@ -22,7 +22,6 @@ const hasOnlineRole = roles => { const onlineRoles = [ DB_ADMIN_ROLE, ONLINE_ROLE, - 'national_admin', // kept for backwards compatibility ]; return roles.some(role => onlineRoles.includes(role)); }; diff --git a/webapp/src/js/bootstrapper/index.js b/webapp/src/js/bootstrapper/index.js index 5a767716ffe..f0136478a51 100644 --- a/webapp/src/js/bootstrapper/index.js +++ b/webapp/src/js/bootstrapper/index.js @@ -80,9 +80,7 @@ }; const hasFullDataAccess = function(userCtx) { - return hasRole(userCtx, '_admin') || - hasRole(userCtx, 'national_admin') || // kept for backwards compatibility - hasRole(userCtx, ONLINE_ROLE); + return hasRole(userCtx, '_admin') || hasRole(userCtx, ONLINE_ROLE); }; /* pouch db set up function */ diff --git a/webapp/src/ts/services/session.service.ts b/webapp/src/ts/services/session.service.ts index 288df3ccfa8..2e7ccfd05ba 100644 --- a/webapp/src/ts/services/session.service.ts +++ b/webapp/src/ts/services/session.service.ts @@ -115,8 +115,7 @@ export class SessionService { } isAdmin(userCtx?) { - return this.isDbAdmin(userCtx) || - this.hasRole('national_admin', userCtx); // deprecated: kept for backwards compatibility: #4525 + return this.isDbAdmin(userCtx); } isDbAdmin(userCtx?) { From 1a1d3af1f45f41e9585d3e27163d904220be9e82 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Fri, 8 Sep 2023 11:05:54 +0300 Subject: [PATCH 59/71] fix unit tests Signed-off-by: Diana Barsan --- admin/tests/unit/services/session.spec.js | 4 ++-- shared-libs/user-management/test/unit/roles.spec.js | 10 +++++----- .../tests/karma/ts/services/db-sync.service.spec.ts | 12 ------------ .../tests/karma/ts/services/session.service.spec.ts | 4 ++-- 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/admin/tests/unit/services/session.spec.js b/admin/tests/unit/services/session.spec.js index a83a9fe41e3..0a532b78221 100644 --- a/admin/tests/unit/services/session.spec.js +++ b/admin/tests/unit/services/session.spec.js @@ -144,10 +144,10 @@ describe('Session service', function() { done(); }); - it('returns true for national_admin', function(done) { + it('returns false for national_admin', function(done) { ipCookie.returns({ roles: [ 'national_admin', 'some_other_role' ] }); const actual = service.isAdmin(); - chai.expect(actual).to.equal(true); + chai.expect(actual).to.equal(false); done(); }); diff --git a/shared-libs/user-management/test/unit/roles.spec.js b/shared-libs/user-management/test/unit/roles.spec.js index e56c4bdb68b..4350b8b6257 100644 --- a/shared-libs/user-management/test/unit/roles.spec.js +++ b/shared-libs/user-management/test/unit/roles.spec.js @@ -23,6 +23,9 @@ describe('roles', () => { ['some_role'], ['one_role', 'district_manager', 'admin'], ['one_role', 'not_district_admin', 'not_admin'], + ['not_chw', 'national_admin'], + ['random', 'national_admin'], + ['national_admin'], ]; scenarios.forEach(userRoles => { const message = `hasOnlineRole failed for ${userRoles}`; @@ -35,9 +38,6 @@ describe('roles', () => { ['_admin'], ['_admin', 'other_role'], ['chw', '_admin'], - ['not_chw', 'national_admin'], - ['random', 'national_admin'], - ['national_admin'], ['mm-online'], ['mm-online', 'other'], ['not-mm-online', 'mm-online'], @@ -57,8 +57,8 @@ describe('roles', () => { }); it('checks "national_admin" role', () => { - chai.expect(roles.isOnlineOnly({ roles: ['national_admin'] })).to.equal(true); - chai.expect(roles.isOnlineOnly({ roles: ['national_admin', 'chw'] })).to.equal(true); + chai.expect(roles.isOnlineOnly({ roles: ['national_admin'] })).to.equal(false); + chai.expect(roles.isOnlineOnly({ roles: ['national_admin', 'chw'] })).to.equal(false); }); it('should check for "mm-online" role', () => { diff --git a/webapp/tests/karma/ts/services/db-sync.service.spec.ts b/webapp/tests/karma/ts/services/db-sync.service.spec.ts index e093401ddd8..cbd42bc3545 100644 --- a/webapp/tests/karma/ts/services/db-sync.service.spec.ts +++ b/webapp/tests/karma/ts/services/db-sync.service.spec.ts @@ -898,18 +898,6 @@ describe('DBSync service', () => { }); describe('on change', () => { - // todo!!! - xit('"changes" from handle calls RulesEngine.monitorExternalChanges', () => { - isOnlineOnly.returns(false); - hasAuth.resolves(true); - const replicationResult = { this: 'is', a: 'replication result' }; - return service.sync().then(() => { - expect(rulesEngine.monitorExternalChanges.callCount).to.equal(1); - expect(rulesEngine.monitorExternalChanges.args[0]).to.deep.equal([replicationResult]); - expectSyncCall(1); - }); - }); - describe('replicate meta', () => { beforeEach(() => { hasAuth.resolves(true); diff --git a/webapp/tests/karma/ts/services/session.service.spec.ts b/webapp/tests/karma/ts/services/session.service.spec.ts index d317835f49d..998967fb7c1 100644 --- a/webapp/tests/karma/ts/services/session.service.spec.ts +++ b/webapp/tests/karma/ts/services/session.service.spec.ts @@ -161,10 +161,10 @@ describe('Session service', () => { expect(actual).to.equal(true); }); - it('returns true for national_admin', () => { + it('returns false for national_admin', () => { cookieGet.returns(JSON.stringify({ roles: [ 'national_admin', 'some_other_role' ] })); const actual = service.isAdmin(); - expect(actual).to.equal(true); + expect(actual).to.equal(false); }); it('returns false for everyone else', () => { From 62436177c629325c73c794a64d6991e5d2752a9a Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Fri, 8 Sep 2023 15:21:04 +0300 Subject: [PATCH 60/71] fix integration tests Signed-off-by: Diana Barsan --- tests/integration/api/controllers/users.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/api/controllers/users.spec.js b/tests/integration/api/controllers/users.spec.js index e18f334e13f..d00153aa0a3 100644 --- a/tests/integration/api/controllers/users.spec.js +++ b/tests/integration/api/controllers/users.spec.js @@ -567,7 +567,7 @@ describe('Users API', () => { it('should throw error when requesting for online roles', () => { const params = { - role: 'national_admin', + role: JSON.stringify(['national_admin', 'mm-online']), facility_id: 'fixture:offline' }; onlineRequestOptions.path += '?' + querystring.stringify(params); @@ -582,7 +582,7 @@ describe('Users API', () => { it('should throw error for array roles of online user', () => { const params = { - role: JSON.stringify(['random', 'national_admin', 'random']), + role: JSON.stringify(['random', 'national_admin', 'mm-online']), facility_id: 'fixture:offline' }; onlineRequestOptions.path += '?' + querystring.stringify(params); From 747ddd422c09ed4515799a2aa53f3e57132e376c Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sat, 9 Sep 2023 01:36:19 +0300 Subject: [PATCH 61/71] fix validate doc update test Signed-off-by: Diana Barsan --- haproxy/tests/test_helper | 1 - nginx/tests/test_helper | 1 - webapp/tests/mocha/unit/validate_doc_update.spec.js | 13 ++++++++----- 3 files changed, 8 insertions(+), 7 deletions(-) delete mode 120000 haproxy/tests/test_helper delete mode 120000 nginx/tests/test_helper diff --git a/haproxy/tests/test_helper b/haproxy/tests/test_helper deleted file mode 120000 index cd72da7c705..00000000000 --- a/haproxy/tests/test_helper +++ /dev/null @@ -1 +0,0 @@ -../../scripts/test/bats/test_helper \ No newline at end of file diff --git a/nginx/tests/test_helper b/nginx/tests/test_helper deleted file mode 120000 index cd72da7c705..00000000000 --- a/nginx/tests/test_helper +++ /dev/null @@ -1 +0,0 @@ -../../scripts/test/bats/test_helper \ No newline at end of file diff --git a/webapp/tests/mocha/unit/validate_doc_update.spec.js b/webapp/tests/mocha/unit/validate_doc_update.spec.js index 659814d646f..10aeb28ddcf 100644 --- a/webapp/tests/mocha/unit/validate_doc_update.spec.js +++ b/webapp/tests/mocha/unit/validate_doc_update.spec.js @@ -39,12 +39,12 @@ describe('validate doc update', () => { const clientValidateDocUpdate = function() { const fn = fs.readFileSync('./ddocs/medic-db/medic-client/validate_doc_update.js'); - eval('(' + fn + ')').apply(null, arguments); + eval('log = console.log; (' + fn + ')').apply(null, arguments); }; const serverValidateDocUpdate = function() { const fn = fs.readFileSync('./ddocs/medic-db/medic/validate_doc_update.js'); - eval('(' + fn + ')').apply(null, arguments); + eval('log = console.log; (' + fn + ')').apply(null, arguments); }; const allowedOnServer = _.partial(checkFn, serverValidateDocUpdate); @@ -62,10 +62,13 @@ describe('validate doc update', () => { 'partners': { _id: 'partners' } }).forEach(([ name, doc ]) => { it(name, done => { - assert.isOk(allowedOnServer(userCtx({roles: [ '_admin' ]}), doc)); - assert.isOk(allowedOnServer(userCtx({roles: [ 'national_admin' ]}), doc)); + assert.equal(allowedOnServer(userCtx( {roles: [ '_admin' ] }), doc), true); assert.deepEqual( - allowedOnServer(userCtx({roles: [ ]}), doc), + allowedOnServer(userCtx({ roles: [ 'national_admin' ] }), doc), + { forbidden: 'You are not authorized to edit admin only docs' } + ); + assert.deepEqual( + allowedOnServer(userCtx({ roles: [ 'test' ] }), doc), disallowed('You are not authorized to edit admin only docs') ); done(); From 16d1769f1b3fbd1318ba490d1d9db8048c53054f Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sat, 9 Sep 2023 02:13:37 +0300 Subject: [PATCH 62/71] revert haproxy / nginx symlink removal Signed-off-by: Diana Barsan --- haproxy/tests/test_helper | 1 + nginx/tests/test_helper | 1 + 2 files changed, 2 insertions(+) create mode 120000 haproxy/tests/test_helper create mode 120000 nginx/tests/test_helper diff --git a/haproxy/tests/test_helper b/haproxy/tests/test_helper new file mode 120000 index 00000000000..8b4d3026da4 --- /dev/null +++ b/haproxy/tests/test_helper @@ -0,0 +1 @@ +scripts/test/bats/test_helper/ \ No newline at end of file diff --git a/nginx/tests/test_helper b/nginx/tests/test_helper new file mode 120000 index 00000000000..8b4d3026da4 --- /dev/null +++ b/nginx/tests/test_helper @@ -0,0 +1 @@ +scripts/test/bats/test_helper/ \ No newline at end of file From d55eb9e2b4820e1d02e49dca137d531b925010d4 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sat, 9 Sep 2023 07:11:04 +0300 Subject: [PATCH 63/71] just add all files haproxy / nginx symlink removal Signed-off-by: Diana Barsan --- haproxy/tests/test_helper | 2 +- nginx/tests/test_helper | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/haproxy/tests/test_helper b/haproxy/tests/test_helper index 8b4d3026da4..2e447ff72ca 120000 --- a/haproxy/tests/test_helper +++ b/haproxy/tests/test_helper @@ -1 +1 @@ -scripts/test/bats/test_helper/ \ No newline at end of file +/home/diana/projects/medic-third/scripts/test/bats/test_helper \ No newline at end of file diff --git a/nginx/tests/test_helper b/nginx/tests/test_helper index 8b4d3026da4..2e447ff72ca 120000 --- a/nginx/tests/test_helper +++ b/nginx/tests/test_helper @@ -1 +1 @@ -scripts/test/bats/test_helper/ \ No newline at end of file +/home/diana/projects/medic-third/scripts/test/bats/test_helper \ No newline at end of file From bc7c65f87e1f8c2d4245986aec9f777053b3707e Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sat, 9 Sep 2023 07:13:23 +0300 Subject: [PATCH 64/71] omg symlink hell Signed-off-by: Diana Barsan --- haproxy/tests/test_helper | 1 - nginx/tests/test_helper | 1 - 2 files changed, 2 deletions(-) delete mode 120000 haproxy/tests/test_helper delete mode 120000 nginx/tests/test_helper diff --git a/haproxy/tests/test_helper b/haproxy/tests/test_helper deleted file mode 120000 index 2e447ff72ca..00000000000 --- a/haproxy/tests/test_helper +++ /dev/null @@ -1 +0,0 @@ -/home/diana/projects/medic-third/scripts/test/bats/test_helper \ No newline at end of file diff --git a/nginx/tests/test_helper b/nginx/tests/test_helper deleted file mode 120000 index 2e447ff72ca..00000000000 --- a/nginx/tests/test_helper +++ /dev/null @@ -1 +0,0 @@ -/home/diana/projects/medic-third/scripts/test/bats/test_helper \ No newline at end of file From 403fc5aebef4a3540b0ce89802ec781a85dfad29 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sat, 9 Sep 2023 07:14:04 +0300 Subject: [PATCH 65/71] omg symlink hell Signed-off-by: Diana Barsan --- haproxy/tests/test_helper | 1 + scripts/test/bats/test_helper/test_helper | 1 + 2 files changed, 2 insertions(+) create mode 120000 haproxy/tests/test_helper create mode 120000 scripts/test/bats/test_helper/test_helper diff --git a/haproxy/tests/test_helper b/haproxy/tests/test_helper new file mode 120000 index 00000000000..cd72da7c705 --- /dev/null +++ b/haproxy/tests/test_helper @@ -0,0 +1 @@ +../../scripts/test/bats/test_helper \ No newline at end of file diff --git a/scripts/test/bats/test_helper/test_helper b/scripts/test/bats/test_helper/test_helper new file mode 120000 index 00000000000..cd72da7c705 --- /dev/null +++ b/scripts/test/bats/test_helper/test_helper @@ -0,0 +1 @@ +../../scripts/test/bats/test_helper \ No newline at end of file From b7777a4fb1940bae258398f206970606db759a1b Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sat, 9 Sep 2023 11:26:16 +0300 Subject: [PATCH 66/71] omg symlink hell Signed-off-by: Diana Barsan --- {scripts/test/bats/test_helper => nginx/tests}/test_helper | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {scripts/test/bats/test_helper => nginx/tests}/test_helper (100%) diff --git a/scripts/test/bats/test_helper/test_helper b/nginx/tests/test_helper similarity index 100% rename from scripts/test/bats/test_helper/test_helper rename to nginx/tests/test_helper From 8c0c4f817e62333cf18f4dbe25d8adc9d6e01b53 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Sat, 9 Sep 2023 12:35:13 +0300 Subject: [PATCH 67/71] fix linting Signed-off-by: Diana Barsan --- api/tests/mocha/services/config-watcher.spec.js | 2 +- .../db/db-access-for-custom-roles.wdio-spec.js | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/api/tests/mocha/services/config-watcher.spec.js b/api/tests/mocha/services/config-watcher.spec.js index 293ce7f23e5..7b0a259c535 100644 --- a/api/tests/mocha/services/config-watcher.spec.js +++ b/api/tests/mocha/services/config-watcher.spec.js @@ -247,7 +247,7 @@ describe('Configuration', () => { }); sinon.stub(environment, 'db').get(() => 'medicdb'); - return emitChange({ id: 'settings' }).then(() => { + return dbWatcher.medic.args[0][0]({ id: 'settings' }).then(() => { chai.expect(settingsService.update.callCount).to.equal(1); chai.expect(settingsService.get.callCount).to.equal(1); chai.expect(config.set.callCount).to.equal(1); diff --git a/tests/e2e/default/db/db-access-for-custom-roles.wdio-spec.js b/tests/e2e/default/db/db-access-for-custom-roles.wdio-spec.js index f87e2fb9142..90b1bb1ed8b 100644 --- a/tests/e2e/default/db/db-access-for-custom-roles.wdio-spec.js +++ b/tests/e2e/default/db/db-access-for-custom-roles.wdio-spec.js @@ -10,14 +10,13 @@ const placeFactory = require('@factories/cht/contacts/place'); const places = placeFactory.generateHierarchy(); const clinic = places.get('clinic'); -const contact = personFactory.build( - { - parent: { - _id: clinic._id, - parent: clinic.parent - }, - phone: '+254712345670' - }); +const contact = personFactory.build({ + parent: { + _id: clinic._id, + parent: clinic.parent + }, + phone: '+254712345670' +}); const docs = [...places.values(), contact]; const newRole = 'new_chw'; From bb5b951b97694158c6c12b0d172dbc4e5844e282 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Mon, 11 Sep 2023 14:37:14 +0300 Subject: [PATCH 68/71] fix all permissions e2e test error. Signed-off-by: Diana Barsan --- .../offline-user/all-permissions.wdio-spec.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e/default/more-options-menu/offline-user/all-permissions.wdio-spec.js b/tests/e2e/default/more-options-menu/offline-user/all-permissions.wdio-spec.js index 203d92af00c..de2430bc553 100644 --- a/tests/e2e/default/more-options-menu/offline-user/all-permissions.wdio-spec.js +++ b/tests/e2e/default/more-options-menu/offline-user/all-permissions.wdio-spec.js @@ -68,8 +68,6 @@ describe('More Options Menu - Offline User', async () => { afterEach(async () => await commonPage.goToBase()); - after(async () => await utils.revertSettings(true)); - describe('all permissions enabled', async () => { it('- Message tab', async () => { await commonPage.goToMessages(); @@ -125,8 +123,8 @@ describe('More Options Menu - Offline User', async () => { const allPermissions = ['can_edit', 'can_delete_contacts', 'can_export_all', 'can_export_contacts', 'can_export_messages', 'can_delete_reports', 'can_update_reports']; - await utils.updatePermissions(offlineUser.roles, [], allPermissions); - await commonPage.closeReloadModal(); + await utils.updatePermissions(offlineUser.roles, [], allPermissions, true); + await commonPage.sync(true); }); after(async () => await utils.revertSettings(true)); From ffc7eafb2d07c1e72c07c245077cf5da8fd2e29e Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Mon, 11 Sep 2023 23:19:49 +0300 Subject: [PATCH 69/71] remove unnecessary env set Signed-off-by: Diana Barsan --- api/tests/mocha/db.spec.js | 1 - 1 file changed, 1 deletion(-) diff --git a/api/tests/mocha/db.spec.js b/api/tests/mocha/db.spec.js index f13ef2f8b6b..a354f274307 100644 --- a/api/tests/mocha/db.spec.js +++ b/api/tests/mocha/db.spec.js @@ -3,7 +3,6 @@ require('chai').use(require('chai-as-promised')); const { expect } = require('chai'); const rewire = require('rewire'); const rpn = require('request-promise-native'); -process.env.UNIT_TEST_ENV = 1; let db; let unitTestEnv; From de879d2ad414b5651056af833f0df98d3e86cb9c Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Mon, 11 Sep 2023 23:21:12 +0300 Subject: [PATCH 70/71] fix bad merge Signed-off-by: Diana Barsan --- tests/integration/api/server.spec.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/api/server.spec.js b/tests/integration/api/server.spec.js index 25c6c213cd9..dab44698b7d 100644 --- a/tests/integration/api/server.spec.js +++ b/tests/integration/api/server.spec.js @@ -211,8 +211,6 @@ describe('server', () => { await utils.startApi(); await utils.delayPromise(1000); - await utils.delayPromise(1000); - await utils.request('/'); await utils.stopHaproxy(); From e8f02fa0afa81d599d66b6f881f01c2890810fc7 Mon Sep 17 00:00:00 2001 From: Diana Barsan Date: Mon, 11 Sep 2023 23:23:12 +0300 Subject: [PATCH 71/71] wtb compiler plis Signed-off-by: Diana Barsan --- tests/integration/cht-conf/cht-conf-actions.spec.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/integration/cht-conf/cht-conf-actions.spec.js b/tests/integration/cht-conf/cht-conf-actions.spec.js index 513d818e57f..f251a1e48ce 100644 --- a/tests/integration/cht-conf/cht-conf-actions.spec.js +++ b/tests/integration/cht-conf/cht-conf-actions.spec.js @@ -35,7 +35,6 @@ describe('cht-conf actions tests', () => { const result = await runCommand('upload-app-settings', configPath); expect(result).to.contain(`INFO Settings updated successfully`); const settings = await utils.getDoc('settings'); - console.log('newVersion', settings._rev); const newVersion = Number(settings._rev.charAt(0)); expect(newVersion).to.be.greaterThanOrEqual(originalVersion); expect(settings.settings.roles).to.include.all.keys('program_officer', 'chw_supervisor', 'chw'); @@ -51,7 +50,7 @@ describe('cht-conf actions tests', () => { it('should upload branding', async () => { const branding = await utils.getDoc('branding').catch(error => { if (error){ - console.err(error); + console.error(error); } }); expect(branding.title).to.equal('Medic'); @@ -63,7 +62,7 @@ describe('cht-conf actions tests', () => { method: 'GET' }).catch(error => { if (error){ - console.log(error); + console.error(error); } }); expect(forms).to.deep.equal([ @@ -92,7 +91,7 @@ describe('cht-conf actions tests', () => { it('should upload resources', async () => { const resources = await utils.getDoc('resources').catch(error => { if (error){ - console.log(error); + console.error(error); } }); expect(resources.resources).to.deep.equal({