diff --git a/docs/NooBaaNonContainerized/CI&Tests.md b/docs/NooBaaNonContainerized/CI&Tests.md index 1ebc40fda1..160cc8ebfd 100644 --- a/docs/NooBaaNonContainerized/CI&Tests.md +++ b/docs/NooBaaNonContainerized/CI&Tests.md @@ -88,7 +88,7 @@ Run `NC mocha tests` with root permissions - #### NC mocha tests The following is a list of `NC mocha test` files - 1. `test_nc_nsfs_cli.js` - Tests NooBaa CLI. -2. `test_nc_nsfs_health` - Tests NooBaa Health CLI. +2. `test_nc_health` - Tests NooBaa Health CLI. 3. `test_nsfs_glacier_backend.js` - Tests NooBaa Glacier Backend. 4. `test_nc_with_a_couple_of_forks.js` - Tests the `bucket_namespace_cache` when running with a couple of forks. Please notice that it uses `nc_coretest` with setup that includes a couple of forks. diff --git a/docs/NooBaaNonContainerized/Health.md b/docs/NooBaaNonContainerized/Health.md index d3184256c2..08bf0c981b 100644 --- a/docs/NooBaaNonContainerized/Health.md +++ b/docs/NooBaaNonContainerized/Health.md @@ -34,6 +34,14 @@ For more details about NooBaa RPM installation, see - [Getting Started](./Gettin - Iterating buckets under the config directory. - Confirming the existence of the bucket's configuration file and its validity as a JSON file. - Verifying that the underlying storage path of a bucket exists. + - `Config directory health` + - checks if config system and directory data exists + - returns the config directory status + - `Config directory upgrade health` + - checks if config system and directory data exists + - checks if there is ongoing upgrade + - returns error if there is no ongoing upgrade, but the config directory phase is locked + - returns message if there is no ongoing upgrade and the config directory is unlocked * Health CLI requires root permissions. @@ -148,6 +156,11 @@ The output of the Health CLI is a JSON object containing the following propertie - Enum: 'PERSISTENT' | 'TEMPORARY' - Description: For TEMPORARY error types, NooBaa attempts multiple retries before updating the status to reflect an error. Currently, TEMPORARY error types are only observed in checks for invalid NooBaa endpoints. +- `config_directory` + - Type: Object {"phase": "CONFIG_DIR_UNLOCKED" | "CONFIG_DIR_LOCKED","config_dir_version": String, + "upgrade_package_version": String, "upgrade_status": Object, "error": Object }. + - Description: An object that consists config directory information, config directory upgrade information etc. + - Example: { "phase": "CONFIG_DIR_UNLOCKED", "config_dir_version": "1.0.0", "upgrade_package_version": "5.18.0", "upgrade_status": { "message": "there is no in-progress upgrade" }} ## Example ```sh @@ -225,6 +238,14 @@ Output: } ], "error_type": "PERSISTENT" + }, + "config_directory": { + "phase": "CONFIG_DIR_UNLOCKED", + "config_dir_version": "1.0.0", + "upgrade_package_version": "5.18.0", + "upgrade_status": { + "message": "there is no in-progress upgrade" + } } } } @@ -243,7 +264,8 @@ Output: - The config file of bucket1 is invalid. Therefore, NooBaa health reports INVALID_CONFIG. - The underlying file system directory of bucket3 is missing. Therefore, NooBaa health reports STORAGE_NOT_EXIST. - +- config_directory: + - the config directory phase is unlocked, config directory version is "1.0.0", matching source code/package version is "5.18.0" and there is no ongoing upgrade. ## Health Errors @@ -365,4 +387,16 @@ The following error codes will be associated with a specific Bucket or Account s - Reasons: - Bucket missing owner account. - Resolutions: - - Check for owner_account property in bucket config file. \ No newline at end of file + - Check for owner_account property in bucket config file. + +#### 8. Config Directory is invalid + - Error code: `INVALID_CONFIG_DIR` + - Error message: Config directory is invalid + - Reasons: + - System.json is missing - NooBaa was never started + - Config directory property is missing in system.json - the user didn't run config directory upgrade when upgrading from 5.17.z to 5.18.0 + - Config directory upgrade error. + - Resolutions: + - Start NooBaa service + - Run `noobaa-cli upgrade` + - Check the in_progress_upgrade the exact reason for the failure. diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js index 25629b0273..65ccf24da3 100644 --- a/src/manage_nsfs/health.js +++ b/src/manage_nsfs/health.js @@ -14,6 +14,7 @@ const { TYPES } = require('./manage_nsfs_constants'); const { get_boolean_or_string_value, throw_cli_error, write_stdout_response, get_bucket_owner_account_by_id } = require('./manage_nsfs_cli_utils'); const { ManageCLIResponse } = require('./manage_nsfs_cli_responses'); const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError; +const { CONFIG_DIR_LOCKED, CONFIG_DIR_UNLOCKED } = require('../upgrade/nc_upgrade_manager'); const HOSTNAME = 'localhost'; @@ -60,6 +61,10 @@ const health_errors = { error_code: 'MISSING_ACCOUNT_OWNER', error_message: 'Bucket account owner not found', }, + INVALID_CONFIG_DIR: { + error_code: 'INVALID_CONFIG_DIR', + error_message: 'Config directory is invalid', + }, UNKNOWN_ERROR: { error_code: 'UNKNOWN_ERROR', error_message: 'An unknown error occurred', @@ -117,12 +122,16 @@ class NSFSHealth { endpoint_state = await this.get_endpoint_response(); memory = await this.get_service_memory_usage(); } + // TODO: add more health status based on system.json, e.g. RPM upgrade issues + const system_data = await this.config_fs.get_system_config_file({ silent_if_missing: true }); + const config_directory_status = this._get_config_dir_status(system_data); + let bucket_details; let account_details; - const response_code = endpoint_state ? endpoint_state.response.response_code : 'NOT_RUNNING'; - const service_health = service_status !== 'active' || pid === '0' || response_code !== 'RUNNING' ? 'NOTOK' : 'OK'; - - const error_code = await this.get_error_code(service_status, pid, response_code); + const endpoint_response_code = (endpoint_state && endpoint_state.response?.response_code) || 'UNKNOWN_ERROR'; + const health_check_params = { service_status, pid, endpoint_response_code, config_directory_status }; + const service_health = this._calc_health_status(health_check_params); + const error_code = this.get_error_code(health_check_params); if (this.all_bucket_details) bucket_details = await this.get_bucket_status(); if (this.all_account_details) account_details = await this.get_account_status(); const health = { @@ -136,6 +145,7 @@ class NSFSHealth { endpoint_state, error_type: health_errors_tyes.TEMPORARY, }, + config_directory_status, accounts_status: { invalid_accounts: account_details === undefined ? undefined : account_details.invalid_storages, valid_accounts: account_details === undefined ? undefined : account_details.valid_storages, @@ -161,7 +171,7 @@ class NSFSHealth { delay_ms: config.NC_HEALTH_ENDPOINT_RETRY_DELAY, func: async () => { endpoint_state = await this.get_endpoint_fork_response(); - if (endpoint_state.response.response_code === fork_response_code.NOT_RUNNING.response_code) { + if (endpoint_state.response?.response_code === fork_response_code.NOT_RUNNING.response_code) { throw new Error('Noobaa endpoint is not running, all the retries failed'); } } @@ -173,13 +183,23 @@ class NSFSHealth { return endpoint_state; } - async get_error_code(nsfs_status, pid, endpoint_response_code) { - if (nsfs_status !== 'active' || pid === '0') { + /** + * get_error_code returns the error code per the failed check + * @param {{service_status: String, + * pid: string, + * endpoint_response_code: string, + * config_directory_status: Object }} health_check_params + * @returns {Object} + */ + get_error_code({ service_status, pid, endpoint_response_code, config_directory_status }) { + if (service_status !== 'active' || pid === '0') { return health_errors.NOOBAA_SERVICE_FAILED; } else if (endpoint_response_code === 'NOT_RUNNING') { return health_errors.NOOBAA_ENDPOINT_FAILED; } else if (endpoint_response_code === 'MISSING_FORKS') { return health_errors.NOOBAA_ENDPOINT_FORK_MISSING; + } else if (config_directory_status.error) { + return health_errors.CONFIG_DIR_ERROR; } } @@ -239,7 +259,7 @@ class NSFSHealth { const fork_count_response = await this.make_endpoint_health_request(url_path); if (!fork_count_response) { return { - response_code: fork_response_code.NOT_RUNNING, + response: fork_response_code.NOT_RUNNING, total_fork_count: total_fork_count, running_workers: worker_ids, }; @@ -421,6 +441,60 @@ class NSFSHealth { err_obj }; } + + /** + * _get_config_dir_status returns the config directory phase, version, + * matching package_version, upgrade_status and error if occured. + * @param {Object} system_data + * @returns {Object} + */ + _get_config_dir_status(system_data) { + if (!system_data) return { error: 'system data is missing' }; + const config_dir_data = system_data.config_directory; + if (!config_dir_data) return { error: 'config directory data is missing, must upgrade config directory' }; + const config_dir_upgrade_status = this._get_config_dir_upgrade_status(config_dir_data); + return { + phase: config_dir_data.phase, + config_dir_version: config_dir_data.config_dir_version, + upgrade_package_version: config_dir_data.upgrade_package_version, + upgrade_status: config_dir_upgrade_status, + error: config_dir_upgrade_status.error || undefined + }; + } + + /** + * _get_config_dir_upgrade_status returns one of the following + * 1. the status of an ongoing upgrade, if valid it returns an object with upgrade details + * 2. if upgrade is not ongoing but config dir is locked, the error details of the upgrade's last_failure will return + * 3. if upgrade is not ongoing and config dir is unlocked, a corresponding message will return + * @param {Object} config_dir_data + * @returns {Object} + */ + _get_config_dir_upgrade_status(config_dir_data) { + if (config_dir_data.in_progress_upgrade) return { in_progress_upgrade: config_dir_data.in_progress_upgrade }; + if (config_dir_data.phase === CONFIG_DIR_LOCKED) { + return { error: 'last_upgrade_failed', last_failure: config_dir_data.upgrade_history.last_failure }; + } + if (config_dir_data.phase === CONFIG_DIR_UNLOCKED) { + return { message: 'there is no in-progress upgrade' }; + } + } + + /** + * _calc_health_status calcs the overall health status of NooBaa NC + * @param {{service_status: String, + * pid: string, + * endpoint_response_code: string, + * config_directory_status: Object }} health_check_params + * @returns {'OK' | 'NOTOK'} + */ + _calc_health_status({ service_status, pid, endpoint_response_code, config_directory_status }) { + const is_unhealthy = service_status !== 'active' || + pid === '0' || + endpoint_response_code !== 'RUNNING' || + config_directory_status.error; + return is_unhealthy ? 'NOTOK' : 'OK'; + } } async function get_health_status(argv, config_fs) { diff --git a/src/test/unit_tests/nc_index.js b/src/test/unit_tests/nc_index.js index 5497e05b9b..49cc875263 100644 --- a/src/test/unit_tests/nc_index.js +++ b/src/test/unit_tests/nc_index.js @@ -11,7 +11,7 @@ require('./test_chunk_fs'); require('./test_namespace_fs_mpu'); require('./test_nb_native_fs'); require('./test_nc_nsfs_cli'); -require('./test_nc_nsfs_health'); +require('./test_nc_health'); require('./test_nsfs_access'); require('./test_nsfs_integration'); require('./test_bucketspace_fs'); diff --git a/src/test/unit_tests/sudo_index.js b/src/test/unit_tests/sudo_index.js index fa3017e028..37068a51d6 100644 --- a/src/test/unit_tests/sudo_index.js +++ b/src/test/unit_tests/sudo_index.js @@ -21,5 +21,5 @@ require('./test_bucketspace_versioning'); require('./test_bucketspace_fs'); require('./test_nsfs_versioning'); require('./test_nc_nsfs_cli'); -require('./test_nc_nsfs_health'); +require('./test_nc_health'); diff --git a/src/test/unit_tests/test_nc_nsfs_health.js b/src/test/unit_tests/test_nc_health.js similarity index 68% rename from src/test/unit_tests/test_nc_nsfs_health.js rename to src/test/unit_tests/test_nc_health.js index e8100a8f60..5d4889335b 100644 --- a/src/test/unit_tests/test_nc_nsfs_health.js +++ b/src/test/unit_tests/test_nc_health.js @@ -1,5 +1,5 @@ /* Copyright (C) 2016 NooBaa */ -/*eslint max-lines-per-function: ["error", 700]*/ +/*eslint max-lines-per-function: ['error', 700]*/ 'use strict'; @@ -8,6 +8,7 @@ const mocha = require('mocha'); const sinon = require('sinon'); const assert = require('assert'); const config = require('../../../config'); +const pkg = require('../../../package.json'); const fs_utils = require('../../util/fs_utils'); const nb_native = require('../../util/nb_native'); const { ConfigFS } = require('../../sdk/config_fs'); @@ -18,11 +19,30 @@ const { get_process_fs_context } = require('../../util/native_fs_utils'); const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors'); const { TYPES, DIAGNOSE_ACTIONS, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); const { TMP_PATH, create_fs_user_by_platform, delete_fs_user_by_platform, exec_manage_cli } = require('../system_tests/test_utils'); +const { CONFIG_DIR_UNLOCKED, CONFIG_DIR_LOCKED } = require('../../upgrade/nc_upgrade_manager'); const tmp_fs_path = path.join(TMP_PATH, 'test_nc_health'); const DEFAULT_FS_CONFIG = get_process_fs_context(); const bucket_storage_path = path.join(tmp_fs_path, 'bucket_storage_path'); +const os = require('os'); +const hostname = os.hostname(); + +const valid_system_json = { + [hostname]: { + 'current_version': pkg.version, + 'upgrade_history': { 'successful_upgrades': [] }, + }, +}; + +const get_service_state_mock_default_response = [{ service_status: 'active', pid: 1000 }, { service_status: 'active', pid: 2000 }]; +const get_endpoint_response_mock_default_response = [{ response: { response_code: 'RUNNING', total_fork_count: 0 } }]; +const get_system_config_mock_default_response = [{ + ...valid_system_json, config_directory: { + phase: CONFIG_DIR_UNLOCKED, config_dir_version: '1.0.0', + package_version: pkg.version, upgrade_history: [] +} }]; +const default_mock_upgrade_status = { message: 'there is no in-progress upgrade' }; mocha.describe('nsfs nc health', function() { @@ -45,6 +65,10 @@ mocha.describe('nsfs nc health', function() { }); mocha.describe('nsfs nc health cli validations', function() { + mocha.afterEach(() => { + restore_health_if_needed(Health); + }); + mocha.it('https_port flag type validation - should fail', async function() { try { await exec_manage_cli(TYPES.DIAGNOSE, DIAGNOSE_ACTIONS.HEALTH, { 'http_port': '' }); @@ -137,8 +161,7 @@ mocha.describe('nsfs nc health', function() { await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, {config_root, ...account1_options}); await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, {config_root, ...bucket1_options}); await fs_utils.file_must_exist(path.join(config_root, 'master_keys.json')); - const get_service_memory_usage = sinon.stub(Health, "get_service_memory_usage"); - get_service_memory_usage.onFirstCall().returns(Promise.resolve(100)); + set_mock_functions(Health, { get_service_memory_usage: [100]}); for (const user of Object.values(fs_users)) { await create_fs_user_by_platform(user.distinguished_name, user.distinguished_name, user.uid, user.gid); } @@ -158,62 +181,64 @@ mocha.describe('nsfs nc health', function() { mocha.afterEach(async () => { await fs_utils.file_delete(config_fs.config_json_path); + restore_health_if_needed(Health); }); mocha.it('Health all condition is success', async function() { - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + valid_system_json.config_directory = { + 'config_dir_version': config_fs.config_dir_version, + 'upgrade_package_version': pkg.version, + 'phase': CONFIG_DIR_UNLOCKED + }; + await Health.config_fs.create_system_config_file(JSON.stringify(valid_system_json)); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + }); Health.all_account_details = true; Health.all_bucket_details = true; const health_status = await Health.nc_nsfs_health(); + await fs_utils.file_delete(Health.config_fs.system_json_path); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 0); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.valid_accounts[0].name, 'account1'); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); assert.strictEqual(health_status.checks.buckets_status.valid_buckets[0].name, 'bucket1'); + assert_config_dir_status(health_status, valid_system_json.config_directory); }); mocha.it('NooBaa service is inactive', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'inactive', pid: 0 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: [{ service_status: 'inactive', pid: 0 }], + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'NOTOK'); assert.strictEqual(health_status.error.error_code, 'NOOBAA_SERVICE_FAILED'); }); mocha.it('NooBaa endpoint return error response is inactive', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'MISSING_FORKS', total_fork_count: 3, running_workers: ['1', '3']}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: [{ response: { response_code: 'MISSING_FORKS', total_fork_count: 3, running_workers: ['1', '3'] } }], + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'NOTOK'); assert.strictEqual(health_status.error.error_code, 'NOOBAA_ENDPOINT_FORK_MISSING'); }); mocha.it('NSFS account with invalid storage path', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); // create it manually because we can not skip invalid storage path check on the CLI const account_invalid_options = { _id: mongo_utils.mongoObjectId(), name: 'account_invalid', nsfs_account_config: { new_buckets_path: path.join(new_buckets_path, '/invalid') } }; await test_utils.write_manual_config_file(TYPES.ACCOUNT, config_fs, account_invalid_options); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); @@ -223,23 +248,21 @@ mocha.describe('nsfs nc health', function() { mocha.it('NSFS bucket with invalid storage path', async function() { this.timeout(5000);// eslint-disable-line no-invalid-this - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); const resp = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account2_options }); const parsed_res = JSON.parse(resp).response.reply; // create it manually because we can not skip invalid storage path check on the CLI const bucket_invalid = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid', path: new_buckets_path + '/bucket1/invalid', owner_account: parsed_res._id }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, bucket_invalid.name); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "STORAGE_NOT_EXIST"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'STORAGE_NOT_EXIST'); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: bucket_invalid.name}); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account2_options.name}); }); @@ -250,22 +273,20 @@ mocha.describe('nsfs nc health', function() { await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, {config_root, ...bucket_inaccessible_options}); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, bucket_inaccessible_options.name); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_options.name); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: bucket_inaccessible_options.name}); @@ -277,18 +298,16 @@ mocha.describe('nsfs nc health', function() { //create bucket manually, cli wont allow bucket with invalid owner const bucket_invalid_owner = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_account', path: new_buckets_path + '/bucket_account', owner_account: 'invalid_account' }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_owner); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "INVALID_ACCOUNT_OWNER"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'INVALID_ACCOUNT_OWNER'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_account'); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: 'bucket_invalid_account'}); @@ -299,35 +318,30 @@ mocha.describe('nsfs nc health', function() { //create bucket manually, cli wont allow bucket with empty owner const bucket_invalid_owner = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_account', path: new_buckets_path + '/bucket_account' }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_owner); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, "MISSING_ACCOUNT_OWNER"); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].code, 'MISSING_ACCOUNT_OWNER'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_account'); await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, { config_root, name: 'bucket_invalid_account'}); }); mocha.it('NSFS invalid bucket schema json', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); // create it manually because we can not skip json schema check on the CLI const bucket_invalid_schema = { _id: mongo_utils.mongoObjectId(), name: 'bucket_invalid_schema', path: new_buckets_path }; await test_utils.write_manual_config_file(TYPES.BUCKET, config_fs, bucket_invalid_schema, 'invalid'); - - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); @@ -337,16 +351,14 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('NSFS invalid account schema json', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); // create it manually because we can not skip json schema check on the CLI const account_invalid_schema = { _id: mongo_utils.mongoObjectId(), name: 'account_invalid_schema', path: new_buckets_path, bla: 5 }; await test_utils.write_manual_config_file(TYPES.ACCOUNT, config_fs, account_invalid_schema, 'invalid'); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); @@ -356,15 +368,13 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Health all condition is success, all_account_details is false', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = false; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 0); @@ -374,15 +384,13 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Health all condition is success, all_bucket_details is false', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = false; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status, undefined); @@ -392,18 +400,16 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Config root path without bucket and account folders', async function() { - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; Health.config_root = config_root_invalid; const old_config_fs = Health.config_fs; Health.config_fs = new ConfigFS(config_root_invalid); - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 100 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 200 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 0); @@ -419,20 +425,18 @@ mocha.describe('nsfs nc health', function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_options.name); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_inaccessible_options.name}); }); @@ -441,15 +445,13 @@ mocha.describe('nsfs nc health', function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, debug: 5, ...account_inaccessible_options}); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); config.NC_DISABLE_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); config.NC_DISABLE_ACCESS_CHECK = false; @@ -466,15 +468,13 @@ mocha.describe('nsfs nc health', function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_options }); await config_fs.delete_config_json_file(); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); config.NC_DISABLE_HEALTH_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); config.NC_DISABLE_HEALTH_ACCESS_CHECK = false; @@ -490,36 +490,32 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with inaccessible path - dn', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_inaccessible_dn_options.name }); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, 'ACCESS_DENIED'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_dn_options.name); }); mocha.it('Account with inaccessible path - dn - NC_DISABLE_ACCESS_CHECK: true - should be valid', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({ response: { response_code: 'RUNNING', total_fork_count: 0 } })); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); config.NC_DISABLE_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); config.NC_DISABLE_ACCESS_CHECK = false; @@ -535,15 +531,13 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with inaccessible path - dn - NC_DISABLE_HEALTH_ACCESS_CHECK: true - should be valid', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_inaccessible_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({ response: { response_code: 'RUNNING', total_fork_count: 0 } })); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); config.NC_DISABLE_HEALTH_ACCESS_CHECK = true; const health_status = await Health.nc_nsfs_health(); config.NC_DISABLE_HEALTH_ACCESS_CHECK = false; @@ -559,15 +553,13 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with invalid dn', async function() { await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...invalid_account_dn_options }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = true; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: invalid_account_dn_options.name }); assert.strictEqual(health_status.checks.buckets_status.valid_buckets.length, 1); @@ -576,21 +568,19 @@ mocha.describe('nsfs nc health', function() { const found = health_status.checks.accounts_status.invalid_accounts.filter(account => account.name === invalid_account_dn_options.name); assert.strictEqual(found.length, 1); - assert.strictEqual(found[0].code, "INVALID_DISTINGUISHED_NAME"); + assert.strictEqual(found[0].code, 'INVALID_DISTINGUISHED_NAME'); }); mocha.it('Account with new_buckets_path missing and allow_bucket_creation false, valid account', async function() { const account_valid = { name: 'account_valid', uid: 999, gid: 999, allow_bucket_creation: false }; await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_valid }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = false; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_valid.name }); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); @@ -601,19 +591,107 @@ mocha.describe('nsfs nc health', function() { mocha.it('Account with new_buckets_path missing and allow_bucket_creation true, invalid account', async function() { const account_invalid = { name: 'account_invalid', uid: 999, gid: 999, allow_bucket_creation: true }; await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, { config_root, ...account_invalid }); - Health.get_service_state.restore(); - Health.get_endpoint_response.restore(); Health.all_account_details = true; Health.all_bucket_details = false; - const get_service_state = sinon.stub(Health, "get_service_state"); - get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) - .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); - const get_endpoint_response = sinon.stub(Health, "get_endpoint_response"); - get_endpoint_response.onFirstCall().returns(Promise.resolve({response: {response_code: 'RUNNING', total_fork_count: 0}})); + set_mock_functions(Health, { + get_service_state: get_service_state_mock_default_response, + get_endpoint_response: get_endpoint_response_mock_default_response, + get_system_config_file: get_system_config_mock_default_response + }); const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.DELETE, { config_root, name: account_invalid.name }); }); + + mocha.it('Health all condition - failed config directory upgrade status', async function() { + valid_system_json.config_directory = { + 'config_dir_version': config_fs.config_dir_version, + 'upgrade_package_version': pkg.version, + 'phase': CONFIG_DIR_LOCKED, + upgrade_history: { + successful_upgrades: [], + last_failure: { error: 'mock error'} + } + }; + await Health.config_fs.create_system_config_file(JSON.stringify(valid_system_json)); + const health_status = await Health.nc_nsfs_health(); + await fs_utils.file_delete(Health.config_fs.system_json_path); + assert_config_dir_status(health_status, valid_system_json.config_directory); + }); + + mocha.it('Health all condition - valid ongoing config directory upgrade', async function() { + valid_system_json.config_directory = { + config_dir_version: config_fs.config_dir_version, + upgrade_package_version: pkg.version, + phase: CONFIG_DIR_LOCKED, + upgrade_history: { + successful_upgrades: [], + }, + in_progress_upgrade: { from_version: '5.17.8', to_version: '5.18.0'} + }; + await Health.config_fs.create_system_config_file(JSON.stringify(valid_system_json)); + const health_status = await Health.nc_nsfs_health(); + await fs_utils.file_delete(Health.config_fs.system_json_path); + assert_config_dir_status(health_status, valid_system_json.config_directory); + }); }); }); + +/** + * assert_config_dir_status asserts config directory status + * @param {Object} health_status + * @param {Object} expected_config_dir + */ +function assert_config_dir_status(health_status, expected_config_dir) { + const actual_config_dir_health = health_status.checks.config_directory_status; + assert.strictEqual(actual_config_dir_health.phase, expected_config_dir.phase); + assert.strictEqual(actual_config_dir_health.config_dir_version, expected_config_dir.config_dir_version); + assert.strictEqual(actual_config_dir_health.upgrade_package_version, expected_config_dir.upgrade_package_version); + const expected_upgrade_status = expected_config_dir.in_progress_upgrade || expected_config_dir.upgrade_history?.last_failure; + assert_upgrade_status(actual_config_dir_health.upgrade_status, expected_upgrade_status); +} + +/** + * assert_upgrade_status asserts there the actual and expected upgrade_status + * @param {Object} actual_upgrade_status + * @param {Object} [expected_upgrade_status] + */ +function assert_upgrade_status(actual_upgrade_status, expected_upgrade_status = default_mock_upgrade_status.message) { + const actual_upgrade_status_res = actual_upgrade_status.in_progress_upgrade || + actual_upgrade_status.last_failure || + actual_upgrade_status.message; + assert.deepStrictEqual(actual_upgrade_status_res, expected_upgrade_status); +} + +/** + * restore_health_if_needed restores health obj functions if needed + * @param {*} health_obj + */ +function restore_health_if_needed(health_obj) { + if (health_obj?.get_service_state?.restore) health_obj.get_service_state.restore(); + if (health_obj?.get_endpoint_response?.restore) health_obj.get_endpoint_response.restore(); + if (health_obj?.config_fs?.get_system_config_file?.restore) health_obj.config_fs.get_system_config_file.restore(); + if (health_obj?.get_service_memory_usage?.restore) health_obj.get_service_memory_usage.restore(); +} + +/** + * set_mock_functions sets mock functions used by the health script + * the second param is an object having the name of the mock functions as the keys and + * the value is an array of responses by the order of their call + * @param {Object} Health + * @param {{get_endpoint_response?: Object[], get_service_state?: Object[], + * get_system_config_file?: Object[], get_service_memory_usage?: Object[]}} mock_function_responses + */ +function set_mock_functions(Health, mock_function_responses) { + for (const mock_function_name of Object.keys(mock_function_responses)) { + const mock_function_responses_arr = mock_function_responses[mock_function_name]; + const obj_to_stub = mock_function_name === 'get_system_config_file' ? Health.config_fs : Health; + + if (obj_to_stub[mock_function_name]?.restore) obj_to_stub[mock_function_name]?.restore(); + const stub = sinon.stub(obj_to_stub, mock_function_name); + for (let i = 0; i < mock_function_responses_arr.length; i++) { + stub.onCall(i).returns(Promise.resolve(mock_function_responses_arr[i])); + } + } +} diff --git a/src/upgrade/nc_upgrade_manager.js b/src/upgrade/nc_upgrade_manager.js index 83bde5fa57..e0630fe94b 100644 --- a/src/upgrade/nc_upgrade_manager.js +++ b/src/upgrade/nc_upgrade_manager.js @@ -6,10 +6,9 @@ const _ = require('lodash'); const path = require('path'); const util = require('util'); const pkg = require('../../package.json'); -const dbg = require('../util/debug_module')('UPGRADE'); +const dbg = require('../util/debug_module')(__filename); const { should_upgrade, run_upgrade_scripts, version_compare } = require('./upgrade_utils'); -dbg.set_process_name('Upgrade'); const hostname = os.hostname(); const CONFIG_DIR_LOCKED = 'CONFIG_DIR_LOCKED'; diff --git a/src/upgrade/upgrade_utils.js b/src/upgrade/upgrade_utils.js index 453548eb1c..ff34af4d15 100644 --- a/src/upgrade/upgrade_utils.js +++ b/src/upgrade/upgrade_utils.js @@ -4,9 +4,7 @@ const fs = require('fs'); const _ = require('lodash'); const path = require('path'); -const dbg = require('../util/debug_module')('UPGRADE'); -dbg.set_process_name('Upgrade'); - +const dbg = require('../util/debug_module')(__filename); /** * @param {string} ver