Merge pull request #8627 from romayalon/romy-add-config-dir-to-hostna…

…me-system-json

NC | Online Upgrade | add host config dir version to system.json | Health - add blocked hosts check

src/cmd/nsfs.js

@@ -330,7 +330,7 @@ async function main(argv = minimist(process.argv.slice(2))) {
                 const nc_upgrade_manager = new NCUpgradeManager(config_fs);
                 await nc_upgrade_manager.update_rpm_upgrade();
             } else {
-                system_data = await config_fs.init_nc_system();
+                system_data = await config_fs.register_hostname_in_system_json();
             }
         }
 

src/manage_nsfs/health.js

@@ -450,15 +450,17 @@ class NSFSHealth {
      */
     _get_config_dir_status(system_data) {
         if (!system_data) return { error: 'system data is missing' };
+        const blocked_hosts_status = this._get_blocked_hosts_status(this.config_fs, system_data);
         const config_dir_data = system_data.config_directory;
-        if (!config_dir_data) return { error: 'config directory data is missing, must upgrade config directory' };
+        if (!config_dir_data) return { error: 'config directory data is missing, must upgrade config directory', blocked_hosts: blocked_hosts_status };
         const config_dir_upgrade_status = this._get_config_dir_upgrade_status(config_dir_data);
         return {
             phase: config_dir_data.phase,
             config_dir_version: config_dir_data.config_dir_version,
             upgrade_package_version: config_dir_data.upgrade_package_version,
             upgrade_status: config_dir_upgrade_status,
-            error: config_dir_upgrade_status.error || undefined
+            error: config_dir_upgrade_status.error || undefined,
+            blocked_hosts: blocked_hosts_status
         };
     }
 
@@ -480,6 +482,47 @@ class NSFSHealth {
         }
     }
 
+    /**
+     * _get_blocked_hosts_status checks if there are any hosts blocked for updating the config directory
+     * 1. if only config dir was upgraded (>=5.18.0) - host's config_dir_version does not exist (<5.18.0) and system's config_dir_version exists (>=5.18.0)
+     * it means it's not blocked because the source code won't include _throw_if_config_dir_locked() but it still can create invalid config files, therefore including it in the blocked list
+     * 2. if system's config_dir_version wasn't upgraded yet and hosts's config_dir_version exist (>= 5.18.0)
+     * it means updates to the config directory from this host are blocked
+     * 3. if system's config dir version does not match the hosts's config_dir_version - updates to the config directory from this host are blocked
+     * @param {import('../sdk/config_fs').ConfigFS} config_fs
+     * @param {Object} system_data 
+     * @returns {Object}
+     */
+    _get_blocked_hosts_status(config_fs, system_data) {
+        const system_config_dir_version = system_data.config_directory?.config_dir_version;
+        const hosts_data = config_fs.get_hosts_data(system_data);
+        let res;
+        for (const host_name of Object.keys(hosts_data)) {
+            const host_data = hosts_data[host_name];
+            let version_compare_err;
+            const only_config_dir_upgraded = !host_data.config_dir_version && system_config_dir_version;
+            const only_host_upgraded = host_data.config_dir_version && !system_config_dir_version;
+            if (only_config_dir_upgraded) {
+                version_compare_err = `host's config_dir_version is undefined, system's config_dir_version already upgraded to ${system_config_dir_version}, updates to the config directory via the host will result with invalid config_dir files until the host source code upgrade`;
+            } else if (only_host_upgraded) {
+                version_compare_err = `host's config_dir_version is ${host_data.config_dir_version}, system's config_dir_version is undefined, updates to the config directory will be blocked until the config dir upgrade`;
+            } else {
+                version_compare_err = host_data.config_dir_version && system_config_dir_version &&
+                    config_fs.compare_host_and_config_dir_version(host_data.config_dir_version, system_config_dir_version);
+            }
+            if (version_compare_err !== undefined) {
+                res = Object.assign(res || {}, {
+                    [host_name]: {
+                        host_version: host_data.current_version,
+                        host_config_dir_version: host_data.config_dir_version,
+                        error: version_compare_err
+                    }
+                });
+            }
+        }
+        return res;
+    }
+
     /**
      *  _calc_health_status calcs the overall health status of NooBaa NC
      * @param {{service_status: String, 

src/sdk/config_fs.js

@@ -1070,15 +1070,15 @@ class ConfigFS {
     }
 
     /**
-     * init_nc_system creates/updates system.json file
+     * register_hostname_in_system_json creates/updates system.json file
      * if system.json does not exist (a new system) - host and config dir data will be set on the newly created file
      * else -
      *  1. if the host data already exist in system.json - return
-     *  2. set the host data on system.json data and update the file
+     *  2. update the host data on system.json
      * Note - config directory data on upgraded systems will be set by nc_upgrade_manager
-     * @returns 
+     * @returns {Promise<Object>}
      */
-    async init_nc_system() {
+    async register_hostname_in_system_json() {
         const system_data = await this.get_system_config_file({silent_if_missing: true});
 
         let updated_system_json = system_data || {};
@@ -1176,21 +1176,40 @@ class ConfigFS {
         const system_data = await this.get_system_config_file({ silent_if_missing: true });
         // if system was never created, currently we allow using the CLI without creating system
         // we should consider changing it to throw on this scenario as well
+        // https://github.com/noobaa/noobaa-core/issues/8468
         if (!system_data) return;
         if (!system_data.config_directory) {
             throw new RpcError('CONFIG_DIR_VERSION_MISMATCH', `config_directory data is missing in system.json, any updates to the config directory are blocked until the config dir upgrade`);
         }
         const running_code_config_dir_version = this.config_dir_version;
         const system_config_dir_version = system_data.config_directory.config_dir_version;
+        const ver_comparison_err = this.compare_host_and_config_dir_version(running_code_config_dir_version, system_config_dir_version);
+        if (ver_comparison_err !== undefined) {
+            throw new RpcError('CONFIG_DIR_VERSION_MISMATCH', ver_comparison_err);
+        }
+    }
+
+    /**
+     * compare_host_and_config_dir_version compares the version of the config dir in the system.json file 
+     * with the config dir version of the running host
+     * if compare result is 0 - undefined will be returned
+     * else - an appropriate error string will be returned
+     * @param {String} running_code_config_dir_version 
+     * @param {String} system_config_dir_version 
+     * @returns {String | Undefined}
+     */
+    compare_host_and_config_dir_version(running_code_config_dir_version, system_config_dir_version) {
         const ver_comparison = version_compare(running_code_config_dir_version, system_config_dir_version);
+        dbg.log0(`config_fs.compare_host_and_config_dir_version: ver_comparison ${ver_comparison} running_code_config_dir_version ${running_code_config_dir_version} system_config_dir_version ${system_config_dir_version}`);
         if (ver_comparison > 0) {
-            throw new RpcError('CONFIG_DIR_VERSION_MISMATCH', `running code config_dir_version=${running_code_config_dir_version} is higher than the config dir version` +
-                `mentioned in system.json =${system_config_dir_version}, any updates to the config directory are blocked until the config dir upgrade`);
+            return `running code config_dir_version=${running_code_config_dir_version} is higher than the config dir version ` +
+                `mentioned in system.json=${system_config_dir_version}, any updates to the config directory are blocked until the config dir upgrade`;
         }
         if (ver_comparison < 0) {
-            throw new RpcError('CONFIG_DIR_VERSION_MISMATCH', `running code config_dir_version=${running_code_config_dir_version} is lower than the config dir version` +
-                `mentioned in system.json =${system_config_dir_version}, any updates to the config directory are blocked until the source code upgrade`);
+            return `running code config_dir_version=${running_code_config_dir_version} is lower than the config dir version ` +
+                `mentioned in system.json=${system_config_dir_version}, any updates to the config directory are blocked until the source code upgrade`;
         }
+        return undefined;
     }
 
     /**
@@ -1201,6 +1220,7 @@ class ConfigFS {
         return {
             [os.hostname()]: {
                 current_version: pkg.version,
+                config_dir_version: this.config_dir_version,
                 upgrade_history: {
                     successful_upgrades: []
                 },
@@ -1226,6 +1246,15 @@ class ConfigFS {
             }
         };
     }
+
+    /**
+     * get_hosts_data recieves system_data and returns only the hosts data
+     * @param {Object} system_data 
+     * @returns {Object}
+     */
+    get_hosts_data(system_data) {
+        return _.omit(system_data, 'config_directory');
+    }
 }
 
 // EXPORTS

src/test/unit_tests/jest_tests/test_config_fs.test.js

@@ -1,8 +1,10 @@
 /* Copyright (C) 2024 NooBaa */
 'use strict';
 
+const os = require('os');
 const path = require('path');
 const config = require('../../../../config');
+const pkg = require('../../../../package.json');
 const { TMP_PATH } = require('../../system_tests/test_utils');
 const { get_process_fs_context } = require('../../../util/native_fs_utils');
 const { ConfigFS } = require('../../../sdk/config_fs');
@@ -29,3 +31,43 @@ describe('adjust_bucket_with_schema_updates', () => {
         expect(bucket).not.toHaveProperty('bucket_owner');
     });
 });
+
+describe('_get_new_hostname_data', () => {
+    it('_get_new_hostname_data - happy path', () => {
+        const new_hostname_data = config_fs._get_new_hostname_data();
+        expect(new_hostname_data).toStrictEqual({
+            [os.hostname()]: {
+                current_version: pkg.version,
+                config_dir_version: config_fs.config_dir_version,
+                upgrade_history: {
+                    successful_upgrades: []
+                },
+            }
+        });
+    });
+});
+
+describe('compare_host_and_config_dir_version', () => {
+    it('running code config_dir_version equals to system.json config_dir_version', () => {
+        const running_code_config_dir_version = '0.0.0';
+        const system_config_dir_version = '0.0.0';
+        const ver_compare_err = config_fs.compare_host_and_config_dir_version(running_code_config_dir_version, system_config_dir_version);
+        expect(ver_compare_err).toBeUndefined();
+    });
+
+    it('running code config_dir_version higher than system.json config_dir_version', () => {
+        const running_code_config_dir_version = '1.0.0';
+        const system_config_dir_version = '0.0.0';
+        const ver_compare_err = config_fs.compare_host_and_config_dir_version(running_code_config_dir_version, system_config_dir_version);
+        expect(ver_compare_err).toBe(`running code config_dir_version=${running_code_config_dir_version} is higher than the config dir version ` +
+                `mentioned in system.json=${system_config_dir_version}, any updates to the config directory are blocked until the config dir upgrade`);
+    });
+
+    it('running code config_dir_version lower than system.json config_dir_version', () => {
+        const running_code_config_dir_version = '0.0.0';
+        const system_config_dir_version = '1.0.0';
+        const ver_compare_err = config_fs.compare_host_and_config_dir_version(running_code_config_dir_version, system_config_dir_version);
+        expect(ver_compare_err).toBe(`running code config_dir_version=${running_code_config_dir_version} is lower than the config dir version ` +
+                `mentioned in system.json=${system_config_dir_version}, any updates to the config directory are blocked until the source code upgrade`);
+    });
+});

src/test/unit_tests/jest_tests/test_nc_upgrade_manager.test.js

@@ -59,60 +59,61 @@ module.exports = {
 `;
 const old_expected_system_json = {
     [hostname]: {
-        'current_version': '5.17.0',
-        'upgrade_history': {
-            'successful_upgrades': [{
-                'timestamp': 1724687496424,
-                'from_version': '5.16.0',
-                'to_version': '5.17.0'
+        current_version: '5.17.0',
+        upgrade_history: {
+            successful_upgrades: [{
+                timestamp: 1724687496424,
+                from_version: '5.16.0',
+                to_version: '5.17.0'
             }]
         },
     }
 };
 
 const old_expected_system_json_has_config_directory = {
     [hostname]: {
-        'current_version': '5.18.1',
-        'upgrade_history': {
-            'successful_upgrades': [{
-                'timestamp': 1724687496424,
-                'from_version': '5.18.0',
-                'to_version': '5.18.1'
+        current_version: '5.18.1',
+        upgrade_history: {
+            successful_upgrades: [{
+                timestamp: 1724687496424,
+                from_version: '5.18.0',
+                to_version: '5.18.1'
             }]
         },
     },
     config_directory: {
-        'config_dir_version': '1.0.0',
-        'upgrade_package_version': '5.18.0',
-        'phase': CONFIG_DIR_PHASES.CONFIG_DIR_UNLOCKED,
-        'upgrade_history': {
-            'successful_upgrades': [{
-                'timestamp': 1724687496424,
-                'completed_scripts': [],
-                'package_from_version': '5.17.0',
-                'package_to_version': '5.18.0'
+        config_dir_version: '1.0.0',
+        upgrade_package_version: '5.18.0',
+        phase: CONFIG_DIR_PHASES.CONFIG_DIR_UNLOCKED,
+        upgrade_history: {
+            successful_upgrades: [{
+                timestamp: 1724687496424,
+                completed_scripts: [],
+                package_from_version: '5.17.0',
+                package_to_version: '5.18.0'
             }]
         }
     }
 };
 
 const old_expected_system_json_no_successful_upgrades = {
     [hostname]: {
-        'current_version': '5.17.0',
-        'upgrade_history': {
-            'successful_upgrades': []
+        current_version: '5.17.0',
+        upgrade_history: {
+            successful_upgrades: []
         },
     }
 };
 
 const current_expected_system_json = {
     [hostname]: {
-        'current_version': pkg.version,
-        'upgrade_history': {
-            'successful_upgrades': [{
-                'timestamp': 1724687496424,
-                'from_version': '5.17.0',
-                'to_version': pkg.version
+        current_version: pkg.version,
+        config_dir_version: config_fs.config_dir_version,
+        upgrade_history: {
+            successful_upgrades: [{
+                timestamp: 1724687496424,
+                from_version: '5.17.0',
+                to_version: pkg.version
             }]
         },
     }
@@ -121,9 +122,10 @@ const current_expected_system_json = {
 
 const current_expected_system_json_no_successful_upgrades = {
     [hostname]: {
-        'current_version': pkg.version,
-        'upgrade_history': {
-            'successful_upgrades': []
+        current_version: pkg.version,
+        config_dir_version: config_fs.config_dir_version,
+        upgrade_history: {
+            successful_upgrades: []
         },
     }
 };
@@ -207,8 +209,10 @@ describe('nc upgrade manager - upgrade RPM', () => {
         await nc_upgrade_manager.update_rpm_upgrade(config_fs);
         const system_data_after_upgrade_run = await config_fs.get_system_config_file();
         const new_version = pkg.version;
+        const new_config_dir_version = config_fs.config_dir_version;
         const host_data_after_upgrade = system_data_after_upgrade_run[hostname];
         expect(host_data_after_upgrade.current_version).toStrictEqual(new_version);
+        expect(host_data_after_upgrade.config_dir_version).toStrictEqual(new_config_dir_version);
         expect(host_data_after_upgrade.upgrade_history.successful_upgrades[0].from_version).toStrictEqual(
             old_expected_system_json[hostname].current_version);
         expect(host_data_after_upgrade.upgrade_history.successful_upgrades[0].to_version).toStrictEqual(new_version);
@@ -219,8 +223,10 @@ describe('nc upgrade manager - upgrade RPM', () => {
         await nc_upgrade_manager.update_rpm_upgrade(config_fs);
         const system_data_after_upgrade_run = await config_fs.get_system_config_file();
         const new_version = pkg.version;
+        const new_config_dir_version = config_fs.config_dir_version;
         const host_data_after_upgrade = system_data_after_upgrade_run[hostname];
         expect(host_data_after_upgrade.current_version).toStrictEqual(new_version);
+        expect(host_data_after_upgrade.config_dir_version).toStrictEqual(new_config_dir_version);
         expect(host_data_after_upgrade.upgrade_history.successful_upgrades[0].from_version).toStrictEqual(
             old_expected_system_json_no_successful_upgrades[hostname].current_version);
         expect(host_data_after_upgrade.upgrade_history.successful_upgrades[0].to_version).toStrictEqual(new_version);