diff --git a/HISTORY.rst b/HISTORY.rst index 0a57323bc..177176ca7 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,10 @@ Release History =============== +0.1.57 +++++++ +* `azdev command-change meta-diff`: Remove duplicated meta-diff code and use azure-cli-diff-tool + 0.1.56 ++++++ * Fix cmdcov issue(#410): Get config from github when get_cli_repo_path failed diff --git a/azdev/__init__.py b/azdev/__init__.py index fb9d5d835..5d7476e79 100644 --- a/azdev/__init__.py +++ b/azdev/__init__.py @@ -4,4 +4,4 @@ # license information. # ----------------------------------------------------------------------------- -__VERSION__ = '0.1.56' +__VERSION__ = '0.1.57' diff --git a/azdev/operations/command_change/__init__.py b/azdev/operations/command_change/__init__.py index e3ea965b8..7759b4612 100644 --- a/azdev/operations/command_change/__init__.py +++ b/azdev/operations/command_change/__init__.py @@ -6,20 +6,18 @@ # pylint: disable=no-else-return -import json -import os import time -from deepdiff import DeepDiff from knack.log import get_logger - +import azure_cli_diff_tool from azdev.utilities import display, require_azure_cli, heading, get_path_table, filter_by_git_diff -from .custom import MetaChangeDetects, DiffExportFormat -from .util import export_meta_changes_to_json, gen_commands_meta, get_commands_meta +from .custom import DiffExportFormat, get_commands_meta +from .util import export_commands_meta from ..statistics import _create_invoker_and_load_cmds, _get_command_source, \ _command_codegen_info # pylint: disable=protected-access from ..statistics.util import filter_modules + logger = get_logger(__name__) @@ -117,28 +115,9 @@ def export_command_meta(modules=None, git_source=None, git_target=None, git_repo commands_info.append(command_info) commands_meta = get_commands_meta(command_loader.command_group_table, commands_info, with_help, with_example) - gen_commands_meta(commands_meta, meta_output_path) + export_commands_meta(commands_meta, meta_output_path) display(f"Total Commands: {len(commands_info)} from {', '.join(selected_mod_names)} have been generated.") def cmp_command_meta(base_meta_file, diff_meta_file, only_break=False, output_type="text", output_file=None): - if not os.path.exists(base_meta_file): - raise Exception("base meta file needed") - if not os.path.exists(diff_meta_file): - raise Exception("diff meta file needed") - start = time.time() - with open(base_meta_file, "r") as g: - command_tree_before = json.load(g) - with open(diff_meta_file, "r") as g: - command_tree_after = json.load(g) - stop = time.time() - logger.info('Command meta files loaded in %i sec', stop - start) - diff = DeepDiff(command_tree_before, command_tree_after) - if not diff: - display(f"No meta diffs from {diff_meta_file} to {base_meta_file}") - return export_meta_changes_to_json(None, output_file) - else: - detected_changes = MetaChangeDetects(diff, command_tree_before, command_tree_after) - detected_changes.check_deep_diffs() - result = detected_changes.export_meta_changes(only_break, output_type) - return export_meta_changes_to_json(result, output_file) + return azure_cli_diff_tool.meta_diff(base_meta_file, diff_meta_file, only_break, output_type, output_file) diff --git a/azdev/operations/command_change/custom.py b/azdev/operations/command_change/custom.py index eb4fe8d72..6350f2433 100644 --- a/azdev/operations/command_change/custom.py +++ b/azdev/operations/command_change/custom.py @@ -3,16 +3,11 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # ----------------------------------------------------------------------------- -import os.path + from enum import Enum + from knack.log import get_logger -from azdev.utilities import (CMD_PROPERTY_ADD_BREAK_LIST, CMD_PROPERTY_REMOVE_BREAK_LIST, - CMD_PROPERTY_UPDATE_BREAK_LIST, PARA_PROPERTY_REMOVE_BREAK_LIST, - PARA_PROPERTY_ADD_BREAK_LIST, PARA_PROPERTY_UPDATE_BREAK_LIST) -from .util import extract_cmd_name, extract_cmd_property, extract_subgroup_name, ChangeType from .util import get_command_tree -from .meta_changes import (CmdAdd, CmdRemove, CmdPropAdd, CmdPropRemove, CmdPropUpdate, ParaAdd, ParaRemove, - ParaPropAdd, ParaPropRemove, ParaPropUpdate, SubgroupAdd, SubgroupRemove) logger = get_logger(__name__) @@ -23,352 +18,151 @@ class DiffExportFormat(Enum): TREE = "tree" -class MetaChangeDetects: - - EXPORTED_META_PROPERTY = ["rule_id", "rule_link_url", "is_break", "rule_message", "suggest_message", - "cmd_name", "subgroup_name"] - CHECKED_PARA_PROPERTY = ["name", "options", "required", "choices", "id_part", "nargs", "default", "desc", - "aaz_type", "type", "aaz_default", "aaz_choices"] - - def __init__(self, deep_diff=None, base_meta=None, diff_meta=None): - self.deep_diff = {} - if deep_diff: - self.deep_diff = deep_diff - else: - logger.info("None diffs from cmd meta json") - assert base_meta["module_name"] == diff_meta["module_name"] - self.module_name = base_meta["module_name"] - self.base_meta = base_meta - self.diff_meta = diff_meta - self.diff_objs = [] - self.cmd_set_with_parameter_change = set() - self.meta_change_whitelist = set() - self.__get_meta_change_whitelist__() - - def __get_meta_change_whitelist__(self): - white_list_file = os.path.dirname(os.path.realpath(__file__)) + "/data/meta_change_whitelist.txt" - if not os.path.exists(white_list_file): - logger.info("meta_change_whitelist.txt not exist, skipped") - return - with open(white_list_file, "r") as f_in: - for line in f_in: - white_key = line.rstrip() - self.meta_change_whitelist.add(white_key) - - @staticmethod - def __search_cmd_obj(cmd_name, search_meta): - command_tree = get_command_tree(cmd_name) - meta_search = search_meta - while True: - if "is_group" not in command_tree: - break - if command_tree["is_group"]: - group_name = command_tree["group_name"] - meta_search = meta_search["sub_groups"][group_name] - command_tree = command_tree["sub_info"] - else: - cmd_name = command_tree["cmd_name"] - meta_search = meta_search["commands"][cmd_name] - break - return meta_search - - def __log_cmd_with_parameter_change(self, cmd_name): - self.cmd_set_with_parameter_change.add(cmd_name) - - def __iter_dict_items(self, dict_items, diff_type): - if diff_type not in [ChangeType.REMOVE, ChangeType.ADD]: - raise Exception("Unsupported dict item type") - - for dict_key in dict_items: - has_cmd, cmd_name = extract_cmd_name(dict_key) - if not has_cmd or not cmd_name: - has_subgroup, subgroup_name = extract_subgroup_name(dict_key) - if not has_subgroup or not subgroup_name: - continue - if diff_type == ChangeType.REMOVE: - diff_obj = SubgroupRemove(subgroup_name) - else: - diff_obj = SubgroupAdd(subgroup_name) - self.diff_objs.append(diff_obj) - continue - - has_cmd_key, cmd_property = extract_cmd_property(dict_key, cmd_name) - if has_cmd_key: - if cmd_property == "parameters": - self.__log_cmd_with_parameter_change(cmd_name) - elif diff_type == ChangeType.ADD: - if cmd_property in CMD_PROPERTY_ADD_BREAK_LIST: - diff_obj = CmdPropAdd(cmd_name, cmd_property, True) - else: - diff_obj = CmdPropAdd(cmd_name, cmd_property, False) - self.diff_objs.append(diff_obj) - else: - if cmd_property in CMD_PROPERTY_REMOVE_BREAK_LIST: - diff_obj = CmdPropRemove(cmd_name, cmd_property, True) - else: - diff_obj = CmdPropRemove(cmd_name, cmd_property, False) - self.diff_objs.append(diff_obj) - else: - if diff_type == ChangeType.REMOVE: - diff_obj = CmdRemove(cmd_name) - else: - diff_obj = CmdAdd(cmd_name) - - self.diff_objs.append(diff_obj) - - def __iter_list_items(self, list_items, diff_type): - """ - ['parameters'][0]['options'][1] - ['parameters'][0]['choices'][0] - ['parameters'][3] - """ - if diff_type not in [ChangeType.REMOVE, ChangeType.ADD]: - raise Exception("Unsupported dict item type") - - for key, _ in list_items.items(): - has_cmd, cmd_name = extract_cmd_name(key) - if not has_cmd or not cmd_name: - print("extract cmd failed for " + key) - continue - has_cmd_key, cmd_property = extract_cmd_property(key, cmd_name) - if not has_cmd_key: +def process_aaz_argument(az_arguments_schema, argument_settings, para): + from azure.cli.core.aaz import has_value # pylint: disable=import-error + _fields = az_arguments_schema._fields # pylint: disable=protected-access + aaz_type = _fields.get(argument_settings["dest"], None) + if aaz_type: + para["aaz_type"] = aaz_type.__class__.__name__ + if aaz_type._type_in_help and aaz_type._type_in_help.lower() != "undefined": # pylint: disable=protected-access + para["type"] = aaz_type._type_in_help # pylint: disable=protected-access + if has_value(aaz_type._default): # pylint: disable=protected-access + para["aaz_default"] = aaz_type._default # pylint: disable=protected-access + if para["aaz_type"] in ["AAZArgEnum"] and aaz_type.get("enum", None) and aaz_type.enum.get("items", None): + para["aaz_choices"] = aaz_type.enum["items"] + + +def process_arg_options(argument_settings, para): + para["options"] = [] + if not argument_settings.get("options_list", None): + return + raw_options_list = argument_settings["options_list"] + option_list = set() + for opt in raw_options_list: + opt_type = opt.__class__.__name__ + if opt_type == "str": + option_list.add(opt) + elif opt_type == "Deprecated": + if hasattr(opt, "hide") and opt.hide: continue - if cmd_property == "parameters": - self.__log_cmd_with_parameter_change(cmd_name) - - def check_dict_item_remove(self): - if not self.deep_diff.get("dictionary_item_removed", None): - return - dict_item_removed = self.deep_diff["dictionary_item_removed"] - self.__iter_dict_items(dict_item_removed, ChangeType.REMOVE) - - def check_dict_item_add(self): - if not self.deep_diff.get("dictionary_item_added", None): - return - dict_item_added = self.deep_diff["dictionary_item_added"] - self.__iter_dict_items(dict_item_added, ChangeType.ADD) - - def check_list_item_remove(self): - if not self.deep_diff.get("iterable_item_removed", None): - return - - list_item_remove = self.deep_diff["iterable_item_removed"] - self.__iter_list_items(list_item_remove, ChangeType.REMOVE) - - def check_list_item_add(self): - if not self.deep_diff.get("iterable_item_added", None): - return - list_item_add = self.deep_diff["iterable_item_added"] - self.__iter_list_items(list_item_add, ChangeType.ADD) - - def check_value_change(self): - if not self.deep_diff.get("values_changed", None): - return - value_changes = self.deep_diff["values_changed"] - for key, value_obj in value_changes.items(): - old_value = value_obj["old_value"] - new_value = value_obj["new_value"] - has_cmd, cmd_name = extract_cmd_name(key) - if not has_cmd or not cmd_name: - print("extract cmd failed for " + key) - continue - - has_cmd_prop, cmd_property = extract_cmd_property(key, cmd_name) - if not has_cmd_prop: - return - if cmd_property == "parameters": - self.__log_cmd_with_parameter_change(cmd_name) - else: - if cmd_property in CMD_PROPERTY_UPDATE_BREAK_LIST: - diff_obj = CmdPropUpdate(cmd_name, cmd_property, True, old_value, new_value) - else: - diff_obj = CmdPropUpdate(cmd_name, cmd_property, False, old_value, new_value) - self.diff_objs.append(diff_obj) - - @staticmethod - def __search_para_with_name_and_options(base_para_obj, search_parameters): - para_name = base_para_obj["name"] - para_option_set = set(base_para_obj["options"]) - for para_obj in search_parameters: - name = para_obj["name"] - option_set = set(para_obj.get("options", [])) - if para_name == name or para_option_set.issubset(option_set): - # parameter obj which has the same name or new option list contains old option list, - # is same parameter obj - # if name is changed and new option list lost element in old option list, then is different - return para_obj - return None - - def check_cmd_parameter_diff(self, cmd_name, base_parameters, cmp_parameters): - """check cmd parameter diff""" - for base_para_obj in base_parameters: - base_para_obj["checked"] = True - cmp_para_obj = self.__search_para_with_name_and_options(base_para_obj, cmp_parameters) - if cmp_para_obj is None: - # cmd lost parameter obj, is break - diff_obj = ParaRemove(cmd_name, base_para_obj["name"], True) - # add flag to avoid duplicate compare - self.diff_objs.append(diff_obj) - continue - cmp_para_obj["checked"] = True - for prop in self.CHECKED_PARA_PROPERTY: - if prop not in base_para_obj and prop not in cmp_para_obj: - continue - if prop in base_para_obj and prop not in cmp_para_obj: - # prop dropped in new para obj - if prop in PARA_PROPERTY_REMOVE_BREAK_LIST: - diff_obj = ParaPropRemove(cmd_name, base_para_obj["name"], prop, True) - else: - diff_obj = ParaPropRemove(cmd_name, base_para_obj["name"], prop, False) - self.diff_objs.append(diff_obj) - continue - if prop not in base_para_obj and prop in cmp_para_obj: - # prop added in new para obj - if prop in PARA_PROPERTY_ADD_BREAK_LIST: - diff_obj = ParaPropAdd(cmd_name, base_para_obj["name"], prop, True) - else: - diff_obj = ParaPropAdd(cmd_name, base_para_obj["name"], prop, False) - self.diff_objs.append(diff_obj) - continue - # prop exists in both new and old para obj, value needs to be checked - base_val = base_para_obj[prop] - cmp_val = cmp_para_obj[prop] - - if base_val != cmp_val: - if isinstance(base_val, list) and isinstance(cmp_val, list): - if set(base_val).issubset(set(cmp_val)): - diff_obj = ParaPropUpdate(cmd_name, base_para_obj["name"], prop, False, base_val, cmp_val) - else: - diff_obj = ParaPropUpdate(cmd_name, base_para_obj["name"], prop, True, base_val, cmp_val) - elif prop in PARA_PROPERTY_UPDATE_BREAK_LIST: - diff_obj = ParaPropUpdate(cmd_name, base_para_obj["name"], prop, True, base_val, cmp_val) - else: - diff_obj = ParaPropUpdate(cmd_name, base_para_obj["name"], prop, False, base_val, cmp_val) - self.diff_objs.append(diff_obj) - - # check added parameter obj, if obj is required, then is break - for cmp_para_obj in cmp_parameters: - if "checked" in cmp_para_obj and cmp_para_obj["checked"]: + if hasattr(opt, "target"): + option_list.add(opt.target) + else: + logger.warning("Unsupported option type: %i", opt_type) + para["options"] = sorted(option_list) + + +def process_arg_type(argument_settings, para): + if not argument_settings.get("type", None): + return + configured_type = argument_settings["type"] + raw_type = None + if hasattr(configured_type, "__name__"): + raw_type = configured_type.__name__ + elif hasattr(configured_type, "__class__"): + raw_type = configured_type.__class__.__name__ + else: + print("unsupported type", configured_type) + return + para["type"] = raw_type if raw_type in ["str", "int", "float", "bool", "file_type"] else "custom_type" + + +def gen_command_meta(command_info, with_help=False, with_example=False): + stored_property_when_exist = ["confirmation", "supports_no_wait", "is_preview"] + command_meta = { + "name": command_info["name"], + "is_aaz": command_info["is_aaz"], + } + for prop in stored_property_when_exist: + if command_info[prop]: + command_meta[prop] = command_info[prop] + if with_example: + try: + command_meta["examples"] = command_info["help"]["examples"] + except AttributeError: + pass + if with_help: + try: + command_meta["desc"] = command_info["help"]["short-summary"] + except AttributeError: + pass + parameters = [] + for _, argument in command_info["arguments"].items(): + if argument.type is None: + continue + settings = argument.type.settings + if settings.get("action", None): + action = settings["action"] + if hasattr(action, "__name__") and action.__name__ == "IgnoreAction": + # ignore argument like: cmd continue - para_name = cmp_para_obj["name"] - required = cmp_para_obj.get("required", None) - if required: - diff_obj = ParaAdd(cmd_name, para_name, True) + para = { + "name": settings["dest"], + } + process_arg_options(settings, para) + process_arg_type(settings, para) + if settings.get("required", False): + para["required"] = True + if settings.get("choices", None): + para["choices"] = sorted(list(settings["choices"])) + if settings.get("id_part", None): + para["id_part"] = settings["id_part"] + if settings.get("nargs", None): + para["nargs"] = settings["nargs"] + if settings.get("default", None): + if not isinstance(settings["default"], (float, int, str, list, bool)): + para["default"] = str(settings["default"]) else: - diff_obj = ParaAdd(cmd_name, para_name, False) - self.diff_objs.append(diff_obj) - - def check_cmds_parameter_diff(self): - """ - deal with command parameter diffs - """ - for cmd_name in self.cmd_set_with_parameter_change: - cmd_base = self.__search_cmd_obj(cmd_name, self.base_meta) - cmd_cmp = self.__search_cmd_obj(cmd_name, self.diff_meta) - base_parameters = cmd_base.get("parameters", []) - cmp_parameters = cmd_cmp.get("parameters", []) - self.check_cmd_parameter_diff(cmd_name, base_parameters, cmp_parameters) - - def filter_diffs_by_whitelist(self): - """ - filter_diffs_by_whitelist - """ - new_diff_objs = [] - for obj in self.diff_objs: - if obj.filter_key and obj.is_break and "\t".join(obj.filter_key) in self.meta_change_whitelist: - continue - new_diff_objs.append(obj) - self.diff_objs = new_diff_objs - - def check_deep_diffs(self): - self.check_dict_item_remove() - self.check_dict_item_add() - self.check_list_item_remove() - self.check_list_item_add() - self.check_value_change() - self.check_cmds_parameter_diff() - self.filter_diffs_by_whitelist() - - @staticmethod - def fill_subgroup_rules(obj, ret_mod, rule): - command_group_info = ret_mod - group_name_arr = obj.subgroup_name.split(" ") - start_level = 1 - while start_level < len(group_name_arr): - group_name = " ".join(group_name_arr[:start_level]) - if group_name not in command_group_info["sub_groups"]: - command_group_info["sub_groups"][group_name] = { - "name": group_name, - "commands": {}, - "sub_groups": {} - } - start_level += 1 - command_group_info = command_group_info["sub_groups"][group_name] - group_name = obj.subgroup_name - group_rules = [] - if group_name not in command_group_info["sub_groups"]: - command_group_info["sub_groups"] = {group_name: group_rules} - group_rules = command_group_info["sub_groups"][group_name] - group_rules.append(rule) - command_group_info["sub_groups"][group_name] = group_rules - - @staticmethod - def fill_cmd_rules(obj, ret_mod, rule): - command_tree = get_command_tree(obj.cmd_name) - command_group_info = ret_mod + para["default"] = settings["default"] + if with_help: + para["desc"] = settings["help"] + if command_info["is_aaz"] and command_info["az_arguments_schema"]: + process_aaz_argument(command_info["az_arguments_schema"], settings, para) + parameters.append(para) + command_meta["parameters"] = parameters + return command_meta + + +def get_commands_meta(command_group_table, commands_info, with_help, with_example): + commands_meta = {} + + for command_info in commands_info: # pylint: disable=too-many-nested-blocks + module_name = command_info["source"]["module"] + command_name = command_info["name"] + if module_name not in commands_meta: + commands_meta[module_name] = { + "module_name": module_name, + "name": "az", + "commands": {}, + "sub_groups": {} + } + command_group_info = commands_meta[module_name] + command_tree = get_command_tree(command_name) while True: if "is_group" not in command_tree: break if command_tree["is_group"]: group_name = command_tree["group_name"] if group_name not in command_group_info["sub_groups"]: + group_info = command_group_table.get(group_name, None) command_group_info["sub_groups"][group_name] = { "name": group_name, "commands": {}, "sub_groups": {} } + if with_help: + try: + command_group_info["sub_groups"][group_name]["desc"] = group_info.help["short-summary"] + except AttributeError: + pass + command_tree = command_tree["sub_info"] command_group_info = command_group_info["sub_groups"][group_name] else: - cmd_name = command_tree["cmd_name"] - command_rules = [] - if cmd_name in command_group_info["commands"]: - command_rules = command_group_info["commands"][cmd_name] - command_rules.append(rule) - command_group_info["commands"][cmd_name] = command_rules + if command_name in command_group_info["commands"]: + logger.warning("repeated command: %i", command_name) + break + command_meta = gen_command_meta(command_info, with_help, with_example) + command_group_info["commands"][command_name] = command_meta break - - def export_meta_changes(self, only_break, output_type="text"): - ret_objs = [] - ret_mod = { - "module_name": self.module_name, - "name": "az", - "commands": {}, - "sub_groups": {} - } - for obj in self.diff_objs: - if only_break and not obj.is_break: - continue - if obj.is_ignore: - continue - ret = {} - for prop in self.EXPORTED_META_PROPERTY: - if hasattr(obj, prop): - ret[prop] = getattr(obj, prop) - ret["rule_name"] = obj.__class__.__name__ - if output_type == "dict": - ret_objs.append(ret) - elif output_type == "text": - ret_objs.append(str(obj)) - if output_type != "tree": - continue - if not hasattr(obj, "cmd_name") and not hasattr(obj, "subgroup_name"): - logger.info("unsupported rule ignored") - elif not hasattr(obj, "cmd_name") and hasattr(obj, "subgroup_name"): - self.fill_subgroup_rules(obj, ret_mod, ret) - elif not hasattr(obj, "subgroup_name") and hasattr(obj, "cmd_name"): - self.fill_cmd_rules(obj, ret_mod, ret) - else: - logger.info("unsupported rule ignored") - - return ret_objs if output_type in ["text", "dict"] else ret_mod + return commands_meta diff --git a/azdev/operations/command_change/data/meta_change_whitelist.txt b/azdev/operations/command_change/data/meta_change_whitelist.txt deleted file mode 100644 index 0d1d72023..000000000 --- a/azdev/operations/command_change/data/meta_change_whitelist.txt +++ /dev/null @@ -1,9 +0,0 @@ -1010 ams asset get-sas-urls expiry_time default -1010 mysql flexible-server create administrator_login default -1010 mysql flexible-server restore restore_point_in_time default -1010 mysql flexible-server import create administrator_login default -1010 postgres flexible-server create administrator_login default -1010 postgres flexible-server restore restore_point_in_time default -1010 mysql server create administrator_login default -1010 postgres server create administrator_login default -1010 mariadb server create administrator_login default \ No newline at end of file diff --git a/azdev/operations/command_change/meta_changes.py b/azdev/operations/command_change/meta_changes.py deleted file mode 100644 index edaa07be3..000000000 --- a/azdev/operations/command_change/meta_changes.py +++ /dev/null @@ -1,256 +0,0 @@ -# ----------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# ----------------------------------------------------------------------------- - -# pylint: disable=too-many-instance-attributes - -from azdev.utilities import get_change_rule_template, get_change_suggest_template -from azdev.utilities import CMD_PROPERTY_IGNORED_LIST, PARA_PROPERTY_IGNORED_LIST, PARA_NAME_IGNORED_LIST, \ - PARA_VALUE_IGNORED_LIST, BREAKING_CHANE_RULE_LINK_URL_PREFIX, BREAKING_CHANE_RULE_LINK_URL_SUFFIX - - -class MetaChange: - - def __init__(self, rule_id="1000", is_break=False, rule_message="", suggest_message="", - is_ignore=False, filter_key=None): - self.rule_id = rule_id - self.rule_link_url = BREAKING_CHANE_RULE_LINK_URL_PREFIX + self.rule_id + BREAKING_CHANE_RULE_LINK_URL_SUFFIX - self.is_break = is_break - self.rule_message = rule_message - self.suggest_message = suggest_message - self.is_ignore = is_ignore - self.filter_key = filter_key - - def __str__(self): - res = [self.rule_message] - if self.is_break: - res.append("is_break: {0}".format(self.is_break)) - res.append(self.suggest_message) - return " | ".join([str(a) for a in res]) - - -class SubgroupAdd(MetaChange): - def __init__(self, subgroup_name, is_break=False): - if not subgroup_name: - raise Exception("sub group name needed") - self.rule_id = "1011" - self.subgroup_name = subgroup_name - self.is_break = is_break - self.rule_message = get_change_rule_template(self.rule_id).format(self.subgroup_name) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.subgroup_name) \ - if self.is_break else "" - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message) - - -class SubgroupRemove(MetaChange): - def __init__(self, subgroup_name, is_break=True): - if not subgroup_name: - raise Exception("sub group name needed") - self.rule_id = "1012" - self.subgroup_name = subgroup_name - self.is_break = is_break - self.rule_message = get_change_rule_template(self.rule_id).format(self.subgroup_name) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.subgroup_name) \ - if self.is_break else "" - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message) - - -class CmdAdd(MetaChange): - def __init__(self, cmd_name, is_break=False): - if not cmd_name: - raise Exception("cmd name needed") - self.rule_id = "1001" - self.cmd_name = cmd_name - self.is_break = is_break - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.cmd_name) if self.is_break else "" - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message) - - -class CmdRemove(MetaChange): - def __init__(self, cmd_name, is_break=True): - if not cmd_name: - raise Exception("cmd name needed") - self.cmd_name = cmd_name - self.rule_id = "1002" - self.is_break = is_break - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.cmd_name) if self.is_break else "" - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message) - - -class CmdPropAdd(MetaChange): - def __init__(self, cmd_name, cmd_property, is_break=False): - if not cmd_name or not cmd_property: - raise Exception("cmd name needed") - self.rule_id = "1003" - self.is_ignore = False - self.cmd_name = cmd_name - self.cmd_property = cmd_property - self.is_break = is_break - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.cmd_property) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.cmd_property, self.cmd_name) \ - if self.is_break else "" - if cmd_property in CMD_PROPERTY_IGNORED_LIST: - self.is_ignore = True - self.filter_key = [self.rule_id, self.cmd_name, self.cmd_property] - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message, self.is_ignore, - self.filter_key) - - -class CmdPropRemove(MetaChange): - def __init__(self, cmd_name, cmd_property, is_break=False): - if not cmd_name or not cmd_property: - raise Exception("cmd name needed") - self.rule_id = "1004" - self.is_ignore = False - self.cmd_name = cmd_name - self.cmd_property = cmd_property - self.is_break = is_break - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.cmd_property) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.cmd_property, self.cmd_name) \ - if self.is_break else "" - if cmd_property in CMD_PROPERTY_IGNORED_LIST: - self.is_ignore = True - self.filter_key = [self.rule_id, self.cmd_name, self.cmd_property] - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message, - self.is_ignore, self.filter_key) - - -class CmdPropUpdate(MetaChange): - - def __init__(self, cmd_name, cmd_property, is_break=False, old_value=None, new_value=None): - if not cmd_name or not cmd_property: - raise Exception("cmd name and cmd prop needed") - self.rule_id = "1005" - self.is_ignore = False - self.cmd_name = cmd_name - self.is_break = is_break - self.cmd_prop_updated = cmd_property - self.old_value = "" - self.new_value = "" - - if old_value is not None: - self.old_value = old_value - if new_value is not None: - self.new_value = new_value - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.cmd_prop_updated, - self.old_value, self.new_value) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.cmd_prop_updated, - self.new_value, self.old_value, - self.cmd_name) if self.is_break else "" - if cmd_property in CMD_PROPERTY_IGNORED_LIST: - self.is_ignore = True - self.filter_key = [self.rule_id, self.cmd_name, self.cmd_prop_updated] - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message, - self.is_ignore, self.filter_key) - - -class ParaAdd(MetaChange): - - def __init__(self, cmd_name, para_name, is_break=False): - if not cmd_name or not para_name: - raise Exception("cmd name, parameter name needed") - self.rule_id = "1006" - self.cmd_name = cmd_name - self.para_name = para_name - self.is_break = is_break - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.para_name) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.para_name, - self.cmd_name) if self.is_break else "" - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message) - - -class ParaRemove(MetaChange): - - def __init__(self, cmd_name, para_name, is_break=False): - if not cmd_name or not para_name: - raise Exception("cmd name, parameter name needed") - self.rule_id = "1007" - self.cmd_name = cmd_name - self.para_name = para_name - self.is_break = is_break - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.para_name) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.para_name, - self.cmd_name) if self.is_break else "" - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message) - - -class ParaPropAdd(MetaChange): - - def __init__(self, cmd_name, para_name, para_property, is_break=False): - if not cmd_name or not para_name or not para_property: - raise Exception("cmd name, parameter name and parameter property needed") - self.rule_id = "1008" - self.is_ignore = False - self.cmd_name = cmd_name - self.para_name = para_name - self.para_prop = para_property - self.is_break = is_break - - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.para_name, - self.para_prop) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.para_prop, - self.para_name, - self.cmd_name) if self.is_break else "" - if para_property in PARA_PROPERTY_IGNORED_LIST or para_name in PARA_NAME_IGNORED_LIST: - self.is_ignore = True - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message, self.is_ignore) - - -class ParaPropRemove(MetaChange): - - def __init__(self, cmd_name, para_name, para_property, is_break=False): - if not cmd_name or not para_name or not para_property: - raise Exception("cmd name, parameter name and parameter property needed") - self.rule_id = "1009" - self.is_ignore = False - self.cmd_name = cmd_name - self.para_name = para_name - self.para_prop = para_property - self.is_break = is_break - - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.para_name, - self.para_prop) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.para_prop, - self.para_name, - self.cmd_name) if self.is_break else "" - if para_property in PARA_PROPERTY_IGNORED_LIST or para_name in PARA_NAME_IGNORED_LIST: - self.is_ignore = True - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message, self.is_ignore) - - -class ParaPropUpdate(MetaChange): - - def __init__(self, cmd_name, para_name, para_property, is_break=False, old_value=None, new_value=None): - if not cmd_name or not para_name or not para_property: - raise Exception("cmd name, parameter name and parameter property needed") - self.rule_id = "1010" - self.is_ignore = False - self.cmd_name = cmd_name - self.para_name = para_name - self.para_prop_updated = para_property - self.is_break = is_break - self.old_value = None - self.new_value = None - if old_value is not None: - self.old_value = old_value - if new_value is not None: - self.new_value = new_value - - self.rule_message = get_change_rule_template(self.rule_id).format(self.cmd_name, self.para_name, - self.para_prop_updated, - self.old_value, self.new_value) - self.suggest_message = get_change_suggest_template(self.rule_id).format(self.para_prop_updated, - self.new_value, self.old_value, - self.para_name, - self.cmd_name) if self.is_break else "" - if para_property in PARA_PROPERTY_IGNORED_LIST or para_name in PARA_NAME_IGNORED_LIST: - self.is_ignore = True - if self.new_value in PARA_VALUE_IGNORED_LIST or self.old_value in PARA_VALUE_IGNORED_LIST: - self.is_ignore = True - self.filter_key = [self.rule_id, self.cmd_name, self.para_name, self.para_prop_updated] - super().__init__(self.rule_id, is_break, self.rule_message, self.suggest_message, - self.is_ignore, self.filter_key) diff --git a/azdev/operations/command_change/util.py b/azdev/operations/command_change/util.py index eddad8895..03d09198a 100644 --- a/azdev/operations/command_change/util.py +++ b/azdev/operations/command_change/util.py @@ -3,7 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- - import json import os import re @@ -62,157 +61,7 @@ def get_command_tree(command_name): return ret -def process_aaz_argument(az_arguments_schema, argument_settings, para): - from azure.cli.core.aaz import has_value # pylint: disable=import-error - _fields = az_arguments_schema._fields # pylint: disable=protected-access - aaz_type = _fields.get(argument_settings["dest"], None) - if aaz_type: - para["aaz_type"] = aaz_type.__class__.__name__ - if aaz_type._type_in_help and aaz_type._type_in_help.lower() != "undefined": # pylint: disable=protected-access - para["type"] = aaz_type._type_in_help # pylint: disable=protected-access - if has_value(aaz_type._default): # pylint: disable=protected-access - para["aaz_default"] = aaz_type._default # pylint: disable=protected-access - if para["aaz_type"] in ["AAZArgEnum"] and aaz_type.get("enum", None) and aaz_type.enum.get("items", None): - para["aaz_choices"] = aaz_type.enum["items"] - - -def process_arg_options(argument_settings, para): - para["options"] = [] - if not argument_settings.get("options_list", None): - return - raw_options_list = argument_settings["options_list"] - option_list = set() - for opt in raw_options_list: - opt_type = opt.__class__.__name__ - if opt_type == "str": - option_list.add(opt) - elif opt_type == "Deprecated": - if hasattr(opt, "hide") and opt.hide: - continue - if hasattr(opt, "target"): - option_list.add(opt.target) - else: - logger.warning("Unsupported option type: %i", opt_type) - para["options"] = sorted(option_list) - - -def process_arg_type(argument_settings, para): - if not argument_settings.get("type", None): - return - configured_type = argument_settings["type"] - raw_type = None - if hasattr(configured_type, "__name__"): - raw_type = configured_type.__name__ - elif hasattr(configured_type, "__class__"): - raw_type = configured_type.__class__.__name__ - else: - print("unsupported type", configured_type) - return - para["type"] = raw_type if raw_type in ["str", "int", "float", "bool", "file_type"] else "custom_type" - - -def gen_command_meta(command_info, with_help=False, with_example=False): - stored_property_when_exist = ["confirmation", "supports_no_wait", "is_preview"] - command_meta = { - "name": command_info["name"], - "is_aaz": command_info["is_aaz"], - } - for prop in stored_property_when_exist: - if command_info[prop]: - command_meta[prop] = command_info[prop] - if with_example: - try: - command_meta["examples"] = command_info["help"]["examples"] - except AttributeError: - pass - if with_help: - try: - command_meta["desc"] = command_info["help"]["short-summary"] - except AttributeError: - pass - parameters = [] - for _, argument in command_info["arguments"].items(): - if argument.type is None: - continue - settings = argument.type.settings - if settings.get("action", None): - action = settings["action"] - if hasattr(action, "__name__") and action.__name__ == "IgnoreAction": - # ignore argument like: cmd - continue - para = { - "name": settings["dest"], - } - process_arg_options(settings, para) - process_arg_type(settings, para) - if settings.get("required", False): - para["required"] = True - if settings.get("choices", None): - para["choices"] = sorted(list(settings["choices"])) - if settings.get("id_part", None): - para["id_part"] = settings["id_part"] - if settings.get("nargs", None): - para["nargs"] = settings["nargs"] - if settings.get("default", None): - if not isinstance(settings["default"], (float, int, str, list, bool)): - para["default"] = str(settings["default"]) - else: - para["default"] = settings["default"] - if with_help: - para["desc"] = settings["help"] - if command_info["is_aaz"] and command_info["az_arguments_schema"]: - process_aaz_argument(command_info["az_arguments_schema"], settings, para) - parameters.append(para) - command_meta["parameters"] = parameters - return command_meta - - -def get_commands_meta(command_group_table, commands_info, with_help, with_example): - commands_meta = {} - - for command_info in commands_info: # pylint: disable=too-many-nested-blocks - module_name = command_info["source"]["module"] - command_name = command_info["name"] - if module_name not in commands_meta: - commands_meta[module_name] = { - "module_name": module_name, - "name": "az", - "commands": {}, - "sub_groups": {} - } - command_group_info = commands_meta[module_name] - command_tree = get_command_tree(command_name) - while True: - if "is_group" not in command_tree: - break - if command_tree["is_group"]: - group_name = command_tree["group_name"] - if group_name not in command_group_info["sub_groups"]: - group_info = command_group_table.get(group_name, None) - command_group_info["sub_groups"][group_name] = { - "name": group_name, - "commands": {}, - "sub_groups": {} - } - if with_help: - try: - command_group_info["sub_groups"][group_name]["desc"] = group_info.help["short-summary"] - except AttributeError: - pass - - command_tree = command_tree["sub_info"] - command_group_info = command_group_info["sub_groups"][group_name] - else: - if command_name in command_group_info["commands"]: - logger.warning("repeated command: %i", command_name) - break - command_meta = gen_command_meta(command_info, with_help, with_example) - command_group_info["commands"][command_name] = command_meta - break - return commands_meta - - -def gen_commands_meta(commands_meta, meta_output_path=None): +def export_commands_meta(commands_meta, meta_output_path=None): options = jsbeautifier.default_options() options.indent_size = 4 for key, module_info in commands_meta.items(): @@ -224,46 +73,3 @@ def gen_commands_meta(commands_meta, meta_output_path=None): os.makedirs(file_folder) with open(file_name, "w") as f_out: f_out.write(jsbeautifier.beautify(json.dumps(module_info), options)) - - -def extract_subgroup_name(key): - subgroup_ame_res = re.findall(SUBGROUP_NAME_PATTERN, key) - if not subgroup_ame_res or len(subgroup_ame_res) == 0: - return False, None - return True, subgroup_ame_res[-1] - - -def extract_cmd_name(key): - cmd_name_res = re.findall(CMD_NAME_PATTERN, key) - if not cmd_name_res or len(cmd_name_res) == 0: - return False, None - return True, cmd_name_res[0] - - -def extract_cmd_property(key, cmd_name): - cmd_key_pattern = re.compile(cmd_name + r"\'\]\[\'([a-zA-Z0-9\-\_]+)\'\]") - cmd_key_res = re.findall(cmd_key_pattern, key) - if not cmd_key_res or len(cmd_key_res) == 0: - return False, None - return True, cmd_key_res[0] - - -def extract_para_info(key): - parameters_ind = key.find("['parameters']") - property_ind = key.find("[", parameters_ind + 1) - property_res = re.findall(CMD_PARAMETER_PROPERTY_PATTERN, key[property_ind:]) - if not property_res: - return None - return property_res - - -def export_meta_changes_to_json(output, output_file): - if not output_file: - return output - output_file_folder = os.path.dirname(output_file) - if output_file_folder and not os.path.exists(output_file_folder): - os.makedirs(output_file_folder) - with open(output_file, "w") as f_out: - if output: - f_out.write(json.dumps(output, indent=4)) - return None diff --git a/azdev/operations/tests/test_break_change.py b/azdev/operations/tests/test_break_change.py index 9487c25dc..6a81837ca 100644 --- a/azdev/operations/tests/test_break_change.py +++ b/azdev/operations/tests/test_break_change.py @@ -8,8 +8,7 @@ import unittest import os from azdev.operations.command_change import export_command_meta, cmp_command_meta -from azdev.operations.command_change.util import get_command_tree, extract_cmd_name, \ - extract_cmd_property, extract_para_info, extract_subgroup_name +from azdev.operations.command_change.util import get_command_tree class MyTestCase(unittest.TestCase): @@ -28,27 +27,6 @@ def test_parse_cmd_tree(self): self.assertFalse(ret["sub_info"]["sub_info"]["is_group"], "group parse failed") self.assertTrue(ret["sub_info"]["sub_info"]["cmd_name"] == "monitor log-profiles create", "group parse failed") - def test_diff_dict_key_parse(self): - test_key = "root['sub_groups']['monitor']['sub_groups']['monitor private-link-scope']['sub_groups']" \ - "['monitor private-link-scope scoped-resource']['commands']" \ - "['monitor private-link-scope scoped-resource list']['parameters'][0]['options'][1]" - has_cmd, cmd_name = extract_cmd_name(test_key) - self.assertTrue(has_cmd, "cmd parse failed from diff dict key") - self.assertEqual(cmd_name, "monitor private-link-scope scoped-resource list", "cmd name extract failed") - has_cmd_key, cmd_key = extract_cmd_property(test_key, cmd_name) - self.assertTrue(has_cmd_key, "cmd property parse failed from diff dict key") - self.assertEqual(cmd_key, "parameters", "cmd property extract failed") - para_res = extract_para_info(test_key) - self.assertEqual(para_res[0], "0", "cmd parameter parse failed") - - def test_diff_dict_key_for_subgroups(self): - test_key = "root['sub_groups']['monitor']['sub_groups']['monitor account']" - has_cmd, _ = extract_cmd_name(test_key) - self.assertFalse(has_cmd, "cmd parse error from diff dict key") - has_subgroup, subgroup_name = extract_subgroup_name(test_key) - self.assertTrue(has_subgroup, "sub group parse failed from diff dict key") - self.assertEqual(subgroup_name, "monitor account", "sub group name extract failed") - def test_diff_meta(self): if not os.path.exists("./jsons/az_monitor_meta_before.json") \ or not os.path.exists("./jsons/az_monitor_meta_after.json"): @@ -79,15 +57,6 @@ def test_diff_meta(self): break self.assertTrue(ignored, "ignored message found") - def test_diff_meta_whitelist(self): - if not os.path.exists("./jsons/az_ams_meta_before.json") \ - or not os.path.exists("./jsons/az_ams_meta_after.json"): - return - result = cmp_command_meta(base_meta_file="./jsons/az_ams_meta_before.json", - diff_meta_file="./jsons/az_ams_meta_after.json", - output_type="text") - self.assertEqual(result, [], "returned change isn't empty") - if __name__ == '__main__': unittest.main() diff --git a/azure-cli-diff-tool/tests/__init__.py b/azure-cli-diff-tool/tests/__init__.py new file mode 100644 index 000000000..99c0f28cd --- /dev/null +++ b/azure-cli-diff-tool/tests/__init__.py @@ -0,0 +1,5 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- diff --git a/azdev/operations/tests/jsons/az_ams_meta_after.json b/azure-cli-diff-tool/tests/jsons/az_ams_meta_after.json similarity index 100% rename from azdev/operations/tests/jsons/az_ams_meta_after.json rename to azure-cli-diff-tool/tests/jsons/az_ams_meta_after.json diff --git a/azdev/operations/tests/jsons/az_ams_meta_before.json b/azure-cli-diff-tool/tests/jsons/az_ams_meta_before.json similarity index 100% rename from azdev/operations/tests/jsons/az_ams_meta_before.json rename to azure-cli-diff-tool/tests/jsons/az_ams_meta_before.json diff --git a/azure-cli-diff-tool/tests/jsons/az_monitor_meta_after.json b/azure-cli-diff-tool/tests/jsons/az_monitor_meta_after.json new file mode 100644 index 000000000..52fbee5d0 --- /dev/null +++ b/azure-cli-diff-tool/tests/jsons/az_monitor_meta_after.json @@ -0,0 +1,249 @@ +{ + "module_name": "monitor", + "name": "az", + "commands": {}, + "sub_groups": { + "monitor": { + "name": "monitor", + "commands": {}, + "sub_groups": { + "monitor private-link-scope": { + "name": "monitor private-link-scope", + "commands": {}, + "sub_groups": { + "monitor private-link-scope scoped-resource": { + "name": "monitor private-link-scope scoped-resource", + "commands": { + "monitor private-link-scope scoped-resource list": { + "name": "monitor private-link-scope scoped-resource list", + "is_aaz": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name", "-g"], + "required": true + }] + }, + "monitor private-link-scope scoped-resource create": { + "name": "monitor private-link-scope scoped-resource create", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "resource_name", + "options": ["--resource-name"], + "required": true + }, { + "name": "linked_resource_id", + "options": ["--linked-resource-id"], + "required": true + }] + }, + "monitor private-link-scope scoped-resource delete": { + "name": "monitor private-link-scope scoped-resource delete", + "is_aaz": false, + "confirmation": true, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "resource_name", + "options": ["--resource-name"], + "required": true + }, { + "name": "yes", + "required": true + }] + } + }, + "sub_groups": {} + }, + "monitor private-link-scope private-link-resource": { + "name": "monitor private-link-scope private-link-resource", + "commands": { + "monitor private-link-scope private-link-resource show": { + "name": "monitor private-link-scope private-link-resource show", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "group_name", + "options": ["--group-name"], + "required": true + }] + }, + "monitor private-link-scope private-link-resource list": { + "name": "monitor private-link-scope private-link-resource list", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "cmd", + "options": ["--cmd"], + "nargs": "?" + }] + } + }, + "sub_groups": {} + }, + "monitor private-link-scope private-endpoint-connection": { + "name": "monitor private-link-scope private-endpoint-connection", + "commands": { + "monitor private-link-scope private-endpoint-connection show": { + "name": "monitor private-link-scope private-endpoint-connection show", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "private_endpoint_connection_name", + "options": ["--private-endpoint-connection-name"], + "required": true + }, { + "name": "cmd", + "options": ["--cmd"], + "nargs": "?" + }] + }, + "monitor private-link-scope private-endpoint-connection list": { + "name": "monitor private-link-scope private-endpoint-connection list", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }] + }, + "monitor private-link-scope private-endpoint-connection approve": { + "name": "monitor private-link-scope private-endpoint-connection approve", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "cmd", + "options": ["--cmd"], + "nargs": "?" + }, { + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "private_endpoint_connection_name", + "options": ["--private-endpoint-connection-name"], + "required": true + }, { + "name": "description", + "options": ["--description"] + }] + }, + "monitor private-link-scope private-endpoint-connection reject": { + "name": "monitor private-link-scope private-endpoint-connection reject", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "cmd", + "options": ["--cmd"], + "nargs": "?" + }, { + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "private_endpoint_connection_name", + "options": ["--private-endpoint-connection-name"], + "required": true + }, { + "name": "description", + "options": ["--description"] + }] + }, + "monitor private-link-scope private-endpoint-connection delete": { + "name": "monitor private-link-scope private-endpoint-connection delete", + "is_aaz": false, + "confirmation": true, + "is_preview": true, + "parameters": [{ + "name": "resource_group", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "private_endpoint_connection_name", + "options": ["--private-endpoint-connection-name"], + "required": true + }, { + "name": "yes", + "options": ["--yes", "-y"], + "default": true + }] + } + }, + "sub_groups": {} + } + } + } + } + } + } +} \ No newline at end of file diff --git a/azure-cli-diff-tool/tests/jsons/az_monitor_meta_before.json b/azure-cli-diff-tool/tests/jsons/az_monitor_meta_before.json new file mode 100644 index 000000000..f52dcce39 --- /dev/null +++ b/azure-cli-diff-tool/tests/jsons/az_monitor_meta_before.json @@ -0,0 +1,283 @@ +{ + "module_name": "monitor", + "name": "az", + "commands": {}, + "sub_groups": { + "monitor": { + "name": "monitor", + "commands": {}, + "sub_groups": { + "monitor private-link-scope": { + "name": "monitor private-link-scope", + "commands": {}, + "sub_groups": { + "monitor private-link-scope scoped-resource": { + "name": "monitor private-link-scope scoped-resource", + "commands": { + "monitor private-link-scope scoped-resource show": { + "name": "monitor private-link-scope scoped-resource show", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "resource_name", + "options": ["--resource-name"], + "required": true + }, { + "name": "cmd", + "options": ["--cmd"], + "nargs": "?" + }] + }, + "monitor private-link-scope scoped-resource create": { + "name": "monitor private-link-scope scoped-resource create", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "resource_name", + "options": ["--resource-name"], + "required": true + }, { + "name": "linked_resource_id", + "options": ["--linked-resource-id"], + "required": true + }] + }, + "monitor private-link-scope scoped-resource list": { + "name": "monitor private-link-scope scoped-resource list", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }] + }, + "monitor private-link-scope scoped-resource delete": { + "name": "monitor private-link-scope scoped-resource delete", + "is_aaz": false, + "confirmation": true, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "resource_name", + "options": ["--resource-name"], + "required": true + }, { + "name": "yes", + "options": ["--yes", "-y"] + }] + } + }, + "sub_groups": {} + }, + "monitor private-link-scope private-link-resource": { + "name": "monitor private-link-scope private-link-resource", + "commands": { + "monitor private-link-scope private-link-resource show": { + "name": "monitor private-link-scope private-link-resource show", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "group_name", + "options": ["--group-name"], + "required": true + }, { + "name": "cmd", + "options": ["--cmd"], + "nargs": "?" + }] + }, + "monitor private-link-scope private-link-resource list": { + "name": "monitor private-link-scope private-link-resource list", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }] + } + }, + "sub_groups": {} + }, + "monitor private-link-scope private-endpoint-connection": { + "name": "monitor private-link-scope private-endpoint-connection", + "commands": { + "monitor private-link-scope private-endpoint-connection show": { + "name": "monitor private-link-scope private-endpoint-connection show", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "private_endpoint_connection_name", + "options": ["--private-endpoint-connection-name"], + "required": true + }, { + "name": "cmd", + "options": ["--cmd"], + "nargs": "?" + }] + }, + "monitor private-link-scope private-endpoint-connection list": { + "name": "monitor private-link-scope private-endpoint-connection list", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }] + }, + "monitor private-link-scope private-endpoint-connection approve": { + "name": "monitor private-link-scope private-endpoint-connection approve", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "cmd", + "options": ["--cmd"], + "nargs": "?" + }, { + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "private_endpoint_connection_name", + "options": ["--private-endpoint-connection-name"], + "required": true + }, { + "name": "description", + "options": ["--description"] + }] + }, + "monitor private-link-scope private-endpoint-connection reject": { + "name": "monitor private-link-scope private-endpoint-connection reject", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "cmd", + "options": ["--cmd"], + "nargs": "?" + }, { + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "private_endpoint_connection_name", + "options": ["--private-endpoint-connection-name"], + "required": true + }, { + "name": "description", + "options": ["--description"] + }] + }, + "monitor private-link-scope private-endpoint-connection delete": { + "name": "monitor private-link-scope private-endpoint-connection delete", + "is_aaz": false, + "is_preview": true, + "parameters": [{ + "name": "resource_group_name", + "options": ["--resource-group", "-g"], + "required": true, + "id_part": "resource_group" + }, { + "name": "scope_name", + "options": ["--scope-name"], + "required": true + }, { + "name": "private_endpoint_connection_name", + "options": ["--private-endpoint-connection-name"], + "required": true + }, { + "name": "yes", + "options": ["--yes", "-y"], + "default": false + }] + } + }, + "sub_groups": { + "monitor private-link-scope private-endpoint-connection cust": { + "name": "monitor private-link-scope private-endpoint-connection cust", + "commands": { + "monitor private-link-scope private-endpoint-connection cust show": { + "name": "monitor private-link-scope private-endpoint-connection cust show", + "parameters": [] + } + }, + "sub_groups": {} + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/azure-cli-diff-tool/tests/test_break_change.py b/azure-cli-diff-tool/tests/test_break_change.py new file mode 100644 index 000000000..d61e4b25e --- /dev/null +++ b/azure-cli-diff-tool/tests/test_break_change.py @@ -0,0 +1,86 @@ +# ----------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# ----------------------------------------------------------------------------- + + +import unittest +import os +from azure_cli_diff_tool import meta_diff +from azure_cli_diff_tool.utils import get_command_tree, extract_cmd_name, extract_cmd_property, extract_para_info, \ + extract_subgroup_name + + +class MyTestCase(unittest.TestCase): + + def test_parse_cmd_tree(self): + cmd_name = "monitor log-profiles create" + ret = get_command_tree(cmd_name) + self.assertTrue(ret["is_group"], "group parse failed") + self.assertFalse(ret["sub_info"]["sub_info"]["is_group"], "group parse failed") + self.assertTrue(ret["sub_info"]["sub_info"]["cmd_name"] == "monitor log-profiles create", "group parse failed") + + def test_diff_dict_key_parse(self): + test_key = "root['sub_groups']['monitor']['sub_groups']['monitor private-link-scope']['sub_groups']" \ + "['monitor private-link-scope scoped-resource']['commands']" \ + "['monitor private-link-scope scoped-resource list']['parameters'][0]['options'][1]" + has_cmd, cmd_name = extract_cmd_name(test_key) + self.assertTrue(has_cmd, "cmd parse failed from diff dict key") + self.assertEqual(cmd_name, "monitor private-link-scope scoped-resource list", "cmd name extract failed") + has_cmd_key, cmd_key = extract_cmd_property(test_key, cmd_name) + self.assertTrue(has_cmd_key, "cmd property parse failed from diff dict key") + self.assertEqual(cmd_key, "parameters", "cmd property extract failed") + para_res = extract_para_info(test_key) + self.assertEqual(para_res[0], "0", "cmd parameter parse failed") + + def test_diff_dict_key_for_subgroups(self): + test_key = "root['sub_groups']['monitor']['sub_groups']['monitor account']" + has_cmd, _ = extract_cmd_name(test_key) + self.assertFalse(has_cmd, "cmd parse error from diff dict key") + has_subgroup, subgroup_name = extract_subgroup_name(test_key) + self.assertTrue(has_subgroup, "sub group parse failed from diff dict key") + self.assertEqual(subgroup_name, "monitor account", "sub group name extract failed") + + def test_diff_meta(self): + if not os.path.exists("./jsons/az_monitor_meta_before.json") \ + or not os.path.exists("./jsons/az_monitor_meta_after.json"): + return + result = meta_diff(base_meta_file="./jsons/az_monitor_meta_before.json", + diff_meta_file="./jsons/az_monitor_meta_after.json", + output_type="text") + target_message = [ + "please confirm cmd `monitor private-link-scope scoped-resource show` removed", + "sub group `monitor private-link-scope private-endpoint-connection cust` removed", + ] + for mes in target_message: + found = False + for line in result: + if line.find(mes) > -1: + found = True + break + self.assertTrue(found, "target message not found") + + ignored_message = [ + "updated property `is_aaz` from `False` to `True`" + ] + for mes in ignored_message: + ignored = True + for line in result: + if line.find(mes) > -1: + ignored = False + break + self.assertTrue(ignored, "ignored message found") + + def test_diff_meta_whitelist(self): + if not os.path.exists("./jsons/az_ams_meta_before.json") \ + or not os.path.exists("./jsons/az_ams_meta_after.json"): + return + result = meta_diff(base_meta_file="./jsons/az_ams_meta_before.json", + diff_meta_file="./jsons/az_ams_meta_after.json", + output_type="text") + self.assertEqual(result, [], "returned change isn't empty") + + +if __name__ == '__main__': + unittest.main() diff --git a/azure-pipelines-cli.yml b/azure-pipelines-cli.yml index 31a8e5c27..576ccb430 100644 --- a/azure-pipelines-cli.yml +++ b/azure-pipelines-cli.yml @@ -80,10 +80,10 @@ jobs: name: 'pool-ubuntu-2004' strategy: matrix: - Python37: - python.version: '3.7' Python38: python.version: '3.8' + Python39: + python.version: '3.9' steps: - task: UsePythonVersion@0 displayName: 'Use Python $(python.version)' @@ -115,6 +115,19 @@ jobs: inputs: codeCoverageTool: Cobertura summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml' + - task: UsePythonVersion@0 + displayName: 'azure cli diff tool unittests' + inputs: + versionSpec: '$(python.version)' + - bash: | + set -ev + python -m venv env + . env/bin/activate + # install azure cli diff tool from source code + cd azure-cli-diff-tool + pip install -e . + pip install pytest-cov + python -m pytest tests --cov=azure_cli_diff_tool --cov-report=xml - job: TestAzdevSetup displayName: 'Test azdev setup' @@ -124,10 +137,10 @@ jobs: name: 'pool-ubuntu-2004' strategy: matrix: - Python37: - python.version: '3.7' Python38: python.version: '3.8' + Python39: + python.version: '3.9' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build' @@ -156,10 +169,10 @@ jobs: name: 'pool-ubuntu-2004' strategy: matrix: - Python37: - python.version: '3.7' Python38: python.version: '3.8' + Python39: + python.version: '3.9' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build' @@ -188,10 +201,10 @@ jobs: name: 'pool-ubuntu-2004' strategy: matrix: - Python37: - python.version: '3.7' Python38: python.version: '3.8' + Python39: + python.version: '3.9' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build' @@ -220,10 +233,10 @@ jobs: name: 'pool-ubuntu-2004' strategy: matrix: - Python37: - python.version: '3.7' Python38: python.version: '3.8' + Python39: + python.version: '3.9' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build' @@ -251,10 +264,10 @@ jobs: name: 'pool-ubuntu-2004' strategy: matrix: - Python37: - python.version: '3.7' Python38: python.version: '3.8' + Python39: + python.version: '3.9' steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 58fcd3787..5e83597a5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,9 +12,6 @@ jobs: name: 'pool-ubuntu-2004' strategy: matrix: - Python37: - python.version: '3.7' - tox_env: 'py37' Python38: python.version: '3.8' tox_env: 'py38' diff --git a/setup.py b/setup.py index 6a00e387d..981ec4f13 100644 --- a/setup.py +++ b/setup.py @@ -82,6 +82,7 @@ 'tox', 'jsbeautifier~=1.14.7', 'deepdiff~=6.3.0', + 'azure-cli-diff-tool', 'tqdm', 'wheel==0.30.0' ], @@ -90,7 +91,6 @@ 'azdev.mod_templates': ['*.*'], 'azdev.operations.linter.rules': ['ci_exclusions.yml'], 'azdev.operations.cmdcov': ['*.*'], - 'azdev.operations.command_change': ['data/*'], }, include_package_data=True, entry_points={