From 373e5cc55da12f7fd0fec94adf11784e9962eda5 Mon Sep 17 00:00:00 2001 From: Brandon Minnix Date: Wed, 15 May 2024 15:06:51 -0400 Subject: [PATCH 1/4] Feature/platform mapper (#489) * Creating a Platform Mapping utility for breaking down platforms into their bits --------- Co-authored-by: Brandon Minnix Co-authored-by: Brandon Minnix Co-authored-by: Przemek Rogala Co-authored-by: Ken Celenza Co-authored-by: Brandon Minnix --- development_scripts.py | 12 ++ docs/dev/code_reference/nist.md | 5 + docs/user/include_jinja_list.md | 3 + docs/user/lib_mapper/nist.md | 9 + docs/user/lib_mapper/nist_reverse.md | 9 + docs/user/lib_use_cases.md | 1 + docs/user/lib_use_cases_nist.md | 42 ++++ docs/user/lib_use_cases_os_version.md | 41 ++++ mkdocs.yml | 1 + netutils/lib_mapper.py | 23 +++ netutils/nist.py | 278 ++++++++++++++++++++++++++ netutils/os_version.py | 190 ++++++++++++++++++ netutils/utils.py | 3 + tests/unit/test_nist.py | 40 ++++ tests/unit/test_os_versions.py | 106 ++++++++++ tests/unit/test_utils.py | 6 +- 16 files changed, 768 insertions(+), 1 deletion(-) create mode 100644 docs/dev/code_reference/nist.md create mode 100644 docs/user/lib_mapper/nist.md create mode 100644 docs/user/lib_mapper/nist_reverse.md create mode 100644 docs/user/lib_use_cases_nist.md create mode 100644 docs/user/lib_use_cases_os_version.md create mode 100644 netutils/nist.py create mode 100644 tests/unit/test_nist.py diff --git a/development_scripts.py b/development_scripts.py index b4cbaea6..a95db916 100644 --- a/development_scripts.py +++ b/development_scripts.py @@ -131,6 +131,18 @@ "_dict": lib_mapper.NETUTILSPARSER_LIB_MAPPER_REVERSE, "_file": "docs/user/lib_mapper/netutilsparser_reverse.md", }, + "nist": { + "header_src": "NIST", + "header_dst": "NORMALIZED", + "_dict": lib_mapper.NIST_LIB_MAPPER, + "_file": "docs/user/lib_mapper/nist.md", + }, + "nist_reverse": { + "header_src": "NORMALIZED", + "header_dst": "NIST", + "_dict": lib_mapper.NIST_LIB_MAPPER_REVERSE, + "_file": "docs/user/lib_mapper/nist_reverse.md", + }, "ntctemplates": { "header_src": "NTCTEMPLATES", "header_dst": "NORMALIZED", diff --git a/docs/dev/code_reference/nist.md b/docs/dev/code_reference/nist.md new file mode 100644 index 00000000..e8999ca8 --- /dev/null +++ b/docs/dev/code_reference/nist.md @@ -0,0 +1,5 @@ +# NIST URLs + +::: netutils.nist + options: + show_submodules: True diff --git a/docs/user/include_jinja_list.md b/docs/user/include_jinja_list.md index 3b6addc7..a00ded5f 100644 --- a/docs/user/include_jinja_list.md +++ b/docs/user/include_jinja_list.md @@ -60,9 +60,12 @@ | mac_to_format | netutils.mac.mac_to_format | | mac_to_int | netutils.mac.mac_to_int | | mac_type | netutils.mac.mac_type | +| get_nist_urls | netutils.nist.get_nist_urls | +| get_nist_vendor_platform_urls | netutils.nist.get_nist_vendor_platform_urls | | compare_version_loose | netutils.os_version.compare_version_loose | | compare_version_strict | netutils.os_version.compare_version_strict | | get_upgrade_path | netutils.os_version.get_upgrade_path | +| version_metadata | netutils.os_version.version_metadata | | compare_cisco_type5 | netutils.password.compare_cisco_type5 | | compare_cisco_type7 | netutils.password.compare_cisco_type7 | | compare_cisco_type9 | netutils.password.compare_cisco_type9 | diff --git a/docs/user/lib_mapper/nist.md b/docs/user/lib_mapper/nist.md new file mode 100644 index 00000000..1af6fa7a --- /dev/null +++ b/docs/user/lib_mapper/nist.md @@ -0,0 +1,9 @@ +| NIST | | NORMALIZED | +| ---------- | -- | ------ | +| adaptive_security_appliance_software | → | cisco_asa | +| eos | → | arista_eos | +| ios | → | cisco_ios | +| ios_xe | → | cisco_xe | +| ios_xr | → | cisco_xr | +| junos | → | juniper_junos | +| nx-os | → | cisco_nxos | \ No newline at end of file diff --git a/docs/user/lib_mapper/nist_reverse.md b/docs/user/lib_mapper/nist_reverse.md new file mode 100644 index 00000000..1619f1df --- /dev/null +++ b/docs/user/lib_mapper/nist_reverse.md @@ -0,0 +1,9 @@ +| NORMALIZED | | NIST | +| ---------- | -- | ------ | +| arista_eos | → | eos | +| cisco_asa | → | adaptive_security_appliance_software | +| cisco_ios | → | ios | +| cisco_nxos | → | nx-os | +| cisco_xe | → | ios_xe | +| cisco_xr | → | ios_xr | +| juniper_junos | → | junos | \ No newline at end of file diff --git a/docs/user/lib_use_cases.md b/docs/user/lib_use_cases.md index 5f7b7bb8..ada1e9fc 100644 --- a/docs/user/lib_use_cases.md +++ b/docs/user/lib_use_cases.md @@ -21,6 +21,7 @@ Functions are grouped with like functions, such as IP or MAC address based funct - Library Helpers - Provides helpers to pull useful information, e.g. NAPALM getters. - Library Mapper - Provides mappings in expected vendor names between Netmiko, NAPALM, pyntc, ntc-templates, pyats, and scrapli. - MAC Address - Provides the ability to work with MAC addresses such as validating or converting to integer. +- NIST - Provides the ability to obtain a URL formatted for NIST CPE Query. - OS Version - Provides the ability to work with OS version, such as defining an upgrade path. - Password - Provides the ability to compare and encrypt common password schemas such as type5 and type7 Cisco passwords. - Ping - Provides the ability to ping, currently only tcp ping. diff --git a/docs/user/lib_use_cases_nist.md b/docs/user/lib_use_cases_nist.md new file mode 100644 index 00000000..4fd080fb --- /dev/null +++ b/docs/user/lib_use_cases_nist.md @@ -0,0 +1,42 @@ +# NIST + +The NIST utility is used for functionality based around NIST DB Queries, and is primarily used to create URLs for the API based queries. + +## Requirements + +In order to use the URLs generated by `netutils.nist.get_nist_urls*`, you will need an api key provided by NIST [here]('https://nvd.nist.gov/developers/request-an-api-key'). This key will need to be passed in as an additional header in your request in the form of `{"apiKey": ""}` as stated by NIST in their [Getting Started]('https://nvd.nist.gov/developers/start-here') section. + + +## Custom URLs + +The largest caveat in this functionality is the consistency of the URL values needed to obtain the CVE information. NIST NVD has specific parameters that can be used for standardization, however this does not mean that entries are standardized. Manually combing through a large amount of CPE Vendor submissions has shown that there are variations in how CPE Vendor data is presented. + +For this reason, for certain Vendor/OS combinations, a custom URL needs to be built. +- **Cisco IOS CPE String** - `cpe:2.3:o:cisco:ios:15.5\\(2\\)s1c:*` + - `15.5\\(2\\)s1c:*` - As seen here, Cisco uses CPE strings that do not include the `:` delimiter, which can be queried using escape characters in the search string. **This is the format of ALL "generic" OS/Other platforms that do not have their own custom NIST URL builder when querying NIST.** + - Default URL Output - `'https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:cisco:ios:15.5\\(2\\)s1c:*'` + +- **Juniper JunOS CPE String** - `cpe:2.3:o:juniper:junos:10.2:r2:*:*:*:*:*:*` + - `10.2:r2:*:*:*:*:*:*` - As noted here, one of the provided URLs to query for this Juniper JunOS OS platform includes additional values that follow NIST delimiter structures. In the case where the parser provides multiple URLs, they will both be evaluated and the CVE from both will be added and associated. + - Custom URL Output - `['https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:10.2r2:*:*:*:*:*:*:*', 'https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:10.2:r2:*:*:*:*:*:*']` + + +## Examples +Here are a few examples showing how to use this in your python code. + +```python + +from netutils.nist import get_nist_urls + +# Get NIST URL for the Cisco IOS object +get_nist_urls("cisco_ios", "15.5(2)S1c") +# ['https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:cisco:ios:15.5\\(2\\)s1c:*'] + +# Get NIST URL(s) for the Juniper JunOS object +get_nist_urls("juniper_junos", "10.2R2.11") +# ['https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:10.2r2:*:*:*:*:*:*:*', 'https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:10.2:r2:*:*:*:*:*:*'] +``` + +Currently known OS/Other Platform types that require a custom NIST URL: + +- Juniper JunOS diff --git a/docs/user/lib_use_cases_os_version.md b/docs/user/lib_use_cases_os_version.md new file mode 100644 index 00000000..f945ab1c --- /dev/null +++ b/docs/user/lib_use_cases_os_version.md @@ -0,0 +1,41 @@ +# OS Version Tools + +The OS Version Tools are used for working with versioning systems. + +## Version Parsing/Deconstruction +Version parsing takes the software version given as a string, and deconstructs that value into the standards of the vendor. The version parsing takes place in the `netutils.os_version` module. This is necessary when specific values or flags from a software version are required to make a logical decision. + +Current Version Parsers: + +- Default Parser +- Juniper JunOS + +**See the following Default and Juniper JunOS parsed versions:** + +```python +>>> from netutils.os_version import version_metadata + +>>> version_metadata("Cisco", "IOS", "15.5") +{ + "major": "15", + "minor": "5", + "vendor_metadata": False, +} +>>> version_metadata("juniper", "junos", "12.4R") +{ + "isservice": False, + "ismaintenance": False, + "isfrs": True, + "isspecial": False, + "service": None, + "service_build": None, + "service_respin": None, + "main": "12", + "minor": "4", + "type": "R", + "build": None, + "major": "12", + "patch": None, + "vendor_metadata": True, +} +``` diff --git a/mkdocs.yml b/mkdocs.yml index abd07488..7db9c5a4 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -149,6 +149,7 @@ nav: - Library Helpers: "dev/code_reference/lib_helpers.md" - Library Mapping: "dev/code_reference/lib_mapping.md" - Mac Address: "dev/code_reference/mac.md" + - NIST: "dev/code_reference/nist.md" - OS Version: "dev/code_reference/os_version.md" - Password: "dev/code_reference/password.md" - Ping: "dev/code_reference/ping.md" diff --git a/netutils/lib_mapper.py b/netutils/lib_mapper.py index 3ca6b1cd..ea8c6b7e 100644 --- a/netutils/lib_mapper.py +++ b/netutils/lib_mapper.py @@ -407,6 +407,17 @@ "SRX": "juniper_junos", # no reverse } +# NIST | Normalized +NIST_LIB_MAPPER = { + "adaptive_security_appliance_software": "cisco_asa", + "nx-os": "cisco_nxos", + "ios_xr": "cisco_xr", + "ios_xe": "cisco_xe", + "eos": "arista_eos", + "ios": "cisco_ios", + "junos": "juniper_junos", +} + # Normalized | NAPALM NAPALM_LIB_MAPPER_REVERSE: t.Dict[str, str] = { "arista_eos": "eos", @@ -557,6 +568,18 @@ "paloalto_panos": "PAN_OS", } +# Normalized | NIST +NIST_LIB_MAPPER_REVERSE = { + "cisco_asa": "adaptive_security_appliance_software", + "cisco_nxos": "nx-os", + "cisco_xr": "ios_xr", + "cisco_xe": "ios_xe", + "arista_eos": "eos", + "cisco_ios": "ios", + "juniper_junos": "junos", +} + + # Deep copy the reverse, where there is no actual translation happening with special # consideration for OS's not in netmiko. _MAIN_LIB_MAPPER = copy.deepcopy(NETMIKO_LIB_MAPPER) diff --git a/netutils/nist.py b/netutils/nist.py new file mode 100644 index 00000000..63cf44ce --- /dev/null +++ b/netutils/nist.py @@ -0,0 +1,278 @@ +"""Classes and functions used for building NIST URLs from the os platform values.""" + +import abc +import dataclasses +import re +import typing as t + +from netutils.os_version import version_metadata + +# Setting up the dataclass values for specific parsers +PLATFORM_FIELDS: t.Dict[str, t.Any] = { + "default": [ + ("vendor", str), + ("os_type", str), + ("version_string", str), + ("major", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("minor", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("patch", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("prerelease", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("buildmetadata", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("vendor_metadata", bool, dataclasses.field(default=False)), # pylint: disable=[E3701] + ], + "juniper": { + "junos": [ + ("main", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("type", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("build", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("service", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("service_build", int, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("service_respin", str, dataclasses.field(default=None)), # pylint: disable=[E3701] + ("isservice", bool, dataclasses.field(default=False)), # pylint: disable=[E3701] + ("ismaintenance", bool, dataclasses.field(default=False)), # pylint: disable=[E3701] + ("isfrs", bool, dataclasses.field(default=False)), # pylint: disable=[E3701] + ("isspecial", bool, dataclasses.field(default=False)), # pylint: disable=[E3701] + ] + }, +} + + +class OsPlatform(metaclass=abc.ABCMeta): + """Base class for dynamically generated vendor specific platform data classes.""" + + def asdict(self) -> t.Dict[str, t.Any]: + """Returns dictionary representation of the class attributes.""" + return dataclasses.asdict(self) # type: ignore + + @abc.abstractmethod + def get_nist_urls(self) -> t.List[str]: + """Returns list of NIST URLs for the platform.""" + + def get(self, key: str) -> t.Any: + """Return value of the attribute matching provided name or None if no attribute is found.""" + return getattr(self, key, None) + + def keys(self) -> t.KeysView[t.Any]: + """Return attributes and their values as dict keys.""" + # Disabling pylint no-member due to BUG: https://github.com/pylint-dev/pylint/issues/7126 + return self.__annotations__.keys() # pylint: disable=no-member + + def __getitem__(self, key: str) -> t.Any: + """Allow retrieving attributes using subscript notation.""" + return getattr(self, key) + + +def _get_nist_urls_juniper_junos(os_platform_data: t.Dict[str, t.Any]) -> t.List[str]: # pylint: disable=R0911 + """Create a list of possible NIST Url strings for JuniperPlatform. + + Returns: + List of NIST CPE URLs that may contain platform data. + """ + nist_urls = [] + base_url = f'{"https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos"}' + + # BASE + _main = os_platform_data.get("main") + _minor = os_platform_data.get("minor") + if os_platform_data["type"]: + _type = os_platform_data["type"].lower() + _build = os_platform_data.get("build") + + # SERVICE + if os_platform_data["service"]: + _service = os_platform_data["service"].lower() + _service_build = os_platform_data.get("service_build") + _service_respin = os_platform_data.get("service_respin") + + # EXTRAS + delim_six = ":*" * 6 + delim_seven = ":*" * 7 + + if os_platform_data["isspecial"]: + # e.g. base_ext = juniper:junos:12.1x47 + base_ext = f"{base_url}:{_main}.{_minor}{_type}{_build}" + else: + # e.g. base_ext = juniper:junos:12.1 + base_ext = f"{base_url}:{_main}.{_minor}" + + # X Series (Special) Examples: 12.1x47:d40, 12.2x50:d41.1 + if os_platform_data["isspecial"] and os_platform_data["service_respin"]: # pylint: disable=R1705 + # nist_urls.append(juniper:junos:12.2x50:d41.1:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}:{_service}{_service_build}.{_service_respin}{delim_six}") + + # nist_urls.append(juniper:junos:12.2x50-d41.1:*:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}-{_service}{_service_build}.{_service_respin}{delim_seven}") + + return nist_urls + + elif os_platform_data["isspecial"]: + # nist_urls.append(juniper:junos:12.1x47:d40:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}:{_service}{_service_build}{delim_six}") + + # nist_urls.append(juniper:junos:12.1x47-d40:*:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}-{_service}{_service_build}{delim_seven}") + + return nist_urls # + + if not os_platform_data.get("type"): + # nist_urls.append(juniper:junos:12.1:-:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}:-{delim_six}") + + return nist_urls + + if not os_platform_data.get("build"): + # nist_urls.append(juniper:junos:10.4s:*:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}{_type}{delim_seven}") + + return nist_urls + + if os_platform_data.get("build") and not os_platform_data.get("service"): + # nist_urls.append(juniper:junos:12.3r12:*:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}{_type}{_build}{delim_seven}") + + # nist_urls.append(juniper:junos:12.2:r1:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}:{_type}{_build}{delim_six}") + + return nist_urls + + if os_platform_data.get("service") and os_platform_data.get("service_respin"): + # nist_urls.append(juniper:junos:11.4r13:s2.1:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}{_type}{_build}:{_service}{_service_build}.{_service_respin}{delim_six}") + + # nist_urls.append(juniper:junos:12.2:r8-s2.1:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}{_type}{_build}-{_service}{_service_build}.{_service_respin}{delim_seven}") + + return nist_urls + + if os_platform_data.get("service"): + # nist_urls.append(juniper:junos:11.4r13:s2:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}{_type}{_build}:{_service}{_service_build}{delim_six}") + + # nist_urls.append(juniper:junos:12.2:r8-s2:*:*:*:*:*:*) + nist_urls.append(f"{base_ext}{_type}{_build}-{_service}{_service_build}{delim_seven}") + + return nist_urls + + raise ValueError("Failure creating Juniper JunOS Version. Format is unknown.") + + +def _get_nist_urls_default(os_platform_data: t.Dict[str, t.Any]) -> t.List[str]: + r"""Create a list of possible NIST Url strings. + + Child models with NIST URL customizations need their own "get_nist_urls" method. + + Returns: + List of NIST CPE URLs that may contain platform data. + """ + nist_urls = [] + escape_list = [r"\(", r"\)"] + base_url = f'{"https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:"}' + + os_platform_data = {"base_url": base_url, **os_platform_data} + os_platform_data["version_string"] = os_platform_data.get("version_string").replace("-", ":") # type: ignore + + version_string = os_platform_data.get("version_string", "").lower() + for escape_char in escape_list: + version_string = re.sub(escape_char, "\\" + escape_char, version_string) + + os_platform_data["version_string"] = version_string + + nist_urls.append( + f"{base_url}{os_platform_data['vendor']}:{os_platform_data['os_type']}:{os_platform_data['version_string']}:*" + ) + + return nist_urls + + +def _os_platform_object_builder(vendor: str, platform: str, version: str) -> object: + """Creates a platform object relative to its need and definition. + + Args: + vendor (str): Name of vendor + platform (str): Name of os/other platform + version (str): Version value + + Returns: + object: Platform object + + Examples: + >>> jp = _os_platform_object_builder("juniper", "junos", "12.1R3-S4.1") + >>> jp.get_nist_urls() + ['https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:12.1r3:s4.1:*:*:*:*:*:*', 'https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:12.1r3-s4.1:*:*:*:*:*:*:*'] + """ + platform = platform.lower() + vendor = vendor.lower() + + class_fields = [*PLATFORM_FIELDS["default"]] + vendor_platform_fields = PLATFORM_FIELDS.get(vendor, {}).get(platform, []) + class_fields.extend(vendor_platform_fields) + + version_parser = version_metadata(vendor, platform, version) + + field_values = { + "vendor": vendor, + "os_type": platform, + "version_string": version, + } + + if version_parser: + field_values.update(version_parser) + + class_name = f"{vendor.capitalize()}{platform.capitalize()}" + get_nist_urls_func = get_nist_url_funcs.get(vendor, {}).get(platform) or get_nist_url_funcs["default"] + + platform_cls = dataclasses.make_dataclass( + cls_name=class_name, fields=class_fields, bases=(OsPlatform,), namespace={"get_nist_urls": get_nist_urls_func} + ) + + return platform_cls(**field_values) + + +get_nist_url_funcs: t.Dict[str, t.Any] = { + "default": _get_nist_urls_default, + "juniper": {"junos": _get_nist_urls_juniper_junos}, +} + + +def get_nist_vendor_platform_urls(vendor: str, platform: str, version: str) -> t.List[str]: + """Generate list of possible NIST URLs for the Vendor, OS Platform, and Version. + + Args: + vendor (str): OS Software Platform Vendor/Manufacturer + platform (str): OS Software Platform Name + version (str): OS Software Platform Version + + Returns: + t.List[str]: NIST URLs to search for possible CVE matches + """ + platform_data = _os_platform_object_builder(vendor, platform, version).__dict__ + + if vendor.lower() == "juniper" and platform.lower() == "junos": + return _get_nist_urls_juniper_junos(platform_data) + return _get_nist_urls_default(platform_data) + + +def get_nist_urls(network_driver: str, version: str) -> t.List[str]: + """Generate list of possible NIST URLs for the Network Driver, and Version. + + Args: + network_driver (str): Value of device network_driver (Ex: cisco_ios, arista_eos) + version (str): OS Software Platform Version + + Returns: + t.List[str]: NIST URLs to search for possible CVE matches + """ + # DICTIONARY FOR VENDOR/PLATFORM TO NETWORK_DRIVER; UPDATE AS NEEDED + network_driver_mappings = { + "arista_eos": {"vendor": "arista", "os_name": "eos"}, + "cisco_ios": {"vendor": "cisco", "os_name": "ios"}, + "cisco_nxos": {"vendor": "cisco", "os_name": "nxos"}, + "cisco_xe": {"vendor": "cisco", "os_name": "xe"}, + "cisco_xr": {"vendor": "cisco", "os_name": "xr"}, + "cisco_asa": {"vendor": "cisco", "os_name": "asa"}, + "juniper_junos": {"vendor": "juniper", "os_name": "junos"}, + } + + vendor_os = network_driver_mappings[network_driver] + + return get_nist_vendor_platform_urls(vendor_os["vendor"], vendor_os["os_name"], version) diff --git a/netutils/os_version.py b/netutils/os_version.py index c363bf28..2065d115 100644 --- a/netutils/os_version.py +++ b/netutils/os_version.py @@ -1,5 +1,6 @@ """Functions for working with OS Versions.""" +import re import typing as t from netutils._private.version import LooseVersion, StrictVersion # type: ignore @@ -128,3 +129,192 @@ def compare_version_strict(current_version: str, comparison: str, target_version >>> """ return _compare_version(current_version, comparison, target_version, "strict") + + +def _juniper_junos_version_metadata(version: str) -> t.Dict[str, t.Any]: + """Parses JunOS Version into usable bits matching JunOS Standards. + + Args: + version + + Returns: + A dictionary containing parsed version information + + Examples: + >>> _juniper_junos_version_metadata("12.3R4") + {'isservice': False, 'ismaintenance': True, 'isfrs': False, 'isspecial': False, 'service': None, 'service_build': None, 'service_respin': None, 'main': '12', 'minor': '3', 'type': 'R', 'build': '4', 'major': '12', 'patch': '4'} + """ + # Use regex to group the main, minor, type and build into useable pieces + # re_main_minor_type_build = re.search(r"^(\d+)\.(\d+)([xXrRsS])?(\d+)?", split_version[0]) + re_main_minor_type_build: re.Pattern[str] = re.compile( + r""" + ^ + (?P
\d+) # main train + \. # dot separator + (?P\d+) # minor version + (?P[xXrRsS])? # version type (optional) + (?P\d+)? # build (optional) + """, + re.VERBOSE, + ) + re_service_build_respin: re.Pattern[str] = re.compile( + r""" + (?P[sSdD])? # service (optional) + (?P\d+)? # service build (optional) + \.? + (?P\d+)? # service respin (optional) + """, + re.VERBOSE, + ) + # Set empty params for service pieces and complete them if a second indice exists from the version split + # Define isservice, isfrs, isspecial, ismaintenance + parsed_version: t.Dict[str, t.Any] = { + "isservice": False, + "ismaintenance": False, + "isfrs": False, + "isspecial": False, + "service": None, + "service_build": None, + "service_respin": None, + } + + # Juniper junos marks the division between main, minor, type and build from the service build and respin with a - + version_core_part, *version_service_part = re.split("-|:", version) + + # Parse out junos into sections that can be used for logic + parsed_version.update(re_main_minor_type_build.search(version_core_part).groupdict()) # type:ignore + + # Adding additional keys for standard major/minor/patch references + parsed_version.update({"major": parsed_version["main"], "patch": parsed_version.get("build")}) + + if version_service_part: + parsed_version.update(re_service_build_respin.search(version_service_part[0]).groupdict()) # type:ignore + if parsed_version.get("service") is not None and parsed_version.get("service", "").lower() == "s": + parsed_version["isservice"] = True + # Juniper looks at the D in special releases like it's the R in normal releases; Use it as the frs identifier + elif ( + parsed_version.get("service") is not None + and parsed_version["service"].lower() == "d" + and (parsed_version.get("service_build") is None or int(parsed_version.get("service_build", 1)) <= 1) + ): + parsed_version["isfrs"] = True + + if parsed_version.get("type") is None: + return parsed_version + + if parsed_version["type"].lower() == "x": + parsed_version["isspecial"] = True + elif parsed_version["type"].lower() == "s": + parsed_version["isservice"] = True + + if parsed_version["type"].lower() == "r" and ( + parsed_version.get("build") is None or int(parsed_version.get("build")) <= 1 # type:ignore + ): + parsed_version["isfrs"] = True + elif parsed_version["type"].lower() == "r": + parsed_version["ismaintenance"] = True + + return parsed_version + + +def _basic_version_metadata(version: str) -> t.Dict[str, t.Any]: + """Parses version value using SemVer 2.0.0 standards. https://semver.org/spec/v2.0.0.html. + + Args: + version (str): String representation of version + + Returns: + A dictionary containing parsed version information + + Examples: + >>> _basic_version_metadata("10.20.30") + {'major': '10', 'minor': '20', 'patch': '30', 'prerelease': None, 'buildmetadata': None} + + >>> _basic_version_metadata("1.0.0-alpha.beta.1") + {'major': '1', 'minor': '0', 'patch': '0', 'prerelease': 'alpha.beta.1', 'buildmetadata': None} + + >>> _basic_version_metadata("1.0.0-alpha-a.b-c-somethinglong+build.1-aef.1-its-okay") + {'major': '1', 'minor': '0', 'patch': '0', 'prerelease': 'alpha-a.b-c-somethinglong', 'buildmetadata': 'build.1-aef.1-its-okay'} + + """ + # Use regex with named groups. REGEX Pattern Provided by SemVer https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + semver_regex: re.Pattern[str] = re.compile( + r""" + ^ + (?P0|[1-9]\d*) + \. + (?P0|[1-9]\d*) + \. + (?P0|[1-9]\d*) + (?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))? + (?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ + """, + re.VERBOSE, + ) + + # If version is not SemVer 2.0.0, attempt to find major/minor only. + basic_regex: re.Pattern[str] = re.compile( + r""" + ^ + (?P0|[1-9]\d*) + \. + (?P0|[1-9]\d*)? + .*$ + """, + re.VERBOSE, + ) + + # Perform regex match against provided version string + try: + parsed_version = semver_regex.match(version) + except AttributeError: + return {"Error": "Empty Version."} + + if not parsed_version: + parsed_version = basic_regex.match(version) + + if parsed_version: + return parsed_version.groupdict() + + return {"Error": "Unable to evaluate the version number entered."} + + +version_metadata_parsers = { + "default": _basic_version_metadata, + "juniper": { + "junos": _juniper_junos_version_metadata, + }, +} + + +def version_metadata(vendor: str, os_type: str, version: str) -> t.Dict[str, t.Any]: + """If a custom version parser is avaialable, use it. + + Args: + vendor (str): Vendor name (Ex: "Juniper") + os_type (str): OS Type (Ex: "JunOS") + version (str): OS Version (Ex: "12.4R") + + Returns: + dict: Dict of broken down version into vendor standards. + + Examples: + >>> from netutils.os_version import version_metadata + >>> version_metadata("Cisco", "IOS", "15.5") + {'major': '15', 'minor': '5', 'vendor_metadata': False} + >>> version_metadata("juniper", "junos", "12.4R") + {'isservice': False, 'ismaintenance': False, 'isfrs': True, 'isspecial': False, 'service': None, 'service_build': None, 'service_respin': None, 'main': '12', 'minor': '4', 'type': 'R', 'build': None, 'major': '12', 'patch': None, 'vendor_metadata': True} + """ + parsed_version = {} + if vendor in version_metadata_parsers: + try: + parsed_version = version_metadata_parsers[vendor][os_type](version) # type:ignore + parsed_version.update({"vendor_metadata": True}) + except KeyError: + parsed_version = version_metadata_parsers["default"](version) # type:ignore + parsed_version.update({"vendor_metadata": False}) + else: + parsed_version = version_metadata_parsers["default"](version) # type:ignore + parsed_version.update({"vendor_metadata": False}) + + return parsed_version diff --git a/netutils/utils.py b/netutils/utils.py index 602324e2..813dad83 100644 --- a/netutils/utils.py +++ b/netutils/utils.py @@ -93,6 +93,9 @@ "get_upgrade_path": "os_version.get_upgrade_path", "hash_data": "hash.hash_data", "get_ips_sorted": "ip.get_ips_sorted", + "version_metadata": "os_version.version_metadata", + "get_nist_urls": "nist.get_nist_urls", + "get_nist_vendor_platform_urls": "nist.get_nist_vendor_platform_urls", } diff --git a/tests/unit/test_nist.py b/tests/unit/test_nist.py new file mode 100644 index 00000000..b381b1e0 --- /dev/null +++ b/tests/unit/test_nist.py @@ -0,0 +1,40 @@ +"""Test functions for NIST URL Utility""" + +import pytest + +from netutils.nist import get_nist_urls + +platform_nist_urls = [ + { + "sent": {"network_driver": "cisco_ios", "version": "15.5"}, + "received": ["https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:cisco:ios:15.5:*"], + }, + { + "sent": {"network_driver": "arista_eos", "version": "4.15.3f"}, + "received": [ + "https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:arista:eos:4.15.3f:*" + ], + }, + # Juniper platforms receive multiple URLs to try as they are not very standardized and some return info on both + { + "sent": {"network_driver": "juniper_junos", "version": "12.3R12-S15"}, + "received": [ + "https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:12.3r12:s15:*:*:*:*:*:*", + "https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:12.3r12-s15:*:*:*:*:*:*:*", + ], + }, + { + "sent": {"network_driver": "juniper_junos", "version": "12.3x48:d25"}, + "received": [ + "https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:12.3x48:d25:*:*:*:*:*:*", + "https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:12.3x48-d25:*:*:*:*:*:*:*", + ], + }, +] + + +# Testing the composition of the nist url(s) created for a platform +@pytest.mark.parametrize("data", platform_nist_urls) +def test_platform_nist(data): + platform_obj = get_nist_urls(data["sent"]["network_driver"], data["sent"]["version"]) + assert platform_obj == data["received"] diff --git a/tests/unit/test_os_versions.py b/tests/unit/test_os_versions.py index 5e1101e0..11d23332 100755 --- a/tests/unit/test_os_versions.py +++ b/tests/unit/test_os_versions.py @@ -32,6 +32,103 @@ {"sent": {"current_version": "1.0.0", "comparison": "!=", "target_version": "2.0.0"}, "received": True}, ] +PLATFORM_VERSION_METADATA = [ + # Cisco and Arista use the generic parsing + { + "sent": {"vendor": "cisco", "platform": "ios", "version": "15.7(2.0z)M"}, + "received": { + "major": "15", + "minor": "7", + "vendor_metadata": False, + }, + }, + { + "sent": {"vendor": "arista", "platform": "eos", "version": "4.15.3f"}, + "received": { + "major": "4", + "minor": "15", + "vendor_metadata": False, + }, + }, + # Juniper Junos uses a custom parser + { + "sent": {"vendor": "juniper", "platform": "junos", "version": "12.4R"}, + "received": { + "isservice": False, + "ismaintenance": False, + "isfrs": True, + "isspecial": False, + "main": "12", + "minor": "4", + "type": "R", + "build": None, + "service": None, + "service_build": None, + "service_respin": None, + "vendor_metadata": True, + "major": "12", + "patch": None, + }, + }, + { + "sent": {"vendor": "juniper", "platform": "junos", "version": "12.3x48-d80"}, + "received": { + "isservice": False, + "ismaintenance": False, + "isfrs": False, + "isspecial": True, + "main": "12", + "minor": "3", + "type": "x", + "build": "48", + "service": "d", + "service_build": "80", + "service_respin": None, + "vendor_metadata": True, + "major": "12", + "patch": "48", + }, + }, + { + "sent": {"vendor": "juniper", "platform": "junos", "version": "12.3x48:d80"}, + "received": { + "isservice": False, + "ismaintenance": False, + "isfrs": False, + "isspecial": True, + "main": "12", + "minor": "3", + "type": "x", + "build": "48", + "service": "d", + "service_build": "80", + "service_respin": None, + "vendor_metadata": True, + "major": "12", + "patch": "48", + }, + }, + { + "sent": {"vendor": "juniper", "platform": "junos", "version": "12.3R12-S15"}, + "received": { + "isservice": True, + "ismaintenance": True, + "isfrs": False, + "isspecial": False, + "main": "12", + "minor": "3", + "type": "R", + "build": "12", + "service": "S", + "service_build": "15", + "service_respin": None, + "vendor_metadata": True, + "major": "12", + "patch": "12", + }, + }, +] + def test_get_upgrade_path(): return_values = ["9.1.15-h1", "10.0.0", "10.0.12", "10.1.0", "10.1.9"] @@ -68,3 +165,12 @@ def test_compare_strict_bad_version(): os_version.compare_version_strict("3.3.3b", "==", "3.3.3b") with pytest.raises(ValueError, match="Invalid comparison operator:"): os_version.compare_version_strict("3.3.3", "not-an-operator", "3.3.3") + + +# Testing the parsing of a Vendor, Platform, Version into vendor standardized sections +@pytest.mark.parametrize("data", PLATFORM_VERSION_METADATA) +def test_platform_parsing(data): + assert ( + os_version.version_metadata(data["sent"]["vendor"], data["sent"]["platform"], data["sent"]["version"]) + == data["received"] + ) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 2901d129..a1a1d145 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -24,7 +24,11 @@ _EXCLUDED_DECORATOR_FUNCTIONS = ["wraps", "total_ordering", "abstractmethod"] -_EXCLUDED_FUNCTIONS = ["jinja2_convenience_function", "import_module", "get_network_driver"] +_EXCLUDED_FUNCTIONS = [ + "jinja2_convenience_function", + "import_module", + "get_network_driver", +] @pytest.fixture From 11f5e88963873e929c292d929a9c3fc3e142a0ce Mon Sep 17 00:00:00 2001 From: Justin Drew <2396364+jdrew82@users.noreply.github.com> Date: Wed, 15 May 2024 22:52:34 -0500 Subject: [PATCH 2/4] Add Mappings for DNA Center (#519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: ✨ Add forward and reverse mapping for DNA Center --- development_scripts.py | 12 ++++++++++++ docs/user/lib_mapper/dna_center.md | 6 ++++++ docs/user/lib_mapper/dna_center_reverse.md | 5 +++++ netutils/lib_mapper.py | 15 +++++++++++++++ 4 files changed, 38 insertions(+) mode change 100644 => 100755 development_scripts.py create mode 100644 docs/user/lib_mapper/dna_center.md create mode 100644 docs/user/lib_mapper/dna_center_reverse.md diff --git a/development_scripts.py b/development_scripts.py old mode 100644 new mode 100755 index a95db916..3932fe69 --- a/development_scripts.py +++ b/development_scripts.py @@ -71,6 +71,18 @@ "_dict": lib_mapper.CAPIRCA_LIB_MAPPER_REVERSE, "_file": "docs/user/lib_mapper/capirca_reverse.md", }, + "dna_center": { + "header_src": "DNA_CENTER", + "header_dst": "NORMALIZED", + "_dict": lib_mapper.DNA_CENTER_LIB_MAPPER, + "_file": "docs/user/lib_mapper/dna_center.md", + }, + "dna_center_reverse": { + "header_src": "NORMALIZED", + "header_dst": "DNA_CENTER", + "_dict": lib_mapper.DNA_CENTER_LIB_MAPPER_REVERSE, + "_file": "docs/user/lib_mapper/dna_center_reverse.md", + }, "forwardnetworks": { "header_src": "FORWARDNETWORKS", "header_dst": "NORMALIZED", diff --git a/docs/user/lib_mapper/dna_center.md b/docs/user/lib_mapper/dna_center.md new file mode 100644 index 00000000..712ec3d9 --- /dev/null +++ b/docs/user/lib_mapper/dna_center.md @@ -0,0 +1,6 @@ +| DNA_CENTER | | NORMALIZED | +| ---------- | -- | ------ | +| IOS | → | cisco_ios | +| IOS-XE | → | cisco_ios | +| IOS-XR | → | cisco_xr | +| NX-OS | → | cisco_nxos | \ No newline at end of file diff --git a/docs/user/lib_mapper/dna_center_reverse.md b/docs/user/lib_mapper/dna_center_reverse.md new file mode 100644 index 00000000..6bca4292 --- /dev/null +++ b/docs/user/lib_mapper/dna_center_reverse.md @@ -0,0 +1,5 @@ +| NORMALIZED | | DNA_CENTER | +| ---------- | -- | ------ | +| cisco_ios | → | IOS | +| cisco_nxos | → | NX-OS | +| cisco_xr | → | IOS-XR | \ No newline at end of file diff --git a/netutils/lib_mapper.py b/netutils/lib_mapper.py index ea8c6b7e..cba7a81f 100644 --- a/netutils/lib_mapper.py +++ b/netutils/lib_mapper.py @@ -124,6 +124,21 @@ "windows": "windows", } +# DNA Center | Normalized +DNA_CENTER_LIB_MAPPER = { + "IOS": "cisco_ios", + "IOS-XE": "cisco_ios", + "NX-OS": "cisco_nxos", + "IOS-XR": "cisco_xr", +} + +# Normalized | DNA Center +DNA_CENTER_LIB_MAPPER_REVERSE = { + "cisco_ios": "IOS", + "cisco_nxos": "NX-OS", + "cisco_xr": "IOS-XR", +} + # Normalized | Netmiko NETMIKO_LIB_MAPPER: t.Dict[str, str] = { "a10": "a10", From be99b5a57a036f7a0d5dd080cd654e6c6a5ff667 Mon Sep 17 00:00:00 2001 From: Ken Celenza Date: Thu, 27 Jun 2024 15:26:43 -0400 Subject: [PATCH 3/4] Provide more descriptive error message when there is a duplicate line to be parsed (#539) --- netutils/config/parser.py | 9 ++++++++- tests/unit/test_parser.py | 11 +++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/netutils/config/parser.py b/netutils/config/parser.py index 2681e551..0a51da91 100644 --- a/netutils/config/parser.py +++ b/netutils/config/parser.py @@ -621,7 +621,14 @@ def _update_same_line_children_configs(self) -> None: new_config_lines: t.List[ConfigLine] = [] for line in self.config_lines: if line in self.same_line_children: - previous_line = new_config_lines[-1] + try: + previous_line = new_config_lines[-1] + except IndexError as error: + raise IndexError( + f"This error is likely from a duplicate line detected at the line `{line.config_line}`, " + "see https://netutils.readthedocs.io/en/latest/dev/dev_config/#duplicate-line-detection " + f"for more details.\nOriginal Error: {error}" + ) previous_config_line = previous_line.config_line current_parents = previous_line.parents + (previous_config_line,) line = ConfigLine(line.config_line, current_parents) diff --git a/tests/unit/test_parser.py b/tests/unit/test_parser.py index dd15e840..c7ff747e 100644 --- a/tests/unit/test_parser.py +++ b/tests/unit/test_parser.py @@ -68,3 +68,14 @@ def test_incorrect_banner_ios(): ) with pytest.raises(ValueError): compliance.parser_map["cisco_ios"](banner_cfg).config_lines # pylint: disable=expression-not-assigned + + +def test_duplicate_line(): + logging = ( + "!\n" + "snmp-server community <> RO SNMP_ACL_RO\n" + "snmp-server community <> RO SNMP_ACL_RO\n" + "snmp-server community <> RW SNMP_ACL_RW\n" + ) + with pytest.raises(IndexError, match=r".*This error is likely from a duplicate line detected.*"): + compliance.parser_map["cisco_ios"](logging).config_lines # pylint: disable=expression-not-assigned From 43fabf07eacdf82e6e542e1586246e7a8fa658d7 Mon Sep 17 00:00:00 2001 From: Justin Drew <2396364+jdrew82@users.noreply.github.com> Date: Tue, 9 Jul 2024 10:35:08 -0500 Subject: [PATCH 4/4] Release v1.9.0 (#545) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * build: 🏗️ Bump to 1.9.0 --- docs/admin/release_notes/version_1.9.md | 18 ++++++++++++++++++ mkdocs.yml | 1 + pyproject.toml | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100755 docs/admin/release_notes/version_1.9.md diff --git a/docs/admin/release_notes/version_1.9.md b/docs/admin/release_notes/version_1.9.md new file mode 100755 index 00000000..3acad549 --- /dev/null +++ b/docs/admin/release_notes/version_1.9.md @@ -0,0 +1,18 @@ +# v1.9 Release Notes + +## Release Overview + +- Added NIST URL creation and platform mapper. +- Added DNA Center platform mappings. +- Improved error message when duplicate line is parsed. + +## [v1.9.0] 2024-07 + +### Added + +- [489](https://github.com/networktocode/netutils/pull/489) Added NIST URL creation and platform mapper. +- [519](https://github.com/networktocode/netutils/pull/519) Added DNA Center forward and reverse platform mappings. + +### Changed + +- [539](https://github.com/networktocode/netutils/pull/539) Provide more descriptive error message when duplicate line is parsed. diff --git a/mkdocs.yml b/mkdocs.yml index 7db9c5a4..f4678ee7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -117,6 +117,7 @@ nav: - Uninstall: "admin/uninstall.md" - Release Notes: - "admin/release_notes/index.md" + - v1.9: "admin/release_notes/version_1.9.md" - v1.8: "admin/release_notes/version_1.8.md" - v1.7: "admin/release_notes/version_1.7.md" - v1.6: "admin/release_notes/version_1.6.md" diff --git a/pyproject.toml b/pyproject.toml index 582dc652..55e1575a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "netutils" -version = "1.8.1" +version = "1.9.0" description = "Common helper functions useful in network automation." authors = ["Network to Code, LLC "] license = "Apache-2.0"