Merge pull request #546 from networktocode/develop

Merge 1.9.0 into main

development_scripts.py

@@ -71,6 +71,18 @@
             "_dict": lib_mapper.CAPIRCA_LIB_MAPPER_REVERSE,
             "_file": "docs/user/lib_mapper/capirca_reverse.md",
         },
+        "dna_center": {
+            "header_src": "DNA_CENTER",
+            "header_dst": "NORMALIZED",
+            "_dict": lib_mapper.DNA_CENTER_LIB_MAPPER,
+            "_file": "docs/user/lib_mapper/dna_center.md",
+        },
+        "dna_center_reverse": {
+            "header_src": "NORMALIZED",
+            "header_dst": "DNA_CENTER",
+            "_dict": lib_mapper.DNA_CENTER_LIB_MAPPER_REVERSE,
+            "_file": "docs/user/lib_mapper/dna_center_reverse.md",
+        },
         "forwardnetworks": {
             "header_src": "FORWARDNETWORKS",
             "header_dst": "NORMALIZED",
@@ -131,6 +143,18 @@
             "_dict": lib_mapper.NETUTILSPARSER_LIB_MAPPER_REVERSE,
             "_file": "docs/user/lib_mapper/netutilsparser_reverse.md",
         },
+        "nist": {
+            "header_src": "NIST",
+            "header_dst": "NORMALIZED",
+            "_dict": lib_mapper.NIST_LIB_MAPPER,
+            "_file": "docs/user/lib_mapper/nist.md",
+        },
+        "nist_reverse": {
+            "header_src": "NORMALIZED",
+            "header_dst": "NIST",
+            "_dict": lib_mapper.NIST_LIB_MAPPER_REVERSE,
+            "_file": "docs/user/lib_mapper/nist_reverse.md",
+        },
         "ntctemplates": {
             "header_src": "NTCTEMPLATES",
             "header_dst": "NORMALIZED",

docs/admin/release_notes/version_1.9.md

@@ -0,0 +1,18 @@
+# v1.9 Release Notes
+
+## Release Overview
+
+- Added NIST URL creation and platform mapper.
+- Added DNA Center platform mappings.
+- Improved error message when duplicate line is parsed.
+
+## [v1.9.0] 2024-07
+
+### Added
+
+- [489](https://github.com/networktocode/netutils/pull/489) Added NIST URL creation and platform mapper.
+- [519](https://github.com/networktocode/netutils/pull/519) Added DNA Center forward and reverse platform mappings.
+
+### Changed
+
+- [539](https://github.com/networktocode/netutils/pull/539) Provide more descriptive error message when duplicate line is parsed.

docs/dev/code_reference/nist.md

@@ -0,0 +1,5 @@
+# NIST URLs
+
+::: netutils.nist
+    options:
+        show_submodules: True

docs/user/include_jinja_list.md

@@ -60,9 +60,12 @@
 | mac_to_format | netutils.mac.mac_to_format |
 | mac_to_int | netutils.mac.mac_to_int |
 | mac_type | netutils.mac.mac_type |
+| get_nist_urls | netutils.nist.get_nist_urls |
+| get_nist_vendor_platform_urls | netutils.nist.get_nist_vendor_platform_urls |
 | compare_version_loose | netutils.os_version.compare_version_loose |
 | compare_version_strict | netutils.os_version.compare_version_strict |
 | get_upgrade_path | netutils.os_version.get_upgrade_path |
+| version_metadata | netutils.os_version.version_metadata |
 | compare_cisco_type5 | netutils.password.compare_cisco_type5 |
 | compare_cisco_type7 | netutils.password.compare_cisco_type7 |
 | compare_cisco_type9 | netutils.password.compare_cisco_type9 |

docs/user/lib_mapper/dna_center.md

@@ -0,0 +1,6 @@
+| DNA_CENTER | | NORMALIZED |
+| ---------- | -- | ------ |
+| IOS | → | cisco_ios |
+| IOS-XE | → | cisco_ios |
+| IOS-XR | → | cisco_xr |
+| NX-OS | → | cisco_nxos |

docs/user/lib_mapper/dna_center_reverse.md

@@ -0,0 +1,5 @@
+| NORMALIZED | | DNA_CENTER |
+| ---------- | -- | ------ |
+| cisco_ios | → | IOS |
+| cisco_nxos | → | NX-OS |
+| cisco_xr | → | IOS-XR |

docs/user/lib_mapper/nist.md

@@ -0,0 +1,9 @@
+| NIST | | NORMALIZED |
+| ---------- | -- | ------ |
+| adaptive_security_appliance_software | → | cisco_asa |
+| eos | → | arista_eos |
+| ios | → | cisco_ios |
+| ios_xe | → | cisco_xe |
+| ios_xr | → | cisco_xr |
+| junos | → | juniper_junos |
+| nx-os | → | cisco_nxos |

docs/user/lib_mapper/nist_reverse.md

@@ -0,0 +1,9 @@
+| NORMALIZED | | NIST |
+| ---------- | -- | ------ |
+| arista_eos | → | eos |
+| cisco_asa | → | adaptive_security_appliance_software |
+| cisco_ios | → | ios |
+| cisco_nxos | → | nx-os |
+| cisco_xe | → | ios_xe |
+| cisco_xr | → | ios_xr |
+| juniper_junos | → | junos |

docs/user/lib_use_cases.md

@@ -21,6 +21,7 @@ Functions are grouped with like functions, such as IP or MAC address based funct
 - Library Helpers - Provides helpers to pull useful information, e.g. NAPALM getters.
 - Library Mapper - Provides mappings in expected vendor names between Netmiko, NAPALM, pyntc, ntc-templates, pyats, and scrapli.
 - MAC Address - Provides the ability to work with MAC addresses such as validating or converting to integer.
+- NIST - Provides the ability to obtain a URL formatted for NIST CPE Query.
 - OS Version - Provides the ability to work with OS version, such as defining an upgrade path.
 - Password - Provides the ability to compare and encrypt common password schemas such as type5 and type7 Cisco passwords.
 - Ping - Provides the ability to ping, currently only tcp ping.

docs/user/lib_use_cases_nist.md

@@ -0,0 +1,42 @@
+# NIST
+
+The NIST utility is used for functionality based around NIST DB Queries, and is primarily used to create URLs for the API based queries.
+
+## Requirements
+
+In order to use the URLs generated by `netutils.nist.get_nist_urls*`, you will need an api key provided by NIST [here]('https://nvd.nist.gov/developers/request-an-api-key').  This key will need to be passed in as an additional header in your request in the form of `{"apiKey": "<key_value>"}` as stated by NIST in their [Getting Started]('https://nvd.nist.gov/developers/start-here') section.
+
+
+## Custom URLs
+
+The largest caveat in this functionality is the consistency of the URL values needed to obtain the CVE information.  NIST NVD has specific parameters that can be used for standardization, however this does not mean that entries are standardized.  Manually combing through a large amount of CPE Vendor submissions has shown that there are variations in how CPE Vendor data is presented.
+
+For this reason, for certain Vendor/OS combinations, a custom URL needs to be built.
+- **Cisco IOS CPE String** - `cpe:2.3:o:cisco:ios:15.5\\(2\\)s1c:*`
+    - `15.5\\(2\\)s1c:*` - As seen here, Cisco uses CPE strings that do not include the `:` delimiter, which can be queried using escape characters in the search string.  **This is the format of ALL "generic" OS/Other platforms that do not have their own custom NIST URL builder when querying NIST.**
+    - Default URL Output - `'https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:cisco:ios:15.5\\(2\\)s1c:*'`
+
+- **Juniper JunOS CPE String** - `cpe:2.3:o:juniper:junos:10.2:r2:*:*:*:*:*:*` 
+    - `10.2:r2:*:*:*:*:*:*` - As noted here, one of the provided URLs to query for this Juniper JunOS OS platform includes additional values that follow NIST delimiter structures.  In the case where the parser provides multiple URLs, they will both be evaluated and the CVE from both will be added and associated.
+    - Custom URL Output - `['https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:10.2r2:*:*:*:*:*:*:*', 'https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:10.2:r2:*:*:*:*:*:*']`
+
+
+## Examples
+Here are a few examples showing how to use this in your python code.
+
+```python
+
+from netutils.nist import get_nist_urls
+
+# Get NIST URL for the Cisco IOS object
+get_nist_urls("cisco_ios", "15.5(2)S1c")
+# ['https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:cisco:ios:15.5\\(2\\)s1c:*']
+
+# Get NIST URL(s) for the Juniper JunOS object
+get_nist_urls("juniper_junos", "10.2R2.11")
+# ['https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:10.2r2:*:*:*:*:*:*:*', 'https://services.nvd.nist.gov/rest/json/cves/2.0?virtualMatchString=cpe:2.3:o:juniper:junos:10.2:r2:*:*:*:*:*:*']
+```
+
+Currently known OS/Other Platform types that require a custom NIST URL:
+
+- Juniper JunOS

docs/user/lib_use_cases_os_version.md

@@ -0,0 +1,41 @@
+# OS Version Tools
+
+The OS Version Tools are used for working with versioning systems.
+
+## Version Parsing/Deconstruction
+Version parsing takes the software version given as a string, and deconstructs that value into the standards of the vendor.  The version parsing takes place in the `netutils.os_version` module.  This is necessary when specific values or flags from a software version are required to make a logical decision.  
+
+Current Version Parsers:
+
+- Default Parser
+- Juniper JunOS
+
+**See the following Default and Juniper JunOS parsed versions:**
+
+```python
+>>> from netutils.os_version import version_metadata
+
+>>> version_metadata("Cisco", "IOS", "15.5")
+{
+    "major": "15",
+    "minor": "5",
+    "vendor_metadata": False,
+}
+>>> version_metadata("juniper", "junos", "12.4R")
+{
+    "isservice": False,
+    "ismaintenance": False,
+    "isfrs": True,
+    "isspecial": False,
+    "service": None,
+    "service_build": None,
+    "service_respin": None,
+    "main": "12",
+    "minor": "4",
+    "type": "R",
+    "build": None,
+    "major": "12",
+    "patch": None,
+    "vendor_metadata": True,
+}
+```

mkdocs.yml

@@ -117,6 +117,7 @@ nav:
       - Uninstall: "admin/uninstall.md"
       - Release Notes:
           - "admin/release_notes/index.md"
+          - v1.9: "admin/release_notes/version_1.9.md"
           - v1.8: "admin/release_notes/version_1.8.md"
           - v1.7: "admin/release_notes/version_1.7.md"
           - v1.6: "admin/release_notes/version_1.6.md"
@@ -149,6 +150,7 @@ nav:
           - Library Helpers: "dev/code_reference/lib_helpers.md"
           - Library Mapping: "dev/code_reference/lib_mapping.md"
           - Mac Address: "dev/code_reference/mac.md"
+          - NIST: "dev/code_reference/nist.md"
           - OS Version: "dev/code_reference/os_version.md"
           - Password: "dev/code_reference/password.md"
           - Ping: "dev/code_reference/ping.md"

netutils/config/parser.py

@@ -621,7 +621,14 @@ def _update_same_line_children_configs(self) -> None:
         new_config_lines: t.List[ConfigLine] = []
         for line in self.config_lines:
             if line in self.same_line_children:
-                previous_line = new_config_lines[-1]
+                try:
+                    previous_line = new_config_lines[-1]
+                except IndexError as error:
+                    raise IndexError(
+                        f"This error is likely from a duplicate line detected at the line `{line.config_line}`, "
+                        "see https://netutils.readthedocs.io/en/latest/dev/dev_config/#duplicate-line-detection "
+                        f"for more details.\nOriginal Error: {error}"
+                    )
                 previous_config_line = previous_line.config_line
                 current_parents = previous_line.parents + (previous_config_line,)
                 line = ConfigLine(line.config_line, current_parents)

netutils/lib_mapper.py

@@ -124,6 +124,21 @@
     "windows": "windows",
 }
 
+# DNA Center | Normalized
+DNA_CENTER_LIB_MAPPER = {
+    "IOS": "cisco_ios",
+    "IOS-XE": "cisco_ios",
+    "NX-OS": "cisco_nxos",
+    "IOS-XR": "cisco_xr",
+}
+
+# Normalized | DNA Center
+DNA_CENTER_LIB_MAPPER_REVERSE = {
+    "cisco_ios": "IOS",
+    "cisco_nxos": "NX-OS",
+    "cisco_xr": "IOS-XR",
+}
+
 # Normalized | Netmiko
 NETMIKO_LIB_MAPPER: t.Dict[str, str] = {
     "a10": "a10",
@@ -407,6 +422,17 @@
     "SRX": "juniper_junos",  # no reverse
 }
 
+# NIST | Normalized
+NIST_LIB_MAPPER = {
+    "adaptive_security_appliance_software": "cisco_asa",
+    "nx-os": "cisco_nxos",
+    "ios_xr": "cisco_xr",
+    "ios_xe": "cisco_xe",
+    "eos": "arista_eos",
+    "ios": "cisco_ios",
+    "junos": "juniper_junos",
+}
+
 # Normalized | NAPALM
 NAPALM_LIB_MAPPER_REVERSE: t.Dict[str, str] = {
     "arista_eos": "eos",
@@ -557,6 +583,18 @@
     "paloalto_panos": "PAN_OS",
 }
 
+# Normalized | NIST
+NIST_LIB_MAPPER_REVERSE = {
+    "cisco_asa": "adaptive_security_appliance_software",
+    "cisco_nxos": "nx-os",
+    "cisco_xr": "ios_xr",
+    "cisco_xe": "ios_xe",
+    "arista_eos": "eos",
+    "cisco_ios": "ios",
+    "juniper_junos": "junos",
+}
+
+
 # Deep copy the reverse, where there is no actual translation happening with special
 # consideration for OS's not in netmiko.
 _MAIN_LIB_MAPPER = copy.deepcopy(NETMIKO_LIB_MAPPER)