diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index c5a5c5743..2a296c680 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -111,7 +111,7 @@ def to_dict(self): def from_dict(cls, ref: dict): return cls( reference_id=ref["reference_id"], - reference_type=ref["reference_type"], + reference_type=ref.get("reference_type") or "", url=ref["url"], severities=[ VulnerabilitySeverity.from_dict(severity) for severity in ref["severities"] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 4db674e3e..9b6df7c13 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -7,14 +7,23 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import csv import hashlib import json import logging +import xml.etree.ElementTree as ET from contextlib import suppress from functools import cached_property +from itertools import groupby +from operator import attrgetter from typing import Union +from cvss.exceptions import CVSS2MalformedError +from cvss.exceptions import CVSS3MalformedError +from cvss.exceptions import CVSS4MalformedError from cwe2.database import Database +from cwe2.mappings import xml_database_path +from cwe2.weakness import Weakness as DBWeakness from django.contrib.auth import get_user_model from django.contrib.auth.models import UserManager from django.core import exceptions @@ -41,8 +50,8 @@ from univers.version_range import AlpineLinuxVersionRange from univers.versions import Version -from aboutcode import hashid from vulnerabilities import utils +from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import normalize_purl from vulnerabilities.utils import purl_to_dict @@ -371,6 +380,127 @@ def get_related_purls(self): """ return [p.package_url for p in self.packages.distinct().all()] + def aggregate_fixed_and_affected_packages(self): + from vulnerabilities.utils import get_purl_version_class + + sorted_fixed_by_packages = self.fixed_by_packages.filter(is_ghost=False).order_by( + "type", "namespace", "name", "qualifiers", "subpath" + ) + + if sorted_fixed_by_packages: + sorted_fixed_by_packages.first().calculate_version_rank + + sorted_affected_packages = self.affected_packages.all() + + if sorted_affected_packages: + sorted_affected_packages.first().calculate_version_rank + + grouped_fixed_by_packages = { + key: list(group) + for key, group in groupby( + sorted_fixed_by_packages, + key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"), + ) + } + + all_affected_fixed_by_matches = [] + + for sorted_affected_package in sorted_affected_packages: + affected_fixed_by_matches = { + "affected_package": sorted_affected_package, + "matched_fixed_by_packages": [], + } + + # Build the key to find matching group + key = ( + sorted_affected_package.type, + sorted_affected_package.namespace, + sorted_affected_package.name, + sorted_affected_package.qualifiers, + sorted_affected_package.subpath, + ) + + # Get matching group from pre-grouped fixed_by_packages + matching_fixed_packages = grouped_fixed_by_packages.get(key, []) + + # Get version classes for comparison + affected_version_class = get_purl_version_class(sorted_affected_package) + affected_version = affected_version_class(sorted_affected_package.version) + + # Compare versions and filter valid matches + matched_fixed_by_packages = [ + fixed_by_package.purl + for fixed_by_package in matching_fixed_packages + if get_purl_version_class(fixed_by_package)(fixed_by_package.version) + > affected_version + ] + + affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages + all_affected_fixed_by_matches.append(affected_fixed_by_matches) + return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches + + def get_severity_vectors_and_values(self): + """ + Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. + """ + severity_vectors = [] + severity_values = set() + + # Exclude EPSS scoring system + base_severities = self.severities.exclude(scoring_system=EPSS.identifier) + + # QuerySet for severities with valid scoring_elements and scoring_system in SCORING_SYSTEMS + valid_scoring_severities = base_severities.filter( + scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() + ) + + for severity in valid_scoring_severities: + try: + vector_values = SCORING_SYSTEMS[severity.scoring_system].get( + severity.scoring_elements + ) + if vector_values: + severity_vectors.append(vector_values) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ) as e: + logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") + + valid_value_severities = base_severities.filter(value__isnull=False).exclude(value="") + + severity_values.update(valid_value_severities.values_list("value", flat=True)) + + return severity_vectors, severity_values + + +def get_cwes(self): + """Yield CWE Weakness objects""" + for cwe_category in self.cwe_files: + cwe_category.seek(0) + reader = csv.DictReader(cwe_category) + for row in reader: + yield DBWeakness(*list(row.values())[0:-1]) + tree = ET.parse(xml_database_path) + root = tree.getroot() + for tag_num in [1, 2]: # Categories , Views + tag = root[tag_num] + for child in tag: + yield DBWeakness( + *[ + child.attrib["ID"], + child.attrib.get("Name"), + None, + child.attrib.get("Status"), + child[0].text, + ] + ) + + +Database.get_cwes = get_cwes + class Weakness(models.Model): """ @@ -379,7 +509,15 @@ class Weakness(models.Model): cwe_id = models.IntegerField(help_text="CWE id") vulnerabilities = models.ManyToManyField(Vulnerability, related_name="weaknesses") - db = Database() + + cwe_by_id = {} + + def get_cwe(self, cwe_id): + if not self.cwe_by_id: + db = Database() + for weakness in db.get_cwes(): + self.cwe_by_id[str(weakness.cwe_id)] = weakness + return self.cwe_by_id[cwe_id] @property def cwe(self): @@ -391,7 +529,7 @@ def weakness(self): Return a queryset of Weakness for this vulnerability. """ try: - weakness = self.db.get(self.cwe_id) + weakness = self.get_cwe(str(self.cwe_id)) return weakness except Exception as e: logger.warning(f"Could not find CWE {self.cwe_id}: {e}") diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index e9e58c79e..7001c8f3b 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -33,10 +33,10 @@ Essentials -
  • +
  • - Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }}) + Severities ({{ severities|length }})
    • @@ -48,12 +48,12 @@
  • - - - Severities vectors ({{ severity_vectors|length }}) - - -
    • + + + Severity details ({{ severity_vectors|length }}) + + + {% if vulnerability.exploits %}
  • @@ -66,11 +66,11 @@ {% endif %}
  • - - - EPSS - - + + + EPSS + +
  • @@ -152,156 +152,72 @@ {{ vulnerability.risk_score }} - - - - - -
    - Severity ({{ severities|length }}) -
    -
    - - - - - - - {% for severity in severities %} - - - - - - {% empty %} - - - - {% endfor %} -
    System Score Found at
    {{ severity.scoring_system }}{{ severity.value }} - {{ severity.url }} -
    - There are no known severity scores. -
    -
    - -
    - Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }}) -
    -
    - - - - + + - - - {% for package in affected_packages|slice:":3" %} + +
    AffectedFixed byAffected and Fixed Packages + + Package Details + +
    +
    + Weaknesses ({{ weaknesses|length }}) +
    +
    + + {% for weakness in weaknesses %} - - + + {% empty %} - {% endfor %} - {% if affected_packages|length > 3 %} - - - - {% endif %} - -
    - {{ package.purl }} - - {% for match in all_affected_fixed_by_matches %} - {% if match.affected_package == package %} - {% if match.matched_fixed_by_packages|length > 0 %} - {% for pkg in match.matched_fixed_by_packages %} - {{ pkg }} -
    - {% endfor %} - {% else %} - There are no reported fixed by versions. - {% endif %} - {% endif %} - {% endfor %} +     		CWE-{{ weakness.cwe_id }} + + {{ weakness.name }} +
    - This vulnerability is not known to affect any packages. + + There are no known CWE.
    - See Affected/Fixed by packages tab for more -
    -
    - -
    - Weaknesses ({{ weaknesses|length }}) -
    -
    - - {% for weakness in weaknesses %} - - - - - - {% empty %} - - - - {% endfor %} -
    CWE-{{ weakness.cwe_id }} - - {{ weakness.name }} - -
    - There are no known CWE. -
    + +
    -
    - - - - - - - - - {% for package in affected_packages %} - - - - - {% empty %} - - - - {% endfor %} - + +
    +
    AffectedFixed by
    - {{ package.purl }} - - - {% for match in all_affected_fixed_by_matches %} - {% if match.affected_package == package %} - {% if match.matched_fixed_by_packages|length > 0 %} - {% for pkg in match.matched_fixed_by_packages %} - {{ pkg }} -
    - {% endfor %} - {% else %} - There are no reported fixed by versions. - {% endif %} - {% endif %} - {% endfor %} - -
    - This vulnerability is not known to affect any packages. -
    + + + + + + {% for severity in severities %} + + + + + + {% empty %} + + + + {% endfor %}
    System Score Found at
    {{ severity.scoring_system }}{{ severity.value }} + {{ severity.url }} +
    + There are no known severity scores. +
    @@ -341,103 +257,6 @@ -
    - {% for severity_vector in severity_vectors %} - {% if severity_vector.version == '2.0' %} - Vector: {{ severity_vector.vectorString }} - - - - - - - - - - - - - - - - - - - -
    Exploitability (E)Access Vector (AV)Access Complexity (AC)Authentication (Au)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.exploitability|cvss_printer:"high,functional,unproven,proof_of_concept,not_defined" }}{{ severity_vector.accessVector|cvss_printer:"local,adjacent_network,network" }}{{ severity_vector.accessComplexity|cvss_printer:"high,medium,low" }}{{ severity_vector.authentication|cvss_printer:"multiple,single,none" }}{{ severity_vector.confidentialityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.integrityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.availabilityImpact|cvss_printer:"none,partial,complete" }}
    - {% elif severity_vector.version == '3.1' or severity_vector.version == '3.0'%} - Vector: {{ severity_vector.vectorString }} - - - - - - - - - - - - - - - - - - - - - -
    Attack Vector (AV)Attack Complexity (AC)Privileges Required (PR)User Interaction (UI)Scope (S)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent_network,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,required"}}{{ severity_vector.scope|cvss_printer:"unchanged,changed" }}{{ severity_vector.confidentialityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.integrityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.availabilityImpact|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.version == '4' %} - Vector: {{ severity_vector.vectorString }} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Attack Vector (AV)Attack Complexity (AC)Attack Requirements (AT)Privileges Required (PR)User Interaction (UI)Vulnerable System Impact Confidentiality (VC)Vulnerable System Impact Integrity (VI)Vulnerable System Impact Availability (VA)Subsequent System Impact Confidentiality (SC)Subsequent System Impact Integrity (SI)Subsequent System Impact Availability (SA)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.attackRequirement|cvss_printer:"none,present" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,passive,active"}}{{ severity_vector.vulnerableSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactAvailability|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactAvailability|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.version == 'ssvc' %} -
    - Vector: {{ severity_vector.vectorString }} -
    - {% endif %} - {% empty %} - - - There are no known vectors. - - - {% endfor %} -
    - -
    {% for exploit in vulnerability.exploits.all %} @@ -586,108 +405,192 @@ {% endfor %} + +
    + {% for severity_vector in severity_vectors %} + {% if severity_vector.vector.version == '2.0' %} + Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} +
    + + + + + + + + + + + + + + + + + + +
    Exploitability (E)Access Vector (AV)Access Complexity (AC)Authentication (Au)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.vector.exploitability|cvss_printer:"high,functional,unproven,proof_of_concept,not_defined" }}{{ severity_vector.vector.accessVector|cvss_printer:"local,adjacent_network,network" }}{{ severity_vector.vector.accessComplexity|cvss_printer:"high,medium,low" }}{{ severity_vector.vector.authentication|cvss_printer:"multiple,single,none" }}{{ severity_vector.vector.confidentialityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.vector.integrityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.vector.availabilityImpact|cvss_printer:"none,partial,complete" }}
    + {% elif severity_vector.vector.version == '3.1' or severity_vector.vector.version == '3.0'%} + Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} + + + + + + + + + + + + + + + + + + + + + +
    Attack Vector (AV)Attack Complexity (AC)Privileges Required (PR)User Interaction (UI)Scope (S)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.vector.attackVector|cvss_printer:"network,adjacent_network,local,physical"}}{{ severity_vector.vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.vector.userInteraction|cvss_printer:"none,required"}}{{ severity_vector.vector.scope|cvss_printer:"unchanged,changed" }}{{ severity_vector.vector.confidentialityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.vector.integrityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.vector.availabilityImpact|cvss_printer:"high,low,none" }}
    + {% elif severity_vector.vector.version == '4' %} + Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Attack Vector (AV)Attack Complexity (AC)Attack Requirements (AT)Privileges Required (PR)User Interaction (UI)Vulnerable System Impact Confidentiality (VC)Vulnerable System Impact Integrity (VI)Vulnerable System Impact Availability (VA)Subsequent System Impact Confidentiality (SC)Subsequent System Impact Integrity (SI)Subsequent System Impact Availability (SA)
    {{ severity_vector.vector.attackVector|cvss_printer:"network,adjacent,local,physical"}}{{ severity_vector.vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.vector.attackRequirement|cvss_printer:"none,present" }}{{ severity_vector.vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.vector.userInteraction|cvss_printer:"none,passive,active"}}{{ severity_vector.vector.vulnerableSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vector.vulnerableSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vector.vulnerableSystemImpactAvailability|cvss_printer:"high,low,none" }}{{ severity_vector.vector.subsequentSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vector.subsequentSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vector.subsequentSystemImpactAvailability|cvss_printer:"high,low,none" }}
    + {% elif severity_vector.vector.version == 'ssvc' %} +
    + Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} +
    + {% endif %} + {% empty %} + + + There are no known vectors. + + + {% endfor %} +
    - {% for severity in severities %} - {% if severity.scoring_system == 'epss' %} -
    +
    + {% if epss_data %}
    - Exploit Prediction Scoring System + Exploit Prediction Scoring System (EPSS)
    - - - - - - - - - - - - {% if severity.published_at %} - - - - + + + + + + + + + + {% if epss_data.published_at %} + + + + {% endif %} - - +
    - - Percentile - - {{ severity.scoring_elements }}
    - - EPSS score - - {{ severity.value }}
    - - Published at - - {{ severity.published_at }}
    + + Percentile + + {{ epss_data.percentile }}
    + + EPSS Score + + {{ epss_data.score }}
    + + Published At + + {{ epss_data.published_at }}
    -
    + {% else %} +

    No EPSS data available for this vulnerability.

    {% endif %} - {% empty %} -
    - - - There are no EPSS available. - - -
    - {% endfor %} +
    -
    - - - - - - - - - - - {% for log in history %} +
    +
    - - Date - - Actor - Action Source - VulnerableCode Version -
    + - - - - - + + + + + - {% empty %} - - - - {% endfor %} -
    {{ log.get_iso_time }}{{ log.actor_name }}{{ log.get_action_type_label }} {{log.source_url }} {{ log.software_version }} + + Date + + Actor + Action Source + VulnerableCode Version +
    - There are no relevant records. -
    -
    + + {% for log in history %} + + {{ log.get_iso_time }} + {{ log.actor_name }} + {{ log.get_action_type_label }} + {{log.source_url }} + {{ log.software_version }} + + {% empty %} + + + There are no relevant records. + + + {% endfor %} + + @@ -711,5 +614,4 @@ } - -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/vulnerabilities/templates/vulnerability_package_details.html b/vulnerabilities/templates/vulnerability_package_details.html new file mode 100644 index 000000000..21fb52192 --- /dev/null +++ b/vulnerabilities/templates/vulnerability_package_details.html @@ -0,0 +1,88 @@ +{% extends "base.html" %} +{% load humanize %} +{% load widget_tweaks %} +{% load static %} +{% load show_cvss %} +{% load url_filters %} + +{% block title %} +VulnerableCode Vulnerability Package Details - {{ vulnerability.vulnerability_id }} +{% endblock %} + +{% block content %} + +{% if vulnerability %} +
    +
    +
    +
    + Vulnerable and Fixing Package details for Vulnerability: + + {{ vulnerability.vulnerability_id }} + +
    +
    +
    + + + + + + + + + {% for package in affected_packages %} + + + + + {% empty %} + + + + {% endfor %} + +
    AffectedFixed by
    + {{ package.purl }} + + + {% for match in all_affected_fixed_by_matches %} + {% if match.affected_package == package %} + {% if match.matched_fixed_by_packages|length > 0 %} + {% for pkg in match.matched_fixed_by_packages %} + {{ pkg }} +
    + {% endfor %} + {% else %} + There are no reported fixed by versions. + {% endif %} + {% endif %} + {% endfor %} + +
    + This vulnerability is not known to affect any packages. +
    +
    +
    +
    +{% endif %} + + + + + +{% endblock %} \ No newline at end of file diff --git a/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt b/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt index 886119bfd..7faf4c22a 100644 --- a/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt +++ b/vulnerabilities/tests/test_data/package_sort/sorted_purls.txt @@ -21,10 +21,10 @@ pkg:conan/capnproto@0.15.2 pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u7?distro=stretch pkg:deb/debian/jackson-databind@2.8.6-1%2Bdeb9u10?distro=stretch pkg:deb/debian/jackson-databind@2.9.8-3%2Bdeb10u4?distro=sid -pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1?distro=sid -pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid +pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=stretch +pkg:deb/debian/jackson-databind@2.13.2.2-1?distro=sid pkg:deb/debian/jackson-databind@2.14.0-1?distro=sid pkg:deb/ubuntu/dpkg@1.13.11ubuntu7~proposed pkg:deb/ubuntu/dpkg@1.13.11ubuntu7.2 @@ -94,10 +94,10 @@ pkg:pypi/jinja2@2.1.1 pkg:pypi/jinja2@2.2 pkg:pypi/jinja2@2.2.1 pkg:pypi/jinja2@2.10 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 -pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=9 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=7 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=5 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=2 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=13 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=12 +pkg:rpm/redhat/openssl@1.0.1e-30.el6_6?arch=11 diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index 98a555294..3b32ee31c 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -23,10 +23,9 @@ from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilitySeverity from vulnerabilities.templatetags.url_filters import url_quote_filter +from vulnerabilities.utils import get_purl_version_class from vulnerabilities.views import PackageDetails from vulnerabilities.views import PackageSearch -from vulnerabilities.views import get_purl_version_class -from vulnerabilities.views import purl_sort_key BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DIR = os.path.join(BASE_DIR, "test_data/package_sort") @@ -202,12 +201,13 @@ def setUp(self): for pkg in input_purls: real_purl = PackageURL.from_string(pkg) attrs = {k: v for k, v in real_purl.to_dict().items() if v} - Package.objects.create(**attrs) + pkg = Package.objects.create(**attrs) + pkg.calculate_version_rank def test_sorted_queryset(self): qs_all = Package.objects.all() pkgs_qs_all = list(qs_all) - sorted_pkgs_qs_all = sorted(pkgs_qs_all, key=purl_sort_key) + sorted_pkgs_qs_all = pkgs_qs_all pkg_package_urls = [obj.package_url for obj in sorted_pkgs_qs_all] sorted_purls = os.path.join(TEST_DIR, "sorted_purls.txt") diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 969a08f2f..d9a3c7e04 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -32,6 +32,7 @@ from packageurl import PackageURL from packageurl.contrib.django.utils import without_empty_values from univers.version_range import RANGE_CLASS_BY_SCHEMES +from univers.version_range import AlpineLinuxVersionRange from univers.version_range import NginxVersionRange from univers.version_range import VersionRange @@ -536,3 +537,12 @@ def normalize_purl(purl: Union[PackageURL, str]): if isinstance(purl, PackageURL): purl = str(purl) return PackageURL.from_string(purl) + + +def get_purl_version_class(purl): + RANGE_CLASS_BY_SCHEMES["apk"] = AlpineLinuxVersionRange + purl_version_class = None + check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) + if check_version_class: + purl_version_class = check_version_class.version_class + return purl_version_class diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 77f75238d..a2df48634 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -14,6 +14,7 @@ from django.contrib import messages from django.core.exceptions import ValidationError from django.core.mail import send_mail +from django.db.models import Prefetch from django.http.response import Http404 from django.shortcuts import redirect from django.shortcuts import render @@ -29,39 +30,14 @@ from vulnerabilities.forms import ApiUserCreationForm from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import VulnerabilitySearchForm -from vulnerabilities.models import VulnerabilityStatusType from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import get_severity_range from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import env PAGE_SIZE = 20 -def purl_sort_key(purl: models.Package): - """ - Return a sort key for the built-in sorted() function when sorting a list - of Package objects. If the Package ``type`` is supported by univers, apply - the univers version class to the Package ``version``, and otherwise use the - ``version`` attribute as is. - """ - purl_version_class = get_purl_version_class(purl) - purl_sort_version = purl.version - if purl_version_class: - purl_sort_version = purl_version_class(purl.version) - return (purl.type, purl.namespace, purl.name, purl_sort_version, purl.qualifiers, purl.subpath) - - -def get_purl_version_class(purl: models.Package): - RANGE_CLASS_BY_SCHEMES["apk"] = AlpineLinuxVersionRange - purl_version_class = None - check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) - if check_version_class: - purl_version_class = check_version_class.version_class - return purl_version_class - - class PackageSearch(ListView): model = models.Package template_name = "packages.html" @@ -159,90 +135,93 @@ def get_queryset(self): return ( super() .get_queryset() + .select_related() .prefetch_related( - "references", - "aliases", - "weaknesses", - "severities", - "exploits", + Prefetch( + "references", + queryset=models.VulnerabilityReference.objects.only( + "reference_id", "reference_type", "url" + ), + ), + Prefetch( + "aliases", + queryset=models.Alias.objects.only("alias"), + ), + Prefetch( + "weaknesses", + queryset=models.Weakness.objects.only("cwe_id"), + ), + Prefetch( + "severities", + queryset=models.VulnerabilitySeverity.objects.only( + "scoring_system", "value", "url", "scoring_elements", "published_at" + ), + ), + Prefetch( + "exploits", + queryset=models.Exploit.objects.only( + "data_source", "description", "required_action", "due_date", "notes" + ), + ), ) ) def get_context_data(self, **kwargs): + """ + Build context with preloaded QuerySets and minimize redundant queries. + """ context = super().get_context_data(**kwargs) - weaknesses = self.object.weaknesses.all() + vulnerability = self.object + + # Pre-fetch and process data in Python instead of the template weaknesses_present_in_db = [ - weakness_object for weakness_object in weaknesses if weakness_object.weakness + weakness_object + for weakness_object in vulnerability.weaknesses.all() + if weakness_object.weakness ] - status = self.object.get_status_label + + valid_severities = self.object.severities.exclude(scoring_system=EPSS.identifier).filter( + scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() + ) severity_vectors = [] - severity_values = set() - for s in self.object.severities.all(): - if s.scoring_system == EPSS.identifier: - continue - - if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS: - try: - vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) - severity_vectors.append(vector_values) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ): - logging.error(f"CVSSMalformedError for {s.scoring_elements}") - - if s.value: - severity_values.add(s.value) - - sorted_affected_packages = sorted(self.object.affected_packages.all(), key=purl_sort_key) - sorted_fixed_by_packages = sorted(self.object.fixed_by_packages.all(), key=purl_sort_key) - - all_affected_fixed_by_matches = [] - for sorted_affected_package in sorted_affected_packages: - affected_fixed_by_matches = {} - affected_fixed_by_matches["affected_package"] = sorted_affected_package - matched_fixed_by_packages = [] - for fixed_by_package in sorted_fixed_by_packages: - - # Ghost Package can't fix vulnerability. - if fixed_by_package.is_ghost: - continue - - sorted_affected_version_class = get_purl_version_class(sorted_affected_package) - fixed_by_version_class = get_purl_version_class(fixed_by_package) - if ( - (fixed_by_package.type == sorted_affected_package.type) - and (fixed_by_package.namespace == sorted_affected_package.namespace) - and (fixed_by_package.name == sorted_affected_package.name) - and (fixed_by_package.qualifiers == sorted_affected_package.qualifiers) - and (fixed_by_package.subpath == sorted_affected_package.subpath) - and ( - fixed_by_version_class(fixed_by_package.version) - > sorted_affected_version_class(sorted_affected_package.version) - ) - ): - matched_fixed_by_packages.append(fixed_by_package.purl) - affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages - all_affected_fixed_by_matches.append(affected_fixed_by_matches) + + for severity in valid_severities: + try: + vector_values = SCORING_SYSTEMS[severity.scoring_system].get( + severity.scoring_elements + ) + if vector_values: + severity_vectors.append({"vector": vector_values, "origin": severity.url}) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ): + logging.error(f"CVSSMalformedError for {severity.scoring_elements}") + + epss_severity = vulnerability.severities.filter(scoring_system="epss").first() + epss_data = None + if epss_severity: + epss_data = { + "percentile": epss_severity.scoring_elements, + "score": epss_severity.value, + "published_at": epss_severity.published_at, + } context.update( { - "vulnerability": self.object, + "vulnerability": vulnerability, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), - "severities": list(self.object.severities.all()), - "severity_score_range": get_severity_range(severity_values), + "severities": list(vulnerability.severities.all()), "severity_vectors": severity_vectors, - "references": self.object.references.all(), - "aliases": self.object.aliases.all(), - "affected_packages": sorted_affected_packages, - "fixed_by_packages": sorted_fixed_by_packages, + "references": list(vulnerability.references.all()), + "aliases": list(vulnerability.aliases.all()), "weaknesses": weaknesses_present_in_db, - "status": status, - "history": self.object.history, - "all_affected_fixed_by_matches": all_affected_fixed_by_matches, + "status": vulnerability.get_status_label, + "history": vulnerability.history, + "epss_data": epss_data, } ) return context @@ -316,3 +295,54 @@ def form_valid(self, form): def get_success_url(self): return reverse_lazy("api_user_request") + + +class VulnerabilityPackagesDetails(DetailView): + """ + View to display all packages affected by or fixing a specific vulnerability. + URL: /vulnerabilities/{vulnerability_id}/packages + """ + + model = models.Vulnerability + template_name = "vulnerability_package_details.html" + slug_url_kwarg = "vulnerability_id" + slug_field = "vulnerability_id" + + def get_queryset(self): + """ + Prefetch and optimize related data to minimize database hits. + """ + return ( + super() + .get_queryset() + .prefetch_related( + Prefetch( + "affecting_packages", + queryset=models.Package.objects.only("type", "namespace", "name", "version"), + ), + Prefetch( + "fixed_by_packages", + queryset=models.Package.objects.only("type", "namespace", "name", "version"), + ), + ) + ) + + def get_context_data(self, **kwargs): + """ + Build context with preloaded QuerySets and minimize redundant queries. + """ + context = super().get_context_data(**kwargs) + vulnerability = self.object + ( + sorted_fixed_by_packages, + sorted_affected_packages, + all_affected_fixed_by_matches, + ) = vulnerability.aggregate_fixed_and_affected_packages() + context.update( + { + "affected_packages": sorted_affected_packages, + "fixed_by_packages": sorted_fixed_by_packages, + "all_affected_fixed_by_matches": all_affected_fixed_by_matches, + } + ) + return context diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 54540a66d..c6dd3da44 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -28,6 +28,7 @@ from vulnerabilities.views import PackageDetails from vulnerabilities.views import PackageSearch from vulnerabilities.views import VulnerabilityDetails +from vulnerabilities.views import VulnerabilityPackagesDetails from vulnerabilities.views import VulnerabilitySearch from vulnerablecode.settings import DEBUG_TOOLBAR @@ -83,6 +84,11 @@ def __init__(self, *args, **kwargs): VulnerabilityDetails.as_view(), name="vulnerability_details", ), + path( + "vulnerabilities//packages", + VulnerabilityPackagesDetails.as_view(), + name="vulnerability_package_details", + ), path( "api/", include(api_router.urls),