From ad9093e1882dd96e7e22e3947f6d5c1ea8926581 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 15:18:05 +0530 Subject: [PATCH 1/9] Optimize vulnerabilities view Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 85 +++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index fd57acea5..7220b6efc 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -7,7 +7,8 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import logging -from datetime import datetime +from itertools import groupby +from operator import attrgetter from cvss.exceptions import CVSS2MalformedError from cvss.exceptions import CVSS3MalformedError @@ -197,36 +198,7 @@ def get_context_data(self, **kwargs): if s.value: severity_values.add(s.value) - sorted_affected_packages = sorted(self.object.affected_packages.all(), key=purl_sort_key) - sorted_fixed_by_packages = sorted(self.object.fixed_by_packages.all(), key=purl_sort_key) - - all_affected_fixed_by_matches = [] - for sorted_affected_package in sorted_affected_packages: - affected_fixed_by_matches = {} - affected_fixed_by_matches["affected_package"] = sorted_affected_package - matched_fixed_by_packages = [] - for fixed_by_package in sorted_fixed_by_packages: - - # Ghost Package can't fix vulnerability. - if fixed_by_package.is_ghost: - continue - - sorted_affected_version_class = get_purl_version_class(sorted_affected_package) - fixed_by_version_class = get_purl_version_class(fixed_by_package) - if ( - (fixed_by_package.type == sorted_affected_package.type) - and (fixed_by_package.namespace == sorted_affected_package.namespace) - and (fixed_by_package.name == sorted_affected_package.name) - and (fixed_by_package.qualifiers == sorted_affected_package.qualifiers) - and (fixed_by_package.subpath == sorted_affected_package.subpath) - and ( - fixed_by_version_class(fixed_by_package.version) - > sorted_affected_version_class(sorted_affected_package.version) - ) - ): - matched_fixed_by_packages.append(fixed_by_package.purl) - affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages - all_affected_fixed_by_matches.append(affected_fixed_by_matches) + sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches = self.aggregate_fixed_and_affected_packages() context.update( { @@ -247,6 +219,57 @@ def get_context_data(self, **kwargs): ) return context + def aggregate_fixed_and_affected_packages(self): + sorted_fixed_by_packages = self.object.fixed_by_packages.filter(is_ghost=False).order_by( + "type", "namespace", "name", "qualifiers", "subpath" + ) + + sorted_affected_packages = self.object.affected_packages.all() + + grouped_fixed_by_packages = { + key: list(group) + for key, group in groupby( + sorted_fixed_by_packages, + key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"), + ) + } + + all_affected_fixed_by_matches = [] + + for sorted_affected_package in sorted_affected_packages: + affected_fixed_by_matches = { + "affected_package": sorted_affected_package, + "matched_fixed_by_packages": [], + } + + # Build the key to find matching group + key = ( + sorted_affected_package.type, + sorted_affected_package.namespace, + sorted_affected_package.name, + sorted_affected_package.qualifiers, + sorted_affected_package.subpath, + ) + + # Get matching group from pre-grouped fixed_by_packages + matching_fixed_packages = grouped_fixed_by_packages.get(key, []) + + # Get version classes for comparison + affected_version_class = get_purl_version_class(sorted_affected_package) + affected_version = affected_version_class(sorted_affected_package.version) + + # Compare versions and filter valid matches + matched_fixed_by_packages = [ + fixed_by_package.purl + for fixed_by_package in matching_fixed_packages + if get_purl_version_class(fixed_by_package)(fixed_by_package.version) + > affected_version + ] + + affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages + all_affected_fixed_by_matches.append(affected_fixed_by_matches) + return sorted_fixed_by_packages,sorted_affected_packages,all_affected_fixed_by_matches + class HomePage(View): template_name = "index.html" From ac6311ae899aec75c7f4051c1faa6da8374ee0f8 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 15:27:08 +0530 Subject: [PATCH 2/9] Fix formatting Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 7220b6efc..82a6916df 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -198,7 +198,11 @@ def get_context_data(self, **kwargs): if s.value: severity_values.add(s.value) - sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches = self.aggregate_fixed_and_affected_packages() + ( + sorted_fixed_by_packages, + sorted_affected_packages, + all_affected_fixed_by_matches, + ) = self.aggregate_fixed_and_affected_packages() context.update( { @@ -268,7 +272,7 @@ def aggregate_fixed_and_affected_packages(self): affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages all_affected_fixed_by_matches.append(affected_fixed_by_matches) - return sorted_fixed_by_packages,sorted_affected_packages,all_affected_fixed_by_matches + return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches class HomePage(View): From 009595fa1f42970bc53dd4f434c9ecee0aba2c7a Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 15:38:30 +0530 Subject: [PATCH 3/9] Optimize vulnerabilities view Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 51 ++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 82a6916df..96bf955c6 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -177,26 +177,7 @@ def get_context_data(self, **kwargs): ] status = self.object.get_status_label - severity_vectors = [] - severity_values = set() - for s in self.object.severities.all(): - if s.scoring_system == EPSS.identifier: - continue - - if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS: - try: - vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) - severity_vectors.append(vector_values) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ): - logging.error(f"CVSSMalformedError for {s.scoring_elements}") - - if s.value: - severity_values.add(s.value) + severity_vectors, severity_values = self.get_severity_vectors_and_values() ( sorted_fixed_by_packages, @@ -274,6 +255,36 @@ def aggregate_fixed_and_affected_packages(self): all_affected_fixed_by_matches.append(affected_fixed_by_matches) return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches + def get_severity_vectors_and_values(self): + """ + Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. + """ + severity_vectors = [] + severity_values = set() + + severities = self.object.severities.exclude(scoring_system=EPSS.identifier) + + for severity in severities: + if severity.scoring_elements and severity.scoring_system in SCORING_SYSTEMS: + try: + vector_values = SCORING_SYSTEMS[severity.scoring_system].get( + severity.scoring_elements + ) + if vector_values: + severity_vectors.append(vector_values) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ) as e: + logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") + + if severity.value: + severity_values.add(severity.value) + + return severity_vectors, severity_values + class HomePage(View): template_name = "index.html" From f54673a35bdb1099bd9d605bc0878e5e710b93a7 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 15:47:34 +0530 Subject: [PATCH 4/9] Optimize vulnerabilities view Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 46 +++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 96bf955c6..947bb8c0b 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -262,26 +262,32 @@ def get_severity_vectors_and_values(self): severity_vectors = [] severity_values = set() - severities = self.object.severities.exclude(scoring_system=EPSS.identifier) - - for severity in severities: - if severity.scoring_elements and severity.scoring_system in SCORING_SYSTEMS: - try: - vector_values = SCORING_SYSTEMS[severity.scoring_system].get( - severity.scoring_elements - ) - if vector_values: - severity_vectors.append(vector_values) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ) as e: - logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") - - if severity.value: - severity_values.add(severity.value) + # Exclude EPSS scoring system + base_severities = self.object.severities.exclude(scoring_system=EPSS.identifier) + + # QuerySet for severities with valid scoring_elements and scoring_system in SCORING_SYSTEMS + valid_scoring_severities = base_severities.filter( + scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() + ) + + for severity in valid_scoring_severities: + try: + vector_values = SCORING_SYSTEMS[severity.scoring_system].get( + severity.scoring_elements + ) + if vector_values: + severity_vectors.append(vector_values) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ) as e: + logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") + + valid_value_severities = base_severities.filter(value__isnull=False).exclude(value="") + + severity_values.update(valid_value_severities.values_list("value", flat=True)) return severity_vectors, severity_values From cef8754314ac20c6493eecc50d0f05204e4e834d Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 16:05:00 +0530 Subject: [PATCH 5/9] Fix formatting Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 947bb8c0b..7591cba43 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -177,7 +177,7 @@ def get_context_data(self, **kwargs): ] status = self.object.get_status_label - severity_vectors, severity_values = self.get_severity_vectors_and_values() + # severity_vectors, severity_values = self.get_severity_vectors_and_values() ( sorted_fixed_by_packages, @@ -190,8 +190,8 @@ def get_context_data(self, **kwargs): "vulnerability": self.object, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), "severities": list(self.object.severities.all()), - "severity_score_range": get_severity_range(severity_values), - "severity_vectors": severity_vectors, + # "severity_score_range": get_severity_range(severity_values), + # "severity_vectors": severity_vectors, "references": self.object.references.all(), "aliases": self.object.aliases.all(), "affected_packages": sorted_affected_packages, From 59294525c1be0324728f30d22bb1689e3bb9fb12 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 16:06:23 +0530 Subject: [PATCH 6/9] Fix formatting Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 7591cba43..3f791dcd6 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -190,8 +190,8 @@ def get_context_data(self, **kwargs): "vulnerability": self.object, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), "severities": list(self.object.severities.all()), - # "severity_score_range": get_severity_range(severity_values), - # "severity_vectors": severity_vectors, + "severity_score_range": "", + "severity_vectors": [], "references": self.object.references.all(), "aliases": self.object.aliases.all(), "affected_packages": sorted_affected_packages, From 607a90b2b84f438ddabee8d30e5d14a49ad4b675 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Mon, 6 Jan 2025 18:46:10 +0530 Subject: [PATCH 7/9] Fix tests Signed-off-by: Tushar Goel --- vulnerabilities/models.py | 97 ++++++++++++++++++++++++++++++++++++++- vulnerabilities/utils.py | 10 ++++ vulnerabilities/views.py | 69 +--------------------------- 3 files changed, 107 insertions(+), 69 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 6248e1e47..866eb430c 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -10,12 +10,15 @@ import hashlib import json import logging -import typing from contextlib import suppress from functools import cached_property -from typing import Optional +from itertools import groupby +from operator import attrgetter from typing import Union +from cvss.exceptions import CVSS2MalformedError +from cvss.exceptions import CVSS3MalformedError +from cvss.exceptions import CVSS4MalformedError from cwe2.database import Database from django.contrib.auth import get_user_model from django.contrib.auth.models import UserManager @@ -45,6 +48,7 @@ from aboutcode import hashid from vulnerabilities import utils +from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import normalize_purl from vulnerabilities.utils import purl_to_dict @@ -371,6 +375,95 @@ def get_related_purls(self): """ return [p.package_url for p in self.packages.distinct().all()] + def aggregate_fixed_and_affected_packages(self): + from vulnerabilities.views import get_purl_version_class + + sorted_fixed_by_packages = self.fixed_by_packages.filter(is_ghost=False).order_by( + "type", "namespace", "name", "qualifiers", "subpath" + ) + + sorted_affected_packages = self.affected_packages.all() + + grouped_fixed_by_packages = { + key: list(group) + for key, group in groupby( + sorted_fixed_by_packages, + key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"), + ) + } + + all_affected_fixed_by_matches = [] + + for sorted_affected_package in sorted_affected_packages: + affected_fixed_by_matches = { + "affected_package": sorted_affected_package, + "matched_fixed_by_packages": [], + } + + # Build the key to find matching group + key = ( + sorted_affected_package.type, + sorted_affected_package.namespace, + sorted_affected_package.name, + sorted_affected_package.qualifiers, + sorted_affected_package.subpath, + ) + + # Get matching group from pre-grouped fixed_by_packages + matching_fixed_packages = grouped_fixed_by_packages.get(key, []) + + # Get version classes for comparison + affected_version_class = get_purl_version_class(sorted_affected_package) + affected_version = affected_version_class(sorted_affected_package.version) + + # Compare versions and filter valid matches + matched_fixed_by_packages = [ + fixed_by_package.purl + for fixed_by_package in matching_fixed_packages + if get_purl_version_class(fixed_by_package)(fixed_by_package.version) + > affected_version + ] + + affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages + all_affected_fixed_by_matches.append(affected_fixed_by_matches) + return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches + + def get_severity_vectors_and_values(self): + """ + Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. + """ + severity_vectors = [] + severity_values = set() + + # Exclude EPSS scoring system + base_severities = self.severities.exclude(scoring_system=EPSS.identifier) + + # QuerySet for severities with valid scoring_elements and scoring_system in SCORING_SYSTEMS + valid_scoring_severities = base_severities.filter( + scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() + ) + + for severity in valid_scoring_severities: + try: + vector_values = SCORING_SYSTEMS[severity.scoring_system].get( + severity.scoring_elements + ) + if vector_values: + severity_vectors.append(vector_values) + except ( + CVSS2MalformedError, + CVSS3MalformedError, + CVSS4MalformedError, + NotImplementedError, + ) as e: + logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") + + valid_value_severities = base_severities.filter(value__isnull=False).exclude(value="") + + severity_values.update(valid_value_severities.values_list("value", flat=True)) + + return severity_vectors, severity_values + class Weakness(models.Model): """ diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index 969a08f2f..32cfcbc02 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -32,6 +32,7 @@ from packageurl import PackageURL from packageurl.contrib.django.utils import without_empty_values from univers.version_range import RANGE_CLASS_BY_SCHEMES +from univers.version_range import AlpineLinuxVersionRange from univers.version_range import NginxVersionRange from univers.version_range import VersionRange @@ -536,3 +537,12 @@ def normalize_purl(purl: Union[PackageURL, str]): if isinstance(purl, PackageURL): purl = str(purl) return PackageURL.from_string(purl) + + +def get_purl_version_class(purl): + RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange + purl_version_class = None + check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) + if check_version_class: + purl_version_class = check_version_class.version_class + return purl_version_class diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 3f791dcd6..c977d0f9d 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -7,8 +7,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # import logging -from itertools import groupby -from operator import attrgetter from cvss.exceptions import CVSS2MalformedError from cvss.exceptions import CVSS3MalformedError @@ -24,17 +22,14 @@ from django.views import generic from django.views.generic.detail import DetailView from django.views.generic.list import ListView -from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.version_range import AlpineLinuxVersionRange from vulnerabilities import models from vulnerabilities.forms import ApiUserCreationForm from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import VulnerabilitySearchForm -from vulnerabilities.models import VulnerabilityStatusType from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import get_severity_range +from vulnerabilities.utils import get_purl_version_class from vulnerablecode.settings import env PAGE_SIZE = 20 @@ -54,15 +49,6 @@ def purl_sort_key(purl: models.Package): return (purl.type, purl.namespace, purl.name, purl_sort_version, purl.qualifiers, purl.subpath) -def get_purl_version_class(purl: models.Package): - RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange - purl_version_class = None - check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) - if check_version_class: - purl_version_class = check_version_class.version_class - return purl_version_class - - class PackageSearch(ListView): model = models.Package template_name = "packages.html" @@ -183,7 +169,7 @@ def get_context_data(self, **kwargs): sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches, - ) = self.aggregate_fixed_and_affected_packages() + ) = self.object.aggregate_fixed_and_affected_packages() context.update( { @@ -204,57 +190,6 @@ def get_context_data(self, **kwargs): ) return context - def aggregate_fixed_and_affected_packages(self): - sorted_fixed_by_packages = self.object.fixed_by_packages.filter(is_ghost=False).order_by( - "type", "namespace", "name", "qualifiers", "subpath" - ) - - sorted_affected_packages = self.object.affected_packages.all() - - grouped_fixed_by_packages = { - key: list(group) - for key, group in groupby( - sorted_fixed_by_packages, - key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"), - ) - } - - all_affected_fixed_by_matches = [] - - for sorted_affected_package in sorted_affected_packages: - affected_fixed_by_matches = { - "affected_package": sorted_affected_package, - "matched_fixed_by_packages": [], - } - - # Build the key to find matching group - key = ( - sorted_affected_package.type, - sorted_affected_package.namespace, - sorted_affected_package.name, - sorted_affected_package.qualifiers, - sorted_affected_package.subpath, - ) - - # Get matching group from pre-grouped fixed_by_packages - matching_fixed_packages = grouped_fixed_by_packages.get(key, []) - - # Get version classes for comparison - affected_version_class = get_purl_version_class(sorted_affected_package) - affected_version = affected_version_class(sorted_affected_package.version) - - # Compare versions and filter valid matches - matched_fixed_by_packages = [ - fixed_by_package.purl - for fixed_by_package in matching_fixed_packages - if get_purl_version_class(fixed_by_package)(fixed_by_package.version) - > affected_version - ] - - affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages - all_affected_fixed_by_matches.append(affected_fixed_by_matches) - return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches - def get_severity_vectors_and_values(self): """ Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. From 4aa2cc480d9eaa4383fe6c1f8da44d9abbf1b061 Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Tue, 7 Jan 2025 18:32:34 +0530 Subject: [PATCH 8/9] Fix views Signed-off-by: Tushar Goel --- vulnerabilities/views.py | 78 +++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 29 deletions(-) diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index c977d0f9d..4a128e9e8 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -22,6 +22,7 @@ from django.views import generic from django.views.generic.detail import DetailView from django.views.generic.list import ListView +from django.db.models import Prefetch from vulnerabilities import models from vulnerabilities.forms import ApiUserCreationForm @@ -143,68 +144,86 @@ class VulnerabilityDetails(DetailView): slug_field = "vulnerability_id" def get_queryset(self): + """ + Prefetch and optimize related data to minimize database hits. + """ return ( super() .get_queryset() + .select_related() .prefetch_related( "references", "aliases", "weaknesses", "severities", "exploits", + Prefetch( + "affecting_packages", + queryset=models.Vulnerability.objects.only( + "type", "namespace", "name", "version" + ), + ), + Prefetch( + "fixed_by_packages", + queryset=models.Vulnerability.objects.only( + "type", "namespace", "name", "version" + ), + ), ) ) + def get_context_data(self, **kwargs): + """ + Build context with preloaded QuerySets and minimize redundant queries. + """ context = super().get_context_data(**kwargs) - weaknesses = self.object.weaknesses.all() + vulnerability = self.object + + # Pre-fetch and process data in Python instead of the template weaknesses_present_in_db = [ - weakness_object for weakness_object in weaknesses if weakness_object.weakness + weakness_object for weakness_object in vulnerability.weaknesses.all() + if weakness_object.weakness ] - status = self.object.get_status_label - - # severity_vectors, severity_values = self.get_severity_vectors_and_values() - + + # Cache aggregated packages ( sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches, - ) = self.object.aggregate_fixed_and_affected_packages() + ) = vulnerability.aggregate_fixed_and_affected_packages() + + severity_vectors, severity_values = self.get_severity_vectors_and_values(vulnerability) context.update( { - "vulnerability": self.object, + "vulnerability": vulnerability, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), - "severities": list(self.object.severities.all()), + "severities": list(vulnerability.severities.all()), "severity_score_range": "", - "severity_vectors": [], - "references": self.object.references.all(), - "aliases": self.object.aliases.all(), + "severity_vectors": severity_vectors, + "references": list(vulnerability.references.all()), + "aliases": list(vulnerability.aliases.all()), "affected_packages": sorted_affected_packages, "fixed_by_packages": sorted_fixed_by_packages, "weaknesses": weaknesses_present_in_db, - "status": status, - "history": self.object.history, + "status": vulnerability.get_status_label, + "history": vulnerability.history, "all_affected_fixed_by_matches": all_affected_fixed_by_matches, } ) return context - def get_severity_vectors_and_values(self): + def get_severity_vectors_and_values(self, vulnerability): """ - Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. + Collect severity vectors and values, excluding EPSS scoring systems efficiently. """ severity_vectors = [] severity_values = set() - - # Exclude EPSS scoring system - base_severities = self.object.severities.exclude(scoring_system=EPSS.identifier) - - # QuerySet for severities with valid scoring_elements and scoring_system in SCORING_SYSTEMS - valid_scoring_severities = base_severities.filter( - scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() - ) - + + # Use prefetch data if available + valid_scoring_severities = getattr(vulnerability, "prefetched_valid_severities", []) + for severity in valid_scoring_severities: try: vector_values = SCORING_SYSTEMS[severity.scoring_system].get( @@ -219,10 +238,11 @@ def get_severity_vectors_and_values(self): NotImplementedError, ) as e: logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") - - valid_value_severities = base_severities.filter(value__isnull=False).exclude(value="") - - severity_values.update(valid_value_severities.values_list("value", flat=True)) + + # Collect valid values using a pre-filtered queryset + severity_values.update( + vulnerability.severities.exclude(value__isnull=True).exclude(value="").values_list("value", flat=True) + ) return severity_vectors, severity_values From b491e3f453ddb4bb212d02a9a8ef880a305d233f Mon Sep 17 00:00:00 2001 From: Tushar Goel Date: Wed, 8 Jan 2025 17:51:11 +0530 Subject: [PATCH 9/9] Move severities to a different tab Signed-off-by: Tushar Goel --- .../templates/vulnerability_details.html | 564 ++---------------- 1 file changed, 37 insertions(+), 527 deletions(-) diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index e9e58c79e..612789d99 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -40,6 +40,13 @@ +
  • + + + Severities ({{ severities|length }}) + + +
  • @@ -48,12 +55,12 @@
  • - - - Severities vectors ({{ severity_vectors|length }}) - - -
    • + + + Severities vectors ({{ severity_vectors|length }}) + + + {% if vulnerability.exploits %}
  • @@ -66,11 +73,11 @@ {% endif %}
  • - - - EPSS - - + + + EPSS + +
  • @@ -152,542 +159,46 @@ {{ vulnerability.risk_score }} - -
    - Severity ({{ severities|length }}) -
    -
    - - - - - - - {% for severity in severities %} - - - - - - {% empty %} - - - - {% endfor %} -
    System Score Found at
    {{ severity.scoring_system }}{{ severity.value }} - {{ severity.url }} -
    - There are no known severity scores. -
    -
    +
    Affected/Fixed by packages ({{ affected_packages|length }}/{{ fixed_by_packages|length }})
    -
    - - - - - - - - - {% for package in affected_packages|slice:":3" %} - - - - - {% empty %} - - - - {% endfor %} - {% if affected_packages|length > 3 %} - - - - {% endif %} - -
    AffectedFixed by
    - {{ package.purl }} - - {% for match in all_affected_fixed_by_matches %} - {% if match.affected_package == package %} - {% if match.matched_fixed_by_packages|length > 0 %} - {% for pkg in match.matched_fixed_by_packages %} - {{ pkg }} -
    - {% endfor %} - {% else %} - There are no reported fixed by versions. - {% endif %} - {% endif %} - {% endfor %} -
    - This vulnerability is not known to affect any packages. -
    - See Affected/Fixed by packages tab for more -
    -
    - -
    - Weaknesses ({{ weaknesses|length }}) -
    -
    - - {% for weakness in weaknesses %} - - - - - - {% empty %} - - - - {% endfor %} -
    CWE-{{ weakness.cwe_id }} - - {{ weakness.name }} - -
    - There are no known CWE. -
    -
    + -
    - - - - - - - - - {% for package in affected_packages %} - - - - - {% empty %} - - - - {% endfor %} - -
    AffectedFixed by
    - {{ package.purl }} - - - {% for match in all_affected_fixed_by_matches %} - {% if match.affected_package == package %} - {% if match.matched_fixed_by_packages|length > 0 %} - {% for pkg in match.matched_fixed_by_packages %} - {{ pkg }} -
    - {% endfor %} - {% else %} - There are no reported fixed by versions. - {% endif %} - {% endif %} - {% endfor %} - -
    - This vulnerability is not known to affect any packages. -
    -
    - -
    - - - - - - - - - {% for ref in references %} + +
    +
    Reference id Reference type URL
    - {% if ref.reference_id %} - - {% else %} - - {% endif %} - - {% if ref.reference_type %} - - {% else %} - - {% endif %} - - + + + - {% empty %} + {% for severity in severities %} - + + - {% endfor %} -
    {{ ref.reference_id }}{{ ref.get_reference_type_display }}{{ ref.url }} System Score Found at
    - There are no known references. + {{ severity.scoring_system }}{{ severity.value }} + {{ severity.url }}
    -
    - -
    - {% for severity_vector in severity_vectors %} - {% if severity_vector.version == '2.0' %} - Vector: {{ severity_vector.vectorString }} - - - - - - - - - - - - - - - - - - - -
    Exploitability (E)Access Vector (AV)Access Complexity (AC)Authentication (Au)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.exploitability|cvss_printer:"high,functional,unproven,proof_of_concept,not_defined" }}{{ severity_vector.accessVector|cvss_printer:"local,adjacent_network,network" }}{{ severity_vector.accessComplexity|cvss_printer:"high,medium,low" }}{{ severity_vector.authentication|cvss_printer:"multiple,single,none" }}{{ severity_vector.confidentialityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.integrityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.availabilityImpact|cvss_printer:"none,partial,complete" }}
    - {% elif severity_vector.version == '3.1' or severity_vector.version == '3.0'%} - Vector: {{ severity_vector.vectorString }} - - - - - - - - - - - - - - - - - - - - - -
    Attack Vector (AV)Attack Complexity (AC)Privileges Required (PR)User Interaction (UI)Scope (S)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent_network,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,required"}}{{ severity_vector.scope|cvss_printer:"unchanged,changed" }}{{ severity_vector.confidentialityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.integrityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.availabilityImpact|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.version == '4' %} - Vector: {{ severity_vector.vectorString }} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Attack Vector (AV)Attack Complexity (AC)Attack Requirements (AT)Privileges Required (PR)User Interaction (UI)Vulnerable System Impact Confidentiality (VC)Vulnerable System Impact Integrity (VI)Vulnerable System Impact Availability (VA)Subsequent System Impact Confidentiality (SC)Subsequent System Impact Integrity (SI)Subsequent System Impact Availability (SA)
    {{ severity_vector.attackVector|cvss_printer:"network,adjacent,local,physical"}}{{ severity_vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.attackRequirement|cvss_printer:"none,present" }}{{ severity_vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.userInteraction|cvss_printer:"none,passive,active"}}{{ severity_vector.vulnerableSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vulnerableSystemImpactAvailability|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.subsequentSystemImpactAvailability|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.version == 'ssvc' %} -
    - Vector: {{ severity_vector.vectorString }} -
    - {% endif %} - {% empty %} - - - There are no known vectors. - - - {% endfor %} -
    - - -
    - {% for exploit in vulnerability.exploits.all %} - - - - - - - - {% if exploit.date_added %} - - - - - {% endif %} - {% if exploit.description %} - - - - - {% endif %} - {% if exploit.required_action %} - - - - - {% endif %} - {% if exploit.due_date %} - - - - - {% endif %} - {% if exploit.notes %} - - - - - {% endif %} - {% if exploit.known_ransomware_campaign_use is not None %} - - - - - {% endif %} - {% if exploit.source_date_published %} - - - - - {% endif %} - {% if exploit.exploit_type %} - - - - - {% endif %} - {% if exploit.platform %} - - - - - {% endif %} - {% if exploit.source_date_updated %} - - - - - {% endif %} - - {% if exploit.source_url %} - - - - - {% endif %} - -
    Data source {{ exploit.data_source }}
    - - Date added - - {{ exploit.date_added }}
    - - Description - - {{ exploit.description }}
    - - Required action - - {{ exploit.required_action }}
    - - Due date - - {{ exploit.due_date }}
    - - Note - - 
    {{ exploit.notes }}
    - - Ransomware campaign use - - {{ exploit.known_ransomware_campaign_use|yesno:"Known,Unknown" }}
    - - Source publication date - - {{ exploit.source_date_published }}
    - - Exploit type - - {{ exploit.exploit_type }}
    - - Platform - - {{ exploit.platform }}
    - - Source update date - - {{ exploit.source_date_updated }}
    - - Source URL - - {{ exploit.source_url }}
    - {% empty %} - - - No exploits are available. - - - {% endfor %} -
    - - - {% for severity in severities %} - {% if severity.scoring_system == 'epss' %} -
    -
    - Exploit Prediction Scoring System -
    - - - - - - - - - - - - - {% if severity.published_at %} - - - - - {% endif %} - - -
    - - Percentile - - {{ severity.scoring_elements }}
    - - EPSS score - - {{ severity.value }}
    - - Published at - - {{ severity.published_at }}
    -
    - {% endif %} - {% empty %} -
    - - - There are no EPSS available. - - -
    - {% endfor %} - -
    - - - - - - - - - - - {% for log in history %} - - - - - - - {% empty %} - {% endfor %}
    - - Date - - Actor - Action Source - VulnerableCode Version -
    {{ log.get_iso_time }}{{ log.actor_name }}{{ log.get_action_type_label }} {{log.source_url }} {{ log.software_version }}
    - There are no relevant records. + + There are no known severity scores.
    + + + @@ -711,5 +222,4 @@ } - -{% endblock %} +{% endblock %} \ No newline at end of file