Merge branch 'main' into feat/1039/ingest-rust-data-github-api

CHANGELOG.rst

@@ -1,6 +1,12 @@
 Release notes
 =============
 
+Version v34.0.0rc4
+-------------------
+
+- Drop migration for removing duplicated changelogs.
+
+
 Version v34.0.0rc3
 -------------------
 

docs/source/conf.py

@@ -34,6 +34,7 @@
     "https://api.github.com/graphql",  # Requires auth
     "https://anongit.gentoo.org/git/data/glsa.git",  # Git only link
     "https://www.softwaretestinghelp.com/how-to-write-good-bug-report/",  # Cloudflare protection
+    "https://www.openssl.org/news/vulnerabilities.xml",  # OpenSSL legacy advisory URL, not longer available
 ]
 
 # Add any Sphinx extension module names here, as strings. They can be

pyproject.toml

@@ -59,7 +59,6 @@ addopts = [
     "--ignore=vulnerabilities/importers/retiredotnet.py",
     "--ignore=vulnerabilities/importers/ruby.py",
     "--ignore=vulnerabilities/importers/rust.py",
-    "--ignore=vulnerabilities/importers/safety_db.py",
     "--ignore=vulnerabilities/importers/suse_backports.py",
     "--ignore=vulnerabilities/importers/suse_scores.py",
     "--ignore=vulnerabilities/importers/ubuntu_usn.py",

requirements.txt

@@ -11,7 +11,7 @@ beautifulsoup4==4.10.0
 binaryornot==0.4.4
 black==22.3.0
 boolean.py==3.8
-certifi==2023.7.22
+certifi==2024.7.4
 cffi==1.15.0
 chardet==4.0.0
 charset-normalizer==2.0.12
@@ -36,15 +36,15 @@ freezegun==1.2.1
 frozenlist==1.3.0
 gitdb==4.0.9
 GitPython==3.1.41
-gunicorn==20.1.0
+gunicorn==22.0.0
 idna==3.3
 imagesize==1.3.0
 importlib-metadata==4.11.3
 iniconfig==1.1.1
 ipython==8.10.0
 isort==5.10.1
 jedi==0.18.1
-Jinja2==3.1.3
+Jinja2==3.1.4
 jsonschema==3.2.0
 license-expression==21.6.14
 lxml==4.9.1
@@ -81,7 +81,7 @@ python-dateutil==2.8.2
 python-dotenv==0.20.0
 pytz==2022.1
 PyYAML==6.0.1
-requests==2.31.0
+requests==2.32.0
 restructuredtext-lint==1.4.0
 saneyaml==0.6.0
 semantic-version==2.9.0
@@ -98,7 +98,7 @@ sphinxcontrib-htmlhelp==2.0.0
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==1.0.3
 sphinxcontrib-serializinghtml==1.1.5
-sqlparse==0.4.4
+sqlparse==0.5.0
 stack-data==0.2.0
 stevedore==3.5.0
 texttable==1.6.4
@@ -107,11 +107,11 @@ tomli==2.0.1
 traitlets==5.1.1
 typing_extensions==4.1.1
 univers==30.11.0
-urllib3==1.26.18
+urllib3==1.26.19
 wcwidth==0.2.5
 websocket-client==0.59.0
 yarl==1.7.2
-zipp==3.8.0
+zipp==3.19.1
 dateparser==1.1.1
 fetchcode==0.3.0
 cwe2==2.0.0

vulnerabilities/api.py

@@ -25,6 +25,7 @@
 from rest_framework.throttling import UserRateThrottle
 
 from vulnerabilities.models import Alias
+from vulnerabilities.models import Kev
 from vulnerabilities.models import Package
 from vulnerabilities.models import Vulnerability
 from vulnerabilities.models import VulnerabilityReference
@@ -167,6 +168,12 @@ def to_representation(self, instance):
         return representation
 
 
+class KEVSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = Kev
+        fields = ["date_added", "description", "required_action", "due_date", "resources_and_notes"]
+
+
 class VulnerabilitySerializer(BaseResourceSerializer):
     fixed_packages = MinimalPackageSerializer(
         many=True, source="filtered_fixed_packages", read_only=True
@@ -175,6 +182,7 @@ class VulnerabilitySerializer(BaseResourceSerializer):
 
     references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set")
     aliases = AliasSerializer(many=True, source="alias")
+    kev = KEVSerializer(read_only=True)
     weaknesses = WeaknessSerializer(many=True)
 
     def to_representation(self, instance):
@@ -183,6 +191,10 @@ def to_representation(self, instance):
         weaknesses = data.get("weaknesses", [])
         data["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None]
 
+        kev = data.get("kev", None)
+        if not kev:
+            data.pop("kev")
+
         return data
 
     class Meta:
@@ -196,6 +208,7 @@ class Meta:
             "affected_packages",
             "references",
             "weaknesses",
+            "kev",
         ]
 
 

vulnerabilities/improvers/__init__.py

@@ -8,6 +8,7 @@
 #
 
 from vulnerabilities.improvers import valid_versions
+from vulnerabilities.improvers import vulnerability_kev
 from vulnerabilities.improvers import vulnerability_status
 
 IMPROVERS_REGISTRY = [
@@ -27,6 +28,7 @@
     valid_versions.RubyImprover,
     valid_versions.GithubOSVImprover,
     vulnerability_status.VulnerabilityStatusImprover,
+    vulnerability_kev.VulnerabilityKevImprover,
 ]
 
 IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY}

vulnerabilities/improvers/vulnerability_kev.py

@@ -0,0 +1,66 @@
+import logging
+from typing import Iterable
+
+from django.db.models import QuerySet
+from sphinx.util import requests
+
+from vulnerabilities.improver import Improver
+from vulnerabilities.improver import Inference
+from vulnerabilities.models import Advisory
+from vulnerabilities.models import Alias
+from vulnerabilities.models import Kev
+
+logger = logging.getLogger(__name__)
+
+
+class VulnerabilityKevImprover(Improver):
+    """
+    Known Exploited Vulnerabilities Improver
+    """
+
+    @property
+    def interesting_advisories(self) -> QuerySet:
+        # TODO Modify KEV improver to iterate over the vulnerabilities alias, not the advisory
+        return [Advisory.objects.first()]
+
+    def get_inferences(self, advisory_data) -> Iterable[Inference]:
+        """
+        Fetch Kev data, iterate over it to find the vulnerability with the specified alias, and create or update
+        the Kev instance accordingly.
+        """
+
+        kev_url = (
+            "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"
+        )
+        response = requests.get(kev_url)
+        kev_data = response.json()
+        if response.status_code != 200:
+            logger.error(
+                f"Failed to fetch the CISA Catalog of Known Exploited Vulnerabilities: {kev_url}"
+            )
+            return []
+
+        for kev_vul in kev_data.get("vulnerabilities", []):
+            alias = Alias.objects.get_or_none(alias=kev_vul["cveID"])
+            if not alias:
+                continue
+
+            vul = alias.vulnerability
+
+            if not vul:
+                continue
+
+            Kev.objects.update_or_create(
+                vulnerability=vul,
+                defaults={
+                    "description": kev_vul["shortDescription"],
+                    "date_added": kev_vul["dateAdded"],
+                    "required_action": kev_vul["requiredAction"],
+                    "due_date": kev_vul["dueDate"],
+                    "resources_and_notes": kev_vul["notes"],
+                    "known_ransomware_campaign_use": True
+                    if kev_vul["knownRansomwareCampaignUse"] == "Known"
+                    else False,
+                },
+            )
+        return []

vulnerabilities/migrations/0057_alter_packagechangelog_software_version_and_more.py → vulnerabilities/migrations/0055_alter_packagechangelog_software_version_and_more.py

@@ -1,12 +1,12 @@
-# Generated by Django 4.1.13 on 2024-02-26 13:52
+# Generated by Django 4.1.13 on 2024-03-18 08:35
 
 from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
-        ("vulnerabilities", "0056_alter_packagechangelog_unique_together_and_more"),
+        ("vulnerabilities", "0054_alter_packagechangelog_software_version_and_more"),
     ]
 
     operations = [

vulnerabilities/migrations/0056_alter_packagechangelog_software_version_and_more.py

@@ -0,0 +1,31 @@
+# Generated by Django 4.1.13 on 2024-03-18 08:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0055_alter_packagechangelog_software_version_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="packagechangelog",
+            name="software_version",
+            field=models.CharField(
+                default="34.0.0rc4",
+                help_text="Version of the software at the time of change",
+                max_length=100,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="vulnerabilitychangelog",
+            name="software_version",
+            field=models.CharField(
+                default="34.0.0rc4",
+                help_text="Version of the software at the time of change",
+                max_length=100,
+            ),
+        ),
+    ]