From 8fe43f7623d6b86e75b632669ee137d095222aab Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 13 Jan 2025 11:22:21 +0530 Subject: [PATCH 1/4] Fix secret generation on mac Signed-off-by: Keshav Priyadarshi --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 067cb419f..47ce9fcd7 100644 --- a/Makefile +++ b/Makefile @@ -42,6 +42,10 @@ else SUDO_POSTGRES= endif +ifeq ($(UNAME), Darwin) + GET_SECRET_KEY=`head /dev/urandom | base64 | head -c50` +endif + virtualenv: @echo "-> Bootstrap the virtualenv with PYTHON_EXE=${PYTHON_EXE}" @${PYTHON_EXE} ${VIRTUALENV_PYZ} --never-download --no-periodic-update ${VENV} From cc6d3006847514aee950b2462d9013714c0dc4fe Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 13 Jan 2025 17:18:34 +0530 Subject: [PATCH 2/4] Add migration to fix alpine PURLs Signed-off-by: Keshav Priyadarshi --- .../migrations/0088_fix_alpine_purl_type.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 vulnerabilities/migrations/0088_fix_alpine_purl_type.py diff --git a/vulnerabilities/migrations/0088_fix_alpine_purl_type.py b/vulnerabilities/migrations/0088_fix_alpine_purl_type.py new file mode 100644 index 000000000..29339cfd4 --- /dev/null +++ b/vulnerabilities/migrations/0088_fix_alpine_purl_type.py @@ -0,0 +1,103 @@ +from datetime import datetime +from datetime import timezone + +from aboutcode.pipeline import LoopProgress +from django.db import migrations +from packageurl import PackageURL + +CHUNK_SIZE = 50000 +BATCH_SIZE = 500 + + +class Migration(migrations.Migration): + def fix_alpine_purl_type(apps, schema_editor): + """Use proper apk package type for Alpine""" + + Package = apps.get_model("vulnerabilities", "Package") + batch = [] + alpine_packages_query = Package.objects.filter(type="alpine") + + log(f"\nFixing PURL for {alpine_packages_query.count():,d} alpine packages") + progress = LoopProgress( + total_iterations=alpine_packages_query.count(), + progress_step=10, + logger=log, + ) + for package in progress.iter(alpine_packages_query.iterator(chunk_size=CHUNK_SIZE)): + package.type = "apk" + package.namespace = "alpine" + + package.package_url = update_alpine_purl(package.package_url, "apk", "alpine") + package.plain_package_url = update_alpine_purl( + package.plain_package_url, "apk", "alpine" + ) + + batch.append(package) + if len(batch) >= BATCH_SIZE: + bulk_update_package(Package, batch) + batch.clear() + + bulk_update_package(Package, batch) + + def reverse_fix_alpine_purl_type(apps, schema_editor): + Package = apps.get_model("vulnerabilities", "Package") + batch = [] + alpine_packages_query = Package.objects.filter(type="apk", namespace="alpine") + + log(f"\nREVERSE: Fix for {alpine_packages_query.count():,d} alpine packages") + progress = LoopProgress( + total_iterations=alpine_packages_query.count(), + progress_step=10, + logger=log, + ) + for package in progress.iter(alpine_packages_query.iterator(chunk_size=CHUNK_SIZE)): + package.type = "alpine" + package.namespace = "" + + package.package_url = update_alpine_purl(package.package_url, "alpine", "") + package.plain_package_url = update_alpine_purl(package.plain_package_url, "alpine", "") + + batch.append(package) + if len(batch) >= BATCH_SIZE: + bulk_update_package(Package, batch) + batch.clear() + + bulk_update_package(Package, batch) + + dependencies = [ + ("vulnerabilities", "0087_update_alpine_advisory_created_by"), + ] + + operations = [ + migrations.RunPython( + code=fix_alpine_purl_type, + reverse_code=reverse_fix_alpine_purl_type, + ), + ] + + +def bulk_update_package(package, batch): + if batch: + package.objects.bulk_update( + objs=batch, + fields=[ + "type", + "namespace", + "package_url", + "plain_package_url", + ], + ) + + +def update_alpine_purl(purl, purl_type, purl_namespace): + package_url = PackageURL.from_string(purl).to_dict() + package_url["type"] = purl_type + package_url["namespace"] = purl_namespace + return str(PackageURL(**package_url)) + + +def log(message): + now_local = datetime.now(timezone.utc).astimezone() + timestamp = now_local.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + message = f"{timestamp} {message}" + print(message) From 8a55549b7f440c70363a2942c554e5febf5b8488 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 13 Jan 2025 17:50:19 +0530 Subject: [PATCH 3/4] Add test for Alpine data migration Signed-off-by: Keshav Priyadarshi --- vulnerabilities/tests/test_data_migrations.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index 38bf9417f..55bbb71ef 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -19,6 +19,7 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Reference +from vulnerabilities.utils import purl_to_dict class TestMigrations(TestCase): @@ -922,3 +923,32 @@ def test_update_pysec_created_by_field(self): == 0 ) assert adv.filter(created_by="alpine_linux_importer").count() == 1 + + +class TestFixAlpinePURLCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0087_update_alpine_advisory_created_by" + migrate_to = "0088_fix_alpine_purl_type" + + def setUpBeforeMigration(self, apps): + Package = apps.get_model("vulnerabilities", "Package") + purl = str( + PackageURL( + type="alpine", + namespace="", + name="curl", + version="7.83.0-r0", + qualifiers="arch=x86", + ) + ) + package1 = Package.objects.create( + **purl_to_dict(purl=purl), package_url=purl, plain_package_url=purl + ) + + def test_fix_alpine_purl(self): + Package = apps.get_model("vulnerabilities", "Package") + package = Package.objects.all() + print(package) + + assert package.filter(type="alpine").count() == 0 + assert package.filter(type="apk").count() == 1 From 3cb9d3f9b33e5812472d0972867ffcb97e1af2c2 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Mon, 13 Jan 2025 21:32:05 +0530 Subject: [PATCH 4/4] Use proper purl type for Alpine in pipeline, models, and views Signed-off-by: Keshav Priyadarshi --- vulnerabilities/models.py | 4 +- .../pipelines/alpine_linux_importer.py | 6 +- .../test_alpine_linux_importer_pipeline.py | 112 +++++++++--------- .../default_improver/alpine-expected.json | 28 ++--- .../default_improver/alpine-input.json | 28 ++--- vulnerabilities/tests/test_models.py | 12 +- vulnerabilities/tests/test_view.py | 5 +- vulnerabilities/views.py | 2 +- 8 files changed, 95 insertions(+), 102 deletions(-) diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 21b1129a2..4db674e3e 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -10,10 +10,8 @@ import hashlib import json import logging -import typing from contextlib import suppress from functools import cached_property -from typing import Optional from typing import Union from cwe2.database import Database @@ -56,7 +54,7 @@ models.CharField.register_lookup(Trim) # patch univers for missing entry -RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange +RANGE_CLASS_BY_SCHEMES["apk"] = AlpineLinuxVersionRange class BaseQuerySet(models.QuerySet): diff --git a/vulnerabilities/pipelines/alpine_linux_importer.py b/vulnerabilities/pipelines/alpine_linux_importer.py index d29f9bc9b..28736e507 100644 --- a/vulnerabilities/pipelines/alpine_linux_importer.py +++ b/vulnerabilities/pipelines/alpine_linux_importer.py @@ -254,7 +254,8 @@ def load_advisories( affected_packages.append( AffectedPackage( package=PackageURL( - type="alpine", + type="apk", + namespace="alpine", name=pkg_infos["name"], qualifiers=qualifiers, ), @@ -266,7 +267,8 @@ def load_advisories( affected_packages.append( AffectedPackage( package=PackageURL( - type="alpine", + type="apk", + namespace="alpine", name=pkg_infos["name"], qualifiers=qualifiers, ), diff --git a/vulnerabilities/tests/pipelines/test_alpine_linux_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_alpine_linux_importer_pipeline.py index 386f239d8..49182b287 100644 --- a/vulnerabilities/tests/pipelines/test_alpine_linux_importer_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_alpine_linux_importer_pipeline.py @@ -36,8 +36,8 @@ def test_process_record(): affected_packages=[ AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={ @@ -52,8 +52,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "armhf", "distroversion": "v3.11", "reponame": "main"}, @@ -64,8 +64,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "armv7", "distroversion": "v3.11", "reponame": "main"}, @@ -76,8 +76,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={ @@ -92,8 +92,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "s390x", "distroversion": "v3.11", "reponame": "main"}, @@ -104,8 +104,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "x86", "distroversion": "v3.11", "reponame": "main"}, @@ -116,8 +116,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, @@ -143,8 +143,8 @@ def test_process_record(): affected_packages=[ AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={ @@ -159,8 +159,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "armhf", "distroversion": "v3.11", "reponame": "main"}, @@ -171,8 +171,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "armv7", "distroversion": "v3.11", "reponame": "main"}, @@ -183,8 +183,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={ @@ -199,8 +199,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "s390x", "distroversion": "v3.11", "reponame": "main"}, @@ -211,8 +211,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "x86", "distroversion": "v3.11", "reponame": "main"}, @@ -223,8 +223,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="xen", version=None, qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, @@ -250,8 +250,8 @@ def test_process_record(): affected_packages=[ AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={ @@ -266,8 +266,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "armhf", "distroversion": "v3.11", "reponame": "main"}, @@ -278,8 +278,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "armv7", "distroversion": "v3.11", "reponame": "main"}, @@ -290,8 +290,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={ @@ -306,8 +306,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "s390x", "distroversion": "v3.11", "reponame": "main"}, @@ -318,8 +318,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "x86", "distroversion": "v3.11", "reponame": "main"}, @@ -330,8 +330,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, @@ -351,8 +351,8 @@ def test_process_record(): affected_packages=[ AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={ @@ -367,8 +367,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "armhf", "distroversion": "v3.11", "reponame": "main"}, @@ -379,8 +379,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "armv7", "distroversion": "v3.11", "reponame": "main"}, @@ -391,8 +391,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={ @@ -407,8 +407,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "s390x", "distroversion": "v3.11", "reponame": "main"}, @@ -419,8 +419,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "x86", "distroversion": "v3.11", "reponame": "main"}, @@ -431,8 +431,8 @@ def test_process_record(): ), AffectedPackage( package=PackageURL( - type="alpine", - namespace=None, + type="apk", + namespace="alpine", name="apk-tools", version=None, qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, diff --git a/vulnerabilities/tests/test_data/default_improver/alpine-expected.json b/vulnerabilities/tests/test_data/default_improver/alpine-expected.json index 5d8a84930..f9d3caf16 100644 --- a/vulnerabilities/tests/test_data/default_improver/alpine-expected.json +++ b/vulnerabilities/tests/test_data/default_improver/alpine-expected.json @@ -6,8 +6,8 @@ "summary": null, "affected_purls": [], "fixed_purl": { - "type": "alpine", - "namespace": "", + "type": "apk", + "namespace": "alpine", "name": "xen", "version": "4.10.0-r1", "qualifiers": "arch=aarch64&distroversion=v3.11&reponame=main", @@ -30,8 +30,8 @@ "summary": null, "affected_purls": [], "fixed_purl": { - "type": "alpine", - "namespace": "", + "type": "apk", + "namespace": "alpine", "name": "xen", "version": "4.10.0-r1", "qualifiers": "arch=armhf&distroversion=v3.11&reponame=main", @@ -54,8 +54,8 @@ "summary": null, "affected_purls": [], "fixed_purl": { - "type": "alpine", - "namespace": "", + "type": "apk", + "namespace": "alpine", "name": "xen", "version": "4.10.0-r1", "qualifiers": "arch=armv7&distroversion=v3.11&reponame=main", @@ -78,8 +78,8 @@ "summary": null, "affected_purls": [], "fixed_purl": { - "type": "alpine", - "namespace": "", + "type": "apk", + "namespace": "alpine", "name": "xen", "version": "4.10.0-r1", "qualifiers": "arch=ppc64le&distroversion=v3.11&reponame=main", @@ -102,8 +102,8 @@ "summary": null, "affected_purls": [], "fixed_purl": { - "type": "alpine", - "namespace": "", + "type": "apk", + "namespace": "alpine", "name": "xen", "version": "4.10.0-r1", "qualifiers": "arch=s390x&distroversion=v3.11&reponame=main", @@ -126,8 +126,8 @@ "summary": null, "affected_purls": [], "fixed_purl": { - "type": "alpine", - "namespace": "", + "type": "apk", + "namespace": "alpine", "name": "xen", "version": "4.10.0-r1", "qualifiers": "arch=x86&distroversion=v3.11&reponame=main", @@ -150,8 +150,8 @@ "summary": null, "affected_purls": [], "fixed_purl": { - "type": "alpine", - "namespace": "", + "type": "apk", + "namespace": "alpine", "name": "xen", "version": "4.10.0-r1", "qualifiers": "arch=x86_64&distroversion=v3.11&reponame=main", diff --git a/vulnerabilities/tests/test_data/default_improver/alpine-input.json b/vulnerabilities/tests/test_data/default_improver/alpine-input.json index 9ff37ecb8..f2143b32b 100644 --- a/vulnerabilities/tests/test_data/default_improver/alpine-input.json +++ b/vulnerabilities/tests/test_data/default_improver/alpine-input.json @@ -4,8 +4,8 @@ "affected_packages": [ { "package": { - "type": "alpine", - "namespace": null, + "type": "apk", + "namespace": "alpine", "name": "xen", "version": null, "qualifiers": { @@ -20,8 +20,8 @@ }, { "package": { - "type": "alpine", - "namespace": null, + "type": "apk", + "namespace": "alpine", "name": "xen", "version": null, "qualifiers": { @@ -36,8 +36,8 @@ }, { "package": { - "type": "alpine", - "namespace": null, + "type": "apk", + "namespace": "alpine", "name": "xen", "version": null, "qualifiers": { @@ -52,8 +52,8 @@ }, { "package": { - "type": "alpine", - "namespace": null, + "type": "apk", + "namespace": "alpine", "name": "xen", "version": null, "qualifiers": { @@ -68,8 +68,8 @@ }, { "package": { - "type": "alpine", - "namespace": null, + "type": "apk", + "namespace": "alpine", "name": "xen", "version": null, "qualifiers": { @@ -84,8 +84,8 @@ }, { "package": { - "type": "alpine", - "namespace": null, + "type": "apk", + "namespace": "alpine", "name": "xen", "version": null, "qualifiers": { @@ -100,8 +100,8 @@ }, { "package": { - "type": "alpine", - "namespace": null, + "type": "apk", + "namespace": "alpine", "name": "xen", "version": null, "qualifiers": { diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 014754786..a5f8e251c 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -8,25 +8,17 @@ # import urllib.parse -from datetime import datetime from unittest import TestCase -from unittest import mock import pytest -from django.db import transaction -from django.db.models.query import QuerySet -from django.db.utils import IntegrityError -from freezegun import freeze_time from packageurl import PackageURL from univers import versions from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.version_range import AlpineLinuxVersionRange from vulnerabilities import models from vulnerabilities.models import Alias from vulnerabilities.models import Package from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityQuerySet class TestVulnerabilityModel(TestCase): @@ -397,7 +389,9 @@ def test_univers_version_class(self): pypi_package_version = RANGE_CLASS_BY_SCHEMES[pypi_package.type].version_class assert pypi_package_version == versions.PypiVersion - alpine_package = models.Package.objects.create(type="alpine", name="lxml", version="0.9") + alpine_package = models.Package.objects.create( + type="apk", namespace="alpine", name="lxml", version="0.9" + ) alpine_version = RANGE_CLASS_BY_SCHEMES[alpine_package.type].version_class assert alpine_version == versions.AlpineLinuxVersion diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index fd62e94a1..98a555294 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -16,7 +16,6 @@ from packageurl import PackageURL from univers import versions -from vulnerabilities import models from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias from vulnerabilities.models import FixingPackageRelatedVulnerability @@ -249,8 +248,8 @@ class TestCustomFilters: "pkg%3Arpm/redhat/katello-client-bootstrap%401.1.0-2%3Farch%3Del6sat", ), ( - "pkg:alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main", - "pkg%3Aalpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain", + "pkg:apk/alpine/nginx@1.10.3-r1?arch=armhf&distroversion=v3.5&reponame=main", + "pkg%3Aapk/alpine/nginx%401.10.3-r1%3Farch%3Darmhf%26distroversion%3Dv3.5%26reponame%3Dmain", ), ("pkg:nginx/nginx@0.9.0?os=windows", "pkg%3Anginx/nginx%400.9.0%3Fos%3Dwindows"), ( diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index fd57acea5..7d0911c64 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -54,7 +54,7 @@ def purl_sort_key(purl: models.Package): def get_purl_version_class(purl: models.Package): - RANGE_CLASS_BY_SCHEMES["alpine"] = AlpineLinuxVersionRange + RANGE_CLASS_BY_SCHEMES["apk"] = AlpineLinuxVersionRange purl_version_class = None check_version_class = RANGE_CLASS_BY_SCHEMES.get(purl.type, None) if check_version_class: