Workflow Checks

Signed-off-by: Rishi Garg <[email protected]>
aboutcode-org · Nov 2, 2024 · 8ea8c8e · 8ea8c8e
2 parents ee09428 + 939aba3
commit 8ea8c8e
Show file tree

Hide file tree

Showing 14 changed files with 285 additions and 70 deletions.
diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py
@@ -642,17 +642,13 @@ def filter_cpe(self, queryset, name, value):
         return self.queryset.filter(vulnerabilityreference__reference_id__startswith=cpe).distinct()
 
 
-class CPEViewSet(viewsets.ReadOnlyModelViewSet):
-    """
-    Lookup for vulnerabilities by CPE (https://nvd.nist.gov/products/cpe)
-    """
+class CPEViewSet(VulnerabilityViewSet):
+    """Lookup for vulnerabilities by CPE (https://nvd.nist.gov/products/cpe)"""
 
     queryset = Vulnerability.objects.filter(
         vulnerabilityreference__reference_id__startswith="cpe"
     ).distinct()
-    serializer_class = VulnerabilitySerializer
-    filter_backends = (filters.DjangoFilterBackend,)
-    throttle_classes = [StaffUserRateThrottle, AnonRateThrottle]
+
     filterset_class = CPEFilterSet
 
     @action(detail=False, methods=["post"])

diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py
@@ -26,7 +26,6 @@
 from vulnerabilities.importers import oss_fuzz
 from vulnerabilities.importers import postgresql
 from vulnerabilities.importers import project_kb_msr2019
-from vulnerabilities.importers import pysec
 from vulnerabilities.importers import redhat
 from vulnerabilities.importers import retiredotnet
 from vulnerabilities.importers import ruby
@@ -42,9 +41,9 @@
 from vulnerabilities.pipelines import npm_importer
 from vulnerabilities.pipelines import nvd_importer
 from vulnerabilities.pipelines import pypa_importer
+from vulnerabilities.pipelines import pysec_importer
 
 IMPORTERS_REGISTRY = [
-    pysec.PyPIImporter,
     alpine_linux.AlpineImporter,
     openssl.OpensslImporter,
     redhat.RedhatImporter,
@@ -78,6 +77,7 @@
     gitlab_importer.GitLabImporterPipeline,
     github_importer.GitHubAPIImporterPipeline,
     nvd_importer.NVDImporterPipeline,
+    pysec_importer.PyPIImporterPipeline,
 ]
 
 IMPORTERS_REGISTRY = {

diff --git a/vulnerabilities/importers/pysec.py b/vulnerabilities/importers/pysec.py
diff --git a/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py b/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py
@@ -0,0 +1,37 @@
+# Generated by Django 4.2.16 on 2024-10-24 13:51
+
+from django.db import migrations
+
+"""
+Update the created_by field on Advisory from the old qualified_name
+to the new pipeline_id.
+"""
+
+
+def update_created_by(apps, schema_editor):
+    from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline
+
+    Advisory = apps.get_model("vulnerabilities", "Advisory")
+    Advisory.objects.filter(created_by="vulnerabilities.importers.pysec.PyPIImporter").update(
+        created_by=PyPIImporterPipeline.pipeline_id
+    )
+
+
+def reverse_update_created_by(apps, schema_editor):
+    from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline
+
+    Advisory = apps.get_model("vulnerabilities", "Advisory")
+    Advisory.objects.filter(created_by=PyPIImporterPipeline.pipeline_id).update(
+        created_by="vulnerabilities.importers.pysec.PyPIImporter"
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0073_delete_packagerelatedvulnerability"),
+    ]
+
+    operations = [
+        migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by),
+    ]
diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py
@@ -406,7 +406,7 @@ def __str__(self):
     @property
     def is_cpe(self):
         """
-        Return Trueis this is a CPE reference.
+        Return True if this is a CPE reference.
         """
         return self.reference_id.startswith("cpe")
 
@@ -557,7 +557,7 @@ def for_cve(self, cve):
 
     def with_is_vulnerable(self):
         """
-        Annotate Package with ``with_is_vulnerable`` boolean attribute.
+        Annotate Package with ``is_vulnerable`` boolean attribute.
         """
         return self.annotate(
             is_vulnerable=Exists(

diff --git a/vulnerabilities/pipelines/pypa_importer.py b/vulnerabilities/pipelines/pypa_importer.py
@@ -6,7 +6,7 @@
 # See https://github.com/aboutcode-org/vulnerablecode for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
-import logging
+
 from pathlib import Path
 from typing import Iterable
 

diff --git a/vulnerabilities/pipelines/pysec_importer.py b/vulnerabilities/pipelines/pysec_importer.py
@@ -0,0 +1,66 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/aboutcode-org/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+import logging
+from io import BytesIO
+from typing import Iterable
+from zipfile import ZipFile
+
+import requests
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline
+
+
+class PyPIImporterPipeline(VulnerableCodeBaseImporterPipeline):
+    """Collect advisories from PyPI."""
+
+    pipeline_id = "pysec_importer"
+
+    license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE"
+    url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip"
+    spdx_license_expression = "CC-BY-4.0"
+    importer_name = "PyPI Importer"
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.fetch_zip,
+            cls.collect_and_store_advisories,
+            cls.import_new_advisories,
+        )
+
+    def fetch_zip(self):
+        self.log(f"Fetching `{self.url}`")
+        self.advisory_zip = requests.get(self.url).content
+
+    def advisories_count(self) -> int:
+        with ZipFile(BytesIO(self.advisory_zip)) as zip:
+            advisory_count = sum(1 for file in zip.namelist() if file.startswith("PYSEC-"))
+        return advisory_count
+
+    def collect_advisories(self) -> Iterable[AdvisoryData]:
+        """Yield AdvisoryData using a zipped data dump of OSV data"""
+        from vulnerabilities.importers.osv import parse_advisory_data
+
+        with ZipFile(BytesIO(self.advisory_zip)) as zip_file:
+            for file_name in zip_file.namelist():
+                if not file_name.startswith("PYSEC-"):
+                    self.log(
+                        f"Unsupported PyPI advisory data file: {file_name}",
+                        level=logging.ERROR,
+                    )
+                    continue
+                with zip_file.open(file_name) as f:
+                    vul_info = json.load(f)
+                    yield parse_advisory_data(
+                        raw_data=vul_info,
+                        supported_ecosystems=["pypi"],
+                        advisory_url=self.url,
+                    )
diff --git a/vulnerabilities/templates/includes/pagination.html b/vulnerabilities/templates/includes/pagination.html
@@ -1,6 +1,7 @@
 {% if is_paginated %}
 <nav class="pagination is-centered" role="navigation" aria-label="pagination">
     {% if page_obj.has_previous %}
+<<<<<<< HEAD
         <a href="?page={{ page_obj.previous_page_number }}&search={{ search|urlencode }}&page_size={{ page_obj.paginator.per_page }}" 
            class="pagination-previous">Previous</a>
     {% else %}
@@ -12,14 +13,29 @@
            class="pagination-next">Next</a>
     {% else %}
         <span class="pagination-next" disabled>Next</span>
+=======
+        <a href="?page={{ page_obj.previous_page_number }}&search={{ search|urlencode }}&page_size={{ page_size }}" class="pagination-previous">Previous</a>
+    {% else %}
+        <a class="pagination-previous" disabled>Previous</a>
+    {% endif %}
+
+    {% if page_obj.has_next %}
+        <a href="?page={{ page_obj.next_page_number }}&search={{ search|urlencode }}&page_size={{ page_size }}" class="pagination-next">Next</a>
+    {% else %}
+        <a class="pagination-next" disabled>Next</a>
+>>>>>>> 939aba34f0d5961044c4a244f02e183a7be7b2ca
     {% endif %}
 
     <ul class="pagination-list">
         {% if page_obj.number > 1 %}
+<<<<<<< HEAD
             <li>
                 <a href="?page=1&search={{ search|urlencode }}&page_size={{ page_obj.paginator.per_page }}" 
                    class="pagination-link" aria-label="Page 1">1</a>
             </li>
+=======
+            <li><a href="?page=1&search={{ search|urlencode }}&page_size={{ page_size }}" class="pagination-link" aria-label="Goto page 1">1</a></li>
+>>>>>>> 939aba34f0d5961044c4a244f02e183a7be7b2ca
             {% if page_obj.number > 4 %}
                 <li><span class="pagination-ellipsis">&hellip;</span></li>
             {% endif %}
@@ -28,6 +44,7 @@
         {% for i in page_obj.paginator.page_range %}
             {% if i > 1 and i < page_obj.paginator.num_pages %}
                 {% if i >= page_obj.number|add:"-3" and i <= page_obj.number|add:"3" %}
+<<<<<<< HEAD
                     <li>
                         {% if page_obj.number == i %}
                             <span class="pagination-link is-current" aria-current="page">{{ i }}</span>
@@ -36,6 +53,13 @@
                                class="pagination-link" aria-label="Goto page {{ i }}">{{ i }}</a>
                         {% endif %}
                     </li>
+=======
+                    {% if page_obj.number == i %}
+                        <li><a class="pagination-link is-current" aria-label="Page {{ i }}" aria-current="page">{{ i }}</a></li>
+                    {% else %}
+                        <li><a href="?page={{ i }}&search={{ search|urlencode }}&page_size={{ page_size }}" class="pagination-link" aria-label="Goto page {{ i }}">{{ i }}</a></li>
+                    {% endif %}
+>>>>>>> 939aba34f0d5961044c4a244f02e183a7be7b2ca
                 {% endif %}
             {% endif %}
         {% endfor %}
@@ -44,12 +68,16 @@
             {% if page_obj.number < page_obj.paginator.num_pages|add:"-3" %}
                 <li><span class="pagination-ellipsis">&hellip;</span></li>
             {% endif %}
+<<<<<<< HEAD
             <li>
                 <a href="?page={{ page_obj.paginator.num_pages }}&search={{ search|urlencode }}&page_size={{ page_obj.paginator.per_page }}" 
                    class="pagination-link" aria-label="Goto page {{ page_obj.paginator.num_pages }}">
                     {{ page_obj.paginator.num_pages }}
                 </a>
             </li>
+=======
+            <li><a href="?page={{ page_obj.paginator.num_pages }}&search={{ search|urlencode }}&page_size={{ page_size }}" class="pagination-link" aria-label="Goto page {{ page_obj.paginator.num_pages }}">{{ page_obj.paginator.num_pages }}</a></li>
+>>>>>>> 939aba34f0d5961044c4a244f02e183a7be7b2ca
         {% endif %}
     </ul>
 </nav>

diff --git a/vulnerabilities/templates/packages.html b/vulnerabilities/templates/packages.html
@@ -21,7 +21,16 @@
                 </div>
                 <div class="is-flex is-justify-content-center mb-2">
                     <div class="select is-small">
+<<<<<<< HEAD
                         {{ pagination_form.page_size }}
+=======
+                        <select id="itemsPerPage" onchange="changeItemsPerPage(this.value)">
+                            <option value="20" {% if page_obj.paginator.per_page == 20 %}selected{% endif %}>20 per page</option>
+                            <option value="50" {% if page_obj.paginator.per_page == 50 %}selected{% endif %}>50 per page</option>
+                            <option value="100" {% if page_obj.paginator.per_page == 100 %}selected{% endif %}>100 per page</option>
+                            <option value="200" {% if page_obj.paginator.per_page == 200 %}selected{% endif %}>200 per page</option>
+                        </select>
+>>>>>>> 939aba34f0d5961044c4a244f02e183a7be7b2ca
                     </div>
                 </div>
                 {% if is_paginated %}
@@ -85,6 +94,18 @@
         {% endif %}
     </section>
 {% endif %}
+<<<<<<< HEAD
 <script src="{% static 'js/pagination.js' %}"></script>
 {% endblock %}
 
+=======
+<script>
+    function changeItemsPerPage(pageSize) {
+        var urlParams = new URLSearchParams(window.location.search);
+        urlParams.set('page_size', pageSize);
+        window.location.search = urlParams.toString();
+    }
+
+</script>
+{% endblock %}
+>>>>>>> 939aba34f0d5961044c4a244f02e183a7be7b2ca
diff --git a/vulnerabilities/templates/vulnerabilities.html b/vulnerabilities/templates/vulnerabilities.html
@@ -21,12 +21,26 @@
                 </div>
                 <div class="is-flex is-justify-content-center mb-2">
                     <div class="select is-small">
+<<<<<<< HEAD
                         {{ pagination_form.page_size }}
                     </div>
                 </div>
                 {% if is_paginated %}
                     {% include 'includes/pagination.html' with page_obj=page_obj %}
                 {% endif %}
+=======
+                        <select id="itemsPerPage" onchange="changeItemsPerPage(this.value)">
+                            <option value="20" {% if page_obj.paginator.per_page == 20 %}selected{% endif %}>20 per page</option>
+                            <option value="50" {% if page_obj.paginator.per_page == 50 %}selected{% endif %}>50 per page</option>
+                            <option value="100" {% if page_obj.paginator.per_page == 100 %}selected{% endif %}>100 per page</option>
+                            <option value="200" {% if page_obj.paginator.per_page == 200 %}selected{% endif %}>200 per page</option>
+                        </select>
+                    </div>
+                </div>
+                {% if is_paginated %}
+                {% include 'includes/pagination.html' with page_obj=page_obj %}
+              {% endif %}
+>>>>>>> 939aba34f0d5961044c4a244f02e183a7be7b2ca
             </div>
         </section>
     </div>
@@ -80,6 +94,19 @@
         {% endif %}
     </section>
 {% endif %}
+<<<<<<< HEAD
 <script src="{% static 'js/pagination.js' %}"></script>
 {% endblock %}
 
+=======
+<script>
+    function changeItemsPerPage(pageSize) {
+        var urlParams = new URLSearchParams(window.location.search);
+        urlParams.set('page_size', pageSize);
+        window.location.search = urlParams.toString();
+    }
+
+</script>
+
+{% endblock %}
+>>>>>>> 939aba34f0d5961044c4a244f02e183a7be7b2ca