diff --git a/.github/workflows/pypi-release-aboutcode-hashid.yml b/.github/workflows/pypi-release-aboutcode-hashid.yml new file mode 100644 index 000000000..1260afb31 --- /dev/null +++ b/.github/workflows/pypi-release-aboutcode-hashid.yml @@ -0,0 +1,38 @@ +name: Build aboutcode.hashid Python distributions and publish on PyPI + +on: + workflow_dispatch: + push: + tags: + - "aboutcode.hashid/*" + +jobs: + build-and-publish: + name: Build and publish library to PyPI + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.11 + + - name: Install flot + run: python -m pip install flot --user + + - name: Build binary wheel and source tarball + run: python -m flot --pyproject pyproject-aboutcode.hashid.toml --sdist --wheel --output-dir dist/ + + - name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + password: ${{ secrets.PYPI_API_TOKEN_ABOUTCODE_HASHID }} + + - name: Upload built archives + uses: actions/upload-artifact@v4 + with: + name: pypi_archives + path: dist/* \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7caadf836..7f6debf44 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,9 +1,118 @@ Release notes ============= -Version (next) + +Version v35.1.0 +--------------------- + +- Use AboutCode mirror for collecting CISA KEV #1685 +- Do not report ghost package as a fix for vulnerability #1679 +- Add pipeline to sort packages #1686 +- Fix urls for API #1678 + + +Version v35.0.0 +--------------------- + +- Add scores in bulk search V1 API #1675 +- Add improver pipeline to flag ghost packages #644 #917 #1395 by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1533 +- Add base pipeline for importers and migrate PyPa importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1559 +- Remove dupe Package.get_non_vulnerable_versions by @pombredanne in https://github.com/aboutcode-org/vulnerablecode/pull/1570 +- Import data from GSD #706 by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/787 +- Add curl advisories importer by @ambuj-1211 in https://github.com/aboutcode-org/vulnerablecode/pull/1439 +- Update dependencies by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1590 +- Bump django from 4.2.0 to 4.2.15 by @dependabot in https://github.com/aboutcode-org/vulnerablecode/pull/1591 +- Bump cryptography from 42.0.4 to 43.0.1 by @dependabot in https://github.com/aboutcode-org/vulnerablecode/pull/1582 +- Bump actions/download-artifact from 3 to 4.1.7 in /.github/workflows by @dependabot in https://github.com/aboutcode-org/vulnerablecode/pull/1581 +- Improve export command by @pombredanne in https://github.com/aboutcode-org/vulnerablecode/pull/1571 +- Fix typo in Kev requests import by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/1594 +- Prepare for release v34.0.1 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1595 +- Bump upload-artifact to v4 by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1596 +- Migrate Npm importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1574 +- Use correct regex for CVE by @pombredanne in https://github.com/aboutcode-org/vulnerablecode/pull/1599 +- Migrate Nginx importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1575 +- Migrate GitLab importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1580 +- Migrate GitHub importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1584 +- Migrate NVD importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1587 +- Match affected and fixed-by Packages by @johnmhoran in https://github.com/aboutcode-org/vulnerablecode/pull/1528 +- Add management command to commit exported data by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1600 +- Add support to Exploits model by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/1562 +- Fix 500 Server Error with DRF browsable API and resolve blank Swagger API documentation by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1603 +- Release v34.0.2 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1604 +- Bump VCIO version by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1605 +- Bump django from 4.2.15 to 4.2.16 by @dependabot in https://github.com/aboutcode-org/vulnerablecode/pull/1608 +- Bump fetchcode from v0.3.0 to v0.6.0 by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1607 +- Use 4-tier system for storing package metadata by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1609 +- Fix vers range crash by @pombredanne in https://github.com/aboutcode-org/vulnerablecode/pull/1598 +- Add GitHub action to publish aboutcode.hashid PyPI by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1615 +- Segregate PackageRelatedVulnerability model to new models by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1612 +- Add documentation for new pipeline design by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1621 +- Fix 500 error in /api/cpes endpoint by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1629 +- Migrate pysec importer to aboutcode pipeline by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1628 +- Avoid memory exhaustion during data migration by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1630 +- Add support for Calculating Risk in VulnerableCode by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/1593 +- Bulk create in migrations by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1640 +- Update README.rst by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1641 +- Prepare for release v34.1.0 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1642 +- Add V2 API endpoints by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1631 +- Prepare for release v34.2.0 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1647 +- Refactor severity score model and fix incorrect suse scores by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1636 +- Add bulk search in v2 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1649 +- Prepare release v34.3.0 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1652 +- Add `on_failure` to handle cleanup during pipeline failure by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1651 +- Fix API bug by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1654 +- Add reference score to package endpoint by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1655 +- Prepare for release v34.3.2 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1656 +- Add support for storing exploitability and weighted severity by @ziadhany in https://github.com/aboutcode-org/vulnerablecode/pull/1646 +- Avoid migrations on version bumps by @keshav-space in https://github.com/aboutcode-org/vulnerablecode/pull/1660 +- Prepare v35.0.0rc1 by @TG1999 in https://github.com/aboutcode-org/vulnerablecode/pull/1664 + + + +Version v35.0.0rc1 +--------------------- + +- Add support for storing exploitability and weighted severity #1646 +- Avoid migrations on version bumps #1660 + + +Version v34.3.2 +---------------- + +- HOTFIX: Add reference score to package endpoint #1655 + + +Version v34.3.1 +---------------- + +- HOTFIX: Fix API bug #1654 + + +Version v34.3.0 +----------------- + +- Add bulk search in v2 #1649 +- Refactor severity score model and fix incorrect suse scores #1636 + + +Version v34.2.0 ------------------- +- Add V2 API endpoints #1631 + + +Version v34.1.0 +------------------- + +- Add support for Calculating Package Vulnerability Risk #1593 +- Migrate pysec importer to aboutcode pipeline #1628 +- Fix 500 error in /api/cpes endpoint #1629 +- Add documentation for new pipeline design #1621 +- Segregate PackageRelatedVulnerability model to new models #1612 +- Add GitHub action to publish aboutcode.hashid PyPI #1615 +- Fix vers range crash #1598 +- Use 4-tier system for storing package metadata #1609 + Version v34.0.2 ------------------- diff --git a/Makefile b/Makefile index 2bd782b6b..067cb419f 100644 --- a/Makefile +++ b/Makefile @@ -125,7 +125,7 @@ bump: docs: rm -rf docs/_build/ - @${ACTIVATE} sphinx-build docs/ docs/_build/ + @${ACTIVATE} sphinx-build docs/source docs/_build/ docker-images: @echo "-> Build Docker services" diff --git a/README.rst b/README.rst index 1652e0fcd..12be81e57 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,6 @@ we are trying to change this and evolve the status quo in a few other areas! Package URL themselves were designed first in ScanCode and VulnerableCode and are now a de-facto standard for vulnerability management and package references. - See https://github.com/package-url/purl-spec The VulnerableCode project is a FOSS community resource to help improve the @@ -47,26 +46,29 @@ security of the open source software ecosystem and its users at large. VulnerableCode consists of a database and the tools to collect, refine and keep the database current. -.. warning:: - VulnerableCode is under active development and is not yet fully - usable. -Read more about VulnerableCode https://vulnerablecode.readthedocs.org/ +.. pull-quote:: + **Warning** + + VulnerableCode is under active development and is not yet fully + usable. + -VulnerableCode is financially supported by NLnet, nexB, Google (through the -GSoC) and the active contributions of several volunteers. +Read more about VulnerableCode https://vulnerablecode.readthedocs.org/ VulnerableCode tech stack is Python, Django, PostgreSQL, nginx and Docker and several libraries. Getting started ---------------- +=============== Run with Docker -^^^^^^^^^^^^^^^^ +--------------- -First install docker, then run:: +First install docker, then run + +.. code:: bash git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode make envfile @@ -74,9 +76,11 @@ First install docker, then run:: docker compose up -d docker compose run vulnerablecode ./manage.py import --list -Then run an importer for nginx advisories (which is small):: +Then run an importer for nginx advisories (which is small) + +.. code:: bash - docker compose exec vulnerablecode ./manage.py import vulnerabilities.importers.nginx.NginxImporter + docker compose exec vulnerablecode ./manage.py import nginx_importer docker compose exec vulnerablecode ./manage.py improve --all At this point, the VulnerableCode app and API should be up and running with @@ -84,50 +88,44 @@ some data at http://localhost Populate VulnerableCode database -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +-------------------------------- VulnerableCode data collection works in two steps: importing data from multiple sources and then refining and improving how package and software vulnerabilities are related. -To run all importers and improvers use this:: +To run all importers and improvers use this + +.. code:: bash ./manage.py import --all + +.. code:: bash + ./manage.py improve --all Local development installation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------ + +On a Debian system, use this -On a Debian system, use this:: +.. code:: bash sudo apt-get install python3-venv python3-dev postgresql libpq-dev build-essential git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode make dev envfile postgres make test source venv/bin/activate - ./manage.py import vulnerabilities.importers.nginx.NginxImporter + ./manage.py import nginx_importer ./manage.py improve --all make run At this point, the VulnerableCode app and API is up at http://127.0.0.1:8001/ -Interface -^^^^^^^^^^ - - -VulnerableCode comes with a minimal web UI: - -.. image:: vulnerablecode-ui.png - -And a JSON API and its minimal web documentation: - -.. image:: vulnerablecode-json-api.png -.. image:: vulnerablecode-api-doc.png - License -^^^^^^^^^^ +======== Copyright (c) nexB Inc. and others. All rights reserved. @@ -147,19 +145,137 @@ See https://github.com/nexB/vulnerablecode for support or download. See https://aboutcode.org for more information about nexB OSS projects. -Acknowledgements -^^^^^^^^^^^^^^^^ -This project was funded through the NGI0 PET Fund, a fund established by -NLnet with financial support from the European Commission's Next Generation -Internet programme, under the aegis of DG Communications Networks, Content -and Technology under grant agreement No 825310. +Acknowledgements, Funding, Support and Sponsoring +================================================= + +This project is funded, supported and sponsored by: + +- Generous support and contributions from users like you! +- the European Commission NGI programme +- the NLnet Foundation +- the Swiss State Secretariat for Education, Research and Innovation (SERI) +- Google, including the Google Summer of Code and the Google Seasons of Doc programmes +- Mercedes-Benz Group +- Microsoft and Microsoft Azure +- AboutCode ASBL +- nexB Inc. + + + +|europa| |dgconnect| + +|ngi| |nlnet| + +|aboutcode| |nexb| + + + +This project was funded through the NGI0 PET Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 825310. + +|ngizeropet| https://nlnet.nl/project/VulnerableCode/ + + +This project was funded through the NGI0 Discovery Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 825322. + +|ngidiscovery| https://nlnet.nl/project/vulnerabilitydatabase/ + + +This project was funded through the NGI0 Core Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 101092990. + +|ngizerocore| https://nlnet.nl/project/VulnerableCode-enhancements/ + + +This project is funded through the NGI0 Entrust Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 101069594. + +|ngizeroentrust| https://nlnet.nl/project/FederatedSoftwareMetadata/ + + +This project was funded through the NGI0 Commons Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 101135429. Additional +funding is made available by the Swiss State Secretariat for Education, Research and Innovation +(SERI). + +|ngizerocommons| |swiss| https://nlnet.nl/project/FederatedCodeNext/ + +This project was funded through the NGI0 Entrust Fund, a fund established by NLnet with financial +support from the European Commission's Next Generation Internet programme, under the aegis of DG +Communications Networks, Content and Technology under grant agreement No 101069594. + +|ngizeroentrust| https://nlnet.nl/project/CRAVEX/ + + + +.. |nlnet| image:: https://nlnet.nl/logo/banner.png + :target: https://nlnet.nl + :height: 50 + :alt: NLnet foundation logo + +.. |ngi| image:: https://ngi.eu/wp-content/uploads/thegem-logos/logo_8269bc6efcf731d34b6385775d76511d_1x.png + :target: https://ngi.eu35 + :height: 50 + :alt: NGI logo + +.. |nexb| image:: https://nexb.com/wp-content/uploads/2022/04/nexB.svg + :target: https://nexb.com + :height: 30 + :alt: nexB logo + +.. |europa| image:: https://ngi.eu/wp-content/uploads/sites/77/2017/10/bandiera_stelle.png + :target: http://ec.europa.eu/index_en.htm + :height: 40 + :alt: Europa logo + +.. |aboutcode| image:: https://aboutcode.org/wp-content/uploads/2023/10/AboutCode.svg + :target: https://aboutcode.org/ + :height: 30 + :alt: AboutCode logo + +.. |swiss| image:: https://www.sbfi.admin.ch/sbfi/en/_jcr_content/logo/image.imagespooler.png/1493119032540/logo.png + :target: https://www.sbfi.admin.ch/sbfi/en/home/seri/seri.html + :height: 40 + :alt: Swiss logo + +.. |dgconnect| image:: https://commission.europa.eu/themes/contrib/oe_theme/dist/ec/images/logo/positive/logo-ec--en.svg + :target: https://commission.europa.eu/about-european-commission/departments-and-executive-agencies/communications-networks-content-and-technology_en + :height: 40 + :alt: EC DG Connect logo + +.. |ngizerocore| image:: https://nlnet.nl/image/logos/NGI0_tag.svg + :target: https://nlnet.nl/core + :height: 40 + :alt: NGI Zero Core Logo + +.. |ngizerocommons| image:: https://nlnet.nl/image/logos/NGI0_tag.svg + :target: https://nlnet.nl/commonsfund/ + :height: 40 + :alt: NGI Zero Commons Logo + +.. |ngizeropet| image:: https://nlnet.nl/image/logos/NGI0PET_tag.svg + :target: https://nlnet.nl/PET + :height: 40 + :alt: NGI Zero PET logo -https://nlnet.nl/project/VulnerableCode/ +.. |ngizeroentrust| image:: https://nlnet.nl/image/logos/NGI0Entrust_tag.svg + :target: https://nlnet.nl/entrust + :height: 38 + :alt: NGI Zero Entrust logo -This project was funded through the NGI0 Discovery Fund, a fund established -by NLnet with financial support from the European Commission's Next Generation -Internet programme, under the aegis of DG Communications Networks, Content -and Technology under grant agreement No 825322. +.. |ngiassure| image:: https://nlnet.nl/image/logos/NGIAssure_tag.svg + :target: https://nlnet.nl/image/logos/NGIAssure_tag.svg + :height: 32 + :alt: NGI Assure logo -https://nlnet.nl/project/vulnerabilitydatabase/ +.. |ngidiscovery| image:: https://nlnet.nl/image/logos/NGI0Discovery_tag.svg + :target: https://nlnet.nl/discovery/ + :height: 40 + :alt: NGI Discovery logo diff --git a/aboutcode/hashid/CHANGELOG.rst b/aboutcode/hashid/CHANGELOG.rst new file mode 100644 index 000000000..2d1f39adf --- /dev/null +++ b/aboutcode/hashid/CHANGELOG.rst @@ -0,0 +1,13 @@ +Changelog +============= + + +v0.2.0 (December 05, 2024) +--------------------------- + +- Use 4-tier system for storing package metadata https://github.com/aboutcode-org/vulnerablecode/pull/1609 + +v0.1.0 (September 12, 2024) +--------------------------- + +- Initial release of the ``aboutcode.hashid`` library. \ No newline at end of file diff --git a/aboutcode/hashid/__init__.py b/aboutcode/hashid/__init__.py index 3cd00ed0a..0d80ffafd 100644 --- a/aboutcode/hashid/__init__.py +++ b/aboutcode/hashid/__init__.py @@ -19,6 +19,9 @@ from packageurl import normalize_qualifiers from packageurl import normalize_subpath +__version__ = "0.2.0" + + """ General purpose utilities to create Vulnerability Ids aka. VCID and content-defined, hash-based paths to store Vulnerability and Package data using these paths in many balanced directories. @@ -28,7 +31,7 @@ which makes every filesystem performance suffer. In addition, when storing these files in Git repositories, we need to avoid creating any repository -with too many files that would make using this repository impactical or exceed the limits of some +with too many files that would make using this repository impractical or exceed the limits of some repository hosting services. Therefore we are storing vulnerability data using a directory tree using the first few characters @@ -46,21 +49,21 @@ def build_vcid(prefix="VCID"): """ Return a new Vulnerable Code ID (aka. VCID) which is a strongly unique vulnerability identifier string using the provided ``prefix``. A VCID is composed of a four letter prefix, and - three segments composed of four letters and dihits each separated by a dash. + three segments composed of four letters and digits each separated by a dash. For example:: >>> import re >>> vcid = build_vcid() >>> assert re.match('VCID(-[a-hjkm-z1-9]{4}){3}', vcid), vcid We were mistakenly not using enough bits. The symptom was that the last - segment of the VCID was always strting with "aaa" This ensure we are now OK: + segment of the VCID was always string with "aaa" This ensure we are now OK: >>> vcids = [build_vcid() for _ in range(50)] >>> assert not any(vid.split("-")[-1].startswith("aaa") for vid in vcids) """ uid = uuid4().bytes - # we keep three segments of 4 base32-encodee bytes, 3*4=12 + # we keep three segments of 4 base32-encoded bytes, 3*4=12 # which corresponds to 60 bits - # becausee each base32 byte can store 5 bits (2**5 = 32) + # because each base32 byte can store 5 bits (2**5 = 32) uid = base32_custom(uid)[:12].decode("utf-8").lower() return f"{prefix}-{uid[:4]}-{uid[4:8]}-{uid[8:12]}" @@ -72,7 +75,7 @@ def get_vcid_yml_file_path(vcid: str): return Path(VULNERABILITY_REPO_NAME) / vulnerability_yml_path(vcid) -# This cuxstom 32 characters alphabet is designed to avoid visually easily confusable characters: +# This custom 32 characters alphabet is designed to avoid visually easily confusable characters: # i and l # 0 and o _base32_alphabet = b"abcdefghjkmnpqrstuvwxyz123456789" @@ -117,7 +120,7 @@ def vulnerability_yml_path(vcid): Return the path to a vulnerability YAML file crafted from the ``vcid`` VCID vulnerability id. The approach is to distribute the files in many directories to avoid having too many files in - any directory and be able to find the path to a vulneravility file given its VCID distributed on + any directory and be able to find the path to a vulnerability file given its VCID distributed on the first two characters of the UUID section of a VCID. The UUID is using a base32 encoding, hence keeping two characters means 32 x 32 = 1024 @@ -140,9 +143,12 @@ def get_package_base_dir(purl: Union[PackageURL, str]): """ Return the base path to a Package directory (ignoring version) for a purl """ + if isinstance(purl, str): + purl = PackageURL.from_string(purl) + path_elements = package_path_elements(purl) phash, core_path, _pversion, _extra_path = path_elements - return Path(f"{PACKAGE_REPOS_NAME_PREFIX}-{phash}") / core_path + return Path(f"{PACKAGE_REPOS_NAME_PREFIX}-{purl.type}-{phash}") / core_path def get_package_purls_yml_file_path(purl: Union[PackageURL, str]): @@ -159,6 +165,52 @@ def get_package_vulnerabilities_yml_file_path(purl: Union[PackageURL, str]): return get_package_base_dir(purl) / VULNERABILITIES_FILENAME +# We use a 4-tier system for storing package metadata. +# The tiers are as follows: +# 1. Super Large Ecosystem (~5M packages): 2^10 = 1,024 git repositories +# 2. Large Ecosystem (~500K packages): 2^7 = 128 git repositories +# 3. Medium Ecosystem (~50K packages): 2^5 = 32 git repositories +# 4. Small Ecosystem (~2K packages): 2^0 = 1 git repository +# See https://github.com/aboutcode-org/federatedcode/issues/3#issuecomment-2388371726 +BIT_COUNT_BY_ECOSYSTEM = { + # Super Large Ecosystem + "github": 10, + "npm": 10, + # Large Ecosystem + "golang": 7, + "maven": 7, + "nuget": 7, + "perl": 7, + "php": 7, + "pypi": 7, + "ruby": 7, + # Medium Ecosystem + "alpm": 5, + "bitbucket": 5, + "cocoapods": 5, + "composer": 5, + "deb": 5, + "docker": 5, + "gem": 5, + "generic": 5, + "huggingface": 5, + "mlflow": 5, + "pub": 5, + "rpm": 5, + # Small Ecosystem + "bitnami": 0, + "cargo": 0, + "conan": 0, + "conda": 0, + "cpan": 0, + "cran": 0, + "hackage": 0, + "hex": 0, + "luarocks": 0, + "swift": 0, +} + + def package_path_elements(purl: Union[PackageURL, str]): """ Return 4-tuple of POSIX path strings crafted from the ``purl`` package PURL string or object. @@ -196,7 +248,7 @@ def package_path_elements(purl: Union[PackageURL, str]): sbom.spdx.2.2.json : a SPDX SBOM .... other files - : one sub directory for each quote-encoded if any + : one sub directory for each quote-encoded if any metadata.yml : ABOUT YAML file with package origin and license metadata for this version scancode-scan.yml : a scancode scan for this package version foo-scan.yml : a scan for this package version created with tool foo @@ -208,15 +260,15 @@ def package_path_elements(purl: Union[PackageURL, str]): We keep the same prefix for different versions:: >>> package_path_elements("pkg:pypi/license_expression@30.3.1") - ('1050', 'pypi/license-expression', '30.3.1', '') + ('50', 'pypi/license-expression', '30.3.1', '') >>> package_path_elements("pkg:pypi/license_expression@10.3.1") - ('1050', 'pypi/license-expression', '10.3.1', '') + ('50', 'pypi/license-expression', '10.3.1', '') We encode with quotes, avoid double encoding of already quoted parts to make subpaths easier for filesystems:: >>> package_path_elements("pkg:pypi/license_expression@30.3.1?foo=bar&baz=bar#sub/path") - ('1050', 'pypi/license-expression', '30.3.1', 'baz%3Dbar%26foo%3Dbar%23sub%2Fpath') + ('50', 'pypi/license-expression', '30.3.1', 'baz%3Dbar%26foo%3Dbar%23sub%2Fpath') >>> purl = PackageURL( ... type="pypi", @@ -225,12 +277,13 @@ def package_path_elements(purl: Union[PackageURL, str]): ... qualifiers=dict(foo="bar"), ... subpath="a/b/c") >>> package_path_elements(purl) - ('1050', 'pypi/license-expression', 'b%23ar%2F%3F30.3.2%21', 'foo%3Dbar%23a%2Fb%2Fc') + ('50', 'pypi/license-expression', 'b%23ar%2F%3F30.3.2%21', 'foo%3Dbar%23a%2Fb%2Fc') """ if isinstance(purl, str): purl = PackageURL.from_string(purl) - purl_hash = get_purl_hash(purl) + bit_count = BIT_COUNT_BY_ECOSYSTEM.get(purl.type, 0) + purl_hash = get_purl_hash(purl=purl, _bit_count=bit_count) if ns := purl.namespace: ns_name = f"{ns}/{purl.name}" @@ -287,17 +340,17 @@ def get_core_purl(purl: Union[PackageURL, str]): return PackageURL(**purld) -def get_purl_hash(purl: Union[PackageURL, str], _bit_count: int = 13) -> str: +def get_purl_hash(purl: Union[PackageURL, str], _bit_count: int = 0) -> str: """ Return a short lower cased hash string from a ``purl`` string or object. The PURL is normalized and we drop its version, qualifiers and subpath. - This function takes a normalized PURL string and a ``_bit_count`` argument defaulting to 13 bits - which represents 2**13 = 8192 possible hash values. It returns a fixed length short hash string + This function takes a normalized PURL string and a ``_bit_count`` argument defaulting to 0 bits + which represents 2**0 = 1 possible hash value. It returns a fixed length short hash string that is left-padded with zeros. The hash length is derived from the bit_count and the number of bits-per-byte stored in an hex - encoding of this bits count. For 13 bits, this means up to 4 characters. + encoding of this bits count. For 10 bits, this means up to 3 characters. The function is carefully designed to be portable across tech stacks and easy to implement in many programming languages: @@ -319,23 +372,23 @@ def get_purl_hash(purl: Union[PackageURL, str], _bit_count: int = 13) -> str: For example:: The hash does not change with version or qualifiers:: - >>> get_purl_hash("pkg:pypi/univers@30.12.0") - '1289' - >>> get_purl_hash("pkg:pypi/univers@10.12.0") - '1289' - >>> get_purl_hash("pkg:pypi/univers@30.12.0?foo=bar#sub/path") - '1289' + >>> get_purl_hash("pkg:pypi/univers@30.12.0", 7) + '09' + >>> get_purl_hash("pkg:pypi/univers@10.12.0", 7) + '09' + >>> get_purl_hash("pkg:pypi/univers@30.12.0?foo=bar#sub/path", 7) + '09' The hash is left padded with zero if it:: - >>> get_purl_hash("pkg:pypi/expressionss") - '0057' + >>> get_purl_hash("pkg:pypi/expressionss", 7) + '57' We normalize the PURL. Here pypi normalization always uses dash for underscore :: - >>> get_purl_hash("pkg:pypi/license_expression") - '1050' - >>> get_purl_hash("pkg:pypi/license-expression") - '1050' + >>> get_purl_hash("pkg:pypi/license_expression", 7) + '50' + >>> get_purl_hash("pkg:pypi/license-expression", 7) + '50' Originally from: https://github.com/nexB/purldb/pull/235/files#diff-a1fd023bd42d73f56019d540f38be711255403547add15108540d70f9948dd40R154 diff --git a/aboutcode/hashid/tests/test_hashid.py b/aboutcode/hashid/tests/test_hashid.py new file mode 100644 index 000000000..f83944984 --- /dev/null +++ b/aboutcode/hashid/tests/test_hashid.py @@ -0,0 +1,54 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# Portions Copyright (c) The Python Software Foundation +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 and Python-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import pytest + +from aboutcode.hashid import package_path_elements + + +@pytest.mark.parametrize( + "purl, purl_hash", + [ + ("pkg:maven/org.apache.commons/io", "4f"), + ("pkg:GOLANG/google.golang.org/genproto@abcdedf#/googleapis/api/annotations/", "4a"), + ("pkg:golang/github.com/nats-io/nats-server/v2/server@v1.2.9", "22"), + ("pkg:bitbucket/birKenfeld/pyGments-main@244fd47e07d1014f0aed9c", "03"), + ("pkg:github/Package-url/purl-Spec@244fd47e07d1004f0aed9c", "095"), + ("pkg:deb/debian/curl@7.50.3-1?arch=i386&distro=jessie", "19"), + ( + "pkg:docker/customer/dockerimage@sha256:244fd47e07d1004f0aed9c?repository_url=gcr.io", + "10", + ), + ("pkg:gem/jruby-launcher@1.1.2?Platform=java", "1e"), + ( + "pkg:Maven/org.apache.xmlgraphics/batik-anim@1.9.1?repositorY_url=repo.spring.io/release&classifier=sources", + "28", + ), + ( + "pkg:Maven/org.apache.xmlgraphics/batik-anim@1.9.1?repositorY_url=repo.spring.io/release&extension=pom", + "28", + ), + ("pkg:Maven/net.sf.jacob-project/jacob@1.14.3?type=dll&classifier=x86", "17"), + ("pkg:npm/%40angular/animation@12.3.1", "323"), + ("pkg:Nuget/EnterpriseLibrary.Common@6.0.1304", "63"), + ("pkg:PYPI/Django_package@1.11.1.dev1", "00"), + ("pkg:composer/guzzlehttp/promises@2.0.2", "1d"), + ("pkg:Rpm/fedora/curl@7.50.3-1.fc25?Arch=i386&Distro=fedora-25", "16"), + ("pkg:maven/HTTPClient/HTTPClient@0.3-3", "4d"), + ("pkg:maven/mygroup/myartifact@1.0.0%20Final?mykey=my%20value", "6f"), + ("pkg:npm/@babel/core#/googleapis/api/annotations/", "0dc"), + ("pkg:npm/@babel/core@1.0.2#/googleapis/api/annotations/", "0dc"), + ("pkg:npm/core@1.0.2#/googleapis/api/annotations/", "23b"), + ("pkg:npm/core#/googleapis/api/annotations/", "23b"), + ], +) +def test_purl_hash(purl, purl_hash): + result_hash, *_ = package_path_elements(purl) + assert result_hash == purl_hash diff --git a/docs/Makefile b/docs/Makefile index d0c3cbf10..788b03961 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,6 +5,7 @@ # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD = sphinx-autobuild SOURCEDIR = source BUILDDIR = build @@ -14,6 +15,13 @@ help: .PHONY: help Makefile +# Run the development server using sphinx-autobuild +docs: + @echo + @echo "Starting up the docs server..." + @echo + $(SPHINXAUTOBUILD) --port 8000 --watch ${SOURCEDIR} $(SOURCEDIR) "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile diff --git a/docs/source/README.gif b/docs/source/README.gif deleted file mode 100644 index 74b7a6512..000000000 Binary files a/docs/source/README.gif and /dev/null differ diff --git a/docs/source/conf.py b/docs/source/conf.py index c0b713034..05cec2924 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -35,6 +35,7 @@ "https://anongit.gentoo.org/git/data/glsa.git", # Git only link "https://www.softwaretestinghelp.com/how-to-write-good-bug-report/", # Cloudflare protection "https://www.openssl.org/news/vulnerabilities.xml", # OpenSSL legacy advisory URL, not longer available + "https://example.org/api/non-existent-packages", ] # Add any Sphinx extension module names here, as strings. They can be diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst index fa6e7075b..ba634ecc8 100644 --- a/docs/source/contributing.rst +++ b/docs/source/contributing.rst @@ -18,9 +18,9 @@ Do Your Homework ---------------- Before adding a contribution or create a new issue, take a look at the project’s -`README `_, read through our +`README `_, read through our `documentation `_, -and browse existing `issues `_, +and browse existing `issues `_, to develop some understanding of the project and confirm whether a given issue/feature has previously been discussed. @@ -35,7 +35,7 @@ First Timers You are here to help, but you are a new contributor! No worries, we always welcome newcomer contributors. We maintain some -`good first issues `_ +`good first issues `_ and encourage new contributors to work on those issues for a smooth start. .. tip:: @@ -47,15 +47,20 @@ Code Contributions For more established contributors, you can contribute to the codebase in several ways: -- Report a `bug `_; just remember to be as +- Report a `bug `_; just remember to be as specific as possible. -- Submit a `bug fix `_ for any existing +- Submit a `bug fix `_ for any existing issue. -- Create a `new issue `_ to request a +- Create a `new issue `_ to request a feature, submit a feedback, or ask a question. +* Want to add support for a new importer pipeline? See the detailed tutorial here: + :ref:`tutorial_add_importer_pipeline`. +* Interested adding a new improver pipeline? Check out the tutorial here: + :ref:`tutorial_add_improver_pipeline`. + .. note:: - Make sure to check existing `issues `_, + Make sure to check existing `issues `_, to confirm whether a given issue or a question has previously been discussed. @@ -84,582 +89,12 @@ Helpful Resources - Review our `comprehensive guide `_ for more details on how to add quality contributions to our codebase and documentation -- Check this free resource on `how to contribute to an open source project on github `_ +- Check this free resource on `How to contribute to an open source project on github `_ - Follow `this wiki page `_ on how to write good commit messages - `Pro Git book `_ - `How to write a good bug report `_ -.. _tutorial_add_a_new_importer: - -Add a new importer -------------------- - -This tutorial contains all the things one should know to quickly implement an importer. -Many internal details about importers can be found inside the -:file:`vulnerabilites/importer.py` file. -Make sure to go through :ref:`importer-overview` before you begin writing one. - -TL;DR -------- - -#. Create a new :file:`vulnerabilities/importers/{importer_name.py}` file. -#. Create a new importer subclass inheriting from the ``Importer`` superclass defined in - ``vulnerabilites.importer``. It is conventional to end an importer name with *Importer*. -#. Specify the importer license. -#. Implement the ``advisory_data`` method to process the data source you are - writing an importer for. -#. Add the newly created importer to the importers registry at - ``vulnerabilites/importers/__init__.py`` - -.. _tutorial_add_a_new_importer_prerequisites: - -Prerequisites --------------- - -Before writing an importer, it is important to familiarize yourself with the following concepts. - -PackageURL -^^^^^^^^^^^^ - -VulnerableCode extensively uses Package URLs to identify a package. See the -`PackageURL specification `_ and its `Python implementation -`_ for more details. - -**Example usage:** - -.. code:: python - - from packageurl import PackageURL - purl = PackageURL(name="ffmpeg", type="deb", version="1.2.3") - - -AdvisoryData -^^^^^^^^^^^^^ - -``AdvisoryData`` is an intermediate data format: -it is expected that your importer will convert the raw scraped data into ``AdvisoryData`` objects. -All the fields in ``AdvisoryData`` dataclass are optional; it is the importer's resposibility to -ensure that it contains meaningful information about a vulnerability. - -AffectedPackage -^^^^^^^^^^^^^^^^ - -``AffectedPackage`` data type is used to store a range of affected versions and a fixed version of a -given package. For all version-related data, `univers `_ library -is used. - -Univers -^^^^^^^^ - -`univers `_ is a Python implementation of the `vers specification `_. -It can parse and compare all the package versions and all the ranges, -from debian, npm, pypi, ruby and more. -It processes all the version range specs and expressions. - -Importer -^^^^^^^^^ - -All the generic importers need to implement the ``Importer`` class. -For ``Git`` or ``Oval`` data source, ``GitImporter`` or ``OvalImporter`` could be implemented. - -.. note:: - - ``GitImporter`` and ``OvalImporter`` need a complete rewrite. - Interested in :ref:`contributing` ? - -Writing an importer ---------------------- - -Create Importer Source File -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -All importers are located in the :file:`vulnerabilites/importers` directory. -Create a new file to put your importer code in. -Generic importers are implemented by writing a subclass for the ``Importer`` superclass and -implementing the unimplemented methods. - -Specify the Importer License -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Importers scrape data off the internet. In order to make sure the data is useable, a license -must be provided. -Populate the ``spdx_license_expression`` with the appropriate value. -The SPDX license identifiers can be found at https://spdx.org/licenses/. - -.. note:: - An SPDX license identifier by itself is a valid licence expression. In case you need more complex - expressions, see https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/ - -Implement the ``advisory_data`` Method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The ``advisory_data`` method scrapes the advisories from the data source this importer is -targeted at. -It is required to return an *Iterable of AdvisoryData objects*, and thus it is a good idea to yield -from this method after creating each AdvisoryData object. - -At this point, an example importer will look like this: - -:file:`vulnerabilites/importers/example.py` - -.. code-block:: python - - from typing import Iterable - - from packageurl import PackageURL - - from vulnerabilities.importer import AdvisoryData - from vulnerabilities.importer import Importer - - - class ExampleImporter(Importer): - - spdx_license_expression = "BSD-2-Clause" - - def advisory_data(self) -> Iterable[AdvisoryData]: - return [] - -This importer is only a valid skeleton and does not import anything at all. - -Let us implement another dummy importer that actually imports some data. - -Here we have a ``dummy_package`` which follows ``NginxVersionRange`` and ``SemverVersion`` for -version management from `univers `_. - -.. note:: - - It is possible that the versioning scheme you are targeting has not yet been - implemented in the `univers `_ library. - If this is the case, you will need to head over there and implement one. - -.. code-block:: python - - from datetime import datetime - from datetime import timezone - from typing import Iterable - - import requests - from packageurl import PackageURL - from univers.version_range import NginxVersionRange - from univers.versions import SemverVersion - - from vulnerabilities.importer import AdvisoryData - from vulnerabilities.importer import AffectedPackage - from vulnerabilities.importer import Importer - from vulnerabilities.importer import Reference - from vulnerabilities.importer import VulnerabilitySeverity - from vulnerabilities.severity_systems import SCORING_SYSTEMS - - - class ExampleImporter(Importer): - - spdx_license_expression = "BSD-2-Clause" - - def advisory_data(self) -> Iterable[AdvisoryData]: - raw_data = fetch_advisory_data() - for data in raw_data: - yield parse_advisory_data(data) - - - def fetch_advisory_data(): - return [ - { - "id": "CVE-2021-23017", - "summary": "1-byte memory overwrite in resolver", - "advisory_severity": "medium", - "vulnerable": "0.6.18-1.20.0", - "fixed": "1.20.1", - "reference": "http://mailman.nginx.org/pipermail/nginx-announce/2021/000300.html", - "published_on": "14-02-2021 UTC", - }, - { - "id": "CVE-2021-1234", - "summary": "Dummy advisory", - "advisory_severity": "high", - "vulnerable": "0.6.18-1.20.0", - "fixed": "1.20.1", - "reference": "http://example.com/cve-2021-1234", - "published_on": "06-10-2021 UTC", - }, - ] - - - def parse_advisory_data(raw_data) -> AdvisoryData: - purl = PackageURL(type="example", name="dummy_package") - affected_version_range = NginxVersionRange.from_native(raw_data["vulnerable"]) - fixed_version = SemverVersion(raw_data["fixed"]) - affected_package = AffectedPackage( - package=purl, affected_version_range=affected_version_range, fixed_version=fixed_version - ) - severity = VulnerabilitySeverity( - system=SCORING_SYSTEMS["generic_textual"], value=raw_data["advisory_severity"] - ) - references = [Reference(url=raw_data["reference"], severities=[severity])] - date_published = datetime.strptime(raw_data["published_on"], "%d-%m-%Y %Z").replace( - tzinfo=timezone.utc - ) - - return AdvisoryData( - aliases=[raw_data["id"]], - summary=raw_data["summary"], - affected_packages=[affected_package], - references=references, - date_published=date_published, - ) - - -.. note:: - - | Use ``make valid`` to format your new code using black and isort automatically. - | Use ``make check`` to check for formatting errors. - -Register the Importer -^^^^^^^^^^^^^^^^^^^^^^ - -Finally, register your importer in the importer registry at -:file:`vulnerabilites/importers/__init__.py` - -.. code-block:: python - :emphasize-lines: 1, 4 - - from vulnerabilities.importers import example - from vulnerabilities.importers import nginx - - IMPORTERS_REGISTRY = [nginx.NginxImporter, example.ExampleImporter] - - IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY} - -Congratulations! You have written your first importer. - -Run Your First Importer -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If everything went well, you will see your importer in the list of available importers. - -.. code-block:: console - :emphasize-lines: 5 - - $ ./manage.py import --list - - Vulnerability data can be imported from the following importers: - vulnerabilities.importers.nginx.NginxImporter - vulnerabilities.importers.example.ExampleImporter - -Now, run the importer. - -.. code-block:: console - - $ ./manage.py import vulnerabilities.importers.example.ExampleImporter - - Importing data using vulnerabilities.importers.example.ExampleImporter - Successfully imported data using vulnerabilities.importers.example.ExampleImporter - -See :ref:`command_line_interface` for command line usage instructions. -Enable Debug Logging (Optional) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -For more visibility, turn on debug logs in :file:`vulnerablecode/settings.py`. - -.. code-block:: python - - DEBUG = True - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - }, - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', - }, - } - -Invoke the import command now and you will see (in a fresh database): - -.. code-block:: console - - $ ./manage.py import vulnerabilities.importers.example.ExampleImporter - - Importing data using vulnerabilities.importers.example.ExampleImporter - Starting import for vulnerabilities.importers.example.ExampleImporter - [*] New Advisory with aliases: ['CVE-2021-23017'], created_by: vulnerabilities.importers.example.ExampleImporter - [*] New Advisory with aliases: ['CVE-2021-1234'], created_by: vulnerabilities.importers.example.ExampleImporter - Finished import for vulnerabilities.importers.example.ExampleImporter. Imported 2 advisories. - Successfully imported data using vulnerabilities.importers.example.ExampleImporter - -.. _tutorial_add_a_new_improver: - -Add a new improver ---------------------- - -This tutorial contains all the things one should know to quickly -implement an improver. -Many internal details about improvers can be found inside the -:file:`vulnerabilites/improver.py` file. -Make sure to go through :ref:`improver-overview` before you begin writing one. - -TL;DR -------- - -#. Locate the importer that this improver will be improving data of at - :file:`vulnerabilities/importers/{importer_name.py}` file. -#. Create a new improver subclass inheriting from the ``Improver`` superclass defined in - ``vulnerabilites.improver``. It is conventional to end an improver name with *Improver*. -#. Implement the ``interesting_advisories`` property to return a QuerySet of imported data - (``Advisory``) you are interested in. -#. Implement the ``get_inferences`` method to return an iterable of ``Inference`` objects for the - given ``AdvisoryData``. -#. Add the newly created improver to the improvers registry at - ``vulnerabilites/improvers/__init__.py``. - -Prerequisites --------------- - -Before writing an improver, it is important to familiarize yourself with the following concepts. - -Importer -^^^^^^^^^^ - -Importers are responsible for scraping vulnerability data from various data sources without creating -a complete relational model between vulnerabilites and their fixes and storing them in a structured -fashion. These data are stored in the ``Advisory`` model and can be converted to an equivalent -``AdvisoryData`` for various use cases. -See :ref:`importer-overview` for a brief overview on importers. - -Importer Prerequisites -^^^^^^^^^^^^^^^^^^^^^^^ - -Improvers consume data produced by importers, and thus it is important to familiarize yourself with -:ref:`Importer Prerequisites `. - -Inference -^^^^^^^^^^^ - -Inferences express the contract between the improvers and the improve runner framework. -An inference is intended to contain data points about a vulnerability without any uncertainties, -which means that one inference will target one vulnerability with the specific relevant affected and -fixed packages (in the form of `PackageURLs `_). -There is no notion of version ranges here: all package versions must be explicitly specified. - -Because this concrete relationship is rarely available anywhere upstream, we have to *infer* -these values, thus the name. -As inferring something is not always perfect, an Inference also comes with a confidence score. - -Improver -^^^^^^^^^ - -All the Improvers must inherit from ``Improver`` superclass and implement the -``interesting_advisories`` property and the ``get_inferences`` method. - -Writing an improver ---------------------- - -Locate the Source File -^^^^^^^^^^^^^^^^^^^^^^^^ - -If the improver will be working on data imported by a specific importer, it will be located in -the same file at :file:`vulnerabilites/importers/{importer-name.py}`. Otherwise, if it is a -generic improver, create a new file :file:`vulnerabilites/improvers/{improver-name.py}`. - -Explore Package Managers (Optional) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your Improver depends on the discrete versions of a package, the package managers' VersionAPI -located at :file:`vulnerabilites/package_managers.py` could come in handy. You will need to -instantiate the relevant ``VersionAPI`` in the improver's constructor and use it later in the -implemented methods. See an already implemented improver (NginxBasicImprover) for an example usage. - -Implement the ``interesting_advisories`` Property -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This property is intended to return a QuerySet of ``Advisory`` on which the ``Improver`` is -designed to work. - -For example, if the improver is designed to work on Advisories imported by ``ExampleImporter``, -the property can be implemented as - -.. code-block:: python - - class ExampleBasicImprover(Improver): - - @property - def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=ExampleImporter.qualified_name) - -Implement the ``get_inferences`` Method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The framework calls ``get_inferences`` method for every ``AdvisoryData`` that is obtained from -the ``Advisory`` QuerySet returned by the ``interesting_advisories`` property. - -It is expected to return an iterable of ``Inference`` objects for the given ``AdvisoryData``. To -avoid storing a lot of Inferences in memory, it is preferable to yield from this method. - -A very simple Improver that processes all Advisories to create the minimal relationships that can -be obtained by existing data can be found at :file:`vulnerabilites/improvers/default.py`, which is -an example of a generic improver. For a more sophisticated and targeted example, you can look -at an already implemented improver (e.g., :file:`vulnerabilites/importers/nginx.py`). - -Improvers are not limited to improving discrete versions and may also improve ``aliases``. -One such example, improving the importer written in the :ref:`importer tutorial -`, is shown below. - -.. code-block:: python - - from datetime import datetime - from datetime import timezone - from typing import Iterable - - import requests - from django.db.models.query import QuerySet - from packageurl import PackageURL - from univers.version_range import NginxVersionRange - from univers.versions import SemverVersion - - from vulnerabilities.importer import AdvisoryData - from vulnerabilities.improver import MAX_CONFIDENCE - from vulnerabilities.improver import Improver - from vulnerabilities.improver import Inference - from vulnerabilities.models import Advisory - from vulnerabilities.severity_systems import SCORING_SYSTEMS - - - class ExampleImporter(Importer): - ... - - - class ExampleAliasImprover(Improver): - @property - def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=ExampleImporter.qualified_name) - - def get_inferences(self, advisory_data) -> Iterable[Inference]: - for alias in advisory_data.aliases: - new_aliases = fetch_additional_aliases(alias) - aliases = new_aliases + [alias] - yield Inference(aliases=aliases, confidence=MAX_CONFIDENCE) - - - def fetch_additional_aliases(alias): - alias_map = { - "CVE-2021-23017": ["PYSEC-1337", "CERTIN-1337"], - "CVE-2021-1234": ["ANONSEC-1337", "CERTDES-1337"], - } - return alias_map.get(alias) - - -.. note:: - - | Use ``make valid`` to format your new code using black and isort automatically. - | Use ``make check`` to check for formatting errrors. - -Register the Improver -^^^^^^^^^^^^^^^^^^^^^^ - -Finally, register your improver in the improver registry at -:file:`vulnerabilites/improvers/__init__.py`. - -.. code-block:: python - :emphasize-lines: 7 - - from vulnerabilities import importers - from vulnerabilities.improvers import default - - IMPROVERS_REGISTRY = [ - default.DefaultImprover, - importers.nginx.NginxBasicImprover, - importers.example.ExampleAliasImprover, - ] - - IMPROVERS_REGISTRY = {x.qualified_name: x for x in IMPROVERS_REGISTRY} - -Congratulations! You have written your first improver. - -Run Your First Improver -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If everything went well, you will see your improver in the list of available improvers. - -.. code-block:: console - :emphasize-lines: 6 - - $ ./manage.py improve --list - - Vulnerability data can be processed by these available improvers: - vulnerabilities.improvers.default.DefaultImprover - vulnerabilities.importers.nginx.NginxBasicImprover - vulnerabilities.importers.example.ExampleAliasImprover - -Before running the improver, make sure you have imported the data. An improver cannot improve if -there is nothing imported. - -.. code-block:: console - - $ ./manage.py import vulnerabilities.importers.example.ExampleImporter - - Importing data using vulnerabilities.importers.example.ExampleImporter - Successfully imported data using vulnerabilities.importers.example.ExampleImporter - -Now, run the improver. - -.. code-block:: console - - $ ./manage.py improve vulnerabilities.importers.example.ExampleAliasImprover - - Improving data using vulnerabilities.importers.example.ExampleAliasImprover - Successfully improved data using vulnerabilities.importers.example.ExampleAliasImprover - -See :ref:`command_line_interface` for command line usage instructions. - -Enable Debug Logging (Optional) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -For more visibility, turn on debug logs in :file:`vulnerablecode/settings.py`. - -.. code-block:: python - - DEBUG = True - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - }, - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG', - }, - } - -Invoke the improve command now and you will see (in a fresh database, after importing): - -.. code-block:: console - - $ ./manage.py improve vulnerabilities.importers.example.ExampleAliasImprover - - Improving data using vulnerabilities.importers.example.ExampleAliasImprover - Running improver: vulnerabilities.importers.example.ExampleAliasImprover - Improving advisory id: 1 - New alias for : PYSEC-1337 - New alias for : CVE-2021-23017 - New alias for : CERTIN-1337 - Improving advisory id: 2 - New alias for : CERTDES-1337 - New alias for : ANONSEC-1337 - New alias for : CVE-2021-1234 - Finished improving using vulnerabilities.importers.example.ExampleAliasImprover. - Successfully improved data using vulnerabilities.importers.example.ExampleAliasImprover - -.. note:: - Even though CVE-2021-23017 and CVE-2021-1234 are not supplied by this improver, the output above shows them - because we left out running the ``DefaultImprover`` in the example. The ``DefaultImprover`` - inserts minimal data found via the importers in the database (here, the above two CVEs). Run - importer, DefaultImprover and then your improver in this sequence to avoid this anomaly. diff --git a/docs/source/index.rst b/docs/source/index.rst index be51eca80..b20b1c9b5 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -29,6 +29,14 @@ In this documentation you will find information on: faq misc +.. toctree:: + :maxdepth: 2 + :caption: Tutorial + + tutorial_add_importer_pipeline + tutorial_add_improver_pipeline + + .. toctree:: :maxdepth: 2 :caption: Reference Documentation diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 76139e4d9..655504d3d 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -6,7 +6,7 @@ Installation .. warning:: VulnerableCode is going through a major structural change and the installations are likely to not produce enough results. - This is tracked in https://github.com/nexB/vulnerablecode/issues/597 + This is tracked in https://github.com/aboutcode-org/vulnerablecode/issues/597 Welcome to **VulnerableCode** installation guide! This guide describes how to install VulnerableCode on various platforms. @@ -40,10 +40,10 @@ Build the Image VulnerableCode is distributed with ``Dockerfile`` and ``docker-compose.yml`` files required for the creation of the Docker image. -Clone the git `VulnerableCode repo `_, +Clone the git `VulnerableCode repo `_, create an environment file, and build the Docker image:: - git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode + git clone https://github.com/aboutcode-org/vulnerablecode.git && cd vulnerablecode make envfile docker compose build @@ -146,9 +146,9 @@ Make sure those are installed:: Clone and Configure ^^^^^^^^^^^^^^^^^^^ -Clone the `VulnerableCode Git repository `_:: +Clone the `VulnerableCode Git repository `_:: - git clone https://github.com/nexB/vulnerablecode.git && cd vulnerablecode + git clone https://github.com/aboutcode-org/vulnerablecode.git && cd vulnerablecode Install the required dependencies:: diff --git a/docs/source/introduction.rst b/docs/source/introduction.rst index ca61bdb0f..5143343a6 100644 --- a/docs/source/introduction.rst +++ b/docs/source/introduction.rst @@ -110,4 +110,4 @@ How can I contribute to VulnerableCode? --------------------------------------- Please get in touch on our `Gitter channel `__. -You can review or get the code and report issues at our `GitHub repo `__. +You can review or get the code and report issues at our `GitHub repo `__. diff --git a/docs/source/soc_gsoc21.rst b/docs/source/soc_gsoc21.rst index 3ccd2b039..84de393de 100644 --- a/docs/source/soc_gsoc21.rst +++ b/docs/source/soc_gsoc21.rst @@ -4,7 +4,7 @@ Google Summer of Code 2021 Final Report Organization - `AboutCode `_ ----------------------------------------------------------- | `Hritik Vijay `_ -| Project: `VulnerableCode `_ +| Project: `VulnerableCode `_ Overview --------- @@ -30,7 +30,7 @@ structure:: Yielding an average of 93% reduction in time (14x faster) -More: https://github.com/nexB/vulnerablecode/pull/478 +More: https://github.com/aboutcode-org/vulnerablecode/pull/478 Speed up upstream tests ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -50,7 +50,7 @@ usage cap. In the end, this is a trade off between resource usage and data accuracy. This brings major performance improvement during the test. | Before: ~6hrs, now ~9 minutes -| More: https://github.com/nexB/vulnerablecode/pull/490 +| More: https://github.com/aboutcode-org/vulnerablecode/pull/490 Improve Docker Configuration ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -63,8 +63,8 @@ using a docker image. The current configuration makes use of files like over any unnecessary files for deployment. | More: -| https://github.com/nexB/vulnerablecode/pull/497 -| https://github.com/nexB/vulnerablecode/pull/521 +| https://github.com/aboutcode-org/vulnerablecode/pull/497 +| https://github.com/aboutcode-org/vulnerablecode/pull/521 Add Makefile ^^^^^^^^^^^^^ @@ -78,8 +78,8 @@ relevant part of the documentation and updated settings to reject insecure deployments. | More: -| https://github.com/nexB/vulnerablecode/pull/497 -| https://github.com/nexB/vulnerablecode/pull/523 +| https://github.com/aboutcode-org/vulnerablecode/pull/497 +| https://github.com/aboutcode-org/vulnerablecode/pull/523 Use svn to collects tags in GitHubTagsAPI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -87,11 +87,11 @@ Surprisingly, GitHub allows svn requests to repositories. Now we can have all the tags with a single request. This is much more efficient and gentle to the APIs. This was as issue since the importers based on GithubDataSource were `failing -`_ because of being rate +`_ because of being rate limited by GitHub. | `Philippe `_, thank you so much for the suggestion -| More: https://github.com/nexB/vulnerablecode/pull/508 +| More: https://github.com/aboutcode-org/vulnerablecode/pull/508 Separate import and improve operations - WIP ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -106,14 +106,14 @@ As a bonus, writing importers will be very easy and welcome more contributors to the project. As of writing this report, this remains a work in progress which will be finished very soon. -More: https://github.com/nexB/vulnerablecode/pull/525 +More: https://github.com/aboutcode-org/vulnerablecode/pull/525 Others ^^^^^^^ -- helper: split_markdown_front_matter: https://github.com/nexB/vulnerablecode/pull/443 -- Dump yaml in favor of saneyaml https://github.com/nexB/vulnerablecode/pull/452 -- Refactor package_managers https://github.com/nexB/vulnerablecode/pull/495/commits -- Importers bugfix https://github.com/nexB/vulnerablecode/pull/505 +- helper: split_markdown_front_matter: https://github.com/aboutcode-org/vulnerablecode/pull/443 +- Dump yaml in favor of saneyaml https://github.com/aboutcode-org/vulnerablecode/pull/452 +- Refactor package_managers https://github.com/aboutcode-org/vulnerablecode/pull/495/commits +- Importers bugfix https://github.com/aboutcode-org/vulnerablecode/pull/505 Pre GSoC ---------- @@ -123,14 +123,14 @@ exploring the codebase, I realized that there is a lot of room for improvement. Thus I looked for simple improvements and bugs to fix in the early stage, which were: -- `Correct API docs path and fix pytest invocation `_ -- `Explicity provide lxml parser to beautifulsoup `_ -- `Make sure vulnerability id is_cve or is_vulcoid `_ -- `Fix istio importer `_ (cleared a huge confusion about the codebase) -- `Add me to AUTHORS `_ (Should've done this a lot earlier) -- `Add unspecified scoring system `_ -- `Fix redhat import failure `_ (This one took a *lot* of effort to pinpoint) -- `expose find_all_cve helper `_ +- `Correct API docs path and fix pytest invocation `_ +- `Explicity provide lxml parser to beautifulsoup `_ +- `Make sure vulnerability id is_cve or is_vulcoid `_ +- `Fix istio importer `_ (cleared a huge confusion about the codebase) +- `Add me to AUTHORS `_ (Should've done this a lot earlier) +- `Add unspecified scoring system `_ +- `Fix redhat import failure `_ (This one took a *lot* of effort to pinpoint) +- `expose find_all_cve helper `_ Post GSoC - Future Plans and what's left ------------------------------------------- @@ -167,7 +167,7 @@ I really enjoyed working on the project. There were ups and downs when I met some weird bugs but every one of them taught me something new about Python, Django and programming in general. The best part of working with my amazing mentors - Philippe and Shivam - were the `weekly meets -`_ +`_ where we would together try to figure out how to proceed with the development. I learned something new with every call and interaction we had. Thank you so much my mentors for providing a very smooth experience and Google for showing diff --git a/docs/source/tutorial_add_importer_pipeline.rst b/docs/source/tutorial_add_importer_pipeline.rst new file mode 100644 index 000000000..3f5b6b785 --- /dev/null +++ b/docs/source/tutorial_add_importer_pipeline.rst @@ -0,0 +1,371 @@ +.. _tutorial_add_importer_pipeline: + +Add a new pipeline to import advisories +======================================== + + +TL;DR +------- + +#. Create a new file ``{name}_importer.py`` inside **vulnerabilities/pipelines/**. +#. Create a new importer pipeline by inheriting **VulnerableCodeBaseImporterPipeline** + defined in **vulnerabilities.pipelines**. By convention the importer pipeline + class should end with **ImporterPipeline**. +#. Specify the license of upstream data being imported. +#. Implement the ``advisories_count`` and ``collect_advisories`` methods. +#. Add the newly created importer pipeline to the importers registry at + **vulnerabilities/importers/__init__.py** + + +Pipeline +-------- + +We use `aboutcode.pipeline `_ +for importing and improving data. At a very high level, a working pipeline contains classmethod +``steps`` that defines what steps to run and in what order. These steps are essentially just +functions. Pipeline provides an easy and effective way to log events inside these steps (it +automatically handles rendering and dissemination for these logs.) + +It also includes built-in progress indicator, which is essential since some of the jobs we run +in the pipeline are long-running tasks that require proper progress indicators. Pipeline provides +way to seamlessly records the progress (it automatically takes care of rendering and dissemination +of these progress). + +Additionally, the pipeline offers a consistent structure, making it easy to run these pipeline steps +with message queue like RQ and store all events related to a particular pipeline for +debugging/improvements. + +This tutorial contains all the things one should know to quickly implement an importer pipeline. +Many internal details about importer pipeline can be found inside the +`vulnerabilities/pipelines/__init__.py +`_ file. + + +.. _tutorial_add_importer_pipeline_prerequisites: + +Prerequisites +-------------- + +Before writing pipeline to import advisories, it is important to familiarize yourself with +the following concepts. + +PackageURL +~~~~~~~~~~ + +VulnerableCode extensively uses Package URLs to identify a package. See the +`PackageURL specification `_ and its `Python implementation +`_ for more details. + +**Example usage:** + +.. code:: python + + from packageurl import PackageURL + purl = PackageURL(name="ffmpeg", type="deb", version="1.2.3") + + +AdvisoryData +~~~~~~~~~~~~~ + +``AdvisoryData`` is an intermediate data format: +it is expected that your importer will convert the raw scraped data into ``AdvisoryData`` objects. +All the fields in ``AdvisoryData`` dataclass are optional; it is the importer's responsibility to +ensure that it contains meaningful information about a vulnerability. + +AffectedPackage +~~~~~~~~~~~~~~~ + +``AffectedPackage`` data type is used to store a range of affected versions and a fixed version of a +given package. For all version-related data, `univers `_ library +is used. + +Univers +~~~~~~~ + +`univers `_ is a Python implementation of the `vers specification `_. +It can parse and compare all the package versions and all the ranges, +from debian, npm, pypi, ruby and more. +It processes all the version range specs and expressions. + + +Writing an Importer Pipeline +----------------------------- + + +Create file for the new importer pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All pipelines, including the importer pipeline, are located in the +`vulnerabilities/pipelines/ +`_ directory. + +The importer pipeline is implemented by subclassing **VulnerableCodeBaseImporterPipeline** +and implementing the unimplemented methods. Since most tasks, such as inserting **AdvisoryData** +into the database and creating package-vulnerability relationships, are the same regardless of +the source of the advisory, these tasks are already taken care of in the base importer pipeline, +i.e., **VulnerableCodeBaseImporterPipeline**. You can simply focus on collecting the raw data and +parsing it to create proper **AdvisoryData** objects. + + +Specify the importer license +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The pipeline scrape data off the internet. In order to make sure the data is useable, a license +must be provided. + +Populate the ``spdx_license_expression`` with the appropriate value. The SPDX license identifiers +can be found at `ScanCode LicenseDB `_. + +.. note:: + An SPDX license identifier by itself is a valid license expression. In case you need more + complex expressions, see https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/ + + +Implement the ``advisories_count`` method +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``advisories_count`` method returns the total number of advisories that will be collected by +this pipeline. + +Suppose the upstream data is a single JSON file containing a list of security advisories; +in that case, you can simply return the count of security advisories in the JSON file, +and that's it. + +.. note:: + In some cases, it could be difficult to get the exact total number of advisories that would + be collected without actually processing the advisories. In such case returning the best + estimate will also work. + + **advisories_count** is used to enable a proper progress indicator and is not used beyond that. + If it is impossible (a super rare case) to compute the total advisory count beforehand, + just return ``0``. + + +Implement the ``collect_advisories`` method +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``collect_advisories`` method collects and parses the advisories from the data source and +yield an *AdvisoryData*. + +At this point, an example importer will look like this: + +.. code-block:: python + :caption: vulnerabilities/pipelines/example_importer.py + :linenos: + :emphasize-lines: 16-17, 20-21, 23-24 + + from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline + + class ExampleImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories Example.""" + + pipeline_id = "example_importer" + + root_url = "https://example.org/path/to/advisories/" + license_url = "https://exmaple.org/license/" + spdx_license_expression = "CC-BY-4.0" + importer_name = "Example Importer" + + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + def advisories_count(self) -> int: + raise NotImplementedError + + def collect_advisories(self) -> Iterable[AdvisoryData]: + raise NotImplementedError + + +This pipeline is only a valid skeleton and does not import anything at all. + +Let us implement a working pipeline that actually imports some data. + +Here we have a ``dummy_package`` which follows ``NginxVersionRange`` and ``SemverVersion`` for +version management from `univers `_. + +.. note:: + + It is possible that the versioning scheme you are targeting has not yet been + implemented in the `univers `_ library. + If this is the case, you will need to head over there and implement one. + +.. code-block:: python + :caption: vulnerabilities/pipelines/example_importer.py + :linenos: + :emphasize-lines: 34-35, 37-40 + + from datetime import datetime + from datetime import timezone + from typing import Iterable + + from packageurl import PackageURL + from univers.version_range import NginxVersionRange + from univers.versions import SemverVersion + + from vulnerabilities.importer import AdvisoryData + from vulnerabilities.importer import AffectedPackage + from vulnerabilities.importer import Reference + from vulnerabilities.importer import VulnerabilitySeverity + from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline + from vulnerabilities.severity_systems import SCORING_SYSTEMS + + + class ExampleImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories Example.""" + + pipeline_id = "example_importer" + + root_url = "https://example.org/path/to/advisories/" + license_url = "https://example.org/license/" + spdx_license_expression = "CC-BY-4.0" + importer_name = "Example Importer" + + @classmethod + def steps(cls): + return ( + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + def advisories_count(self) -> int: + return len(fetch_advisory_data()) + + def collect_advisories(self) -> Iterable[AdvisoryData]: + raw_data = fetch_advisory_data() + for data in raw_data: + yield parse_advisory_data(data) + + + def fetch_advisory_data(): + return [ + { + "id": "CVE-2021-23017", + "summary": "1-byte memory overwrite in resolver", + "advisory_severity": "medium", + "vulnerable": "0.6.18-1.20.0", + "fixed": "1.20.1", + "reference": "http://mailman.nginx.org/pipermail/nginx-announce/2021/000300.html", + "published_on": "14-02-2021 UTC", + }, + { + "id": "CVE-2021-1234", + "summary": "Dummy advisory", + "advisory_severity": "high", + "vulnerable": "0.6.18-1.20.0", + "fixed": "1.20.1", + "reference": "http://example.org/cve-2021-1234", + "published_on": "06-10-2021 UTC", + }, + ] + + + def parse_advisory_data(raw_data) -> AdvisoryData: + purl = PackageURL(type="example", name="dummy_package") + affected_version_range = NginxVersionRange.from_native(raw_data["vulnerable"]) + fixed_version = SemverVersion(raw_data["fixed"]) + affected_package = AffectedPackage( + package=purl, affected_version_range=affected_version_range, fixed_version=fixed_version + ) + severity = VulnerabilitySeverity( + system=SCORING_SYSTEMS["generic_textual"], value=raw_data["advisory_severity"] + ) + references = [Reference(url=raw_data["reference"], severities=[severity])] + date_published = datetime.strptime(raw_data["published_on"], "%d-%m-%Y %Z").replace( + tzinfo=timezone.utc + ) + advisory_url = f"https://example.org/advisory/{raw_data['id']}" + + return AdvisoryData( + aliases=[raw_data["id"]], + summary=raw_data["summary"], + affected_packages=[affected_package], + references=references, + url=advisory_url, + date_published=date_published, + ) + + +.. important:: + Steps should include ``collect_and_store_advisories`` and ``import_new_advisories`` + in the order shown above. They are defined in **VulnerableCodeBaseImporterPipeline**. + + It is the **collect_and_store_advisories** that is responsible for making calls to + **collect_advisories** and **advisories_count**, and hence **collect_advisories** and + **advisories_count** should never be directly added in steps. + + +.. attention:: + + Implement ``on_failure`` to handle cleanup in case of pipeline failure. + Cleanup of downloaded archives or cloned repos is necessary to avoid potential resource leakage. + +.. note:: + + | Use ``make valid`` to format your new code using black and isort automatically. + | Use ``make check`` to check for formatting errors. + +Register the Importer Pipeline +------------------------------ + +Finally, register your pipeline in the importer registry at +`vulnerabilities/importers/__init__.py +`_ + +.. code-block:: python + :caption: vulnerabilities/importers/__init__.py + :linenos: + :emphasize-lines: 1, 6 + + from vulnerabilities.pipelines import example_importer + from vulnerabilities.pipelines import nginx_importer + + IMPORTERS_REGISTRY = [ + nginx_importer.NginxImporterPipeline, + example_importer.ExampleImporterPipeline, + ] + + IMPORTERS_REGISTRY = { + x.pipeline_id if issubclass(x, VulnerableCodeBaseImporterPipeline) else x.qualified_name: x + for x in IMPORTERS_REGISTRY + } + +Congratulations! You have written your first importer pipeline. + +Run Your First Importer Pipeline +-------------------------------- + +If everything went well, you will see your pipeline in the list of available importers. + +.. code-block:: console + :emphasize-lines: 5 + + $ ./manage.py import --list + + Vulnerability data can be imported from the following importers: + nginx_importer + example_importer + +Now, run the importer. + +.. code-block:: console + + $ ./manage.py import example_importer + + Importing data using example_importer + INFO 2024-10-16 10:15:10.483 Pipeline [ExampleImporterPipeline] starting + INFO 2024-10-16 10:15:10.483 Step [collect_and_store_advisories] starting + INFO 2024-10-16 10:15:10.483 Collecting 2 advisories + INFO 2024-10-16 10:15:10.498 Successfully collected 2 advisories + INFO 2024-10-16 10:15:10.498 Step [collect_and_store_advisories] completed in 0 seconds + INFO 2024-10-16 10:15:10.498 Step [import_new_advisories] starting + INFO 2024-10-16 10:15:10.499 Importing 2 new advisories + INFO 2024-10-16 10:15:10.562 Successfully imported 2 new advisories + INFO 2024-10-16 10:15:10.563 Step [import_new_advisories] completed in 0 seconds + INFO 2024-10-16 10:15:10.563 Pipeline completed in 0 seconds + + +See :ref:`command_line_interface` for command line usage instructions. diff --git a/docs/source/tutorial_add_improver_pipeline.rst b/docs/source/tutorial_add_improver_pipeline.rst new file mode 100644 index 000000000..feed3556b --- /dev/null +++ b/docs/source/tutorial_add_improver_pipeline.rst @@ -0,0 +1,275 @@ +.. _tutorial_add_improver_pipeline: + +Add pipeline to improve/enhance data +===================================== + +TL;DR +------- + +#. Create a new file ``{improver_name}.py`` inside **vulnerabilities/pipelines/**. +#. Create a new improver pipeline by inheriting **VulnerableCodePipeline** defined + in **vulnerabilities.pipelines**. +#. Implement ``steps`` **classmethod** to define what function to run and in which order. +#. Implement the individual function defined in ``steps`` +#. Add the newly created pipeline to the improvers registry at + **vulnerabilities/improvers/__init__.py**. + +Pipeline +-------- + +We use `aboutcode.pipeline `_ +for importing and improving data. At a very high level, a working pipeline contains classmethod +``steps`` that defines what steps to run and in what order. These steps are essentially just +functions. Pipeline provides an easy and effective way to log events inside these steps (it +automatically handles rendering and dissemination for these logs.) + +It also includes built-in progress indicator, which is essential since some of the jobs we run +in the pipeline are long-running tasks that require proper progress indicators. Pipeline provides +way to seamlessly records the progress (it automatically takes care of rendering and dissemination +of these progress). + +Additionally, the pipeline offers a consistent structure, making it easy to run these pipeline steps +with message queue like RQ and store all events related to a particular pipeline for +debugging/improvements. + +This tutorial contains all the things one should know to quickly implement an improver pipeline. + + +Prerequisites +------------- + +The new improver design lets you do all sorts of cool improvements and enhancements. +Some of those are: + +* Let's suppose you have a certain number of packages and vulnerabilities in your database, + and you want to make sure that the packages being shown in VulnerableCode do indeed exist + upstream. Oftentimes, we come across advisory data that contains made-up package versions. + We can write (well, we already have) a pipeline that iterates through all the packages in + VulnerableCode and labels them as ghost packages if they don't exist upstream. + + +- A basic security advisory only contains CVE/aliases, summary, fixed/affected version, and + severity. But now we can use the new pipeline to enhance the vulnerability info with exploits from + various sources like ExploitDB, Metasploit, etc. + + +* Likewise, we can have more pipelines to flag malicious/yanked packages. + + +So you see, the new improver pipeline is very powerful in what you can achieve, but as always, with +great power comes great responsibility. By design, the new improver are unconstrained, and you must +be absolutely sure of what you're doing and should have robust tests for these pipelines in place. + + +Writing an Improver Pipeline +----------------------------- + +**Scenario:** Suppose we come around a source that curates and stores the list of packages that +don't exist upstream and makes it available through the REST API endpoint +https://example.org/api/non-existent-packages, which gives a JSON response with a list of +non-existent packages. + +Let's write a pipeline that will use this source to flag these non-existent package as +ghost package. + + +Create file for the new improver pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All pipelines, including the improver pipeline, are located in the +`vulnerabilities/pipelines/ +`_ directory. + +The improver pipeline is implemented by subclassing `VulnerableCodePipeline`. + +Specify the importer license +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the improver pipeline scrapes data off the internet, we need to track the license for +the scraped data to make sure that we can legally use it. + +Populate the ``spdx_license_expression`` with the appropriate value. The SPDX license identifiers +can be found at `ScanCode LicenseDB `_. + +.. note:: + An SPDX license identifier by itself is a valid license expression. In case you need more + complex expressions, see https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/ + + +Add skeleton for new pipeline +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this scenario pipeline needs to do two thing fetch raw data and use that to flag those packages. + +At this point improver will look like this: + +.. code-block:: python + :caption: vulnerabilities/pipelines/flag_ghost_package_with_example_org.py + :linenos: + :emphasize-lines: 14-15, 18-19, 21-22 + + from vulnerabilities.pipelines import VulnerableCodePipeline + + class FlagGhostPackagesWithExampleOrg(VulnerableCodePipeline): + """Example improver pipeline to flag ghost packages.""" + + pipeline_id = "flag_ghost_package_with_example_org" + + license_url = "https://exmaple.org/license/" + spdx_license_expression = "CC-BY-4.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_response, + cls.flag_ghost_packages, + ) + + def fetch_response(self): + raise NotImplementedError + + def flag_ghost_packages(self): + raise NotImplementedError + + +Implement the steps +~~~~~~~~~~~~~~~~~~~ + +We will evolve our high level design by implementing ``fetch_response`` and ``flag_ghost_packages`` +methods. + +.. code-block:: python + :caption: vulnerabilities/pipelines/flag_ghost_package_with_example_org.py + :linenos: + :emphasize-lines: 20-32, 34-42 + + from vulnerabilities.models import Package + from vulnerabilities.pipelines import VulnerableCodePipeline + + + class FlagGhostPackagesWithExampleOrg(VulnerableCodePipeline): + """Example improver pipeline to flag ghost packages.""" + + pipeline_id = "flag_ghost_package_with_example_org" + + license_url = "https://exmaple.org/license/" + spdx_license_expression = "CC-BY-4.0" + + @classmethod + def steps(cls): + return ( + cls.fetch_response, + cls.flag_ghost_packages, + ) + + def fetch_response(self): + # Since this is imaginary source we will mock the response + # In actual implementation you need to use request library to get data. + mock_response = { + "non-existent": [ + "pkg:npm/626@1.1.1", + "pkg:npm/bootstrap-tagsinput@0.8.0", + "pkg:npm/dojo@1.0.0", + "pkg:npm/dojo@1.1.0", + "pkg:npm/electron@1.8.0", + ] + } + self.fetched_data = mock_response + + def flag_ghost_packages(self): + non_existent_packages = self.fetched_data.get("non-existent", []) + + ghost_packages = Package.objects.filter(package_url__in=non_existent_packages) + ghost_package_count = ghost_packages.count() + + ghost_packages.update(is_ghost=True) + + self.log(f"Successfully flagged {ghost_package_count:,d} ghost Packages") + + +.. attention:: + + Implement ``on_failure`` to handle cleanup in case of pipeline failure. + Cleanup of downloaded archives or cloned repos is necessary to avoid potential resource leakage. + +.. note:: + + | Use ``make valid`` to format your new code using black and isort automatically. + | Use ``make check`` to check for formatting errors. + + +Register the Improver Pipeline +------------------------------ + +Finally, register your improver in the improver registry at +`vulnerabilities/improvers/__init__.py +`_ + + +.. code-block:: python + :caption: vulnerabilities/improvers/__init__.py + :linenos: + :emphasize-lines: 2, 6 + + from vulnerabilities.pipeline import enhance_with_kev + from vulnerabilities.pipeline import flag_ghost_package_with_example_org + + IMPROVERS_REGISTRY = [ + enhance_with_kev.VulnerabilityKevPipeline, + flag_ghost_package_with_example_org.FlagGhostPackagesWithExampleOrg, + ] + + IMPROVERS_REGISTRY = { + x.pipeline_id if issubclass(x, VulnerableCodePipeline) else x.qualified_name: x + for x in IMPROVERS_REGISTRY + } + + +Congratulations! You have written your first improver pipeline. + +Run Your First Improver Pipeline +-------------------------------- + +If everything went well, you will see your improver in the list of available improvers. + +.. code-block:: console + :emphasize-lines: 5 + + $ ./manage.py improve --list + + Vulnerability data can be processed by these available improvers: + enhance_with_kev + flag_ghost_package_with_example_org + +Now, run the improver. + +.. code-block:: console + + $ ./manage.py improve flag_ghost_package_with_example_org + + Improving data using flag_ghost_package_with_example_org + INFO 2024-10-17 14:37:54.482 Pipeline [FlagGhostPackagesWithExampleOrg] starting + INFO 2024-10-17 14:37:54.482 Step [fetch_response] starting + INFO 2024-10-17 14:37:54.482 Step [fetch_response] completed in 0 seconds + INFO 2024-10-17 14:37:54.482 Step [flag_ghost_packages] starting + INFO 2024-10-17 14:37:54.488 Successfully flagged 5 ghost Packages + INFO 2024-10-17 14:37:54.488 Step [flag_ghost_packages] completed in 0 seconds + INFO 2024-10-17 14:37:54.488 Pipeline completed in 0 seconds + + +See :ref:`command_line_interface` for command line usage instructions. + +.. tip:: + + If you need to improve package vulnerability relations created using a certain pipeline, + simply use the **pipeline_id** to filter out only those items. For example, if you want + to improve only those **AffectedByPackageRelatedVulnerability** entries that were created + by npm_importer pipeline, you can do so with the following query: + + .. code-block:: python + + AffectedByPackageRelatedVulnerability.objects.filter(created_by=NpmImporterPipeline.pipeline_id) + +.. note:: + + Make sure to use properly optimized query sets, and wherever needed, use paginated query sets. diff --git a/pyproject-aboutcode.hashid.toml b/pyproject-aboutcode.hashid.toml index 705b6015f..a2c304323 100644 --- a/pyproject-aboutcode.hashid.toml +++ b/pyproject-aboutcode.hashid.toml @@ -4,7 +4,7 @@ build-backend = "flot.buildapi" [project] name = "aboutcode.hashid" -version = "0.1.0" +version = "0.2.0" description = "A library for aboutcode hash-based identifiers for VCID, and PURLs" readme = "aboutcode/hashid/README.rst" license = { text = "Apache-2.0 AND Python-2.0" } @@ -66,6 +66,7 @@ excludes = [ "**/*.bak", "**/.ipynb_checkpoints", "aboutcode/hashid/python.LICENSE", + "aboutcode/hashid/tests/**/*", ] metadata_files = ["apache-2.0.LICENSE", "NOTICE", "aboutcode/hashid/python.LICENSE"] diff --git a/requirements.txt b/requirements.txt index 94bf8f0df..84ea22538 100644 --- a/requirements.txt +++ b/requirements.txt @@ -101,6 +101,7 @@ snowballstemmer==2.2.0 soupsieve==2.3.2 Sphinx==4.5.0 sphinxcontrib-applehelp==1.0.2 +sphinx-autobuild==2024.10.3 sphinxcontrib-devhelp==1.0.2 sphinxcontrib-django2==1.5 sphinxcontrib-htmlhelp==2.0.0 diff --git a/setup.cfg b/setup.cfg index 596dc5901..a3db96abd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = vulnerablecode -version = 34.0.2 +version = 35.1.0 license = Apache-2.0 AND CC-BY-SA-4.0 # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 @@ -111,6 +111,7 @@ dev = Sphinx>=4.5.0 sphinx_rtd_theme>=1.0.0 sphinxcontrib-django2>=1.5 + sphinx-autobuild>=2024.10.3 # Tests pytest>=7.0.1 pytest-django>=4.5.2 diff --git a/vulnerabilities/__init__.py b/vulnerabilities/__init__.py index bdac1cd30..20854f2ad 100644 --- a/vulnerabilities/__init__.py +++ b/vulnerabilities/__init__.py @@ -3,6 +3,6 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/admin.py b/vulnerabilities/admin.py index 19e006ae5..eecef0276 100644 --- a/vulnerabilities/admin.py +++ b/vulnerabilities/admin.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -13,7 +13,6 @@ from vulnerabilities.models import ApiUser from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilitySeverity @@ -35,12 +34,6 @@ class PackageAdmin(admin.ModelAdmin): search_fields = ["name"] -@admin.register(PackageRelatedVulnerability) -class PackageRelatedVulnerabilityAdmin(admin.ModelAdmin): - list_filter = ("package__type", "package__namespace") - search_fields = ["vulnerability__vulnerability_id", "package__name"] - - @admin.register(VulnerabilitySeverity) class VulnerabilitySeverityAdmin(admin.ModelAdmin): pass diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py index 5d953db9b..1fd480ce9 100644 --- a/vulnerabilities/api.py +++ b/vulnerabilities/api.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -22,9 +22,7 @@ from rest_framework import viewsets from rest_framework.decorators import action from rest_framework.response import Response -from rest_framework.reverse import reverse from rest_framework.throttling import AnonRateThrottle -from rest_framework.throttling import UserRateThrottle from vulnerabilities.models import Alias from vulnerabilities.models import Exploit @@ -54,13 +52,25 @@ def to_representation(self, instance): class VulnerabilityReferenceSerializer(serializers.ModelSerializer): - scores = VulnerabilitySeveritySerializer(many=True, source="vulnerabilityseverity_set") + scores = serializers.SerializerMethodField() reference_url = serializers.CharField(source="url") class Meta: model = VulnerabilityReference fields = ["reference_url", "reference_id", "reference_type", "scores", "url"] + def get_scores(self, instance): + severities_related_to_reference = [ + severity + for severity in self.context.get("severities", []) + if severity.url == instance.url + ] + + return VulnerabilitySeveritySerializer( + severities_related_to_reference, + many=True, + ).data + class BaseResourceSerializer(serializers.HyperlinkedModelSerializer): """ @@ -143,7 +153,7 @@ class VulnSerializerRefsAndSummary(BaseResourceSerializer): many=True, source="filtered_fixed_packages", read_only=True ) - references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") + references = serializers.SerializerMethodField() aliases = serializers.SerializerMethodField() @@ -151,9 +161,31 @@ def get_aliases(self, obj): # Assuming `obj.aliases` is a queryset of `Alias` objects return [alias.alias for alias in obj.aliases.all()] + def get_references(self, vulnerability): + references = vulnerability.vulnerabilityreference_set.all() + severities = vulnerability.severities.all() + + serialized_references = VulnerabilityReferenceSerializer( + references, + context={"severities": severities}, + many=True, + ).data + + return serialized_references + class Meta: model = Vulnerability - fields = ["url", "vulnerability_id", "summary", "references", "fixed_packages", "aliases"] + fields = [ + "url", + "vulnerability_id", + "summary", + "references", + "fixed_packages", + "aliases", + "risk_score", + "exploitability", + "weighted_severity", + ] class WeaknessSerializer(serializers.HyperlinkedModelSerializer): @@ -199,8 +231,7 @@ class VulnerabilitySerializer(BaseResourceSerializer): many=True, source="filtered_fixed_packages", read_only=True ) affected_packages = MinimalPackageSerializer(many=True, read_only=True) - - references = VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") + references = serializers.SerializerMethodField() aliases = AliasSerializer(many=True, source="alias") exploits = ExploitSerializer(many=True, read_only=True) weaknesses = WeaknessSerializer(many=True) @@ -214,10 +245,22 @@ def to_representation(self, instance): return data + def get_references(self, vulnerability): + references = vulnerability.vulnerabilityreference_set.all() + severities = vulnerability.severities.all() + + serialized_references = VulnerabilityReferenceSerializer( + references, + context={"severities": severities}, + many=True, + ).data + + return serialized_references + def get_severity_range_score(self, instance): severity_vectors = [] severity_values = set() - for s in instance.severities: + for s in instance.severities.all(): if s.scoring_system == EPSS.identifier: continue @@ -251,6 +294,9 @@ class Meta: "weaknesses", "exploits", "severity_range_score", + "exploitability", + "weighted_severity", + "risk_score", ] @@ -287,7 +333,7 @@ def get_fixed_packages(self, package): type=package.type, qualifiers=package.qualifiers, subpath=package.subpath, - packagerelatedvulnerability__fix=True, + fixingpackagerelatedvulnerability__isnull=False, ) .with_is_vulnerable() .distinct() @@ -300,10 +346,13 @@ def get_vulnerabilities_for_a_package(self, package, fix) -> dict: otherwise return vulnerabilities fixed by the `package`. """ fixed_packages = self.get_fixed_packages(package=package) - qs = package.vulnerabilities.filter(packagerelatedvulnerability__fix=fix) + if not fix: + qs = package.affected_by_vulnerabilities.all() + else: + qs = package.fixing_vulnerabilities.all() qs = qs.prefetch_related( Prefetch( - "packages", + "fixed_by_packages", queryset=fixed_packages, to_attr="filtered_fixed_packages", ) @@ -318,6 +367,10 @@ def get_fixing_vulnerabilities(self, package) -> dict: """ Return a mapping of vulnerabilities fixed in the given `package`. """ + # Ghost package should not fix any vulnerability. + if package.is_ghost: + return [] + return self.get_vulnerabilities_for_a_package(package=package, fix=True) def get_affected_vulnerabilities(self, package) -> dict: @@ -356,6 +409,7 @@ class Meta: "latest_non_vulnerable_version", "affected_by_vulnerabilities", "fixing_vulnerabilities", + "risk_score", ] @@ -372,7 +426,6 @@ class Meta: "qualifiers", "subpath", "purl", - "packagerelatedvulnerability__fix", ] def filter_purl(self, queryset, name, value): @@ -590,7 +643,14 @@ def get_fixed_packages_qs(self): Filter the packages that fixes a vulnerability on fields like name, namespace and type. """ - return self.get_packages_qs().filter(packagerelatedvulnerability__fix=True) + return ( + self.get_packages_qs() + .filter( + fixingpackagerelatedvulnerability__isnull=False, + is_ghost=False, + ) + .with_is_vulnerable() + ) def get_packages_qs(self): """ @@ -613,13 +673,12 @@ def get_queryset(self): super() .get_queryset() .prefetch_related( - Prefetch( - "packages", - queryset=self.get_packages_qs(), - ), "weaknesses", + "references", + "exploits", + "severities", Prefetch( - "packages", + "fixed_by_packages", queryset=self.get_fixed_packages_qs(), to_attr="filtered_fixed_packages", ), @@ -640,17 +699,13 @@ def filter_cpe(self, queryset, name, value): return self.queryset.filter(vulnerabilityreference__reference_id__startswith=cpe).distinct() -class CPEViewSet(viewsets.ReadOnlyModelViewSet): - """ - Lookup for vulnerabilities by CPE (https://nvd.nist.gov/products/cpe) - """ +class CPEViewSet(VulnerabilityViewSet): + """Lookup for vulnerabilities by CPE (https://nvd.nist.gov/products/cpe)""" queryset = Vulnerability.objects.filter( vulnerabilityreference__reference_id__startswith="cpe" ).distinct() - serializer_class = VulnerabilitySerializer - filter_backends = (filters.DjangoFilterBackend,) - throttle_classes = [StaffUserRateThrottle, AnonRateThrottle] + filterset_class = CPEFilterSet @action(detail=False, methods=["post"]) diff --git a/vulnerabilities/api_extension.py b/vulnerabilities/api_extension.py index 4b9211c76..7a13baf42 100644 --- a/vulnerabilities/api_extension.py +++ b/vulnerabilities/api_extension.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -84,11 +84,10 @@ class Meta: class V2VulnerabilitySeveritySerializer(ModelSerializer): score = CharField(source="value") - reference = V2VulnerabilityReferenceSerializer() class Meta: model = VulnerabilitySeverity - fields = ("score", "scoring_system", "scoring_elements", "published_at", "reference") + fields = ("url", "score", "scoring_system", "scoring_elements", "published_at") class V2WeaknessSerializer(ModelSerializer): @@ -127,9 +126,9 @@ class V2VulnerabilitySerializer(ModelSerializer): aliases = SerializerMethodField("get_aliases") weaknesses = V2WeaknessSerializer(many=True, source="weaknesses_set") - scores = V2VulnerabilitySeveritySerializer(many=True, source="vulnerabilityseverity_set") references = V2VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") exploits = V2ExploitSerializer(many=True, source="weaknesses") + severities = V2VulnerabilitySeveritySerializer(many=True) def get_aliases(self, vulnerability): return vulnerability.aliases.only("alias").values_list("alias", flat=True) @@ -145,11 +144,11 @@ class Meta: "vulnerability_id", "aliases", "status", - "scores", "weaknesses", "summary", "exploits", "references", + "severities", ) @@ -238,8 +237,6 @@ class Meta: "qualifiers", "subpath", "purl", - # this hurts - "packagerelatedvulnerability__fix", ] def filter_purl(self, queryset, name, value): @@ -360,7 +357,7 @@ def get_queryset(self): .get_queryset() .prefetch_related( "weaknesses", - # "severities", + "severities", # "exploits", ) ) diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py new file mode 100644 index 000000000..2ab782d59 --- /dev/null +++ b/vulnerabilities/api_v2.py @@ -0,0 +1,505 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +from django_filters import rest_framework as filters +from drf_spectacular.utils import OpenApiParameter +from drf_spectacular.utils import extend_schema +from drf_spectacular.utils import extend_schema_view +from packageurl import PackageURL +from rest_framework import serializers +from rest_framework import status +from rest_framework import viewsets +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.reverse import reverse + +from vulnerabilities.api import PackageFilterSet +from vulnerabilities.api import VulnerabilitySeveritySerializer +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import Weakness + + +class WeaknessV2Serializer(serializers.ModelSerializer): + cwe_id = serializers.CharField() + name = serializers.CharField() + description = serializers.CharField() + + class Meta: + model = Weakness + fields = ["cwe_id", "name", "description"] + + +class VulnerabilityReferenceV2Serializer(serializers.ModelSerializer): + url = serializers.CharField() + reference_type = serializers.CharField() + reference_id = serializers.CharField() + + class Meta: + model = VulnerabilityReference + fields = ["url", "reference_type", "reference_id"] + + +class VulnerabilitySeverityV2Serializer(serializers.ModelSerializer): + class Meta: + model = VulnerabilitySeverity + fields = ["url", "value", "scoring_system", "scoring_elements", "published_at"] + + def to_representation(self, instance): + data = super().to_representation(instance) + published_at = data.get("published_at", None) + if not published_at: + data.pop("published_at") + return data + + +class VulnerabilityV2Serializer(serializers.ModelSerializer): + aliases = serializers.SerializerMethodField() + weaknesses = WeaknessV2Serializer(many=True) + references = VulnerabilityReferenceV2Serializer(many=True, source="vulnerabilityreference_set") + severities = VulnerabilitySeverityV2Serializer(many=True) + exploitability = serializers.FloatField(read_only=True) + weighted_severity = serializers.FloatField(read_only=True) + risk_score = serializers.FloatField(read_only=True) + + class Meta: + model = Vulnerability + fields = [ + "vulnerability_id", + "aliases", + "summary", + "severities", + "weaknesses", + "references", + "exploitability", + "weighted_severity", + "risk_score", + ] + + def get_aliases(self, obj): + return [alias.alias for alias in obj.aliases.all()] + + +class VulnerabilityListSerializer(serializers.ModelSerializer): + url = serializers.SerializerMethodField() + + class Meta: + model = Vulnerability + fields = ["vulnerability_id", "url"] + + def get_url(self, obj): + request = self.context.get("request") + return reverse( + "vulnerability-v2-detail", + kwargs={"vulnerability_id": obj.vulnerability_id}, + request=request, + ) + + +@extend_schema_view( + list=extend_schema( + parameters=[ + OpenApiParameter( + name="vulnerability_id", + description="Filter by one or more vulnerability IDs", + required=False, + type={"type": "array", "items": {"type": "string"}}, + location=OpenApiParameter.QUERY, + ), + OpenApiParameter( + name="alias", + description="Filter by alias (CVE or other unique identifier)", + required=False, + type={"type": "array", "items": {"type": "string"}}, + location=OpenApiParameter.QUERY, + ), + ] + ) +) +class VulnerabilityV2ViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Vulnerability.objects.all() + serializer_class = VulnerabilityV2Serializer + lookup_field = "vulnerability_id" + + def get_queryset(self): + queryset = super().get_queryset() + vulnerability_ids = self.request.query_params.getlist("vulnerability_id") + aliases = self.request.query_params.getlist("alias") + + if vulnerability_ids: + queryset = queryset.filter(vulnerability_id__in=vulnerability_ids) + + if aliases: + queryset = queryset.filter(aliases__alias__in=aliases).distinct() + + return queryset + + def get_serializer_class(self): + if self.action == "list": + return VulnerabilityListSerializer + return super().get_serializer_class() + + def list(self, request, *args, **kwargs): + queryset = self.get_queryset() + vulnerability_ids = request.query_params.getlist("vulnerability_id") + + # If exactly one vulnerability_id is provided, return the serialized data + if len(vulnerability_ids) == 1: + try: + vulnerability = queryset.get(vulnerability_id=vulnerability_ids[0]) + serializer = self.get_serializer(vulnerability) + return Response(serializer.data) + except Vulnerability.DoesNotExist: + return Response({"detail": "Not found."}, status=404) + + # Otherwise, return a dictionary of vulnerabilities keyed by vulnerability_id + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + data = serializer.data + vulnerabilities = {item["vulnerability_id"]: item for item in data} + return self.get_paginated_response({"vulnerabilities": vulnerabilities}) + + serializer = self.get_serializer(queryset, many=True) + data = serializer.data + vulnerabilities = {item["vulnerability_id"]: item for item in data} + return Response({"vulnerabilities": vulnerabilities}) + + +class PackageV2Serializer(serializers.ModelSerializer): + purl = serializers.CharField(source="package_url") + risk_score = serializers.FloatField(read_only=True) + affected_by_vulnerabilities = serializers.SerializerMethodField() + fixing_vulnerabilities = serializers.SerializerMethodField() + next_non_vulnerable_version = serializers.CharField(read_only=True) + latest_non_vulnerable_version = serializers.CharField(read_only=True) + + class Meta: + model = Package + fields = [ + "purl", + "affected_by_vulnerabilities", + "fixing_vulnerabilities", + "next_non_vulnerable_version", + "latest_non_vulnerable_version", + "risk_score", + ] + + def get_affected_by_vulnerabilities(self, obj): + return [vuln.vulnerability_id for vuln in obj.affected_by_vulnerabilities.all()] + + def get_fixing_vulnerabilities(self, obj): + # Ghost package should not fix any vulnerability. + if obj.is_ghost: + return [] + return [vuln.vulnerability_id for vuln in obj.fixing_vulnerabilities.all()] + + +class PackageurlListSerializer(serializers.Serializer): + purls = serializers.ListField( + child=serializers.CharField(), + allow_empty=False, + help_text="List of PackageURL strings in canonical form.", + ) + + +class PackageBulkSearchRequestSerializer(PackageurlListSerializer): + purl_only = serializers.BooleanField(required=False, default=False) + plain_purl = serializers.BooleanField(required=False, default=False) + + +class LookupRequestSerializer(serializers.Serializer): + purl = serializers.CharField( + required=True, + help_text="PackageURL strings in canonical form.", + ) + + +class PackageV2FilterSet(filters.FilterSet): + affected_by_vulnerability = filters.CharFilter( + field_name="affected_by_vulnerabilities__vulnerability_id" + ) + fixing_vulnerability = filters.CharFilter(field_name="fixing_vulnerabilities__vulnerability_id") + purl = filters.CharFilter(field_name="package_url") + + +class PackageV2ViewSet(viewsets.ReadOnlyModelViewSet): + queryset = Package.objects.all() + serializer_class = PackageV2Serializer + filter_backends = (filters.DjangoFilterBackend,) + filterset_class = PackageV2FilterSet + + def get_queryset(self): + queryset = super().get_queryset() + package_purls = self.request.query_params.getlist("purl") + affected_by_vulnerability = self.request.query_params.get("affected_by_vulnerability") + fixing_vulnerability = self.request.query_params.get("fixing_vulnerability") + + if package_purls: + queryset = queryset.filter(package_url__in=package_purls) + if affected_by_vulnerability: + queryset = queryset.filter( + affected_by_vulnerabilities__vulnerability_id=affected_by_vulnerability + ) + if fixing_vulnerability: + queryset = queryset.filter( + fixing_vulnerabilities__vulnerability_id=fixing_vulnerability + ) + return queryset.with_is_vulnerable() + + def list(self, request, *args, **kwargs): + queryset = self.get_queryset() + + # Apply pagination + page = self.paginate_queryset(queryset) + if page is not None: + # Collect only vulnerabilities for packages in the current page + vulnerabilities = set() + for package in page: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + # Serialize the vulnerabilities with vulnerability_id as keys + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data + for vuln in vulnerabilities + } + + # Serialize the current page of packages + serializer = self.get_serializer(page, many=True) + data = serializer.data + + # Use 'self.get_paginated_response' to include pagination data + return self.get_paginated_response( + {"vulnerabilities": vulnerability_data, "packages": data} + ) + + # If pagination is not applied, collect vulnerabilities for all packages + vulnerabilities = set() + for package in queryset: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data for vuln in vulnerabilities + } + + # Serialize all packages when pagination is not applied + serializer = self.get_serializer(queryset, many=True) + data = serializer.data + return Response({"vulnerabilities": vulnerability_data, "packages": data}) + + @extend_schema( + request=PackageurlListSerializer, + responses={200: PackageV2Serializer(many=True)}, + ) + @action( + detail=False, + methods=["post"], + serializer_class=PackageurlListSerializer, + filter_backends=[], + pagination_class=None, + ) + def bulk_lookup(self, request): + """ + Return the response for exact PackageURLs requested for. + """ + serializer = self.serializer_class(data=request.data) + if not serializer.is_valid(): + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={ + "error": serializer.errors, + "message": "A non-empty 'purls' list of PURLs is required.", + }, + ) + validated_data = serializer.validated_data + purls = validated_data.get("purls") + + # Fetch packages matching the provided purls + packages = Package.objects.for_purls(purls).with_is_vulnerable() + + # Collect vulnerabilities associated with these packages + vulnerabilities = set() + for package in packages: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + # Serialize vulnerabilities with vulnerability_id as keys + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data for vuln in vulnerabilities + } + + # Serialize packages + package_data = PackageV2Serializer( + packages, + many=True, + context={"request": request}, + ).data + + return Response( + { + "vulnerabilities": vulnerability_data, + "packages": package_data, + } + ) + + @extend_schema( + request=PackageBulkSearchRequestSerializer, + responses={200: PackageV2Serializer(many=True)}, + ) + @action( + detail=False, + methods=["post"], + serializer_class=PackageBulkSearchRequestSerializer, + filter_backends=[], + pagination_class=None, + ) + def bulk_search(self, request): + """ + Lookup for vulnerable packages using many Package URLs at once. + """ + serializer = self.serializer_class(data=request.data) + if not serializer.is_valid(): + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={ + "error": serializer.errors, + "message": "A non-empty 'purls' list of PURLs is required.", + }, + ) + validated_data = serializer.validated_data + purls = validated_data.get("purls") + purl_only = validated_data.get("purl_only", False) + plain_purl = validated_data.get("plain_purl", False) + + if plain_purl: + purl_objects = [PackageURL.from_string(purl) for purl in purls] + plain_purl_objects = [ + PackageURL( + type=purl.type, + namespace=purl.namespace, + name=purl.name, + version=purl.version, + ) + for purl in purl_objects + ] + plain_purls = [str(purl) for purl in plain_purl_objects] + + query = ( + Package.objects.filter(plain_package_url__in=plain_purls) + .order_by("plain_package_url") + .distinct("plain_package_url") + .with_is_vulnerable() + ) + + packages = query + + # Collect vulnerabilities associated with these packages + vulnerabilities = set() + for package in packages: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data + for vuln in vulnerabilities + } + + if not purl_only: + package_data = PackageV2Serializer( + packages, many=True, context={"request": request} + ).data + return Response( + { + "vulnerabilities": vulnerability_data, + "packages": package_data, + } + ) + + # Using order by and distinct because there will be + # many fully qualified purl for a single plain purl + vulnerable_purls = query.vulnerable().only("plain_package_url") + vulnerable_purls = [str(package.plain_package_url) for package in vulnerable_purls] + return Response(data=vulnerable_purls) + + query = Package.objects.filter(package_url__in=purls).distinct().with_is_vulnerable() + packages = query + + # Collect vulnerabilities associated with these packages + vulnerabilities = set() + for package in packages: + vulnerabilities.update(package.affected_by_vulnerabilities.all()) + vulnerabilities.update(package.fixing_vulnerabilities.all()) + + vulnerability_data = { + vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data for vuln in vulnerabilities + } + + if not purl_only: + package_data = PackageV2Serializer( + packages, many=True, context={"request": request} + ).data + return Response( + { + "vulnerabilities": vulnerability_data, + "packages": package_data, + } + ) + + vulnerable_purls = query.vulnerable().only("package_url") + vulnerable_purls = [str(package.package_url) for package in vulnerable_purls] + return Response(data=vulnerable_purls) + + @action(detail=False, methods=["get"]) + def all(self, request): + """ + Return a list of Package URLs of vulnerable packages. + """ + vulnerable_purls = ( + Package.objects.vulnerable() + .only("package_url") + .order_by("package_url") + .distinct() + .values_list("package_url", flat=True) + ) + return Response(vulnerable_purls) + + @extend_schema( + request=LookupRequestSerializer, + responses={200: PackageV2Serializer(many=True)}, + ) + @action( + detail=False, + methods=["post"], + serializer_class=LookupRequestSerializer, + filter_backends=[], + pagination_class=None, + ) + def lookup(self, request): + """ + Return the response for exact PackageURL requested for. + """ + serializer = self.serializer_class(data=request.data) + if not serializer.is_valid(): + return Response( + status=status.HTTP_400_BAD_REQUEST, + data={ + "error": serializer.errors, + "message": "A 'purl' is required.", + }, + ) + validated_data = serializer.validated_data + purl = validated_data.get("purl") + + qs = self.get_queryset().for_purls([purl]).with_is_vulnerable() + return Response(PackageV2Serializer(qs, many=True, context={"request": request}).data) diff --git a/vulnerabilities/forms.py b/vulnerabilities/forms.py index 793936c72..a00885637 100644 --- a/vulnerabilities/forms.py +++ b/vulnerabilities/forms.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py index 5e5937951..0dcafda10 100644 --- a/vulnerabilities/import_runner.py +++ b/vulnerabilities/import_runner.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -21,9 +21,10 @@ from vulnerabilities.improver import Inference from vulnerabilities.improvers.default import DefaultImporter from vulnerabilities.models import Advisory +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityChangeLog from vulnerabilities.models import VulnerabilityReference @@ -179,54 +180,55 @@ def process_inferences(inferences: List[Inference], advisory: Advisory, improver reference_id=ref.reference_id, url=ref.url, ) - if not reference: - continue - VulnerabilityRelatedReference.objects.update_or_create( - reference=reference, - vulnerability=vulnerability, - ) + if reference: + VulnerabilityRelatedReference.objects.update_or_create( + reference=reference, + vulnerability=vulnerability, + ) updated = False for severity in ref.severities: try: published_at = str(severity.published_at) if severity.published_at else None - _vs, updated = VulnerabilitySeverity.objects.update_or_create( + ( + vulnerability_severity, + updated, + ) = VulnerabilitySeverity.objects.update_or_create( scoring_system=severity.system.identifier, - reference=reference, + url=ref.url, + value=severity.value, + scoring_elements=severity.scoring_elements, defaults={ - "value": str(severity.value), - "scoring_elements": str(severity.scoring_elements), "published_at": published_at, }, ) + vulnerability.severities.add(vulnerability_severity) except: logger.error( f"Failed to create VulnerabilitySeverity for: {severity} with error:\n{traceback_format_exc()}" ) if updated: logger.info( - f"Severity updated for reference {ref!r} to value: {severity.value!r} " + f"Severity updated for reference {ref.url!r} to value: {severity.value!r} " f"and scoring_elements: {severity.scoring_elements!r}" ) for affected_purl in inference.affected_purls or []: vulnerable_package, _ = Package.objects.get_or_create_from_purl(purl=affected_purl) - PackageRelatedVulnerability( + AffectedByPackageRelatedVulnerability( vulnerability=vulnerability, package=vulnerable_package, created_by=improver_name, confidence=inference.confidence, - fix=False, ).update_or_create(advisory=advisory) if inference.fixed_purl: fixed_package, _ = Package.objects.get_or_create_from_purl(purl=inference.fixed_purl) - PackageRelatedVulnerability( + FixingPackageRelatedVulnerability( vulnerability=vulnerability, package=fixed_package, created_by=improver_name, confidence=inference.confidence, - fix=True, ).update_or_create(advisory=advisory) if inference.weaknesses and vulnerability: diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index 005534512..c5a5c5743 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -223,18 +223,30 @@ def from_dict(cls, affected_pkg: dict): """ package = PackageURL(**affected_pkg["package"]) affected_version_range = None - if ( - affected_pkg["affected_version_range"] - and affected_pkg["affected_version_range"] != "None" - ): - affected_version_range = VersionRange.from_string( - affected_pkg["affected_version_range"] - ) + affected_range = affected_pkg["affected_version_range"] + + # TODO: "None" is a likely bug + if affected_range and affected_range != "None": + try: + affected_version_range = VersionRange.from_string(affected_range) + except: + tb = traceback.format_exc() + logger.error( + f"Cannot create AffectedPackage with invalid or unknown range: {affected_pkg!r} with error: {tb!r}" + ) + return + fixed_version = affected_pkg["fixed_version"] if fixed_version and affected_version_range: # TODO: revisit after https://github.com/nexB/univers/issues/10 fixed_version = affected_version_range.version_class(fixed_version) + if not fixed_version and not affected_version_range: + logger.error( + f"Cannot create AffectedPackage without fixed version or affected range: {affected_pkg!r}" + ) + return + return cls( package=package, affected_version_range=affected_version_range, @@ -295,7 +307,9 @@ def from_dict(cls, advisory_data): "aliases": advisory_data["aliases"], "summary": advisory_data["summary"], "affected_packages": [ - AffectedPackage.from_dict(pkg) for pkg in advisory_data["affected_packages"] + AffectedPackage.from_dict(pkg) + for pkg in advisory_data["affected_packages"] + if pkg is not None ], "references": [Reference.from_dict(ref) for ref in advisory_data["references"]], "date_published": datetime.datetime.fromisoformat(date_published) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 10eb3686f..b40bdfc3a 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -27,7 +27,6 @@ from vulnerabilities.importers import oss_fuzz from vulnerabilities.importers import postgresql from vulnerabilities.importers import project_kb_msr2019 -from vulnerabilities.importers import pysec from vulnerabilities.importers import redhat from vulnerabilities.importers import retiredotnet from vulnerabilities.importers import ruby @@ -43,9 +42,9 @@ from vulnerabilities.pipelines import npm_importer from vulnerabilities.pipelines import nvd_importer from vulnerabilities.pipelines import pypa_importer +from vulnerabilities.pipelines import pysec_importer IMPORTERS_REGISTRY = [ - pysec.PyPIImporter, alpine_linux.AlpineImporter, openssl.OpensslImporter, redhat.RedhatImporter, @@ -80,6 +79,7 @@ gitlab_importer.GitLabImporterPipeline, github_importer.GitHubAPIImporterPipeline, nvd_importer.NVDImporterPipeline, + pysec_importer.PyPIImporterPipeline, ] IMPORTERS_REGISTRY = { diff --git a/vulnerabilities/importers/alpine_linux.py b/vulnerabilities/importers/alpine_linux.py index 9ad2a79b5..db169184e 100644 --- a/vulnerabilities/importers/alpine_linux.py +++ b/vulnerabilities/importers/alpine_linux.py @@ -4,7 +4,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py index 10a99dd02..3dc286803 100644 --- a/vulnerabilities/importers/apache_httpd.py +++ b/vulnerabilities/importers/apache_httpd.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/apache_kafka.py b/vulnerabilities/importers/apache_kafka.py index 89de85219..27c244b2a 100644 --- a/vulnerabilities/importers/apache_kafka.py +++ b/vulnerabilities/importers/apache_kafka.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py index 50a02a0ec..9d371ee7d 100644 --- a/vulnerabilities/importers/apache_tomcat.py +++ b/vulnerabilities/importers/apache_tomcat.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py index 2e9ef6a87..640fb24dc 100644 --- a/vulnerabilities/importers/archlinux.py +++ b/vulnerabilities/importers/archlinux.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/curl.py b/vulnerabilities/importers/curl.py index 84ab4c82f..a7f5e86fa 100644 --- a/vulnerabilities/importers/curl.py +++ b/vulnerabilities/importers/curl.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py index 94057675f..e29c9b788 100644 --- a/vulnerabilities/importers/debian.py +++ b/vulnerabilities/importers/debian.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/debian_oval.py b/vulnerabilities/importers/debian_oval.py index aa3d6917d..f5a747a11 100644 --- a/vulnerabilities/importers/debian_oval.py +++ b/vulnerabilities/importers/debian_oval.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py index 4fd492a92..3fe0ec15b 100644 --- a/vulnerabilities/importers/elixir_security.py +++ b/vulnerabilities/importers/elixir_security.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import urllib.parse as urlparse diff --git a/vulnerabilities/importers/epss.py b/vulnerabilities/importers/epss.py index 83822fa5d..982229e09 100644 --- a/vulnerabilities/importers/epss.py +++ b/vulnerabilities/importers/epss.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import csv diff --git a/vulnerabilities/importers/fireeye.py b/vulnerabilities/importers/fireeye.py index 453afe10d..34daf97e0 100644 --- a/vulnerabilities/importers/fireeye.py +++ b/vulnerabilities/importers/fireeye.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py index 2c91f7f2f..2f569cdf1 100644 --- a/vulnerabilities/importers/gentoo.py +++ b/vulnerabilities/importers/gentoo.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/github_osv.py b/vulnerabilities/importers/github_osv.py index bef06a8af..f0490044e 100644 --- a/vulnerabilities/importers/github_osv.py +++ b/vulnerabilities/importers/github_osv.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json diff --git a/vulnerabilities/importers/gsd.py b/vulnerabilities/importers/gsd.py index 50d3aa8f6..4d69bd63a 100644 --- a/vulnerabilities/importers/gsd.py +++ b/vulnerabilities/importers/gsd.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py index 9341a76de..8f9f6334a 100644 --- a/vulnerabilities/importers/istio.py +++ b/vulnerabilities/importers/istio.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging diff --git a/vulnerabilities/importers/kaybee.py b/vulnerabilities/importers/kaybee.py index 1b908e4b5..ccee4a68f 100644 --- a/vulnerabilities/importers/kaybee.py +++ b/vulnerabilities/importers/kaybee.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/mattermost.py b/vulnerabilities/importers/mattermost.py index 62eddbeb0..a422ea32a 100644 --- a/vulnerabilities/importers/mattermost.py +++ b/vulnerabilities/importers/mattermost.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py index 11667badc..8eea10370 100644 --- a/vulnerabilities/importers/mozilla.py +++ b/vulnerabilities/importers/mozilla.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/openssl.py b/vulnerabilities/importers/openssl.py index ca69436c9..b71206418 100644 --- a/vulnerabilities/importers/openssl.py +++ b/vulnerabilities/importers/openssl.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/oss_fuzz.py b/vulnerabilities/importers/oss_fuzz.py index e81f06fc2..63b879990 100644 --- a/vulnerabilities/importers/oss_fuzz.py +++ b/vulnerabilities/importers/oss_fuzz.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging diff --git a/vulnerabilities/importers/osv.py b/vulnerabilities/importers/osv.py index 1ee45a1be..90f4200e8 100644 --- a/vulnerabilities/importers/osv.py +++ b/vulnerabilities/importers/osv.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/postgresql.py b/vulnerabilities/importers/postgresql.py index ee6de3976..70ab1bfe9 100644 --- a/vulnerabilities/importers/postgresql.py +++ b/vulnerabilities/importers/postgresql.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py index e099d3f36..a006b1353 100644 --- a/vulnerabilities/importers/project_kb_msr2019.py +++ b/vulnerabilities/importers/project_kb_msr2019.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/pysec.py b/vulnerabilities/importers/pysec.py deleted file mode 100644 index 058747463..000000000 --- a/vulnerabilities/importers/pysec.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import json -import logging -from io import BytesIO -from typing import Iterable -from zipfile import ZipFile - -import requests - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importers.osv import parse_advisory_data - -logger = logging.getLogger(__name__) - - -class PyPIImporter(Importer): - license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" - spdx_license_expression = "CC-BY-4.0" - importer_name = "PyPI Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - """ - Yield AdvisoryData using a zipped data dump of OSV data - """ - url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" - response = requests.get(url).content - with ZipFile(BytesIO(response)) as zip_file: - for file_name in zip_file.namelist(): - if not file_name.startswith("PYSEC-"): - logger.error(f"Unsupported PyPI advisory data file: {file_name}") - continue - with zip_file.open(file_name) as f: - vul_info = json.load(f) - yield parse_advisory_data( - raw_data=vul_info, supported_ecosystems=["pypi"], advisory_url=url - ) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py index a2cc1940b..68e3d5062 100644 --- a/vulnerabilities/importers/redhat.py +++ b/vulnerabilities/importers/redhat.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/retiredotnet.py b/vulnerabilities/importers/retiredotnet.py index 0ed580ba1..139ecd1af 100644 --- a/vulnerabilities/importers/retiredotnet.py +++ b/vulnerabilities/importers/retiredotnet.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py index 6a3b5f3f1..268419587 100644 --- a/vulnerabilities/importers/ruby.py +++ b/vulnerabilities/importers/ruby.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py index a1e97c277..c61907a82 100644 --- a/vulnerabilities/importers/rust.py +++ b/vulnerabilities/importers/rust.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/suse_backports.py b/vulnerabilities/importers/suse_backports.py index 6d661ef3f..e7863e7e7 100644 --- a/vulnerabilities/importers/suse_backports.py +++ b/vulnerabilities/importers/suse_backports.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/suse_oval.py b/vulnerabilities/importers/suse_oval.py index 5ac35e23e..0722682f7 100644 --- a/vulnerabilities/importers/suse_oval.py +++ b/vulnerabilities/importers/suse_oval.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/suse_scores.py b/vulnerabilities/importers/suse_scores.py index f43e69576..b7f2089ac 100644 --- a/vulnerabilities/importers/suse_scores.py +++ b/vulnerabilities/importers/suse_scores.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/ubuntu.py b/vulnerabilities/importers/ubuntu.py index 646b40028..e47515b93 100644 --- a/vulnerabilities/importers/ubuntu.py +++ b/vulnerabilities/importers/ubuntu.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py index 6bef117f9..1aa247ec6 100644 --- a/vulnerabilities/importers/ubuntu_usn.py +++ b/vulnerabilities/importers/ubuntu_usn.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py index 2b28a9771..a0cafa324 100644 --- a/vulnerabilities/importers/xen.py +++ b/vulnerabilities/importers/xen.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py index cb8fb6a04..452e1e2f6 100644 --- a/vulnerabilities/improve_runner.py +++ b/vulnerabilities/improve_runner.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -17,9 +17,10 @@ from vulnerabilities.importers import IMPORTERS_REGISTRY from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityChangeLog from vulnerabilities.models import VulnerabilityReference @@ -135,12 +136,11 @@ def process_inferences( vulnerable_package, created = Package.objects.get_or_create_from_purl( purl=affected_purl ) - PackageRelatedVulnerability( + AffectedByPackageRelatedVulnerability( vulnerability=vulnerability, package=vulnerable_package, created_by=improver_name, confidence=inference.confidence, - fix=False, ).update_or_create( advisory=advisory, ) @@ -149,12 +149,11 @@ def process_inferences( fixed_package, created = Package.objects.get_or_create_from_purl( purl=inference.fixed_purl ) - PackageRelatedVulnerability( + FixingPackageRelatedVulnerability( vulnerability=vulnerability, package=fixed_package, created_by=improver_name, confidence=inference.confidence, - fix=True, ).update_or_create( advisory=advisory, ) diff --git a/vulnerabilities/improver.py b/vulnerabilities/improver.py index b8caeb9ed..0d5e7f13d 100644 --- a/vulnerabilities/improver.py +++ b/vulnerabilities/improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/improvers/__init__.py b/vulnerabilities/improvers/__init__.py index 138488faa..13746dcf6 100644 --- a/vulnerabilities/improvers/__init__.py +++ b/vulnerabilities/improvers/__init__.py @@ -3,13 +3,15 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from vulnerabilities.improvers import valid_versions from vulnerabilities.improvers import vulnerability_status from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipelines import compute_package_risk +from vulnerabilities.pipelines import compute_package_version_rank from vulnerabilities.pipelines import enhance_with_exploitdb from vulnerabilities.pipelines import enhance_with_kev from vulnerabilities.pipelines import enhance_with_metasploit @@ -39,6 +41,8 @@ enhance_with_kev.VulnerabilityKevPipeline, enhance_with_metasploit.MetasploitImproverPipeline, enhance_with_exploitdb.ExploitDBImproverPipeline, + compute_package_risk.ComputePackageRiskPipeline, + compute_package_version_rank.ComputeVersionRankPipeline, ] IMPROVERS_REGISTRY = { diff --git a/vulnerabilities/improvers/add_missing_refid.py b/vulnerabilities/improvers/add_missing_refid.py index 80e7f6eeb..d3d5efa72 100644 --- a/vulnerabilities/improvers/add_missing_refid.py +++ b/vulnerabilities/improvers/add_missing_refid.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py index 720a143e8..f2e9009e8 100644 --- a/vulnerabilities/improvers/default.py +++ b/vulnerabilities/improvers/default.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -105,6 +105,8 @@ def get_exact_purls(affected_package: AffectedPackage) -> Tuple[List[PackageURL] ... ) >>> assert expected == got """ + if not affected_package: + return [], [] try: vr = affected_package.affected_version_range diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py index 3e9d8d6d9..dc994ced7 100644 --- a/vulnerabilities/improvers/valid_versions.py +++ b/vulnerabilities/improvers/valid_versions.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/improvers/vulnerability_status.py b/vulnerabilities/improvers/vulnerability_status.py index 353cca54c..214e6dc35 100644 --- a/vulnerabilities/improvers/vulnerability_status.py +++ b/vulnerabilities/improvers/vulnerability_status.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/lib_oval.py b/vulnerabilities/lib_oval.py index 0b178d47c..30bc7f0c0 100644 --- a/vulnerabilities/lib_oval.py +++ b/vulnerabilities/lib_oval.py @@ -5,7 +5,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/management/commands/create_api_user.py b/vulnerabilities/management/commands/create_api_user.py index db471cac4..86f6bb486 100644 --- a/vulnerabilities/management/commands/create_api_user.py +++ b/vulnerabilities/management/commands/create_api_user.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/management/commands/export.py b/vulnerabilities/management/commands/export.py index 848ffc3cc..08685e33d 100644 --- a/vulnerabilities/management/commands/export.py +++ b/vulnerabilities/management/commands/export.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging @@ -22,20 +22,12 @@ def serialize_severity(sev): - # inlines refs - ref = sev.reference - sevref = { - "url": ref.url, - "reference_type": ref.reference_type, - "reference_id": ref.reference_id, - } - return { "score": sev.value, "scoring_system": sev.scoring_system, "scoring_elements": sev.scoring_elements, "published_at": sev.published_at, - "reference": sevref, + "url": sev.url, } @@ -44,7 +36,7 @@ def serialize_vulnerability(vuln): Return a plain data mapping seralized from ``vuln`` Vulnerability instance. """ aliases = list(vuln.aliases.values_list("alias", flat=True)) - severities = [serialize_severity(sev) for sev in vuln.severities] + severities = [serialize_severity(sev) for sev in vuln.severities.all()] weaknesses = [wkns.cwe for wkns in vuln.weaknesses.all()] references = list( @@ -116,7 +108,7 @@ def export_data(self, base_path: Path): } package_vulnerabilities.append(package_data) - for vuln in pkg_version.vulnerabilities.all(): + for vuln in pkg_version.vulnerabilities: vcid = vuln.vulnerability_id # do not write twice the same file if vcid in seen_vcid: @@ -158,10 +150,14 @@ def packages_by_type_ns_name(): qs = ( Package.objects.order_by("type", "namespace", "name", "version") .prefetch_related( - "vulnerabilities", - "vulnerabilities__references", - "vulnerabilities__weaknesses", - "vulnerabilities__references__vulnerabilityseverity_set", + "affected_by_vulnerabilities", + "affected_by_vulnerabilities__references", + "affected_by_vulnerabilities__weaknesses", + "affected_by_vulnerabilities__severities", + "fixing_vulnerabilities", + "fixing_vulnerabilities__references", + "fixing_vulnerabilities__weaknesses", + "fixing_vulnerabilities__severities", ) .paginated() ) diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index 7779ba833..f4876b11a 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import traceback diff --git a/vulnerabilities/management/commands/improve.py b/vulnerabilities/management/commands/improve.py index 2bed1c47a..10ba07a27 100644 --- a/vulnerabilities/management/commands/improve.py +++ b/vulnerabilities/management/commands/improve.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/management/commands/purl2cpe.py b/vulnerabilities/management/commands/purl2cpe.py index 277348615..dc614242c 100644 --- a/vulnerabilities/management/commands/purl2cpe.py +++ b/vulnerabilities/management/commands/purl2cpe.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/middleware/ban_user_agent.py b/vulnerabilities/middleware/ban_user_agent.py index 6aafc490c..34332194d 100644 --- a/vulnerabilities/middleware/ban_user_agent.py +++ b/vulnerabilities/middleware/ban_user_agent.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/migrations/0038_remove_corrupted_advisories_with_incorrect_refs_and_severity.py b/vulnerabilities/migrations/0038_remove_corrupted_advisories_with_incorrect_refs_and_severity.py index 61cfc0531..17a86f58e 100644 --- a/vulnerabilities/migrations/0038_remove_corrupted_advisories_with_incorrect_refs_and_severity.py +++ b/vulnerabilities/migrations/0038_remove_corrupted_advisories_with_incorrect_refs_and_severity.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/migrations/0041_remove_vulns_with_empty_aliases.py b/vulnerabilities/migrations/0041_remove_vulns_with_empty_aliases.py index d2c44c280..7c3efe166 100644 --- a/vulnerabilities/migrations/0041_remove_vulns_with_empty_aliases.py +++ b/vulnerabilities/migrations/0041_remove_vulns_with_empty_aliases.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py b/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py index 41294f20a..ccec92bde 100644 --- a/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py +++ b/vulnerabilities/migrations/0070_alter_advisory_created_by_and_more.py @@ -36,4 +36,4 @@ class Migration(migrations.Migration): max_length=100, ), ), - ] + ] \ No newline at end of file diff --git a/vulnerabilities/migrations/0071_auto_20241007_1044.py b/vulnerabilities/migrations/0071_auto_20241007_1044.py new file mode 100644 index 000000000..182c2f532 --- /dev/null +++ b/vulnerabilities/migrations/0071_auto_20241007_1044.py @@ -0,0 +1,197 @@ +from django.db import migrations, models +import django.db.models.deletion +from aboutcode.pipeline import LoopProgress + +def split_packagerelatedvulnerability(apps, schema_editor): + PackageRelatedVulnerability = apps.get_model('vulnerabilities', 'PackageRelatedVulnerability') + FixingPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'FixingPackageRelatedVulnerability') + AffectedByPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'AffectedByPackageRelatedVulnerability') + + obsolete_package_relation_query = PackageRelatedVulnerability.objects.all() + obsolete_package_relation_query_count = obsolete_package_relation_query.count() + print(f"\nMigrating {obsolete_package_relation_query_count:,d} old package vulnerability relationship.") + + progress = LoopProgress( + total_iterations=obsolete_package_relation_query_count, + progress_step=1, + logger=print, + ) + fixing_packages = [] + affected_packages = [] + for prv in progress.iter(obsolete_package_relation_query.iterator(chunk_size=10000)): + if prv.fix: + fp = FixingPackageRelatedVulnerability( + package=prv.package, + vulnerability=prv.vulnerability, + created_by=prv.created_by, + confidence=prv.confidence, + ) + fixing_packages.append(fp) + else: + ap = AffectedByPackageRelatedVulnerability( + package=prv.package, + vulnerability=prv.vulnerability, + created_by=prv.created_by, + confidence=prv.confidence, + ) + affected_packages.append(ap) + + FixingPackageRelatedVulnerability.objects.bulk_create(fixing_packages, batch_size=10000) + AffectedByPackageRelatedVulnerability.objects.bulk_create(affected_packages, batch_size=10000) + +def reverse_migration(apps, schema_editor): + FixingPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'FixingPackageRelatedVulnerability') + AffectedByPackageRelatedVulnerability = apps.get_model('vulnerabilities', 'AffectedByPackageRelatedVulnerability') + PackageRelatedVulnerability = apps.get_model('vulnerabilities', 'PackageRelatedVulnerability') + + fixing_package_relation_query = FixingPackageRelatedVulnerability.objects.all() + fixing_package_relation_query_count = fixing_package_relation_query.count() + print(f"\nMigrating {fixing_package_relation_query_count:,d} FixingPackage to old relationship.") + + progress = LoopProgress( + total_iterations=fixing_package_relation_query_count, + progress_step=1, + logger=print, + ) + for fpv in progress.iter(fixing_package_relation_query.iterator(chunk_size=10000)): + PackageRelatedVulnerability.objects.create( + package=fpv.package, + vulnerability=fpv.vulnerability, + created_by=fpv.created_by, + confidence=fpv.confidence, + fix=True, + ) + + affected_package_relation_query = AffectedByPackageRelatedVulnerability.objects.all() + affected_package_relation_query_count = affected_package_relation_query.count() + print(f"\nMigrating {affected_package_relation_query_count:,d} AffectedPackage to old relationship.") + + progress = LoopProgress( + total_iterations=affected_package_relation_query_count, + progress_step=1, + logger=print, + ) + for apv in progress.iter(affected_package_relation_query.iterator(chunk_size=10000)): + PackageRelatedVulnerability.objects.create( + package=apv.package, + vulnerability=apv.vulnerability, + created_by=apv.created_by, + confidence=apv.confidence, + fix=False, + ) + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0070_alter_advisory_created_by_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="advisory", + name="created_by", + field=models.CharField( + help_text="Fully qualified name of the importer prefixed with themodule name importing the advisory. Eg:vulnerabilities.pipeline.nginx_importer.NginxImporterPipeline", + max_length=100, + ), + ), + migrations.CreateModel( + name="FixingPackageRelatedVulnerability", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "created_by", + models.CharField( + blank=True, + help_text="Fully qualified name of the improver prefixed with the module name responsible for creating this relation. Eg: vulnerabilities.importers.nginx.NginxBasicImprover", + max_length=100, + ), + ), + ( + "confidence", + models.PositiveIntegerField( + default=100, + help_text="Confidence score for this relation", + validators=[ + django.core.validators.MinValueValidator(0), + django.core.validators.MaxValueValidator(100), + ], + ), + ), + ( + "package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.package" + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="vulnerabilities.vulnerability", + ), + ), + ], + options={ + "verbose_name_plural": "Fixing Package Related Vulnerabilities", + "ordering": ["package", "vulnerability"], + "abstract": False, + "unique_together": {("package", "vulnerability")}, + }, + ), + migrations.CreateModel( + name="AffectedByPackageRelatedVulnerability", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ( + "created_by", + models.CharField( + blank=True, + help_text="Fully qualified name of the improver prefixed with the module name responsible for creating this relation. Eg: vulnerabilities.importers.nginx.NginxBasicImprover", + max_length=100, + ), + ), + ( + "confidence", + models.PositiveIntegerField( + default=100, + help_text="Confidence score for this relation", + validators=[ + django.core.validators.MinValueValidator(0), + django.core.validators.MaxValueValidator(100), + ], + ), + ), + ( + "package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="vulnerabilities.package" + ), + ), + ( + "vulnerability", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="vulnerabilities.vulnerability", + ), + ), + ], + options={ + "verbose_name_plural": "Affected By Package Related Vulnerabilities", + "ordering": ["package", "vulnerability"], + "abstract": False, + "unique_together": {("package", "vulnerability")}, + }, + ), + migrations.RunPython(split_packagerelatedvulnerability, reverse_migration), + ] diff --git a/vulnerabilities/migrations/0072_remove_package_vulnerabilities_and_more.py b/vulnerabilities/migrations/0072_remove_package_vulnerabilities_and_more.py new file mode 100644 index 000000000..cf15473af --- /dev/null +++ b/vulnerabilities/migrations/0072_remove_package_vulnerabilities_and_more.py @@ -0,0 +1,46 @@ +# Generated by Django 4.2.15 on 2024-10-07 12:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0071_auto_20241007_1044"), + ] + + operations = [ + migrations.RemoveField( + model_name="package", + name="vulnerabilities", + ), + migrations.RemoveField( + model_name="vulnerability", + name="packages", + ), + migrations.AddField( + model_name="package", + name="affected_by_vulnerabilities", + field=models.ManyToManyField( + through="vulnerabilities.AffectedByPackageRelatedVulnerability", + to="vulnerabilities.vulnerability", + ), + ), + migrations.AddField( + model_name="vulnerability", + name="affecting_packages", + field=models.ManyToManyField( + through="vulnerabilities.AffectedByPackageRelatedVulnerability", + to="vulnerabilities.package", + ), + ), + migrations.AddField( + model_name="vulnerability", + name="fixed_by_packages", + field=models.ManyToManyField( + related_name="fixing_vulnerabilities", + through="vulnerabilities.FixingPackageRelatedVulnerability", + to="vulnerabilities.package", + ), + ), + ] diff --git a/vulnerabilities/migrations/0073_delete_packagerelatedvulnerability.py b/vulnerabilities/migrations/0073_delete_packagerelatedvulnerability.py new file mode 100644 index 000000000..a99147a16 --- /dev/null +++ b/vulnerabilities/migrations/0073_delete_packagerelatedvulnerability.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.15 on 2024-10-15 10:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0072_remove_package_vulnerabilities_and_more"), + ] + + operations = [ + migrations.DeleteModel( + name="PackageRelatedVulnerability", + ), + ] diff --git a/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py b/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py new file mode 100644 index 000000000..d0e73181a --- /dev/null +++ b/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py @@ -0,0 +1,37 @@ +# Generated by Django 4.2.16 on 2024-10-24 13:51 + +from django.db import migrations + +""" +Update the created_by field on Advisory from the old qualified_name +to the new pipeline_id. +""" + + +def update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by="vulnerabilities.importers.pysec.PyPIImporter").update( + created_by=PyPIImporterPipeline.pipeline_id + ) + + +def reverse_update_created_by(apps, schema_editor): + from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline + + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter(created_by=PyPIImporterPipeline.pipeline_id).update( + created_by="vulnerabilities.importers.pysec.PyPIImporter" + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0073_delete_packagerelatedvulnerability"), + ] + + operations = [ + migrations.RunPython(update_created_by, reverse_code=reverse_update_created_by), + ] diff --git a/vulnerabilities/migrations/0075_package_risk_score.py b/vulnerabilities/migrations/0075_package_risk_score.py new file mode 100644 index 000000000..72827ae63 --- /dev/null +++ b/vulnerabilities/migrations/0075_package_risk_score.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.16 on 2024-10-29 10:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0074_update_pysec_advisory_created_by"), + ] + + operations = [ + migrations.AddField( + model_name="package", + name="risk_score", + field=models.DecimalField( + decimal_places=2, + help_text="Risk score between 0.00 and 10.00, where higher values indicate greater vulnerability risk for the package.", + max_digits=4, + null=True, + ), + ), + ] diff --git a/vulnerabilities/migrations/0076_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0076_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..6ff76eb90 --- /dev/null +++ b/vulnerabilities/migrations/0076_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-08 07:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0075_package_risk_score"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.1.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.1.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0077_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0077_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..82862b3da --- /dev/null +++ b/vulnerabilities/migrations/0077_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-11 12:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0076_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.2.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.2.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0078_alter_vulnerabilityseverity_options_and_more.py b/vulnerabilities/migrations/0078_alter_vulnerabilityseverity_options_and_more.py new file mode 100644 index 000000000..c3e4aad38 --- /dev/null +++ b/vulnerabilities/migrations/0078_alter_vulnerabilityseverity_options_and_more.py @@ -0,0 +1,220 @@ +# Generated by Django 4.2.16 on 2024-11-01 12:40 + +from django.db import migrations, models +from aboutcode.pipeline import LoopProgress +from datetime import datetime +from datetime import timezone + + +class Migration(migrations.Migration): + """ + Add url field to the VulnerabilitySeverity model and remove reference foreignkey + relationship. Also, add new M2M 'severities' field in Vulnerability and + AffectedByPackageRelatedVulnerability relationship. + + To achieve this following model changes and data migrations are applied in chronological order: + - Make VulnerabilitySeverity reference field nullable (to make the migration reversible). + - Add 'severities' field to AffectedByPackageRelatedVulnerability. + - Add 'severities' field to Vulnerability. + - Add 'url' field to VulnerabilitySeverity. + - Data migration to remove corrupted SUSE scores. + - Data migration to enable reprocessing of old suse Advisory on next import. + - Data migration to populate new VulnerabilitySeverity url field using reference. + - Data migration to populate Vulnerability 'severities' M2M relationship. + - Delete VulnerabilitySeverity reference field. + """ + + def remove_inaccurate_suse_score(apps, schema_editor): + """ + Remove inaccurate suse severity scores. + See https://github.com/aboutcode-org/vulnerablecode/issues/1597 + """ + print(f"\nRemoving inaccurate suse severity scores.") + VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity") + VulnerabilitySeverity.objects.filter( + reference__url="https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml" + ).delete() + + def reverse_remove_inaccurate_suse_score(apps, schema_editor): + """Reverse data migration not needed for inaccurate severity scores.""" + pass + + def reprocess_suse_advisory_on_next_import(apps, schema_editor): + """Clear `date_imported` on old suse advisory to enable reprocessing on next suse import.""" + + print(f"\nEnable reprocess of old suse advisory on next import.") + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter( + created_by="vulnerabilities.importers.suse_scores.SUSESeverityScoreImporter" + ).update(date_imported=None) + + def reverse_reprocess_suse_advisory_on_next_import(apps, schema_editor): + """Populate `date_imported` on old suse advisory to prevent reprocessing on next suse import.""" + Advisory = apps.get_model("vulnerabilities", "Advisory") + Advisory.objects.filter( + created_by="vulnerabilities.importers.suse_scores.SUSESeverityScoreImporter" + ).update(date_imported=datetime.now(timezone.utc)) + + def populate_severity_model_with_url(apps, schema_editor): + """Populate the new VulnerabilitySeverity `url` field using reference url""" + + VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity") + chunk_size = 10000 + batch = [] + vulnerability_severity_query = VulnerabilitySeverity.objects.select_related("reference") + + print( + f"\nPopulating {vulnerability_severity_query.count():,d} VulnerabilitySeverity" + "`url` field using reference url." + ) + progress = LoopProgress( + total_iterations=vulnerability_severity_query.count(), + progress_step=10, + logger=print, + ) + for severity in progress.iter(vulnerability_severity_query.iterator(chunk_size=chunk_size)): + severity.url = severity.reference.url + batch.append(severity) + + if len(batch) >= chunk_size: + VulnerabilitySeverity.objects.bulk_update( + objs=batch, + fields=["url"], + ) + batch.clear() + + VulnerabilitySeverity.objects.bulk_update( + objs=batch, + fields=["url"], + ) + + def reverse_populate_severity_model_reference_using_url(apps, schema_editor): + """Reverse: Populate the reference using `url` to ensure proper reverse migration.""" + VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity") + VulnerabilityReference = apps.get_model("vulnerabilities", "VulnerabilityReference") + chunk_size = 10000 + batch = [] + vulnerability_severity_query = VulnerabilitySeverity.objects.select_related("reference") + + print( + f"\nReverse: Populating {vulnerability_severity_query.count():,d} VulnerabilitySeverity" + "`reference` relation using url." + ) + progress = LoopProgress( + total_iterations=vulnerability_severity_query.count(), + progress_step=10, + logger=print, + ) + for severity in progress.iter(vulnerability_severity_query.iterator(chunk_size=chunk_size)): + severity.reference = VulnerabilityReference.objects.get(url=severity.url) + batch.append(severity) + + if len(batch) >= chunk_size: + VulnerabilitySeverity.objects.bulk_update( + objs=batch, + fields=["reference"], + ) + batch.clear() + + VulnerabilitySeverity.objects.bulk_update( + objs=batch, + fields=["reference"], + ) + + def populate_vulnerability_model_with_severities(apps, schema_editor): + """Populate the new Vulnerability `severities` relation using referenced severity.""" + Vulnerability = apps.get_model("vulnerabilities", "Vulnerability") + chunk_size = 10000 + + vulnerability_query = Vulnerability.objects.prefetch_related( + "references__vulnerabilityseverity_set" + ) + print( + f"\nPopulating {vulnerability_query.count():,d} Vulnerability `severities`" + "relation using referenced severity." + ) + progress = LoopProgress( + total_iterations=vulnerability_query.count(), + progress_step=10, + logger=print, + ) + for vulnerability in progress.iter(vulnerability_query.iterator(chunk_size=chunk_size)): + references = vulnerability.references.all() + severity_ids = [ + severity.id + for reference in references + for severity in reference.vulnerabilityseverity_set.all() + if reference.vulnerabilityseverity_set.exists() + ] + vulnerability.severities.set(severity_ids) + + def reverse_populate_vulnerability_model_with_severities(apps, schema_editor): + """Reverse data migration not needed for new `severities` relationship.""" + pass + + dependencies = [ + ("vulnerabilities", "0077_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + # reference field should nullable to properly support reverse migration. + migrations.AlterField( + model_name="vulnerabilityseverity", + name="reference", + field=models.ForeignKey( + to="vulnerabilities.vulnerabilityreference", + on_delete=models.CASCADE, + null=True, + ), + ), + migrations.AlterModelOptions( + name="vulnerabilityseverity", + options={"ordering": ["url", "scoring_system", "value"]}, + ), + migrations.AlterUniqueTogether( + name="vulnerabilityseverity", + unique_together=set(), + ), + migrations.AddField( + model_name="affectedbypackagerelatedvulnerability", + name="severities", + field=models.ManyToManyField( + related_name="affected_package_vulnerability_relations", + to="vulnerabilities.vulnerabilityseverity", + ), + ), + migrations.AddField( + model_name="vulnerability", + name="severities", + field=models.ManyToManyField( + related_name="vulnerabilities", to="vulnerabilities.vulnerabilityseverity" + ), + ), + migrations.AddField( + model_name="vulnerabilityseverity", + name="url", + field=models.URLField( + help_text="URL to the vulnerability severity", max_length=1024, null=True + ), + ), + migrations.RunPython( + code=remove_inaccurate_suse_score, + reverse_code=reverse_remove_inaccurate_suse_score, + ), + migrations.RunPython( + code=reprocess_suse_advisory_on_next_import, + reverse_code=reverse_reprocess_suse_advisory_on_next_import, + ), + migrations.RunPython( + code=populate_severity_model_with_url, + reverse_code=reverse_populate_severity_model_reference_using_url, + ), + migrations.RunPython( + code=populate_vulnerability_model_with_severities, + reverse_code=reverse_populate_vulnerability_model_with_severities, + ), + migrations.RemoveField( + model_name="vulnerabilityseverity", + name="reference", + ), + ] diff --git a/vulnerabilities/migrations/0079_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0079_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..a9f92c8af --- /dev/null +++ b/vulnerabilities/migrations/0079_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-14 08:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0078_alter_vulnerabilityseverity_options_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.3.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.3.0", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0080_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0080_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..f535e7fe4 --- /dev/null +++ b/vulnerabilities/migrations/0080_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-15 02:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0079_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.3.1", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.3.1", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0081_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0081_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..97126051a --- /dev/null +++ b/vulnerabilities/migrations/0081_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,31 @@ +# Generated by Django 4.2.16 on 2024-11-15 06:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0080_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + default="34.3.2", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + default="34.3.2", + help_text="Version of the software at the time of change", + max_length=100, + ), + ), + ] diff --git a/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py b/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py new file mode 100644 index 000000000..26a55e714 --- /dev/null +++ b/vulnerabilities/migrations/0082_vulnerability_exploitability_and_more.py @@ -0,0 +1,43 @@ +# Generated by Django 4.2.16 on 2024-11-17 13:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0081_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="vulnerability", + name="exploitability", + field=models.DecimalField( + decimal_places=1, + help_text="Exploitability indicates the likelihood that a vulnerability in a software package could be used by malicious actors to compromise systems, applications, or networks. This metric is determined automatically based on the discovery of known exploits.", + max_digits=2, + null=True, + ), + ), + migrations.AddField( + model_name="vulnerability", + name="weighted_severity", + field=models.DecimalField( + decimal_places=1, + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", + max_digits=3, + null=True, + ), + ), + migrations.AlterField( + model_name="package", + name="risk_score", + field=models.DecimalField( + decimal_places=1, + help_text="Risk score between 0.00 and 10.00, where higher values indicate greater vulnerability risk for the package.", + max_digits=3, + null=True, + ), + ), + ] diff --git a/vulnerabilities/migrations/0083_alter_packagechangelog_software_version_and_more.py b/vulnerabilities/migrations/0083_alter_packagechangelog_software_version_and_more.py new file mode 100644 index 000000000..54c5a7b14 --- /dev/null +++ b/vulnerabilities/migrations/0083_alter_packagechangelog_software_version_and_more.py @@ -0,0 +1,27 @@ +# Generated by Django 4.2.16 on 2024-11-15 11:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0082_vulnerability_exploitability_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="packagechangelog", + name="software_version", + field=models.CharField( + help_text="Version of the software at the time of change", max_length=100 + ), + ), + migrations.AlterField( + model_name="vulnerabilitychangelog", + name="software_version", + field=models.CharField( + help_text="Version of the software at the time of change", max_length=100 + ), + ), + ] diff --git a/vulnerabilities/migrations/0084_alter_package_options_package_version_rank.py b/vulnerabilities/migrations/0084_alter_package_options_package_version_rank.py new file mode 100644 index 000000000..6b33c1a59 --- /dev/null +++ b/vulnerabilities/migrations/0084_alter_package_options_package_version_rank.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.16 on 2024-12-04 11:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0083_alter_packagechangelog_software_version_and_more"), + ] + + operations = [ + migrations.AlterModelOptions( + name="package", + options={ + "ordering": [ + "type", + "namespace", + "name", + "version_rank", + "version", + "qualifiers", + "subpath", + ] + }, + ), + migrations.AddField( + model_name="package", + name="version_rank", + field=models.IntegerField( + default=0, + help_text="Rank of the version to support ordering by version. Rank zero means the rank has not been defined yet", + ), + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index fc5eb5e3c..c2e89022f 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -87,7 +87,7 @@ def affecting_vulnerabilities(self): """ Return a queryset of Vulnerability that affect a package. """ - return self.filter(packagerelatedvulnerability__fix=False) + return self.filter(affecting_packages__isnull=False) def with_cpes(self): """ @@ -151,15 +151,47 @@ def search(self, query: str = None): def with_package_counts(self): return self.annotate( - vulnerable_package_count=Count( - "packages", filter=Q(packagerelatedvulnerability__fix=False), distinct=True - ), - patched_package_count=Count( - "packages", filter=Q(packagerelatedvulnerability__fix=True), distinct=True - ), + vulnerable_package_count=Count("affecting_packages", distinct=True), + patched_package_count=Count("fixed_by_packages", distinct=True), ) +class VulnerabilitySeverity(models.Model): + url = models.URLField( + max_length=1024, + null=True, + help_text="URL to the vulnerability severity", + ) + + scoring_system_choices = tuple( + (system.identifier, system.name) for system in SCORING_SYSTEMS.values() + ) + + scoring_system = models.CharField( + max_length=50, + choices=scoring_system_choices, + help_text="Identifier for the scoring system used. Available choices are: {} ".format( + ",\n".join(f"{sid}: {sname}" for sid, sname in scoring_system_choices) + ), + ) + + value = models.CharField(max_length=50, help_text="Example: 9.0, Important, High") + + scoring_elements = models.CharField( + max_length=150, + null=True, + help_text="Supporting scoring elements used to compute the score values. " + "For example a CVSS vector string as used to compute a CVSS score.", + ) + + published_at = models.DateTimeField( + blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity" + ) + + class Meta: + ordering = ["url", "scoring_system", "value"] + + class VulnerabilityStatusType(models.IntegerChoices): """List of vulnerability statuses.""" @@ -191,15 +223,53 @@ class Vulnerability(models.Model): to="VulnerabilityReference", through="VulnerabilityRelatedReference" ) - packages = models.ManyToManyField( + affecting_packages = models.ManyToManyField( + to="Package", + through="AffectedByPackageRelatedVulnerability", + ) + + fixed_by_packages = models.ManyToManyField( to="Package", - through="PackageRelatedVulnerability", + through="FixingPackageRelatedVulnerability", + related_name="fixing_vulnerabilities", # Unique related_name ) status = models.IntegerField( choices=VulnerabilityStatusType.choices, default=VulnerabilityStatusType.PUBLISHED ) + severities = models.ManyToManyField( + VulnerabilitySeverity, + related_name="vulnerabilities", + ) + + exploitability = models.DecimalField( + null=True, + max_digits=2, + decimal_places=1, + help_text="Exploitability indicates the likelihood that a vulnerability in a software package could be used by malicious actors to compromise systems, " + "applications, or networks. This metric is determined automatically based on the discovery of known exploits.", + ) + + weighted_severity = models.DecimalField( + null=True, + max_digits=3, + decimal_places=1, + help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", + ) + + @property + def risk_score(self): + """ + Risk expressed as a number ranging from 0 to 10. + Risk is calculated from weighted severity and exploitability values. + It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 + Risk = min(weighted severity * exploitability, 10) + """ + if self.exploitability and self.weighted_severity: + risk_score = min(float(self.exploitability * self.weighted_severity), 10.0) + return round(risk_score, 1) + objects = VulnerabilityQuerySet.as_manager() class Meta: @@ -213,32 +283,25 @@ def __str__(self): def vcid(self): return self.vulnerability_id - @property - def severities(self): - """ - Return a queryset of VulnerabilitySeverity for this vulnerability. - """ - return VulnerabilitySeverity.objects.filter(reference__in=self.references.all()) - @property def affected_packages(self): """ Return a queryset of packages that are affected by this vulnerability. """ - return self.packages.affected() - - # legacy aliases - vulnerable_packages = affected_packages + return self.affecting_packages.with_is_vulnerable() @property - def fixed_by_packages(self): + def packages_fixing(self): """ Return a queryset of packages that are fixing this vulnerability. """ - return self.packages.fixing() + return self.fixed_by_packages + + # legacy aliases + vulnerable_packages = affected_packages # legacy alias - patched_packages = fixed_by_packages + patched_packages = packages_fixing @property def get_aliases(self): @@ -404,7 +467,7 @@ def __str__(self): @property def is_cpe(self): """ - Return Trueis this is a CPE reference. + Return True if this is a CPE reference. """ return self.reference_id.startswith("cpe") @@ -444,7 +507,7 @@ def get_fixed_by_package_versions(self, purl: PackageURL, fix=True): } if fix: - filter_dict["packagerelatedvulnerability__fix"] = True + filter_dict["fixing_vulnerabilities__isnull"] = False return Package.objects.filter(**filter_dict).distinct() @@ -466,7 +529,7 @@ def affected(self): """ Return only packages affected by a vulnerability. """ - return self.filter(packagerelatedvulnerability__fix=False) + return self.filter(affected_by_vulnerabilities__isnull=False) vulnerable = affected @@ -474,17 +537,15 @@ def fixing(self): """ Return only packages fixing a vulnerability . """ - return self.filter(packagerelatedvulnerability__fix=True) + return self.filter(fixing_vulnerabilities__isnull=False) def with_vulnerability_counts(self): return self.annotate( vulnerability_count=Count( - "vulnerabilities", - filter=Q(packagerelatedvulnerability__fix=False), + "affected_by_vulnerabilities", ), patched_vulnerability_count=Count( - "vulnerabilities", - filter=Q(packagerelatedvulnerability__fix=True), + "fixing_vulnerabilities", ), ) @@ -557,13 +618,12 @@ def for_cve(self, cve): def with_is_vulnerable(self): """ - Annotate Package with ``with_is_vulnerable`` boolean attribute. + Annotate Package with ``is_vulnerable`` boolean attribute. """ return self.annotate( is_vulnerable=Exists( - PackageRelatedVulnerability.objects.filter( + AffectedByPackageRelatedVulnerability.objects.filter( package=OuterRef("pk"), - fix=False, ) ) ) @@ -572,7 +632,7 @@ def only_vulnerable(self): return self._vulnerable(True) def only_non_vulnerable(self): - return self._vulnerable(False) + return self._vulnerable(False).filter(is_ghost=False) def _vulnerable(self, vulnerable=True): """ @@ -580,6 +640,12 @@ def _vulnerable(self, vulnerable=True): """ return self.with_is_vulnerable().filter(is_vulnerable=vulnerable) + def vulnerable(self): + """ + Return only packages that are vulnerable. + """ + return self.filter(affected_by_vulnerabilities__isnull=False) + def get_purl_query_lookups(purl): """ @@ -601,8 +667,15 @@ class Package(PackageURLMixin): # https://github.com/package-url/packageurl-python/pull/67 # gets merged - vulnerabilities = models.ManyToManyField( - to="Vulnerability", through="PackageRelatedVulnerability" + affected_by_vulnerabilities = models.ManyToManyField( + to="Vulnerability", + through="AffectedByPackageRelatedVulnerability", + ) + + fixing_vulnerabilities = models.ManyToManyField( + to="Vulnerability", + through="FixingPackageRelatedVulnerability", + related_name="fixed_by_packages", # Unique related_name ) package_url = models.CharField( @@ -624,6 +697,20 @@ class Package(PackageURLMixin): help_text="True if the package does not exist in the upstream package manager or its repository.", ) + risk_score = models.DecimalField( + null=True, + max_digits=3, + decimal_places=1, + help_text="Risk score between 0.00 and 10.00, where higher values " + "indicate greater vulnerability risk for the package.", + ) + + version_rank = models.IntegerField( + help_text="Rank of the version to support ordering by version. Rank " + "zero means the rank has not been defined yet", + default=0, + ) + objects = PackageQuerySet.as_manager() def save(self, *args, **kwargs): @@ -657,17 +744,40 @@ def purl(self): class Meta: unique_together = ["type", "namespace", "name", "version", "qualifiers", "subpath"] - ordering = ["type", "namespace", "name", "version", "qualifiers", "subpath"] + ordering = ["type", "namespace", "name", "version_rank", "version", "qualifiers", "subpath"] def __str__(self): return self.package_url + @property + def calculate_version_rank(self): + """ + Calculate and return the `version_rank` for a package that does not have one. + If this package already has a `version_rank`, return it. + + The calculated rank will be interpolated between two packages that have + `version_rank` values and are closest to this package in terms of version order. + """ + + group_packages = Package.objects.filter( + type=self.type, + namespace=self.namespace, + name=self.name, + ) + + if any(p.version_rank == 0 for p in group_packages): + sorted_packages = sorted(group_packages, key=lambda p: self.version_class(p.version)) + for rank, package in enumerate(sorted_packages, start=1): + package.version_rank = rank + Package.objects.bulk_update(sorted_packages, fields=["version_rank"]) + return self.version_rank + @property def affected_by(self): """ Return a queryset of vulnerabilities affecting this package. """ - return self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) + return self.affected_by_vulnerabilities.all() # legacy aliases vulnerable_to = affected_by @@ -678,7 +788,7 @@ def fixing(self): """ Return a queryset of vulnerabilities fixed by this package. """ - return self.vulnerabilities.filter(packagerelatedvulnerability__fix=True) + return self.fixing_vulnerabilities.all() # legacy aliases resolved_to = fixing @@ -708,14 +818,6 @@ def get_details_url(self, request): return reverse("package_details", kwargs={"purl": self.purl}, request=request) - def sort_by_version(self, packages): - """ - Return a sequence of `packages` sorted by version. - """ - if not packages: - return [] - return sorted(packages, key=lambda x: self.version_class(x.version)) - @cached_property def version_class(self): range_class = RANGE_CLASS_BY_SCHEMES.get(self.type) @@ -733,6 +835,10 @@ def next_non_vulnerable_version(self): next_non_vulnerable, _ = self.get_non_vulnerable_versions() return next_non_vulnerable.version if next_non_vulnerable else None + @property + def vulnerabilities(self): + return self.affected_by_vulnerabilities.all() | self.fixing_vulnerabilities.all() + @property def latest_non_vulnerable_version(self): """ @@ -746,19 +852,20 @@ def get_non_vulnerable_versions(self): Return a tuple of the next and latest non-vulnerable versions as Package instance. Return a tuple of (None, None) if there is no non-vulnerable version. """ + if self.version_rank == 0: + self.calculate_version_rank non_vulnerable_versions = Package.objects.get_fixed_by_package_versions( self, fix=False ).only_non_vulnerable() - sorted_versions = self.sort_by_version(non_vulnerable_versions) - later_non_vulnerable_versions = [ - non_vuln_ver - for non_vuln_ver in sorted_versions - if self.version_class(non_vuln_ver.version) > self.current_version - ] + later_non_vulnerable_versions = non_vulnerable_versions.filter( + version_rank__gt=self.version_rank + ) + + later_non_vulnerable_versions = list(later_non_vulnerable_versions) if later_non_vulnerable_versions: - sorted_versions = self.sort_by_version(later_non_vulnerable_versions) + sorted_versions = later_non_vulnerable_versions next_non_vulnerable = sorted_versions[0] latest_non_vulnerable = sorted_versions[-1] return next_non_vulnerable, latest_non_vulnerable @@ -787,13 +894,15 @@ def get_affecting_vulnerabilities(self): Return a list of vulnerabilities that affect this package together with information regarding the versions that fix the vulnerabilities. """ + if self.version_rank == 0: + self.calculate_version_rank package_details_vulns = [] fixed_by_packages = Package.objects.get_fixed_by_package_versions(self, fix=True) - package_vulnerabilities = self.vulnerabilities.affecting_vulnerabilities().prefetch_related( + package_vulnerabilities = self.affected_by_vulnerabilities.prefetch_related( Prefetch( - "packages", + "fixed_by_packages", queryset=fixed_by_packages, to_attr="fixed_packages", ) @@ -810,12 +919,13 @@ def get_affecting_vulnerabilities(self): if fixed_version > self.current_version: later_fixed_packages.append(fixed_pkg) - next_fixed_package = None next_fixed_package_vulns = [] sort_fixed_by_packages_by_version = [] if later_fixed_packages: - sort_fixed_by_packages_by_version = self.sort_by_version(later_fixed_packages) + sort_fixed_by_packages_by_version = sorted( + later_fixed_packages, key=lambda p: p.version_rank + ) fixed_by_pkgs = [] @@ -845,16 +955,8 @@ def fixing_vulnerabilities(self): """ Return a queryset of Vulnerabilities that are fixed by this package. """ - return self.vulnerabilities.filter(packagerelatedvulnerability__fix=True) - - @property - def affected_by_vulnerabilities(self): - """ - Return a queryset of Vulnerabilities that affect this package. - """ - return self.vulnerabilities.filter(packagerelatedvulnerability__fix=False) - - affecting_vulnerabilities = affected_by_vulnerabilities + print("A") + return self.fixed_by_vulnerabilities.all() @property def affecting_vulns(self): @@ -862,37 +964,39 @@ def affecting_vulns(self): Return a queryset of Vulnerabilities that affect this `package`. """ fixed_by_packages = Package.objects.get_fixed_by_package_versions(self, fix=True) - return self.vulnerabilities.affecting_vulnerabilities().prefetch_related( + return self.affected_by_vulnerabilities.all().prefetch_related( Prefetch( - "packages", + "fixed_by_packages", queryset=fixed_by_packages, to_attr="fixed_packages", ) ) -class PackageRelatedVulnerability(models.Model): +class PackageRelatedVulnerabilityBase(models.Model): """ - Track the relationship between a Package and Vulnerability. + Abstract base class for package-vulnerability relations. """ - # TODO: Fix related_name package = models.ForeignKey( Package, on_delete=models.CASCADE, + # related_name="%(class)s_set", # Unique related_name per subclass ) vulnerability = models.ForeignKey( Vulnerability, on_delete=models.CASCADE, + # related_name="%(class)s_set", # Unique related_name per subclass ) created_by = models.CharField( max_length=100, blank=True, - help_text="Fully qualified name of the improver prefixed with the" - "module name responsible for creating this relation. Eg:" - "vulnerabilities.importers.nginx.NginxBasicImprover", + help_text=( + "Fully qualified name of the improver prefixed with the module name " + "responsible for creating this relation. Eg: vulnerabilities.importers.nginx.NginxBasicImprover" + ), ) from vulnerabilities.improver import MAX_CONFIDENCE @@ -903,54 +1007,45 @@ class PackageRelatedVulnerability(models.Model): help_text="Confidence score for this relation", ) - fix = models.BooleanField( - default=False, - db_index=True, - help_text="Does this relation fix the specified vulnerability ?", - ) - class Meta: + abstract = True unique_together = ["package", "vulnerability"] - verbose_name_plural = "PackageRelatedVulnerabilities" - indexes = [models.Index(fields=["fix"])] ordering = ["package", "vulnerability"] def __str__(self): - return f"{self.package.package_url} {self.vulnerability.vulnerability_id}" + relation = "fixes" if isinstance(self, FixingPackageRelatedVulnerability) else "affected by" + return f"{self.package.package_url} {relation} {self.vulnerability.vulnerability_id}" def update_or_create(self, advisory): """ - Update if supplied record has more confidence than existing record - Create if doesn't exist + Update if supplied record has more confidence than existing record. + Create if it doesn't exist. """ + model_class = self.__class__ try: - existing = PackageRelatedVulnerability.objects.get( + existing = model_class.objects.get( vulnerability=self.vulnerability, package=self.package ) if self.confidence > existing.confidence: existing.created_by = self.created_by existing.confidence = self.confidence - existing.fix = self.fix existing.save() - # TODO: later we want these to be part of a log field in the DB logger.info( f"Confidence improved for {self.package} R {self.vulnerability}, " f"new confidence: {self.confidence}" ) self.add_package_vulnerability_changelog(advisory=advisory) - - except self.DoesNotExist: - PackageRelatedVulnerability.objects.create( + except model_class.DoesNotExist: + model_class.objects.create( vulnerability=self.vulnerability, created_by=self.created_by, package=self.package, confidence=self.confidence, - fix=self.fix, ) logger.info( f"New relationship {self.package} R {self.vulnerability}, " - f"fix: {self.fix}, confidence: {self.confidence}" + f"confidence: {self.confidence}" ) self.add_package_vulnerability_changelog(advisory=advisory) @@ -960,7 +1055,7 @@ def add_package_vulnerability_changelog(self, advisory): from vulnerabilities.utils import get_importer_name importer_name = get_importer_name(advisory) - if self.fix: + if isinstance(self, FixingPackageRelatedVulnerability): change_logger = PackageChangeLog.log_fixing else: change_logger = PackageChangeLog.log_affected_by @@ -972,37 +1067,20 @@ def add_package_vulnerability_changelog(self, advisory): ) -class VulnerabilitySeverity(models.Model): - reference = models.ForeignKey(VulnerabilityReference, on_delete=models.CASCADE) - - scoring_system_choices = tuple( - (system.identifier, system.name) for system in SCORING_SYSTEMS.values() - ) - - scoring_system = models.CharField( - max_length=50, - choices=scoring_system_choices, - help_text="Identifier for the scoring system used. Available choices are: {} ".format( - ",\n".join(f"{sid}: {sname}" for sid, sname in scoring_system_choices) - ), - ) +class FixingPackageRelatedVulnerability(PackageRelatedVulnerabilityBase): + class Meta(PackageRelatedVulnerabilityBase.Meta): + verbose_name_plural = "Fixing Package Related Vulnerabilities" - value = models.CharField(max_length=50, help_text="Example: 9.0, Important, High") - scoring_elements = models.CharField( - max_length=150, - null=True, - help_text="Supporting scoring elements used to compute the score values. " - "For example a CVSS vector string as used to compute a CVSS score.", - ) +class AffectedByPackageRelatedVulnerability(PackageRelatedVulnerabilityBase): - published_at = models.DateTimeField( - blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity" + severities = models.ManyToManyField( + VulnerabilitySeverity, + related_name="affected_package_vulnerability_relations", ) - class Meta: - unique_together = ["reference", "scoring_system", "value"] - ordering = ["reference", "scoring_system", "value"] + class Meta(PackageRelatedVulnerabilityBase.Meta): + verbose_name_plural = "Affected By Package Related Vulnerabilities" class AliasQuerySet(BaseQuerySet): @@ -1137,7 +1215,9 @@ def to_advisory_data(self) -> "AdvisoryData": return AdvisoryData( aliases=self.aliases, summary=self.summary, - affected_packages=[AffectedPackage.from_dict(pkg) for pkg in self.affected_packages], + affected_packages=[ + AffectedPackage.from_dict(pkg) for pkg in self.affected_packages if pkg + ], references=[Reference.from_dict(ref) for ref in self.references], date_published=self.date_published, weaknesses=self.weaknesses, @@ -1227,7 +1307,8 @@ class ChangeLog(models.Model): software_version = models.CharField( max_length=100, help_text="Version of the software at the time of change", - default=VULNERABLECODE_VERSION, + blank=False, + null=False, ) @property diff --git a/vulnerabilities/oval_parser.py b/vulnerabilities/oval_parser.py index 2a958312c..fd2b114a7 100755 --- a/vulnerabilities/oval_parser.py +++ b/vulnerabilities/oval_parser.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index 0d3589b67..d74db9f35 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -10,11 +10,13 @@ import logging from datetime import datetime from datetime import timezone +from timeit import default_timer as timer from traceback import format_exc as traceback_format_exc from typing import Iterable from aboutcode.pipeline import BasePipeline from aboutcode.pipeline import LoopProgress +from aboutcode.pipeline import humanize_time from vulnerabilities.importer import AdvisoryData from vulnerabilities.improver import MAX_CONFIDENCE @@ -29,6 +31,57 @@ class VulnerableCodePipeline(BasePipeline): pipeline_id = None # Unique Pipeline ID + def on_failure(self): + """ + Tasks to run in the event that pipeline execution fails. + + Implement cleanup or other tasks that need to be performed + on pipeline failure, such as: + - Removing cloned repositories. + - Deleting downloaded archives. + """ + pass + + def execute(self): + """Execute each steps in the order defined on this pipeline class.""" + self.log(f"Pipeline [{self.pipeline_name}] starting") + + steps = self.pipeline_class.get_steps(groups=self.selected_groups) + steps_count = len(steps) + pipeline_start_time = timer() + + for current_index, step in enumerate(steps, start=1): + step_name = step.__name__ + + if self.selected_steps and step_name not in self.selected_steps: + self.log(f"Step [{step_name}] skipped") + continue + + self.set_current_step(f"{current_index}/{steps_count} {step_name}") + self.log(f"Step [{step_name}] starting") + step_start_time = timer() + + try: + step(self) + except Exception as exception: + self.log("Pipeline failed") + on_failure_start_time = timer() + self.log(f"Running [on_failure] tasks") + self.on_failure() + on_failure_run_time = timer() - on_failure_start_time + self.log(f"Completed [on_failure] tasks in {humanize_time(on_failure_run_time)}") + + return 1, self.output_from_exception(exception) + + step_run_time = timer() - step_start_time + self.log(f"Step [{step_name}] completed in {humanize_time(step_run_time)}") + + self.set_current_step("") # Reset the `current_step` field on completion + pipeline_run_time = timer() - pipeline_start_time + self.log(f"Pipeline completed in {humanize_time(pipeline_run_time)}") + + return 0, "" + def log(self, message, level=logging.INFO): """Log the given `message` to the current module logger and execution_log.""" now_local = datetime.now(timezone.utc).astimezone() @@ -51,8 +104,8 @@ class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline): Base importer pipeline for importing advisories. Uses: - Subclass this Pipeline and implement ``advisories_count`` and ``collect_advisories`` method. - Also override the ``steps`` and ``advisory_confidence`` as needed. + Subclass this Pipeline and implement ``advisories_count`` and ``collect_advisories`` + method. Also override the ``steps`` and ``advisory_confidence`` as needed. """ pipeline_id = None # Unique Pipeline ID, this should be the name of pipeline module. diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py new file mode 100644 index 000000000..7ac4de838 --- /dev/null +++ b/vulnerabilities/pipelines/compute_package_risk.py @@ -0,0 +1,147 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from aboutcode.pipeline import LoopProgress +from django.db.models import Prefetch + +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.risk import compute_package_risk +from vulnerabilities.risk import compute_vulnerability_risk_factors + + +class ComputePackageRiskPipeline(VulnerableCodePipeline): + """ + Compute risk score for packages. + + See https://github.com/aboutcode-org/vulnerablecode/issues/1543 + """ + + pipeline_id = "compute_package_risk" + license_expression = None + + @classmethod + def steps(cls): + return ( + cls.compute_and_store_vulnerability_risk_score, + cls.compute_and_store_package_risk_score, + ) + + def compute_and_store_vulnerability_risk_score(self): + affected_vulnerabilities = ( + Vulnerability.objects.filter(affecting_packages__isnull=False) + .prefetch_related( + "references", + "severities", + "exploits", + ) + .distinct() + ) + + self.log( + f"Calculating risk for {affected_vulnerabilities.count():,d} vulnerability with a affected packages records" + ) + + progress = LoopProgress(total_iterations=affected_vulnerabilities.count(), logger=self.log) + + updatables = [] + updated_vulnerability_count = 0 + batch_size = 5000 + + for vulnerability in progress.iter(affected_vulnerabilities.paginated(per_page=batch_size)): + severities = vulnerability.severities.all() + references = vulnerability.references.all() + exploits = vulnerability.exploits.all() + + weighted_severity, exploitability = compute_vulnerability_risk_factors( + references=references, + severities=severities, + exploits=exploits, + ) + vulnerability.weighted_severity = weighted_severity + vulnerability.exploitability = exploitability + + updatables.append(vulnerability) + + if len(updatables) >= batch_size: + updated_vulnerability_count += bulk_update( + model=Vulnerability, + items=updatables, + fields=["weighted_severity", "exploitability"], + logger=self.log, + ) + + updated_vulnerability_count += bulk_update( + model=Vulnerability, + items=updatables, + fields=["weighted_severity", "exploitability"], + logger=self.log, + ) + + self.log( + f"Successfully added risk score for {updated_vulnerability_count:,d} vulnerability" + ) + + def compute_and_store_package_risk_score(self): + affected_packages = ( + Package.objects.filter(affected_by_vulnerabilities__isnull=False).prefetch_related( + Prefetch( + "affectedbypackagerelatedvulnerability_set__vulnerability", + queryset=Vulnerability.objects.only("weighted_severity", "exploitability"), + ), + ) + ).distinct() + + self.log(f"Calculating risk for {affected_packages.count():,d} affected package records") + + progress = LoopProgress( + total_iterations=affected_packages.count(), + logger=self.log, + progress_step=5, + ) + + updatables = [] + updated_package_count = 0 + batch_size = 10000 + + for package in progress.iter(affected_packages.paginated(per_page=batch_size)): + risk_score = compute_package_risk(package) + + if not risk_score: + continue + + package.risk_score = risk_score + updatables.append(package) + + if len(updatables) >= batch_size: + updated_package_count += bulk_update( + model=Package, + items=updatables, + fields=["risk_score"], + logger=self.log, + ) + updated_package_count += bulk_update( + model=Package, + items=updatables, + fields=["risk_score"], + logger=self.log, + ) + self.log(f"Successfully added risk score for {updated_package_count:,d} package") + + +def bulk_update(model, items, fields, logger): + item_count = 0 + if items: + try: + model.objects.bulk_update(objs=items, fields=fields) + item_count += len(items) + except Exception as e: + logger(f"Error updating {model.__name__}: {e}") + items.clear() + return item_count diff --git a/vulnerabilities/pipelines/compute_package_version_rank.py b/vulnerabilities/pipelines/compute_package_version_rank.py new file mode 100644 index 000000000..73d4aa60a --- /dev/null +++ b/vulnerabilities/pipelines/compute_package_version_rank.py @@ -0,0 +1,93 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from itertools import groupby + +from aboutcode.pipeline import LoopProgress +from django.db import transaction +from univers.version_range import RANGE_CLASS_BY_SCHEMES +from univers.versions import Version + +from vulnerabilities.models import Package +from vulnerabilities.pipelines import VulnerableCodePipeline + + +class ComputeVersionRankPipeline(VulnerableCodePipeline): + """ + A pipeline to compute and assign version ranks for all packages. + """ + + pipeline_id = "compute_version_rank" + license_expression = None + + @classmethod + def steps(cls): + return (cls.compute_and_store_version_rank,) + + def compute_and_store_version_rank(self): + """ + Compute and assign version ranks to all packages. + """ + groups = Package.objects.only("type", "namespace", "name").order_by( + "type", "namespace", "name" + ) + + def key(package): + return package.type, package.namespace, package.name + + groups = groupby(groups, key=key) + + groups = [(list(x), list(y)) for x, y in groups] + + total_groups = len(groups) + self.log(f"Calculating `version_rank` for {total_groups:,d} groups of packages.") + + progress = LoopProgress( + total_iterations=total_groups, + logger=self.log, + progress_step=5, + ) + + for group, packages in progress.iter(groups): + type, namespace, name = group + if type not in RANGE_CLASS_BY_SCHEMES: + continue + self.update_version_rank_for_group(packages) + + self.log("Successfully populated `version_rank` for all packages.") + + @transaction.atomic + def update_version_rank_for_group(self, packages): + """ + Update the `version_rank` for all packages in a specific group. + """ + + # Sort the packages by version + sorted_packages = self.sort_packages_by_version(packages) + + # Assign version ranks + updates = [] + for rank, package in enumerate(sorted_packages, start=1): + package.version_rank = rank + updates.append(package) + + # Bulk update to save the ranks + Package.objects.bulk_update(updates, fields=["version_rank"]) + + def sort_packages_by_version(self, packages): + """ + Sort packages by version using `version_class`. + """ + + if not packages: + return [] + version_class = RANGE_CLASS_BY_SCHEMES.get(packages[0].type).version_class + if not version_class: + version_class = Version + return sorted(packages, key=lambda p: version_class(p.version)) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py index 54554f951..b31ac4134 100644 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ b/vulnerabilities/pipelines/enhance_with_exploitdb.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import csv import io import logging diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py index 6372bd3b0..e20f61653 100644 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ b/vulnerabilities/pipelines/enhance_with_kev.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import logging from traceback import format_exc as traceback_format_exc @@ -26,9 +35,7 @@ def steps(cls): ) def fetch_exploits(self): - kev_url = ( - "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json" - ) + kev_url = "https://raw.githubusercontent.com/aboutcode-org/aboutcode-mirror-kev/refs/heads/main/known_exploited_vulnerabilities.json" self.log(f"Fetching {kev_url}") try: diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py index 72897abd0..d6989ab47 100644 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ b/vulnerabilities/pipelines/enhance_with_metasploit.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import logging from traceback import format_exc as traceback_format_exc diff --git a/vulnerabilities/pipelines/flag_ghost_packages.py b/vulnerabilities/pipelines/flag_ghost_packages.py index 46d484167..7daee4115 100644 --- a/vulnerabilities/pipelines/flag_ghost_packages.py +++ b/vulnerabilities/pipelines/flag_ghost_packages.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/pipelines/github_importer.py b/vulnerabilities/pipelines/github_importer.py index 4603b939a..66c457824 100644 --- a/vulnerabilities/pipelines/github_importer.py +++ b/vulnerabilities/pipelines/github_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/pipelines/gitlab_importer.py b/vulnerabilities/pipelines/gitlab_importer.py index 0b76a31f2..4f25c4d94 100644 --- a/vulnerabilities/pipelines/gitlab_importer.py +++ b/vulnerabilities/pipelines/gitlab_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -106,6 +106,9 @@ def clean_downloads(self): self.log(f"Removing cloned repository") self.vcs_response.delete() + def on_failure(self): + self.clean_downloads() + def parse_advisory_path(base_path: Path, file_path: Path) -> Tuple[str, str, str]: """ diff --git a/vulnerabilities/pipelines/npm_importer.py b/vulnerabilities/pipelines/npm_importer.py index 42444ad29..7b6d3aba2 100644 --- a/vulnerabilities/pipelines/npm_importer.py +++ b/vulnerabilities/pipelines/npm_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -163,3 +163,6 @@ def clean_downloads(self): if self.vcs_response: self.log(f"Removing cloned repository") self.vcs_response.delete() + + def on_failure(self): + self.clean_downloads() diff --git a/vulnerabilities/pipelines/nvd_importer.py b/vulnerabilities/pipelines/nvd_importer.py index 38800eb62..bd6f33cf9 100644 --- a/vulnerabilities/pipelines/nvd_importer.py +++ b/vulnerabilities/pipelines/nvd_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/pipelines/pypa_importer.py b/vulnerabilities/pipelines/pypa_importer.py index 29a1283fe..aebafacf4 100644 --- a/vulnerabilities/pipelines/pypa_importer.py +++ b/vulnerabilities/pipelines/pypa_importer.py @@ -3,10 +3,10 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import logging + from pathlib import Path from typing import Iterable @@ -68,3 +68,6 @@ def clean_downloads(self): if self.vcs_response: self.log(f"Removing cloned repository") self.vcs_response.delete() + + def on_failure(self): + self.clean_downloads() diff --git a/vulnerabilities/pipelines/pysec_importer.py b/vulnerabilities/pipelines/pysec_importer.py new file mode 100644 index 000000000..32a9fd896 --- /dev/null +++ b/vulnerabilities/pipelines/pysec_importer.py @@ -0,0 +1,66 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +import json +import logging +from io import BytesIO +from typing import Iterable +from zipfile import ZipFile + +import requests + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline + + +class PyPIImporterPipeline(VulnerableCodeBaseImporterPipeline): + """Collect advisories from PyPI.""" + + pipeline_id = "pysec_importer" + + license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" + url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" + spdx_license_expression = "CC-BY-4.0" + importer_name = "PyPI Importer" + + @classmethod + def steps(cls): + return ( + cls.fetch_zip, + cls.collect_and_store_advisories, + cls.import_new_advisories, + ) + + def fetch_zip(self): + self.log(f"Fetching `{self.url}`") + self.advisory_zip = requests.get(self.url).content + + def advisories_count(self) -> int: + with ZipFile(BytesIO(self.advisory_zip)) as zip: + advisory_count = sum(1 for file in zip.namelist() if file.startswith("PYSEC-")) + return advisory_count + + def collect_advisories(self) -> Iterable[AdvisoryData]: + """Yield AdvisoryData using a zipped data dump of OSV data""" + from vulnerabilities.importers.osv import parse_advisory_data + + with ZipFile(BytesIO(self.advisory_zip)) as zip_file: + for file_name in zip_file.namelist(): + if not file_name.startswith("PYSEC-"): + self.log( + f"Unsupported PyPI advisory data file: {file_name}", + level=logging.ERROR, + ) + continue + with zip_file.open(file_name) as f: + vul_info = json.load(f) + yield parse_advisory_data( + raw_data=vul_info, + supported_ecosystems=["pypi"], + advisory_url=self.url, + ) diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index f33eb4d2b..6637122a3 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -18,8 +18,9 @@ from vulnerabilities.importer import AdvisoryData from vulnerabilities.improver import MAX_CONFIDENCE from vulnerabilities.models import Advisory +from vulnerabilities.models import AffectedByPackageRelatedVulnerability +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilityRelatedReference from vulnerabilities.models import VulnerabilitySeverity @@ -103,25 +104,24 @@ def import_advisory( reference_id=ref.reference_id, url=ref.url, ) - if not reference: - continue - - VulnerabilityRelatedReference.objects.update_or_create( - reference=reference, - vulnerability=vulnerability, - ) + if reference: + VulnerabilityRelatedReference.objects.update_or_create( + reference=reference, + vulnerability=vulnerability, + ) for severity in ref.severities: try: published_at = str(severity.published_at) if severity.published_at else None - _, created = VulnerabilitySeverity.objects.update_or_create( + vulnerability_severity, created = VulnerabilitySeverity.objects.update_or_create( scoring_system=severity.system.identifier, - reference=reference, + url=ref.url, + value=severity.value, + scoring_elements=severity.scoring_elements, defaults={ - "value": str(severity.value), - "scoring_elements": str(severity.scoring_elements), "published_at": published_at, }, ) + vulnerability.severities.add(vulnerability_severity) except: if logger: logger( @@ -131,29 +131,27 @@ def import_advisory( if not created: if logger: logger( - f"Severity updated for reference {ref!r} to value: {severity.value!r} " + f"Severity updated for reference {ref.url!r} to value: {severity.value!r} " f"and scoring_elements: {severity.scoring_elements!r}", level=logging.DEBUG, ) for affected_purl in affected_purls or []: vulnerable_package, _ = Package.objects.get_or_create_from_purl(purl=affected_purl) - PackageRelatedVulnerability( + AffectedByPackageRelatedVulnerability( vulnerability=vulnerability, package=vulnerable_package, created_by=pipeline_id, confidence=confidence, - fix=False, ).update_or_create(advisory=advisory) for fixed_purl in fixed_purls: fixed_package, _ = Package.objects.get_or_create_from_purl(purl=fixed_purl) - PackageRelatedVulnerability( + FixingPackageRelatedVulnerability( vulnerability=vulnerability, package=fixed_package, created_by=pipeline_id, confidence=confidence, - fix=True, ).update_or_create(advisory=advisory) if advisory_data.weaknesses and vulnerability: diff --git a/vulnerabilities/references.py b/vulnerabilities/references.py index 87b45b9f6..47225f520 100644 --- a/vulnerabilities/references.py +++ b/vulnerabilities/references.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py new file mode 100644 index 000000000..a4508a03f --- /dev/null +++ b/vulnerabilities/risk.py @@ -0,0 +1,114 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from urllib.parse import urlparse + +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.severity_systems import EPSS +from vulnerabilities.weight_config import WEIGHT_CONFIG + +DEFAULT_WEIGHT = 5 + + +def get_weighted_severity(severities): + """ + Weighted Severity is the maximum value obtained when each Severity is multiplied + by its associated Weight/10. + Example of Weighted Severity: max(7*(10/10), 8*(3/10), 6*(8/10)) = 7 + """ + if not severities: + return 0 + + score_map = { + "low": 3, + "moderate": 6.9, + "medium": 6.9, + "high": 8.9, + "important": 8.9, + "critical": 10.0, + "urgent": 10.0, + } + + score_list = [] + for severity in severities: + parsed_url = urlparse(severity.url) + severity_source = parsed_url.netloc.replace("www.", "", 1) + weight = WEIGHT_CONFIG.get(severity_source, DEFAULT_WEIGHT) + max_weight = float(weight) / 10 + vul_score = severity.value + try: + vul_score = float(vul_score) + vul_score_value = vul_score * max_weight + except ValueError: + vul_score = vul_score.lower() + vul_score_value = score_map.get(vul_score, 0) * max_weight + + score_list.append(vul_score_value) + + max_score = max(score_list) if score_list else 0 + return round(max_score, 1) + + +def get_exploitability_level(exploits, references, severities): + """ + Exploitability refers to the potential or + probability of a software package vulnerability being exploited by + malicious actors to compromise systems, applications, or networks. + It is determined automatically by discovery of exploits. + """ + # no exploit known ( default .5) + exploit_level = 0.5 + + if exploits: + # Automatable Exploit with PoC script published OR known exploits (KEV) in the wild OR known ransomware + exploit_level = 2 + + elif severities: + # high EPSS. + for severity in severities: + if severity.scoring_system == EPSS.identifier and float(severity.value) > 0.8: + exploit_level = 2 + break + + elif references: + # PoC/Exploit script published + for reference in references: + if reference.reference_type == VulnerabilityReference.EXPLOIT: + exploit_level = 1 + break + + return exploit_level + + +def compute_vulnerability_risk_factors(references, severities, exploits): + """ + Risk may be expressed as a number ranging from 0 to 10. + Risk is calculated from weighted severity and exploitability values. + It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 + + Risk = min(weighted severity * exploitability, 10) + """ + weighted_severity = get_weighted_severity(severities) + exploitability = get_exploitability_level(exploits, references, severities) + return weighted_severity, exploitability + + +def compute_package_risk(package): + """ + Calculate the risk for a package by iterating over all vulnerabilities that affects this package + and determining the associated risk. + """ + result = [] + for relation in package.affectedbypackagerelatedvulnerability_set.all(): + if risk := relation.vulnerability.risk_score: + result.append(float(risk)) + + if not result: + return + + return round(max(result), 1) diff --git a/vulnerabilities/rpm_utils.py b/vulnerabilities/rpm_utils.py index 206f3668f..84d440c9f 100644 --- a/vulnerabilities/rpm_utils.py +++ b/vulnerabilities/rpm_utils.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/severity_systems.py b/vulnerabilities/severity_systems.py index f5be70a5b..946cb6479 100644 --- a/vulnerabilities/severity_systems.py +++ b/vulnerabilities/severity_systems.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html index 26de42fa8..dd72d0500 100644 --- a/vulnerabilities/templates/package_details.html +++ b/vulnerabilities/templates/package_details.html @@ -112,6 +112,20 @@ {% endif %} + + + + Risk + + + + {% if package.risk_score %} + {{ package.risk_score }} + {% endif %} + + diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html index d1f2fb6de..e9e58c79e 100644 --- a/vulnerabilities/templates/vulnerability_details.html +++ b/vulnerabilities/templates/vulnerability_details.html @@ -121,6 +121,38 @@ Status {{ status }} + + + + Exploitability + + {{ vulnerability.exploitability }} + + + + + Weighted Severity + + {{ vulnerability.weighted_severity }} + + + + + Risk + + {{ vulnerability.risk_score }} + + + @@ -139,13 +171,9 @@ {{ severity.scoring_system }} {{ severity.value }} - {% if severity.reference.url %} - - {{ severity.reference.url }} + + {{ severity.url }} - {% else %} - {{ severity.reference.reference_id }} - {% endif %} {% empty %} diff --git a/vulnerabilities/templatetags/__init__.py b/vulnerabilities/templatetags/__init__.py index bdac1cd30..20854f2ad 100644 --- a/vulnerabilities/templatetags/__init__.py +++ b/vulnerabilities/templatetags/__init__.py @@ -3,6 +3,6 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/conftest.py b/vulnerabilities/tests/conftest.py index f9216c742..de75014fb 100644 --- a/vulnerabilities/tests/conftest.py +++ b/vulnerabilities/tests/conftest.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/example_importer_improver.py b/vulnerabilities/tests/example_importer_improver.py index bb74e3a3f..bcaa87c45 100644 --- a/vulnerabilities/tests/example_importer_improver.py +++ b/vulnerabilities/tests/example_importer_improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/pipelines/__init__.py b/vulnerabilities/tests/pipelines/__init__.py index 03cc81e75..451fd4447 100644 --- a/vulnerabilities/tests/pipelines/__init__.py +++ b/vulnerabilities/tests/pipelines/__init__.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/pipelines/test_base_pipeline.py b/vulnerabilities/tests/pipelines/test_base_pipeline.py index aaf01a7f9..02e8c6b09 100644 --- a/vulnerabilities/tests/pipelines/test_base_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_base_pipeline.py @@ -19,6 +19,8 @@ from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import Reference from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.tests.pipelines import TestLogger advisory_data1 = AdvisoryData( aliases=["CVE-2020-13371337"], @@ -47,6 +49,33 @@ def get_advisory1(created_by="test_pipeline"): ) +class TestVulnerableCodePipeline(TestCase): + def test_on_failure(self): + class TestPipeline(VulnerableCodePipeline): + def __init__(self, test_logger): + super().__init__() + self.log = test_logger.write + + @classmethod + def steps(cls): + return (cls.step1,) + + def step1(self): + raise Exception("Something went wrong!") + + def on_failure(self): + self.log("Doing cleanup.") + + logger = TestLogger() + pipeline = TestPipeline(test_logger=logger) + + pipeline.execute() + log_result = logger.getvalue() + + self.assertIn("Pipeline failed", log_result) + self.assertIn("Running [on_failure] tasks", log_result) + + class TestVulnerableCodeBaseImporterPipeline(TestCase): @patch.object( VulnerableCodeBaseImporterPipeline, diff --git a/vulnerabilities/tests/pipelines/test_compute_package_risk.py b/vulnerabilities/tests/pipelines/test_compute_package_risk.py new file mode 100644 index 000000000..a366d32e8 --- /dev/null +++ b/vulnerabilities/tests/pipelines/test_compute_package_risk.py @@ -0,0 +1,34 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# +from decimal import Decimal + +import pytest + +from vulnerabilities.models import AffectedByPackageRelatedVulnerability +from vulnerabilities.models import Package +from vulnerabilities.pipelines.compute_package_risk import ComputePackageRiskPipeline +from vulnerabilities.tests.test_risk import vulnerability + + +@pytest.mark.django_db +def test_simple_risk_pipeline(vulnerability): + pkg = Package.objects.create(type="pypi", name="foo", version="2.3.0") + assert Package.objects.count() == 1 + + improver = ComputePackageRiskPipeline() + improver.execute() + + assert pkg.risk_score is None + + AffectedByPackageRelatedVulnerability.objects.create(package=pkg, vulnerability=vulnerability) + improver = ComputePackageRiskPipeline() + improver.execute() + + pkg = Package.objects.get(type="pypi", name="foo", version="2.3.0") + assert pkg.risk_score == Decimal("3.1") # max( 6.9 * 9/10 , 6.5 * 9/10 ) * .5 = 3.105 diff --git a/vulnerabilities/tests/pipelines/test_exploitdb.py b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py similarity index 75% rename from vulnerabilities/tests/pipelines/test_exploitdb.py rename to vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py index f08f7fec0..f54dad55d 100644 --- a/vulnerabilities/tests/pipelines/test_exploitdb.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import os from unittest import mock from unittest.mock import Mock diff --git a/vulnerabilities/tests/pipelines/test_kev.py b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py similarity index 75% rename from vulnerabilities/tests/pipelines/test_kev.py rename to vulnerabilities/tests/pipelines/test_enhance_with_kev.py index 71583a617..a93c16555 100644 --- a/vulnerabilities/tests/pipelines/test_kev.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import os from unittest import mock from unittest.mock import Mock diff --git a/vulnerabilities/tests/pipelines/test_metasploit.py b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py similarity index 76% rename from vulnerabilities/tests/pipelines/test_metasploit.py rename to vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py index 1116950d2..eea99e0ca 100644 --- a/vulnerabilities/tests/pipelines/test_metasploit.py +++ b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py @@ -1,3 +1,12 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + import os from unittest import mock from unittest.mock import Mock diff --git a/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py b/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py index fa718f78c..192901c36 100644 --- a/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py +++ b/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py index 29e869381..dfa5ea371 100644 --- a/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py index d10413a8b..c3dc7be43 100644 --- a/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py index 549a269bb..5b90ca986 100644 --- a/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py +++ b/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_pysec.py b/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py similarity index 69% rename from vulnerabilities/tests/test_pysec.py rename to vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py index dcba3a776..c38256c01 100644 --- a/vulnerabilities/tests/test_pysec.py +++ b/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py @@ -3,28 +3,28 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # + import json -import os +from pathlib import Path from unittest import TestCase from vulnerabilities.importers.osv import parse_advisory_data from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN from vulnerabilities.tests.util_tests import check_results_against_json -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/pysec") +TEST_DATA = Path(__file__).parent.parent / "test_data" / "pysec" class TestPyPIImporter(TestCase): def test_to_advisories_with_summary(self): - with open(os.path.join(TEST_DATA, "pysec-advisories_with_summary.json")) as f: + with open(TEST_DATA / "pysec-advisories_with_summary.json") as f: mock_response = json.load(f) results = parse_advisory_data(mock_response, ["pypi"], "https://test.com").to_dict() - expected_file = os.path.join(TEST_DATA, "pysec-advisories_with_summary-expected.json") + expected_file = TEST_DATA / "pysec-advisories_with_summary-expected.json" check_results_against_json( results=results, expected_file=expected_file, @@ -32,12 +32,12 @@ def test_to_advisories_with_summary(self): ) def test_to_advisories_without_summary(self): - with open(os.path.join(TEST_DATA, "pysec-advisories_without_summary.json")) as f: + with open(TEST_DATA / "pysec-advisories_without_summary.json") as f: mock_response = json.load(f) results = parse_advisory_data(mock_response, ["pypi"], "https://test.com").to_dict() - expected_file = os.path.join(TEST_DATA, "pysec-advisories_without_summary-expected.json") + expected_file = TEST_DATA / "pysec-advisories_without_summary-expected.json" check_results_against_json( results=results, expected_file=expected_file, @@ -45,14 +45,14 @@ def test_to_advisories_without_summary(self): ) def test_to_advisories_with_cwe(self): - with open(os.path.join(TEST_DATA, "pysec-advisory_with_cwe.json")) as f: + with open(TEST_DATA / "pysec-advisory_with_cwe.json") as f: mock_response = json.load(f) results = parse_advisory_data( raw_data=mock_response, supported_ecosystems=["pypi"], advisory_url="https://tes.com" ).to_dict() - expected_file = os.path.join(TEST_DATA, "pysec-advisories_with_cwe-expected.json") + expected_file = TEST_DATA / "pysec-advisories_with_cwe-expected.json" check_results_against_json( results=results, expected_file=expected_file, diff --git a/vulnerabilities/tests/test_affected_package.py b/vulnerabilities/tests/test_affected_package.py index 2ebbcbddb..f56551d65 100644 --- a/vulnerabilities/tests/test_affected_package.py +++ b/vulnerabilities/tests/test_affected_package.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_alpine.py b/vulnerabilities/tests/test_alpine.py index d65e27401..1ab74a89a 100644 --- a/vulnerabilities/tests/test_alpine.py +++ b/vulnerabilities/tests/test_alpine.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py index a57437d01..853eafbd0 100644 --- a/vulnerabilities/tests/test_apache_httpd.py +++ b/vulnerabilities/tests/test_apache_httpd.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_apache_kafka.py b/vulnerabilities/tests/test_apache_kafka.py index 720196381..92c76c7b1 100644 --- a/vulnerabilities/tests/test_apache_kafka.py +++ b/vulnerabilities/tests/test_apache_kafka.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_apache_tomcat.py b/vulnerabilities/tests/test_apache_tomcat.py index 3ed522597..51d33634f 100644 --- a/vulnerabilities/tests/test_apache_tomcat.py +++ b/vulnerabilities/tests/test_apache_tomcat.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py index 18807a6d7..a5f80aa06 100644 --- a/vulnerabilities/tests/test_api.py +++ b/vulnerabilities/tests/test_api.py @@ -3,13 +3,12 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json import os -from collections import OrderedDict from urllib.parse import quote from django.test import TestCase @@ -20,10 +19,11 @@ from vulnerabilities.api import PackageSerializer from vulnerabilities.api import VulnerabilityReferenceSerializer +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias from vulnerabilities.models import ApiUser +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilityRelatedReference @@ -33,6 +33,7 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DATA = os.path.join(BASE_DIR, "test_data") +TEST_DIR = os.path.join(TEST_DATA, "api") def cleaned_response(response): @@ -210,8 +211,8 @@ def setUp(self): self.pkg1 = Package.objects.create(name="flask", type="pypi", version="0.1.2") self.pkg2 = Package.objects.create(name="flask", type="deb", version="0.1.2") for pkg in [self.pkg1, self.pkg2]: - PackageRelatedVulnerability.objects.create( - package=pkg, vulnerability=self.vulnerability, fix=True + FixingPackageRelatedVulnerability.objects.create( + package=pkg, vulnerability=self.vulnerability ) self.reference1 = VulnerabilityReference.objects.create( @@ -219,8 +220,8 @@ def setUp(self): url="https://.com", ) - VulnerabilitySeverity.objects.create( - reference=self.reference1, + severity = VulnerabilitySeverity.objects.create( + url="https://.com", scoring_system=EPSS.identifier, scoring_elements=".0016", value="0.526", @@ -236,6 +237,7 @@ def setUp(self): cwe_id=10000 ) # cwe not present in weaknesses_db self.invalid_weaknesses.vulnerabilities.add(self.vulnerability) + self.vulnerability.severities.add(severity) def test_api_status(self): response = self.csrf_client.get("/api/vulnerabilities/") @@ -297,6 +299,9 @@ def test_api_with_single_vulnerability(self): }, ], "exploits": [], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, } def test_api_with_single_vulnerability_with_filters(self): @@ -343,8 +348,60 @@ def test_api_with_single_vulnerability_with_filters(self): }, ], "exploits": [], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + } + + def test_api_with_single_vulnerability_no_ghost_fix(self): + self.pkg2.is_ghost = True + self.pkg1.is_ghost = True + self.pkg2.save() + self.pkg1.save() + + response = self.csrf_client.get( + f"/api/vulnerabilities/{self.vulnerability.id}", format="json" + ).data + + expected = { + "url": f"http://testserver/api/vulnerabilities/{self.vulnerability.id}", + "vulnerability_id": self.vulnerability.vulnerability_id, + "summary": "test", + "severity_range_score": None, + "aliases": [], + "resource_url": f"http://testserver/vulnerabilities/{self.vulnerability.vulnerability_id}", + "fixed_packages": [], + "affected_packages": [], + "references": [ + { + "reference_url": "https://.com", + "reference_id": "", + "reference_type": "", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://.com", + } + ], + "weaknesses": [ + { + "cwe_id": 119, + "name": "Improper Restriction of Operations within the Bounds of a Memory Buffer", + "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", + }, + ], + "exploits": [], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, } + assert expected == response + def set_as_affected_by(package, vulnerability): """ @@ -364,11 +421,16 @@ def _set_pkg_as(package, vulnerability, fixing=False): """ Set the ``package`` Package as affected or fixing the ``vulnerability`` Vulnerability. """ - PackageRelatedVulnerability.objects.create( - package=package, - vulnerability=vulnerability, - fix=fixing, - ) + if fixing: + FixingPackageRelatedVulnerability.objects.create( + package=package, + vulnerability=vulnerability, + ) + else: + AffectedByPackageRelatedVulnerability.objects.create( + package=package, + vulnerability=vulnerability, + ) def create_vuln(vcid, aliases=()): @@ -427,6 +489,7 @@ def setUp(self): self.pkg_2_14_0_rc1 = from_purl( "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1" ) + self.pkg_2_12_6.calculate_version_rank set_as_fixing(package=self.pkg_2_12_6, vulnerability=self.vul3) @@ -546,6 +609,23 @@ def setUp(self): self.pkg_2_14_0_rc1 = from_purl( "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1" ) + self.pkg_2_12_6.calculate_version_rank + + self.ref = VulnerabilityReference.objects.create( + reference_type="advisory", reference_id="CVE-xxx-xxx", url="https://example.com" + ) + + self.severity = VulnerabilitySeverity.objects.create( + url="https://example.com", + scoring_system=EPSS.identifier, + scoring_elements=".0016", + value="0.526", + ) + self.vul1.references.add(self.ref) + self.vul1.severities.add(self.severity) + + self.vul3.references.add(self.ref) + self.vul3.severities.add(self.severity) set_as_fixing(package=self.pkg_2_12_6, vulnerability=self.vul3) @@ -561,8 +641,8 @@ def setUp(self): def test_api_with_lesser_and_greater_fixed_by_packages(self): response = self.csrf_client.get(f"/api/packages/{self.pkg_2_13_1.id}", format="json").data - expected_response = { - "url": f"http://testserver/api/packages/{self.pkg_2_13_1.id}", + expected = { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", "type": "maven", "namespace": "com.fasterxml.jackson.core", @@ -574,109 +654,93 @@ def test_api_with_lesser_and_greater_fixed_by_packages(self): "next_non_vulnerable_version": "2.14.0-rc1", "latest_non_vulnerable_version": "2.14.0-rc1", "affected_by_vulnerabilities": [ - OrderedDict( - [ - ( - "url", - f"http://testserver/api/vulnerabilities/{self.vul1.id}", - ), - ("vulnerability_id", "VCID-vul1-vul1-vul1"), - ("summary", "This is VCID-vul1-vul1-vul1"), - ("references", []), - ( - "fixed_packages", - [ - OrderedDict( - [ - ( - "url", - f"http://testserver/api/packages/{self.pkg_2_13_2.id}", - ), - ( - "purl", - "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - ), - ("is_vulnerable", True), - ( - "affected_by_vulnerabilities", - [{"vulnerability": "VCID-vul2-vul2-vul2"}], - ), - ( - "resource_url", - "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", - ), - ] - ) + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul1.id), + "vulnerability_id": "VCID-vul1-vul1-vul1", + "summary": "This is VCID-vul1-vul1-vul1", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } ], - ), - ("aliases", ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"]), - ("resource_url", "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1"), - ] - ) + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_2.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul2-vul2-vul2"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + } + ], + "aliases": ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1", + } ], "fixing_vulnerabilities": [ - OrderedDict( - [ - ( - "url", - f"http://testserver/api/vulnerabilities/{self.vul3.id}", - ), - ("vulnerability_id", "VCID-vul3-vul3-vul3"), - ("summary", "This is VCID-vul3-vul3-vul3"), - ("references", []), - ( - "fixed_packages", - [ - OrderedDict( - [ - ( - "url", - f"http://testserver/api/packages/{self.pkg_2_12_6.id}", - ), - ( - "purl", - "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", - ), - ("is_vulnerable", False), - ("affected_by_vulnerabilities", []), - ( - "resource_url", - "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", - ), - ] - ), - OrderedDict( - [ - ( - "url", - f"http://testserver/api/packages/{self.pkg_2_13_1.id}", - ), - ( - "purl", - "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", - ), - ("is_vulnerable", True), - ( - "affected_by_vulnerabilities", - [{"vulnerability": "VCID-vul1-vul1-vul1"}], - ), - ( - "resource_url", - "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", - ), - ] - ), + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul3.id), + "vulnerability_id": "VCID-vul3-vul3-vul3", + "summary": "This is VCID-vul3-vul3-vul3", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } ], - ), - ("aliases", ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"]), - ("resource_url", "http://testserver/vulnerabilities/VCID-vul3-vul3-vul3"), - ] - ) + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_12_6.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", + "is_vulnerable": False, + "affected_by_vulnerabilities": [], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", + }, + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul1-vul1-vul1"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + }, + ], + "aliases": ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul3-vul3-vul3", + } ], + "risk_score": None, "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", } - assert json.dumps(response, indent=2) == json.dumps(expected_response, indent=2) + assert response == expected def test_is_vulnerable_attribute_only_exists_on_queryset(self): assert not hasattr(self.pkg_2_13_1, "is_vulnerable") @@ -728,6 +792,176 @@ def test_api_with_ignorning_qualifiers(self): == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1" ) + def test_api_with_ghost_package_no_fixing_vulnerabilities(self): + self.pkg_2_13_1.is_ghost = True + self.pkg_2_13_1.save() + + response = self.csrf_client.get(f"/api/packages/{self.pkg_2_13_1.id}", format="json").data + + expected = { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "type": "maven", + "namespace": "com.fasterxml.jackson.core", + "name": "jackson-databind", + "version": "2.13.1", + "qualifiers": {}, + "subpath": "", + "is_vulnerable": True, + "next_non_vulnerable_version": "2.12.6", + "latest_non_vulnerable_version": "2.14.0-rc1", + "affected_by_vulnerabilities": [ + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul1.id), + "vulnerability_id": "VCID-vul1-vul1-vul1", + "summary": "This is VCID-vul1-vul1-vul1", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_2.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul2-vul2-vul2"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + } + ], + "aliases": ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1", + } + ], + "fixing_vulnerabilities": [], + "risk_score": None, + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + } + + assert response == expected + + def test_api_with_ghost_package_no_next_latest_non_vulnerabilities(self): + self.pkg_2_14_0_rc1.is_ghost = True + self.pkg_2_14_0_rc1.save() + + response = self.csrf_client.get(f"/api/packages/{self.pkg_2_13_1.id}", format="json").data + + expected = { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "type": "maven", + "namespace": "com.fasterxml.jackson.core", + "name": "jackson-databind", + "version": "2.13.1", + "qualifiers": {}, + "subpath": "", + "is_vulnerable": True, + "next_non_vulnerable_version": None, + "latest_non_vulnerable_version": None, + "affected_by_vulnerabilities": [ + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul1.id), + "vulnerability_id": "VCID-vul1-vul1-vul1", + "summary": "This is VCID-vul1-vul1-vul1", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_2.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul2-vul2-vul2"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + } + ], + "aliases": ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1", + } + ], + "fixing_vulnerabilities": [ + { + "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul3.id), + "vulnerability_id": "VCID-vul3-vul3-vul3", + "summary": "This is VCID-vul3-vul3-vul3", + "references": [ + { + "reference_url": "https://example.com", + "reference_id": "CVE-xxx-xxx", + "reference_type": "advisory", + "scores": [ + { + "value": "0.526", + "scoring_system": "epss", + "scoring_elements": ".0016", + } + ], + "url": "https://example.com", + } + ], + "fixed_packages": [ + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_12_6.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", + "is_vulnerable": False, + "affected_by_vulnerabilities": [], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", + }, + { + "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), + "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + "is_vulnerable": True, + "affected_by_vulnerabilities": [ + {"vulnerability": "VCID-vul1-vul1-vul1"} + ], + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + }, + ], + "aliases": ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"], + "risk_score": None, + "exploitability": None, + "weighted_severity": None, + "resource_url": "http://testserver/vulnerabilities/VCID-vul3-vul3-vul3", + } + ], + "risk_score": None, + "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", + } + + assert response == expected + class CPEApi(TestCase): def setUp(self): @@ -754,6 +988,46 @@ def test_api_response(self): self.assertEqual(response["count"], 1) +class TestCPEApiWithPackageVulnerabilityRelation(TestCase): + def setUp(self): + self.user = ApiUser.objects.create_api_user(username="e@mail.com") + self.auth = f"Token {self.user.auth_token.key}" + self.csrf_client = APIClient(enforce_csrf_checks=True) + self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) + self.vulnerability = Vulnerability.objects.create(summary="test") + self.affected_package, _ = Package.objects.get_or_create_from_purl( + purl="pkg:nginx/nginx@v3.4" + ) + self.fixed_package, _ = Package.objects.get_or_create_from_purl(purl="pkg:nginx/nginx@v4.0") + AffectedByPackageRelatedVulnerability.objects.create( + vulnerability=self.vulnerability, + created_by="test", + package=self.affected_package, + confidence=100, + ) + FixingPackageRelatedVulnerability.objects.create( + vulnerability=self.vulnerability, + created_by="test", + package=self.fixed_package, + confidence=100, + ) + for i in range(0, 10): + ref, _ = VulnerabilityReference.objects.get_or_create( + reference_id=f"cpe:/a:nginx:{i}", + url=f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query=cpe:/a:nginx:{i}", + ) + VulnerabilityRelatedReference.objects.create( + reference=ref, vulnerability=self.vulnerability + ) + + def test_cpe_api(self): + response = self.csrf_client.get("/api/cpes/", format="json") + self.assertEqual(status.HTTP_200_OK, response.status_code) + + response_data = response.json() + self.assertEqual(1, response_data["count"]) + + class AliasApi(TestCase): def setUp(self): self.user = ApiUser.objects.create_api_user(username="e@mail.com") diff --git a/vulnerabilities/tests/test_api_extension.py b/vulnerabilities/tests/test_api_extension.py index 634016149..fc971a576 100644 --- a/vulnerabilities/tests/test_api_extension.py +++ b/vulnerabilities/tests/test_api_extension.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -16,7 +16,6 @@ from vulnerabilities.api_extension import V2VulnerabilitySeveritySerializer from vulnerabilities.models import Alias from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilityRelatedReference @@ -46,7 +45,7 @@ def vulnerability_severity(vulnerability_reference): scoring_system="cvssv3_vector", value="7.0", scoring_elements="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - reference_id=vulnerability_reference.id, + url=f"https://..", ) @@ -68,10 +67,9 @@ def vulnerability(db, vulnerability_reference, vulnerability_severity): @fixture def package_related_vulnerability(db, package, vulnerability): - PackageRelatedVulnerability.objects.create( + AffectedByPackageRelatedVulnerability.objects.create( package=package, vulnerability=vulnerability, - fix=False, ) return package @@ -88,7 +86,7 @@ def test_V2VulnerabilitySeveritySerializer(vulnerability_severity): results = V2VulnerabilitySeveritySerializer(instance=vulnerability_severity).data expected = { "published_at": None, - "reference": {"reference_id": "fake", "reference_type": "", "reference_url": "https://.."}, + "url": "https://..", "score": "7.0", "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", "scoring_system": "cvssv3_vector", diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py new file mode 100644 index 000000000..fa3b7773c --- /dev/null +++ b/vulnerabilities/tests/test_api_v2.py @@ -0,0 +1,564 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from django.urls import reverse +from packageurl import PackageURL +from rest_framework import status +from rest_framework.test import APIClient +from rest_framework.test import APITestCase + +from vulnerabilities.api_v2 import PackageV2Serializer +from vulnerabilities.api_v2 import VulnerabilityListSerializer +from vulnerabilities.models import Alias +from vulnerabilities.models import ApiUser +from vulnerabilities.models import Package +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import Weakness + + +class VulnerabilityV2ViewSetTest(APITestCase): + def setUp(self): + # Create vulnerabilities + self.vuln1 = Vulnerability.objects.create( + vulnerability_id="VCID-1234", summary="Test vulnerability 1" + ) + self.vuln2 = Vulnerability.objects.create( + vulnerability_id="VCID-5678", summary="Test vulnerability 2" + ) + + # Create aliases + Alias.objects.create(alias="CVE-2021-1234", vulnerability=self.vuln1) + Alias.objects.create(alias="CVE-2021-5678", vulnerability=self.vuln2) + + # Create weaknesses + self.weakness1 = Weakness.objects.create(cwe_id=79) + self.weakness1.vulnerabilities.add(self.vuln1) + + self.weakness2 = Weakness.objects.create(cwe_id=89) + self.weakness2.vulnerabilities.add(self.vuln2) + + # Create references + self.reference1 = VulnerabilityReference.objects.create( + url="https://example.com/ref1", reference_type="advisory", reference_id="REF-1" + ) + self.reference1.vulnerabilities.add(self.vuln1) + + self.reference2 = VulnerabilityReference.objects.create( + url="https://example.com/ref2", reference_type="exploit", reference_id="REF-2" + ) + self.reference2.vulnerabilities.add(self.vuln2) + + self.user = ApiUser.objects.create_api_user(username="e@mail.com") + self.auth = f"Token {self.user.auth_token.key}" + self.client = APIClient(enforce_csrf_checks=True) + self.client.credentials(HTTP_AUTHORIZATION=self.auth) + + def test_list_vulnerabilities(self): + """ + Test listing vulnerabilities without filters. + Should return a paginated response with vulnerabilities dictionary. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertEqual(len(response.data["results"]["vulnerabilities"]), 2) + self.assertIn("VCID-1234", response.data["results"]["vulnerabilities"]) + self.assertIn("VCID-5678", response.data["results"]["vulnerabilities"]) + self.assertTrue("url" in response.data["results"]["vulnerabilities"]["VCID-1234"]) + + def test_retrieve_vulnerability_detail(self): + """ + Test retrieving vulnerability details by vulnerability_id. + """ + url = reverse("vulnerability-v2-detail", kwargs={"vulnerability_id": "VCID-1234"}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["vulnerability_id"], "VCID-1234") + self.assertEqual(response.data["summary"], "Test vulnerability 1") + self.assertEqual(response.data["aliases"], ["CVE-2021-1234"]) + self.assertEqual(len(response.data["weaknesses"]), 1) + self.assertEqual(len(response.data["references"]), 1) + + def test_filter_vulnerability_by_vulnerability_id(self): + """ + Test filtering vulnerabilities by vulnerability_id. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get(url, {"vulnerability_id": "VCID-1234"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data["vulnerability_id"], "VCID-1234") + + def test_filter_vulnerability_by_alias(self): + """ + Test filtering vulnerabilities by alias. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get(url, {"alias": "CVE-2021-5678"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertEqual( + response.data["results"]["vulnerabilities"]["VCID-5678"]["vulnerability_id"], + "VCID-5678", + ) + + def test_filter_vulnerabilities_multiple_ids(self): + """ + Test filtering vulnerabilities by multiple vulnerability_ids. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get( + url, {"vulnerability_id": ["VCID-1234", "VCID-5678"]}, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["vulnerabilities"]), 2) + + def test_filter_vulnerabilities_multiple_aliases(self): + """ + Test filtering vulnerabilities by multiple aliases. + """ + url = reverse("vulnerability-v2-list") + response = self.client.get( + url, {"alias": ["CVE-2021-1234", "CVE-2021-5678"]}, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["vulnerabilities"]), 2) + + def test_invalid_vulnerability_id(self): + """ + Test retrieving a vulnerability with an invalid vulnerability_id. + Should return 404 Not Found. + """ + url = reverse("vulnerability-v2-detail", kwargs={"vulnerability_id": "VCID-9999"}) + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) + + def test_get_url_in_serializer(self): + """ + Test that the serializer correctly includes the URL field. + """ + vulnerability = Vulnerability.objects.get(vulnerability_id="VCID-1234") + serializer = VulnerabilityListSerializer(vulnerability, context={"request": None}) + self.assertIn("url", serializer.data) + self.assertEqual(serializer.data["vulnerability_id"], "VCID-1234") + + def test_list_vulnerabilities_pagination(self): + """ + Test listing vulnerabilities with pagination. + """ + # Create additional vulnerabilities to trigger pagination + for i in range(3, 15): + Vulnerability.objects.create( + vulnerability_id=f"VCID-{i}", summary=f"Test vulnerability {i}" + ) + + url = reverse("vulnerability-v2-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertIn("next", response.data) + self.assertIn("previous", response.data) + # The 'vulnerabilities' dictionary should contain vulnerabilities up to the page limit + self.assertEqual( + len(response.data["results"]["vulnerabilities"]), 10 + ) # Assuming default page size is 10 + + +class PackageV2ViewSetTest(APITestCase): + def setUp(self): + # Create packages + self.package1 = Package.objects.create( + package_url="pkg:pypi/django@3.2", name="django", version="3.2", type="pypi" + ) + self.package2 = Package.objects.create( + package_url="pkg:npm/lodash@4.17.20", name="lodash", version="4.17.20", type="npm" + ) + + # Create vulnerabilities + self.vuln1 = Vulnerability.objects.create( + vulnerability_id="VCID-1234", summary="Test vulnerability 1" + ) + self.vuln2 = Vulnerability.objects.create( + vulnerability_id="VCID-5678", summary="Test vulnerability 2" + ) + + # Associate packages with vulnerabilities + self.package1.affected_by_vulnerabilities.add(self.vuln1) + self.package2.fixing_vulnerabilities.add(self.vuln2) + + self.user = ApiUser.objects.create_api_user(username="e@mail.com") + self.auth = f"Token {self.user.auth_token.key}" + self.client = APIClient(enforce_csrf_checks=True) + self.client.credentials(HTTP_AUTHORIZATION=self.auth) + + def test_list_packages(self): + """ + Test listing packages without filters. + Should return a list of packages with their details and associated vulnerabilities. + """ + url = reverse("package-v2-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("packages", response.data["results"]) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertEqual(len(response.data["results"]["packages"]), 2) + # Verify that vulnerabilities are included + self.assertIsInstance(response.data["results"]["vulnerabilities"], dict) + package_vulns = set() + for package in response.data["results"]["packages"]: + package_vulns.update(package["affected_by_vulnerabilities"]) + package_vulns.update(package["fixing_vulnerabilities"]) + self.assertTrue( + all(vuln_id in response.data["results"]["vulnerabilities"] for vuln_id in package_vulns) + ) + + def test_filter_packages_by_purl(self): + """ + Test filtering packages by one or more PURLs. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"purl": "pkg:pypi/django@3.2"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 1) + self.assertEqual(response.data["results"]["packages"][0]["purl"], "pkg:pypi/django@3.2") + + def test_filter_packages_by_affected_vulnerability(self): + """ + Test filtering packages by affected_by_vulnerability. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"affected_by_vulnerability": "VCID-1234"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 1) + self.assertEqual(response.data["results"]["packages"][0]["purl"], "pkg:pypi/django@3.2") + + def test_filter_packages_by_fixing_vulnerability(self): + """ + Test filtering packages by fixing_vulnerability. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"fixing_vulnerability": "VCID-5678"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 1) + self.assertEqual(response.data["results"]["packages"][0]["purl"], "pkg:npm/lodash@4.17.20") + + def test_package_serializer_fields(self): + """ + Test that the PackageV2Serializer returns the correct fields. + """ + package = Package.objects.get(package_url="pkg:pypi/django@3.2") + serializer = PackageV2Serializer(package) + data = serializer.data + self.assertIn("purl", data) + self.assertIn("affected_by_vulnerabilities", data) + self.assertIn("fixing_vulnerabilities", data) + self.assertIn("next_non_vulnerable_version", data) + self.assertIn("latest_non_vulnerable_version", data) + self.assertEqual(data["purl"], "pkg:pypi/django@3.2") + self.assertEqual(data["affected_by_vulnerabilities"], ["VCID-1234"]) + self.assertEqual(data["fixing_vulnerabilities"], []) + + def test_list_packages_pagination(self): + """ + Test listing packages with pagination. + """ + # Create additional packages to trigger pagination + for i in range(3, 15): + Package.objects.create( + package_url=f"pkg:pypi/package{i}@1.0.{i}", + name=f"package{i}", + version=f"1.0.{i}", + type="pypi", + ) + + url = reverse("package-v2-list") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("results", response.data) + self.assertIn("packages", response.data["results"]) + self.assertIn("vulnerabilities", response.data["results"]) + self.assertIn("next", response.data) + self.assertIn("previous", response.data) + self.assertEqual( + len(response.data["results"]["packages"]), 10 + ) # Assuming default page size is 10 + + def test_invalid_vulnerability_filter(self): + """ + Test filtering packages with an invalid vulnerability ID. + Should return an empty list. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"affected_by_vulnerability": "VCID-9999"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 0) + + def test_invalid_purl_filter(self): + """ + Test filtering packages with an invalid PURL. + Should return an empty list. + """ + url = reverse("package-v2-list") + response = self.client.get(url, {"purl": "pkg:nonexistent/package@1.0.0"}, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["results"]["packages"]), 0) + + def test_get_affected_by_vulnerabilities(self): + """ + Test the get_affected_by_vulnerabilities method in the serializer. + """ + package = Package.objects.get(package_url="pkg:pypi/django@3.2") + serializer = PackageV2Serializer() + vulnerabilities = serializer.get_affected_by_vulnerabilities(package) + self.assertEqual(vulnerabilities, ["VCID-1234"]) + + def test_get_fixing_vulnerabilities(self): + """ + Test the get_fixing_vulnerabilities method in the serializer. + """ + package = Package.objects.get(package_url="pkg:npm/lodash@4.17.20") + serializer = PackageV2Serializer() + vulnerabilities = serializer.get_fixing_vulnerabilities(package) + self.assertEqual(vulnerabilities, ["VCID-5678"]) + + def test_bulk_lookup_with_valid_purls(self): + """ + Test bulk lookup with valid PURLs. + Should return packages and their associated vulnerabilities. + """ + url = reverse("package-v2-bulk-lookup") + data = {"purls": ["pkg:pypi/django@3.2", "pkg:npm/lodash@4.17.20"]} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("packages", response.data) + self.assertIn("vulnerabilities", response.data) + self.assertEqual(len(response.data["packages"]), 2) + # Verify that the returned data matches the packages + purls = [package["purl"] for package in response.data["packages"]] + self.assertIn("pkg:pypi/django@3.2", purls) + self.assertIn("pkg:npm/lodash@4.17.20", purls) + # Verify that vulnerabilities are included + package_vulns = set() + for package in response.data["packages"]: + package_vulns.update(package["affected_by_vulnerabilities"]) + package_vulns.update(package["fixing_vulnerabilities"]) + self.assertTrue( + all(vuln_id in response.data["vulnerabilities"] for vuln_id in package_vulns) + ) + + def test_bulk_lookup_with_invalid_purls(self): + """ + Test bulk lookup with invalid PURLs. + """ + url = reverse("package-v2-bulk-lookup") + data = {"purls": ["pkg:pypi/nonexistent@1.0.0", "pkg:npm/unknown@0.0.1"]} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Since the packages don't exist, the response should be empty + self.assertEqual(len(response.data["packages"]), 0) + self.assertEqual(len(response.data["vulnerabilities"]), 0) + + def test_bulk_lookup_with_empty_purls(self): + """ + Test bulk lookup with empty purls list. + Should return 400 Bad Request. + """ + url = reverse("package-v2-bulk-lookup") + data = {"purls": []} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("error", response.data) + self.assertIn("message", response.data) + self.assertEqual(response.data["message"], "A non-empty 'purls' list of PURLs is required.") + + def test_bulk_search_with_valid_purls(self): + """ + Test bulk search with valid PURLs. + Should return packages and their associated vulnerabilities. + """ + url = reverse("package-v2-bulk-search") + data = {"purls": ["pkg:pypi/django@3.2", "pkg:npm/lodash@4.17.20"]} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("packages", response.data) + self.assertIn("vulnerabilities", response.data) + self.assertEqual(len(response.data["packages"]), 2) + purls = [package["purl"] for package in response.data["packages"]] + self.assertIn("pkg:pypi/django@3.2", purls) + self.assertIn("pkg:npm/lodash@4.17.20", purls) + # Verify that vulnerabilities are included + package_vulns = set() + for package in response.data["packages"]: + package_vulns.update(package["affected_by_vulnerabilities"]) + package_vulns.update(package["fixing_vulnerabilities"]) + self.assertTrue( + all(vuln_id in response.data["vulnerabilities"] for vuln_id in package_vulns) + ) + + def test_bulk_search_with_purl_only_true(self): + """ + Test bulk search with purl_only set to True. + Should return only the PURLs of vulnerable packages. + """ + url = reverse("package-v2-bulk-search") + data = { + "purls": ["pkg:pypi/django@3.2", "pkg:npm/lodash@4.17.20"], + "purl_only": True, + } + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Since purl_only=True, response should be a list of PURLs + self.assertIsInstance(response.data, list) + # Only vulnerable packages should be included + self.assertEqual(len(response.data), 1) + self.assertEqual(response.data, ["pkg:pypi/django@3.2"]) + + def test_bulk_search_with_plain_purl_true(self): + """ + Test bulk search with plain_purl set to True. + Should return packages grouped by plain PURLs. + """ + # Create another package with the same name and version but different qualifiers + Package.objects.create( + name="django", + version="3.2", + type="pypi", + qualifiers={"extension": "tar.gz"}, + ) + + url = reverse("package-v2-bulk-search") + data = { + "purls": ["pkg:pypi/django@3.2", "pkg:pypi/django@3.2?extension=tar.gz"], + "plain_purl": True, + } + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertIn("packages", response.data) + self.assertIn("vulnerabilities", response.data) + # Since plain_purl=True, packages with the same type, namespace, name, version are grouped + self.assertEqual(len(response.data["packages"]), 1) + purl = response.data["packages"][0]["purl"] + self.assertTrue(purl.startswith("pkg:pypi/django@3.2")) + + def test_bulk_search_with_purl_only_and_plain_purl_true(self): + """ + Test bulk search with purl_only and plain_purl both set to True. + Should return only the plain PURLs of vulnerable packages. + """ + url = reverse("package-v2-bulk-search") + data = { + "purls": ["pkg:pypi/django@3.2", "pkg:pypi/django@3.1"], + "purl_only": True, + "plain_purl": True, + } + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Response should be a list of plain PURLs + self.assertIsInstance(response.data, list) + # Only one plain PURL should be returned for vulnerable packages + self.assertEqual(len(response.data), 1) + self.assertEqual(response.data, ["pkg:pypi/django@3.2"]) + + def test_bulk_search_with_invalid_purls(self): + """ + Test bulk search with invalid PURLs. + Should return an empty response. + """ + url = reverse("package-v2-bulk-search") + data = {"purls": ["pkg:pypi/nonexistent@1.0.0", "pkg:npm/unknown@0.0.1"]} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Since the packages don't exist, the response should be empty + self.assertEqual(len(response.data["packages"]), 0) + self.assertEqual(len(response.data["vulnerabilities"]), 0) + + def test_bulk_search_with_empty_purls(self): + """ + Test bulk search with empty purls list. + Should return 400 Bad Request. + """ + url = reverse("package-v2-bulk-search") + data = {"purls": []} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("error", response.data) + self.assertIn("message", response.data) + self.assertEqual(response.data["message"], "A non-empty 'purls' list of PURLs is required.") + + def test_all_vulnerable_packages(self): + """ + Test the 'all' endpoint that returns all vulnerable package URLs. + """ + url = reverse("package-v2-all") + response = self.client.get(url, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # Since package1 is vulnerable, it should be returned + expected_purls = ["pkg:pypi/django@3.2"] + self.assertEqual(sorted(response.data), sorted(expected_purls)) + + def test_lookup_with_valid_purl(self): + """ + Test the 'lookup' endpoint with a valid PURL. + Should return the package and its associated vulnerabilities. + """ + url = reverse("package-v2-lookup") + data = {"purl": "pkg:pypi/django@3.2"} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(1, len(response.data)) + self.assertIn("purl", response.data[0]) + self.assertIn("affected_by_vulnerabilities", response.data[0]) + self.assertIn("fixing_vulnerabilities", response.data[0]) + self.assertIn("next_non_vulnerable_version", response.data[0]) + self.assertIn("latest_non_vulnerable_version", response.data[0]) + self.assertEqual(response.data[0]["purl"], "pkg:pypi/django@3.2") + self.assertEqual(response.data[0]["affected_by_vulnerabilities"], ["VCID-1234"]) + self.assertEqual(response.data[0]["fixing_vulnerabilities"], []) + + def test_lookup_with_invalid_purl(self): + """ + Test the 'lookup' endpoint with a PURL that does not exist. + Should return empty packages and vulnerabilities. + """ + url = reverse("package-v2-lookup") + data = {"purl": "pkg:pypi/nonexistent@1.0.0"} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # No packages or vulnerabilities should be returned + self.assertEqual(len(response.data), 0) + + def test_lookup_with_missing_purl(self): + """ + Test the 'lookup' endpoint without providing a 'purl'. + Should return 400 Bad Request. + """ + url = reverse("package-v2-lookup") + data = {} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + self.assertIn("error", response.data) + self.assertIn("message", response.data) + self.assertEqual(response.data["message"], "A 'purl' is required.") + + def test_lookup_with_invalid_purl_format(self): + """ + Test the 'lookup' endpoint with an invalid PURL format. + Should return empty packages and vulnerabilities. + """ + url = reverse("package-v2-lookup") + data = {"purl": "invalid_purl_format"} + response = self.client.post(url, data, format="json") + self.assertEqual(response.status_code, status.HTTP_200_OK) + # No packages or vulnerabilities should be returned + self.assertEqual(len(response.data), 0) diff --git a/vulnerabilities/tests/test_archlinux.py b/vulnerabilities/tests/test_archlinux.py index f5ee38b75..d8582191c 100644 --- a/vulnerabilities/tests/test_archlinux.py +++ b/vulnerabilities/tests/test_archlinux.py @@ -4,7 +4,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_basics.py b/vulnerabilities/tests/test_basics.py index f56bf2173..c83aea393 100644 --- a/vulnerabilities/tests/test_basics.py +++ b/vulnerabilities/tests/test_basics.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_changelog.py b/vulnerabilities/tests/test_changelog.py index 11912e14d..1d5eedaea 100644 --- a/vulnerabilities/tests/test_changelog.py +++ b/vulnerabilities/tests/test_changelog.py @@ -3,26 +3,27 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from datetime import datetime +from unittest.mock import patch import pytest +from packageurl import PackageURL from univers.version_range import NpmVersionRange from univers.versions import SemverVersion -from vulnerabilities.import_runner import ImportRunner +from vulnerabilities import models from vulnerabilities.importer import AffectedPackage -from vulnerabilities.models import * from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline @pytest.mark.django_db def test_package_changelog(): - pkg, _ = Package.objects.get_or_create_from_purl("pkg:npm/foo@1.0.0") - assert PackageChangeLog.objects.filter(package=pkg).count() == 0 - adv = Advisory.objects.create( + pkg, _ = models.Package.objects.get_or_create_from_purl("pkg:npm/foo@1.0.0") + assert models.PackageChangeLog.objects.filter(package=pkg).count() == 0 + adv = models.Advisory.objects.create( created_by=NpmImporterPipeline.pipeline_id, summary="TEST", date_collected=datetime.now(), @@ -39,16 +40,18 @@ def test_package_changelog(): aliases=["CVE-123"], ) NpmImporterPipeline().import_advisory(advisory=adv) - assert PackageChangeLog.objects.filter(package=pkg).count() == 1 + assert models.PackageChangeLog.objects.filter(package=pkg).count() == 1 NpmImporterPipeline().import_advisory(advisory=adv) - assert PackageChangeLog.objects.filter(package=pkg).count() == 1 + assert models.PackageChangeLog.objects.filter(package=pkg).count() == 1 assert ( - PackageChangeLog.objects.filter(action_type=PackageChangeLog.FIXING, package=pkg).count() + models.PackageChangeLog.objects.filter( + action_type=models.PackageChangeLog.FIXING, package=pkg + ).count() == 1 ) - pkg1, _ = Package.objects.get_or_create_from_purl("pkg:npm/foo@2.0.0") - assert PackageChangeLog.objects.filter(package=pkg1).count() == 0 - adv = Advisory.objects.create( + pkg1, _ = models.Package.objects.get_or_create_from_purl("pkg:npm/foo@2.0.0") + assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 0 + adv = models.Advisory.objects.create( created_by=NpmImporterPipeline.pipeline_id, summary="TEST-1", date_collected=datetime.now(), @@ -65,12 +68,13 @@ def test_package_changelog(): aliases=["CVE-145"], ) NpmImporterPipeline().import_advisory(advisory=adv) - assert PackageChangeLog.objects.filter(package=pkg1).count() == 1 + assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 1 NpmImporterPipeline().import_advisory(advisory=adv) - assert PackageChangeLog.objects.filter(package=pkg1).count() == 1 + assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 1 assert ( - PackageChangeLog.objects.filter( - action_type=PackageChangeLog.AFFECTED_BY, package=pkg1 + models.PackageChangeLog.objects.filter( + action_type=models.PackageChangeLog.AFFECTED_BY, + package=pkg1, ).count() == 1 ) @@ -78,7 +82,7 @@ def test_package_changelog(): @pytest.mark.django_db def test_vulnerability_changelog(): - adv = Advisory.objects.create( + adv = models.Advisory.objects.create( created_by=NpmImporterPipeline.pipeline_id, summary="TEST_1", date_collected=datetime.now(), @@ -97,10 +101,37 @@ def test_vulnerability_changelog(): NpmImporterPipeline().import_advisory(advisory=adv) # 1 Changelogs is expected here: # 1 for importing vuln details - assert VulnerabilityChangeLog.objects.count() == 1 + assert models.VulnerabilityChangeLog.objects.count() == 1 NpmImporterPipeline().import_advisory(advisory=adv) - assert VulnerabilityChangeLog.objects.count() == 1 + assert models.VulnerabilityChangeLog.objects.count() == 1 assert ( - VulnerabilityChangeLog.objects.filter(action_type=VulnerabilityChangeLog.IMPORT).count() + models.VulnerabilityChangeLog.objects.filter( + action_type=models.VulnerabilityChangeLog.IMPORT + ).count() == 1 ) + + +@patch("vulnerabilities.models.VULNERABLECODE_VERSION", "test-version") +@pytest.mark.django_db +def test_vulnerability_changelog_software_version(): + adv = models.Advisory.objects.create( + created_by=NpmImporterPipeline.pipeline_id, + summary="TEST_1", + date_collected=datetime.now(), + url="https://test.com/source", + affected_packages=[ + AffectedPackage( + package=PackageURL( + type="npm", + name="foo", + ), + fixed_version=SemverVersion("1.0"), + ).to_dict() + ], + aliases=["CVE-TEST-1234"], + ) + NpmImporterPipeline().import_advisory(advisory=adv) + npm_vulnerability_log = models.VulnerabilityChangeLog.objects.first() + + assert ("test-version", npm_vulnerability_log.software_version) diff --git a/vulnerabilities/tests/test_compute_package_version_rank.py b/vulnerabilities/tests/test_compute_package_version_rank.py new file mode 100644 index 000000000..12cd172a8 --- /dev/null +++ b/vulnerabilities/tests/test_compute_package_version_rank.py @@ -0,0 +1,59 @@ +from unittest.mock import patch + +import pytest +from univers.versions import Version + +from vulnerabilities.models import Package +from vulnerabilities.pipelines.compute_package_version_rank import ComputeVersionRankPipeline + + +@pytest.mark.django_db +class TestComputeVersionRankPipeline: + @pytest.fixture + def pipeline(self): + return ComputeVersionRankPipeline() + + @pytest.fixture + def packages(self, db): + package_type = "pypi" + namespace = "test_namespace" + name = "test_package" + Package.objects.create(type=package_type, namespace=namespace, name=name, version="1.0.0") + Package.objects.create(type=package_type, namespace=namespace, name=name, version="1.1.0") + Package.objects.create(type=package_type, namespace=namespace, name=name, version="0.9.0") + return Package.objects.filter(type=package_type, namespace=namespace, name=name) + + def test_compute_and_store_version_rank(self, pipeline, packages): + with patch.object(pipeline, "log") as mock_log: + pipeline.compute_and_store_version_rank() + assert mock_log.call_count > 0 + for package in packages: + assert package.version_rank is not None + + def test_update_version_rank_for_group(self, pipeline, packages): + with patch.object(Package.objects, "bulk_update") as mock_bulk_update: + pipeline.update_version_rank_for_group(packages) + mock_bulk_update.assert_called_once() + updated_packages = mock_bulk_update.call_args[0][0] + assert len(updated_packages) == len(packages) + for idx, package in enumerate(sorted(packages, key=lambda p: Version(p.version))): + assert updated_packages[idx].version_rank == idx + + def test_sort_packages_by_version(self, pipeline, packages): + sorted_packages = pipeline.sort_packages_by_version(packages) + versions = [p.version for p in sorted_packages] + assert versions == sorted(versions, key=Version) + + def test_sort_packages_by_version_empty(self, pipeline): + assert pipeline.sort_packages_by_version([]) == [] + + def test_sort_packages_by_version_invalid_scheme(self, pipeline, packages): + for package in packages: + package.type = "invalid" + assert pipeline.sort_packages_by_version(packages) == [] + + def test_compute_and_store_version_rank_invalid_scheme(self, pipeline): + Package.objects.create(type="invalid", namespace="test", name="package", version="1.0.0") + with patch.object(pipeline, "log") as mock_log: + pipeline.compute_and_store_version_rank() + mock_log.assert_any_call("Successfully populated `version_rank` for all packages.") diff --git a/vulnerabilities/tests/test_cpe_reference.py b/vulnerabilities/tests/test_cpe_reference.py index 7f119292f..2c66db2e8 100644 --- a/vulnerabilities/tests/test_cpe_reference.py +++ b/vulnerabilities/tests/test_cpe_reference.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_create_api_user_command.py b/vulnerabilities/tests/test_create_api_user_command.py index 6c54fca24..00faa9a78 100644 --- a/vulnerabilities/tests/test_create_api_user_command.py +++ b/vulnerabilities/tests/test_create_api_user_command.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_curl.py b/vulnerabilities/tests/test_curl.py index 528686e39..6822e9677 100644 --- a/vulnerabilities/tests/test_curl.py +++ b/vulnerabilities/tests/test_curl.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_data/export_command/aboutcode-packages-1ccd/generic/nginx/test/purls.yml b/vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/purls.yml similarity index 100% rename from vulnerabilities/tests/test_data/export_command/aboutcode-packages-1ccd/generic/nginx/test/purls.yml rename to vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/purls.yml diff --git a/vulnerabilities/tests/test_data/export_command/aboutcode-packages-1ccd/generic/nginx/test/vulnerabilities.yml b/vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/vulnerabilities.yml similarity index 100% rename from vulnerabilities/tests/test_data/export_command/aboutcode-packages-1ccd/generic/nginx/test/vulnerabilities.yml rename to vulnerabilities/tests/test_data/export_command/aboutcode-packages-generic-0d/generic/nginx/test/vulnerabilities.yml diff --git a/vulnerabilities/tests/test_data/export_command/aboutcode-vulnerabilities/ps/VCID-pst6-b358-aaap.yml b/vulnerabilities/tests/test_data/export_command/aboutcode-vulnerabilities/ps/VCID-pst6-b358-aaap.yml index 07adaf90e..63ab7f5af 100644 --- a/vulnerabilities/tests/test_data/export_command/aboutcode-vulnerabilities/ps/VCID-pst6-b358-aaap.yml +++ b/vulnerabilities/tests/test_data/export_command/aboutcode-vulnerabilities/ps/VCID-pst6-b358-aaap.yml @@ -7,10 +7,7 @@ severities: scoring_system: cvssv3_vector scoring_elements: CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H published_at: - reference: - url: https://.. - reference_type: - reference_id: fake + url: https://.. weaknesses: - CWE-15 references: diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index d43755980..046c86ce5 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -672,7 +672,7 @@ def setUpBeforeMigration(self, apps): date_collected=timezone.now(), ) - def test_removal_of_duped_purls(self): + def test_update_npm_pypa_created_by_field(self): Advisory = apps.get_model("vulnerabilities", "Advisory") adv = Advisory.objects.all() @@ -714,7 +714,7 @@ def setUpBeforeMigration(self, apps): date_collected=timezone.now(), ) - def test_removal_of_duped_purls(self): + def test_update_nginx_created_by_field(self): Advisory = apps.get_model("vulnerabilities", "Advisory") adv = Advisory.objects.all() @@ -753,7 +753,7 @@ def setUpBeforeMigration(self, apps): date_collected=timezone.now(), ) - def test_removal_of_duped_purls(self): + def test_update_gitlab_created_by_field(self): Advisory = apps.get_model("vulnerabilities", "Advisory") adv = Advisory.objects.all() @@ -794,7 +794,7 @@ def setUpBeforeMigration(self, apps): date_collected=timezone.now(), ) - def test_removal_of_duped_purls(self): + def test_update_github_created_by_field(self): Advisory = apps.get_model("vulnerabilities", "Advisory") adv = Advisory.objects.all() @@ -835,9 +835,48 @@ def setUpBeforeMigration(self, apps): date_collected=timezone.now(), ) - def test_removal_of_duped_purls(self): + def test_update_nvd_created_by_field(self): Advisory = apps.get_model("vulnerabilities", "Advisory") adv = Advisory.objects.all() assert adv.filter(created_by="vulnerabilities.importers.nvd.NVDImporter").count() == 0 assert adv.filter(created_by="nvd_importer").count() == 1 + + +class TestUpdatePysecAdvisoryCreatedByField(TestMigrations): + app_name = "vulnerabilities" + migrate_from = "0073_delete_packagerelatedvulnerability" + migrate_to = "0074_update_pysec_advisory_created_by" + + advisory_data1 = AdvisoryData( + aliases=["CVE-2020-13371337"], + summary="vulnerability description here", + affected_packages=[ + AffectedPackage( + package=PackageURL(type="pypi", name="foobar"), + affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), + ) + ], + references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + date_published=timezone.now(), + url="https://test.com", + ) + + def setUpBeforeMigration(self, apps): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv1 = Advisory.objects.create( + aliases=self.advisory_data1.aliases, + summary=self.advisory_data1.summary, + affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], + references=[ref.to_dict() for ref in self.advisory_data1.references], + url=self.advisory_data1.url, + created_by="vulnerabilities.importers.pysec.PyPIImporter", + date_collected=timezone.now(), + ) + + def test_update_pysec_created_by_field(self): + Advisory = apps.get_model("vulnerabilities", "Advisory") + adv = Advisory.objects.all() + + assert adv.filter(created_by="vulnerabilities.importers.pysec.PyPIImporter").count() == 0 + assert adv.filter(created_by="pysec_importer").count() == 1 diff --git a/vulnerabilities/tests/test_data_source.py b/vulnerabilities/tests/test_data_source.py index b0baf5685..40eeb6b3f 100644 --- a/vulnerabilities/tests/test_data_source.py +++ b/vulnerabilities/tests/test_data_source.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_debian.py b/vulnerabilities/tests/test_debian.py index ad21ef92a..25bbcb04d 100644 --- a/vulnerabilities/tests/test_debian.py +++ b/vulnerabilities/tests/test_debian.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_debian_oval.py b/vulnerabilities/tests/test_debian_oval.py index b8fb0935b..e6c9f9eef 100644 --- a/vulnerabilities/tests/test_debian_oval.py +++ b/vulnerabilities/tests/test_debian_oval.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_default_improver.py b/vulnerabilities/tests/test_default_improver.py index 703e985a4..63408a522 100644 --- a/vulnerabilities/tests/test_default_improver.py +++ b/vulnerabilities/tests/test_default_improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -17,6 +17,7 @@ from vulnerabilities.importer import Reference from vulnerabilities.improver import Inference from vulnerabilities.improvers.default import DefaultImprover +from vulnerabilities.improvers.default import get_exact_purls from vulnerabilities.tests import util_tests BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -136,3 +137,29 @@ def test_default_improver_with_nvd(): for data in list(default_improver.get_inferences(AdvisoryData.from_dict(advisory_data))) ] util_tests.check_results_against_json(result, expected_file) + + +def test_AffectedPackage_from_dict_should_not_crash_with_invalid_version_range(): + package = PackageURL( + type="rpm", + namespace="rpms", + name="python", + qualifiers={}, + subpath=None, + ) + + test_ranges = [ + # foo is a non-existing range + "vers:foo/1.2.3", + # apache was not supported and returned from vulnerabilities.importers.apache_httpd.ApacheHTTPDImporter + "vers:apache/", + None, + ] + for tr in test_ranges: + pkg = { + "package": package.to_dict(), + "affected_version_range": tr, + "fixed_version": None, + } + + assert AffectedPackage.from_dict(pkg) is None diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py index 249181347..2531ed695 100644 --- a/vulnerabilities/tests/test_elixir_security.py +++ b/vulnerabilities/tests/test_elixir_security.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_example.py b/vulnerabilities/tests/test_example.py index bb035b61b..119a8b61d 100644 --- a/vulnerabilities/tests/test_example.py +++ b/vulnerabilities/tests/test_example.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -85,8 +85,8 @@ def test_improve_framework_using_example_improver(self): ImproveRunner(improver_class=ExampleAliasImprover).run() assert models.Package.objects.count() == 3 - assert models.PackageRelatedVulnerability.objects.filter(fix=True).count() == 1 - assert models.PackageRelatedVulnerability.objects.filter(fix=False).count() == 2 + assert models.FixingPackageRelatedVulnerability.objects.count() == 1 + assert models.AffectedByPackageRelatedVulnerability.objects.count() == 2 assert models.VulnerabilitySeverity.objects.count() == 1 assert models.VulnerabilityReference.objects.count() == 1 diff --git a/vulnerabilities/tests/test_export.py b/vulnerabilities/tests/test_export.py index ded0bd4b6..244940261 100644 --- a/vulnerabilities/tests/test_export.py +++ b/vulnerabilities/tests/test_export.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -18,9 +18,9 @@ from pytest import raises from aboutcode import hashid +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilityRelatedReference @@ -50,7 +50,7 @@ def vulnerability_severity(vulnerability_reference): scoring_system="cvssv3_vector", value="7.0", scoring_elements="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - reference_id=vulnerability_reference.id, + url=f"https://..", ) @@ -66,16 +66,16 @@ def vulnerability(db, vulnerability_reference, vulnerability_severity): weakness = Weakness.objects.create(cwe_id=15) vulnerability.weaknesses.add(weakness) + vulnerability.severities.add(vulnerability_severity) return vulnerability @fixture def package_related_vulnerability(db, package, vulnerability): - PackageRelatedVulnerability.objects.create( + AffectedByPackageRelatedVulnerability.objects.create( package=package, vulnerability=vulnerability, - fix=False, ) return package diff --git a/vulnerabilities/tests/test_fireeye.py b/vulnerabilities/tests/test_fireeye.py index 15935728c..f3e3bb862 100644 --- a/vulnerabilities/tests/test_fireeye.py +++ b/vulnerabilities/tests/test_fireeye.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import os diff --git a/vulnerabilities/tests/test_fix_models.py b/vulnerabilities/tests/test_fix_models.py index 10845f353..cc94a41ec 100644 --- a/vulnerabilities/tests/test_fix_models.py +++ b/vulnerabilities/tests/test_fix_models.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -11,8 +11,9 @@ from django.test import TestCase from packageurl import PackageURL +from vulnerabilities.models import AffectedByPackageRelatedVulnerability +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability User = get_user_model() @@ -37,20 +38,18 @@ def setUp(self): ) vuln_package = Package.objects.create(**query_kwargs) # Attaching same package to 2 vulnerabilities - PackageRelatedVulnerability.objects.create( + AffectedByPackageRelatedVulnerability.objects.create( package=vuln_package, vulnerability=vuln1, - fix=False, ) - PackageRelatedVulnerability.objects.create( + FixingPackageRelatedVulnerability.objects.create( package=vuln_package, vulnerability=vuln2, - fix=False, ) def test_get_vulnerable_packages(self): vuln_packages = Package.objects.vulnerable() - assert vuln_packages.count() == 20 + assert vuln_packages.count() == 10 assert vuln_packages.distinct().count() == 10 vuln_purls = [pkg.purl for pkg in vuln_packages.distinct().only(*PackageURL._fields)] assert vuln_purls == [ diff --git a/vulnerabilities/tests/test_forms.py b/vulnerabilities/tests/test_forms.py index 8999970bc..381c9c8eb 100644 --- a/vulnerabilities/tests/test_forms.py +++ b/vulnerabilities/tests/test_forms.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_gentoo.py b/vulnerabilities/tests/test_gentoo.py index 73edfb1ba..74bbaedd8 100644 --- a/vulnerabilities/tests/test_gentoo.py +++ b/vulnerabilities/tests/test_gentoo.py @@ -4,7 +4,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_github_osv.py b/vulnerabilities/tests/test_github_osv.py index 559ba8d15..bcb5fdda5 100644 --- a/vulnerabilities/tests/test_github_osv.py +++ b/vulnerabilities/tests/test_github_osv.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json diff --git a/vulnerabilities/tests/test_gsd.py b/vulnerabilities/tests/test_gsd.py index 34099f947..41bfeff52 100644 --- a/vulnerabilities/tests/test_gsd.py +++ b/vulnerabilities/tests/test_gsd.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import datetime diff --git a/vulnerabilities/tests/test_import_command.py b/vulnerabilities/tests/test_import_command.py index e2f5ac9a0..5734a7942 100644 --- a/vulnerabilities/tests/test_import_command.py +++ b/vulnerabilities/tests/test_import_command.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index 123011266..3f88e0963 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_importer.py b/vulnerabilities/tests/test_importer.py index 7298bc4b6..ebfc8592c 100644 --- a/vulnerabilities/tests/test_importer.py +++ b/vulnerabilities/tests/test_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_improve_command.py b/vulnerabilities/tests/test_improve_command.py index 03b74f3b1..b6b152b40 100644 --- a/vulnerabilities/tests/test_improve_command.py +++ b/vulnerabilities/tests/test_improve_command.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_improve_runner.py b/vulnerabilities/tests/test_improve_runner.py index e4b820c33..347f87c97 100644 --- a/vulnerabilities/tests/test_improve_runner.py +++ b/vulnerabilities/tests/test_improve_runner.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -24,9 +24,10 @@ from vulnerabilities.improver import Improver from vulnerabilities.improver import Inference from vulnerabilities.models import Advisory +from vulnerabilities.models import AffectedByPackageRelatedVulnerability from vulnerabilities.models import Alias +from vulnerabilities.models import FixingPackageRelatedVulnerability from vulnerabilities.models import Package -from vulnerabilities.models import PackageRelatedVulnerability from vulnerabilities.models import Vulnerability from vulnerabilities.models import VulnerabilityReference from vulnerabilities.models import VulnerabilityRelatedReference diff --git a/vulnerabilities/tests/test_improver.py b/vulnerabilities/tests/test_improver.py index 8c05223c2..e526d4ac9 100644 --- a/vulnerabilities/tests/test_improver.py +++ b/vulnerabilities/tests/test_improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py index c3c715e5a..706de1422 100644 --- a/vulnerabilities/tests/test_istio.py +++ b/vulnerabilities/tests/test_istio.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index 6e6eb64eb..014754786 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -51,12 +51,10 @@ def test_package_to_vulnerability(self): p2 = models.Package.objects.create(type="deb", name="git", version="2.31.1") v1 = models.Vulnerability.objects.create(vulnerability_id="CVE-123-2002") - prv1 = models.PackageRelatedVulnerability.objects.create( - package=p1, vulnerability=v1, fix=False - ) - prv2 = models.PackageRelatedVulnerability.objects.create( - package=p2, vulnerability=v1, fix=True + prv1 = models.AffectedByPackageRelatedVulnerability.objects.create( + package=p1, vulnerability=v1 ) + prv2 = models.FixingPackageRelatedVulnerability.objects.create(package=p2, vulnerability=v1) assert p1.fixing_vulnerabilities.count() == 0 @@ -68,12 +66,10 @@ def test_vulnerability_package(self): p2 = models.Package.objects.create(type="deb", name="git", version="2.31.1") v1 = models.Vulnerability.objects.create(vulnerability_id="CVE-123-2002") - prv1 = models.PackageRelatedVulnerability.objects.create( - package=p1, vulnerability=v1, fix=False - ) - prv2 = models.PackageRelatedVulnerability.objects.create( - package=p2, vulnerability=v1, fix=True + prv1 = models.AffectedByPackageRelatedVulnerability.objects.create( + package=p1, vulnerability=v1 ) + prv2 = models.FixingPackageRelatedVulnerability.objects.create(package=p2, vulnerability=v1) assert v1.vulnerable_packages.count() == 1 assert v1.fixed_by_packages.count() == 1 @@ -110,10 +106,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.AffectedByPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_4_1_1, vulnerability=self.vuln_VCID_g2fu_45jw_aaan, - fix=False, ) # aliases @@ -133,10 +128,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.FixingPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_4_3_6, vulnerability=self.vuln_VCID_g2fu_45jw_aaan, - fix=True, ) # vuln for fixed pkg -- and also vuln # 2 for affected pkg @@ -146,10 +140,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.AffectedByPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_4_3_6, vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, - fix=False, ) # aliases @@ -161,10 +154,9 @@ def setUp(self): # vuln # 2 for affected pkg -- already defined above bc also vuln for fixed pkg above! # relationship - models.PackageRelatedVulnerability.objects.create( + models.AffectedByPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_4_1_1, vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, - fix=False, ) # aliases -- already defined above @@ -180,10 +172,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.FixingPackageRelatedVulnerability.objects.create( package=self.package_pypi_redis_5_0_0b1, vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, - fix=True, ) # This vulnerability does not affect any redis packages in this set of tests but does affect a made-up package, self.package_pypi_bogus_1_2_3. @@ -203,10 +194,9 @@ def setUp(self): ) # relationship - models.PackageRelatedVulnerability.objects.create( + models.AffectedByPackageRelatedVulnerability.objects.create( package=self.package_pypi_bogus_1_2_3, vulnerability=self.vuln_VCID_abcd_efgh_1234, - fix=False, ) # This vulnerability does not affect any packages in this set of tests included to test .all(). @@ -433,8 +423,11 @@ def test_sort_by_version(self): version="3.0.0", ) - sorted_pkgs = requesting_package.sort_by_version(vuln_pkg_list) - first_sorted_item = sorted_pkgs[0] + requesting_package.calculate_version_rank + + sorted_pkgs = Package.objects.filter(package_url__in=list_to_sort) + + sorted_pkgs = list(sorted_pkgs) assert sorted_pkgs[0].purl == "pkg:npm/sequelize@3.9.1" assert sorted_pkgs[-1].purl == "pkg:npm/sequelize@3.40.1" diff --git a/vulnerabilities/tests/test_mozilla.py b/vulnerabilities/tests/test_mozilla.py index 8086f0183..15437e22b 100644 --- a/vulnerabilities/tests/test_mozilla.py +++ b/vulnerabilities/tests/test_mozilla.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_msr2019.py b/vulnerabilities/tests/test_msr2019.py index 4af8334e3..16696dd71 100644 --- a/vulnerabilities/tests/test_msr2019.py +++ b/vulnerabilities/tests/test_msr2019.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_openssl.py b/vulnerabilities/tests/test_openssl.py index 7bbeb895e..0effc9515 100644 --- a/vulnerabilities/tests/test_openssl.py +++ b/vulnerabilities/tests/test_openssl.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_oss_fuzz.py b/vulnerabilities/tests/test_oss_fuzz.py index d44c7ab9c..27f5ebfc4 100644 --- a/vulnerabilities/tests/test_oss_fuzz.py +++ b/vulnerabilities/tests/test_oss_fuzz.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import os diff --git a/vulnerabilities/tests/test_osv.py b/vulnerabilities/tests/test_osv.py index 5779d0589..e7505510c 100644 --- a/vulnerabilities/tests/test_osv.py +++ b/vulnerabilities/tests/test_osv.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import datetime diff --git a/vulnerabilities/tests/test_performance.py b/vulnerabilities/tests/test_performance.py index 05f043aed..ba17bfaa9 100644 --- a/vulnerabilities/tests/test_performance.py +++ b/vulnerabilities/tests/test_performance.py @@ -4,7 +4,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import pytest diff --git a/vulnerabilities/tests/test_postgres_workaround.py b/vulnerabilities/tests/test_postgres_workaround.py index 0da44e256..3b0f215ab 100644 --- a/vulnerabilities/tests/test_postgres_workaround.py +++ b/vulnerabilities/tests/test_postgres_workaround.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_postgresql.py b/vulnerabilities/tests/test_postgresql.py index b53d55df3..0e9e71481 100644 --- a/vulnerabilities/tests/test_postgresql.py +++ b/vulnerabilities/tests/test_postgresql.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_redhat_importer.py b/vulnerabilities/tests/test_redhat_importer.py index 7a16e49c1..f5be28dca 100644 --- a/vulnerabilities/tests/test_redhat_importer.py +++ b/vulnerabilities/tests/test_redhat_importer.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_retiredotnet.py b/vulnerabilities/tests/test_retiredotnet.py index 2fa9c0669..52070b2d1 100644 --- a/vulnerabilities/tests/test_retiredotnet.py +++ b/vulnerabilities/tests/test_retiredotnet.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py new file mode 100644 index 000000000..420c8c402 --- /dev/null +++ b/vulnerabilities/tests/test_risk.py @@ -0,0 +1,185 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import pytest + +from vulnerabilities.models import Exploit +from vulnerabilities.models import Vulnerability +from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import VulnerabilityRelatedReference +from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import Weakness +from vulnerabilities.risk import compute_vulnerability_risk_factors +from vulnerabilities.risk import get_exploitability_level +from vulnerabilities.risk import get_weighted_severity +from vulnerabilities.severity_systems import CVSSV3 +from vulnerabilities.severity_systems import EPSS +from vulnerabilities.severity_systems import GENERIC + + +@pytest.fixture +@pytest.mark.django_db +def vulnerability(): + vul = Vulnerability(vulnerability_id="VCID-Existing") + vul.save() + + severity1 = VulnerabilitySeverity.objects.create( + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx1", + scoring_system=CVSSV3.identifier, + scoring_elements="CVSS:3.0/AV:P/AC:H/PR:H/UI:R/S:C/C:H/I:H/A:N/E:H/RL:O/RC:R/CR:H/MAC:H/MC:L", + value="6.5", + ) + + severity2 = VulnerabilitySeverity.objects.create( + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx1", + scoring_system=GENERIC.identifier, + value="MODERATE", # 6.9 + ) + vul.severities.add(severity1) + vul.severities.add(severity2) + + weaknesses = Weakness.objects.create(cwe_id=119) + vul.weaknesses.add(weaknesses) + return vul + + +@pytest.fixture +@pytest.mark.django_db +def exploit(): + vul = Vulnerability(vulnerability_id="VCID-Exploit") + vul.save() + return Exploit.objects.create(vulnerability=vul, description="exploit description") + + +@pytest.fixture +@pytest.mark.django_db +def vulnerability_with_exploit_ref(): + vul = Vulnerability(vulnerability_id="VCID-Exploit-Ref") + vul.save() + + reference_exploit = VulnerabilityReference.objects.create( + reference_id="", + reference_type=VulnerabilityReference.EXPLOIT, + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxxx2", + ) + + VulnerabilityRelatedReference.objects.create(reference=reference_exploit, vulnerability=vul) + return vul + + +@pytest.fixture +@pytest.mark.django_db +def high_epss_score(): + vul = Vulnerability(vulnerability_id="VCID-HIGH-EPSS") + vul.save() + + severity = VulnerabilitySeverity.objects.create( + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx3", + scoring_system=EPSS.identifier, + value=".9", + ) + vul.severities.add(severity) + + return vul.severities.all() + + +@pytest.fixture +@pytest.mark.django_db +def low_epss_score(): + vul = Vulnerability(vulnerability_id="VCID-LOW-EPSS") + vul.save() + + severity = VulnerabilitySeverity.objects.create( + url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx4", + scoring_system=EPSS.identifier, + value=".3", + ) + vul.severities.add(severity) + + return vul.severities.all() + + +@pytest.mark.django_db +def test_exploitability_level( + exploit, vulnerability_with_exploit_ref, high_epss_score, low_epss_score +): + + assert get_exploitability_level(exploit, None, None) == 2 + + assert get_exploitability_level(None, None, high_epss_score) == 2 + + assert get_exploitability_level(None, None, low_epss_score) == 0.5 + + assert ( + get_exploitability_level( + exploits=None, + references=vulnerability_with_exploit_ref.references.all(), + severities=vulnerability_with_exploit_ref.severities.all(), + ) + == 1 + ) + + assert get_exploitability_level(None, None, None) == 0.5 + + +@pytest.mark.django_db +def test_get_weighted_severity(vulnerability): + severities = vulnerability.severities.all() + assert get_weighted_severity(severities) == 6.2 + + severity2 = VulnerabilitySeverity.objects.create( + url="https://security-tracker.debian.org/tracker/CVE-2019-13057", + scoring_system=GENERIC.identifier, + value="CRITICAL", + ) + vulnerability.severities.add(severity2) + + new_severities = vulnerability.severities.all() + assert get_weighted_severity(new_severities) == 7 + + +@pytest.mark.django_db +def test_compute_vulnerability_risk_factors(vulnerability, exploit): + severities = vulnerability.severities.all() + references = vulnerability.references.all() + + assert compute_vulnerability_risk_factors(references, severities, exploit) == ( + 6.2, + 2, + ) + + assert compute_vulnerability_risk_factors(references, severities, None) == (6.2, 0.5) + + assert compute_vulnerability_risk_factors(references, None, exploit) == (0, 2) + + assert compute_vulnerability_risk_factors(None, None, None) == (0, 0.5) + + +@pytest.mark.django_db +def test_get_vulnerability_risk_score(vulnerability): + vulnerability.weighted_severity = 6.0 + vulnerability.exploitability = 2 + + assert vulnerability.risk_score == 10.0 # max risk_score can be reached + + vulnerability.weighted_severity = 6 + vulnerability.exploitability = 0.5 + assert vulnerability.risk_score == 3.0 + + vulnerability.weighted_severity = 5.6 + vulnerability.exploitability = 0.5 + assert vulnerability.risk_score == 2.8 + + vulnerability.weighted_severity = None + vulnerability.exploitability = 0.5 + assert vulnerability.risk_score is None + + vulnerability.weighted_severity = None + vulnerability.exploitability = None + assert vulnerability.risk_score is None diff --git a/vulnerabilities/tests/test_ruby.py b/vulnerabilities/tests/test_ruby.py index 0e06afe1d..e66300512 100644 --- a/vulnerabilities/tests/test_ruby.py +++ b/vulnerabilities/tests/test_ruby.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import json diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py index 8a6d88e47..58b7c4302 100644 --- a/vulnerabilities/tests/test_rust.py +++ b/vulnerabilities/tests/test_rust.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_suse.py b/vulnerabilities/tests/test_suse.py index 5b2691b43..78a59ecc8 100644 --- a/vulnerabilities/tests/test_suse.py +++ b/vulnerabilities/tests/test_suse.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_suse_backports.py b/vulnerabilities/tests/test_suse_backports.py index 6253b7018..aa3737246 100644 --- a/vulnerabilities/tests/test_suse_backports.py +++ b/vulnerabilities/tests/test_suse_backports.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_suse_oval.py b/vulnerabilities/tests/test_suse_oval.py index eba1ca0f2..436cfcf49 100644 --- a/vulnerabilities/tests/test_suse_oval.py +++ b/vulnerabilities/tests/test_suse_oval.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_suse_scores.py b/vulnerabilities/tests/test_suse_scores.py index 0abdd48b1..96a5d0d07 100644 --- a/vulnerabilities/tests/test_suse_scores.py +++ b/vulnerabilities/tests/test_suse_scores.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_throttling.py b/vulnerabilities/tests/test_throttling.py index 364ef487e..dbf8d759d 100644 --- a/vulnerabilities/tests/test_throttling.py +++ b/vulnerabilities/tests/test_throttling.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_ubuntu.py b/vulnerabilities/tests/test_ubuntu.py index fa54fa1af..b31588347 100644 --- a/vulnerabilities/tests/test_ubuntu.py +++ b/vulnerabilities/tests/test_ubuntu.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_ubuntu_usn.py b/vulnerabilities/tests/test_ubuntu_usn.py index cac58e47a..d28d036b7 100644 --- a/vulnerabilities/tests/test_ubuntu_usn.py +++ b/vulnerabilities/tests/test_ubuntu_usn.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_upstream.py b/vulnerabilities/tests/test_upstream.py index ad5f50113..71b6d86af 100644 --- a/vulnerabilities/tests/test_upstream.py +++ b/vulnerabilities/tests/test_upstream.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_utils.py b/vulnerabilities/tests/test_utils.py index e67aa18bf..c9ba98e79 100644 --- a/vulnerabilities/tests/test_utils.py +++ b/vulnerabilities/tests/test_utils.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index 1ba488320..692305f8d 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_vulnerability_status_improver.py b/vulnerabilities/tests/test_vulnerability_status_improver.py index f2eb5ce0f..89a57c0b7 100644 --- a/vulnerabilities/tests/test_vulnerability_status_improver.py +++ b/vulnerabilities/tests/test_vulnerability_status_improver.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/test_xen.py b/vulnerabilities/tests/test_xen.py index d9e913f68..3e5822b1e 100644 --- a/vulnerabilities/tests/test_xen.py +++ b/vulnerabilities/tests/test_xen.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/tests/util_tests.py b/vulnerabilities/tests/util_tests.py index dccc5c9cd..5690fad4b 100644 --- a/vulnerabilities/tests/util_tests.py +++ b/vulnerabilities/tests/util_tests.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/throttling.py b/vulnerabilities/throttling.py index d439a04b2..99b1d7756 100644 --- a/vulnerabilities/throttling.py +++ b/vulnerabilities/throttling.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # from rest_framework.exceptions import Throttled diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index cb16f0eb6..969a08f2f 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index 394dc1c36..fd57acea5 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import logging @@ -119,7 +119,10 @@ def get_context_data(self, **kwargs): package = self.object context["package"] = package context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") - context["fixing_vulnerabilities"] = package.fixing.order_by("vulnerability_id") + # Ghost package should not fix any vulnerability. + context["fixing_vulnerabilities"] = ( + None if package.is_ghost else package.fixing.order_by("vulnerability_id") + ) context["package_search_form"] = PackageSearchForm(self.request.GET) context["fixed_package_details"] = package.fixed_package_details @@ -153,7 +156,17 @@ class VulnerabilityDetails(DetailView): slug_field = "vulnerability_id" def get_queryset(self): - return super().get_queryset().prefetch_related("references", "aliases", "weaknesses") + return ( + super() + .get_queryset() + .prefetch_related( + "references", + "aliases", + "weaknesses", + "severities", + "exploits", + ) + ) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -165,7 +178,7 @@ def get_context_data(self, **kwargs): severity_vectors = [] severity_values = set() - for s in self.object.severities: + for s in self.object.severities.all(): if s.scoring_system == EPSS.identifier: continue @@ -193,6 +206,11 @@ def get_context_data(self, **kwargs): affected_fixed_by_matches["affected_package"] = sorted_affected_package matched_fixed_by_packages = [] for fixed_by_package in sorted_fixed_by_packages: + + # Ghost Package can't fix vulnerability. + if fixed_by_package.is_ghost: + continue + sorted_affected_version_class = get_purl_version_class(sorted_affected_package) fixed_by_version_class = get_purl_version_class(fixed_by_package) if ( @@ -214,7 +232,7 @@ def get_context_data(self, **kwargs): { "vulnerability": self.object, "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), - "severities": list(self.object.severities), + "severities": list(self.object.severities.all()), "severity_score_range": get_severity_range(severity_values), "severity_vectors": severity_vectors, "references": self.object.references.all(), diff --git a/vulnerabilities/weight_config.py b/vulnerabilities/weight_config.py new file mode 100644 index 000000000..f9ad8292d --- /dev/null +++ b/vulnerabilities/weight_config.py @@ -0,0 +1,2919 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +WEIGHT_CONFIG = { + "nvd.nist.gov": 9, + "api.first.org": 9, + "github.com": 9, + "access.redhat.com": 9, + "bugzilla.redhat.com": 9, + "cve.mitre.org": 9, + "people.canonical.com": 9, + "lists.apache.org": 9, + "ubuntu.com": 9, + "openwall.com": 9, + "lists.fedoraproject.org": 9, + "npmjs.com": 9, + "web.archive.org": 9, + "security.archlinux.org": 9, + "mozilla.org": 9, + "huntr.dev": 9, + "snyk.io": 9, + "vuldb.com": 9, + "jenkins.io": 9, + "security.netapp.com": 9, + "securityfocus.com": 9, + "lists.debian.org": 9, + "rustsec.org": 8, + "usn.ubuntu.com": 8, + "packetstormsecurity.com": 8, + "pkg.go.dev": 8, + "hackerone.com": 8, + "git.kernel.org": 8, + "groups.google.com": 8, + "security.snyk.io": 8, + "lists.opensuse.org": 8, + "rhn.redhat.com": 8, + "seclists.org": 8, + "code.google.com": 8, + "moodle.org": 8, + "portal.msrc.microsoft.com": 8, + "issues.apache.org": 7, + "debian.org": 7, + "jira.xwiki.org": 7, + "exchange.xforce.ibmcloud.com": 7, + "typo3.org": 7, + "exploit-db.com": 7, + "gist.github.com": 7, + "plugins.trac.wordpress.org": 7, + "bugs.wireshark.org": 7, + "git.moodle.org": 7, + "secunia.com": 7, + "huntr.com": 7, + "security.gentoo.org": 7, + "bugs.launchpad.net": 7, + "code.wireshark.org": 7, + "svn.apache.org": 7, + "httpd.apache.org": 7, + "securitytracker.com": 7, + "wordfence.com": 7, + "wireshark.org": 7, + "bugs.chromium.org": 7, + "gitlab.com": 7, + "silverstripe.org": 7, + "crbug.com": 7, + "phpmyadmin.net": 7, + "securitylab.github.com": 7, + "curl.se": 7, + "sourceforge.net": 7, + "drupal.org": 7, + "discuss.hashicorp.com": 7, + "pypi.org": 7, + "djangoproject.com": 7, + "bugzilla.mozilla.org": 7, + "raw.githubusercontent.com": 7, + "lists.gnu.org": 7, + "bugs.debian.org": 7, + "security-tracker.debian.org": 7, + "marc.info": 7, + "support.apple.com": 7, + "blogs.gentoo.org": 7, + "bitbucket.org": 7, + "bugzilla.suse.com": 7, + "go.dev": 7, + "pivotal.io": 7, + "whitesourcesoftware.com": 7, + "cdn.kernel.org": 7, + "gitee.com": 7, + "lore.kernel.org": 7, + "ruby-lang.org": 7, + "msrc.microsoft.com": 7, + "wordpress.org": 7, + "plone.org": 7, + "redhat.com": 7, + "oracle.com": 7, + "anonsvn.wireshark.org": 7, + "mail-archives.apache.org": 7, + "xenbits.xen.org": 6, + "postgresql.org": 6, + "twitter.com": 6, + "git.openssl.org": 6, + "jvn.jp": 6, + "rubygems.org": 6, + "symfony.com": 6, + "www-01.ibm.com": 6, + "src.chromium.org": 6, + "tenable.com": 6, + "lists.apple.com": 6, + "packagist.org": 6, + "issues.redhat.com": 6, + "helpx.adobe.com": 6, + "phabricator.wikimedia.org": 6, + "review.openstack.org": 6, + "chromereleases.googleblog.com": 6, + "contao.org": 6, + "discuss.elastic.co": 6, + "security.openstack.org": 6, + "lkml.org": 6, + "medium.com": 6, + "nozominetworks.com": 6, + "grafana.com": 6, + "oval.cisecurity.org": 6, + "review.opendev.org": 6, + "advisory.checkmarx.net": 6, + "vupen.com": 6, + "launchpad.net": 6, + "openssl.org": 6, + "framework.zend.com": 6, + "samba.org": 6, + "talosintelligence.com": 6, + "zerodayinitiative.com": 6, + "bugs.eclipse.org": 6, + "gitlab.gnome.org": 6, + "nifi.apache.org": 6, + "x-stream.github.io": 6, + "tanzu.vmware.com": 6, + "jvndb.jvn.jp": 6, + "opendev.org": 6, + "bugzilla.kernel.org": 6, + "sourceware.org": 6, + "codereview.chromium.org": 6, + "research.jfrog.com": 6, + "discuss.rubyonrails.org": 6, + "spring.io": 6, + "twcert.org.tw": 6, + "googlechromereleases.blogspot.com": 6, + "sec.cloudapps.cisco.com": 6, + "issues.jboss.org": 6, + "simplesamlphp.org": 6, + "mail-archive.com": 6, + "docs.saltstack.com": 6, + "docs.shopware.com": 6, + "kb.isc.org": 6, + "support.f5.com": 6, + "kernel.org": 6, + "osvdb.org": 6, + "cwiki.apache.org": 6, + "hg.moinmo.in": 6, + "fluidattacks.com": 6, + "gitlab.eclipse.org": 6, + "nuget.org": 6, + "lists.openstack.org": 6, + "developer.mozilla.org": 6, + "youtube.com": 6, + "drive.google.com": 6, + "owasp.org": 6, + "vaadin.com": 6, + "mercurial-scm.org": 6, + "mandriva.com": 6, + "archives.neohapsis.com": 6, + "hg.code.sf.net": 6, + "curl.haxx.se": 6, + "pillow.readthedocs.io": 6, + "struts.apache.org": 6, + "vulncheck.com": 6, + "webkitgtk.org": 6, + "documentation.concretecms.org": 6, + "patchstack.com": 6, + "patchwork.kernel.org": 6, + "hg.graphicsmagick.org": 6, + "source.android.com": 6, + "liferay.dev": 6, + "nodesecurity.io": 6, + "web.nvd.nist.gov": 6, + "jira.mongodb.org": 6, + "cve.org": 6, + "core.trac.wordpress.org": 6, + "news.ycombinator.com": 6, + "downloads.asterisk.org": 6, + "tomcat.apache.org": 6, + "kb.cert.org": 6, + "cert-portal.siemens.com": 6, + "devhub.checkmarx.com": 6, + "lists.nongnu.org": 6, + "squid-cache.org": 6, + "ink-desk-28f.notion.site": 6, + "h20566.www2.hpe.com": 6, + "intel.com": 6, + "docs.djangoproject.com": 6, + "go.googlesource.com": 6, + "herolab.usd.de": 6, + "matrix.org": 6, + "talosintel.com": 6, + "vapid.dhs.org": 6, + "ckeditor.com": 6, + "cwe.mitre.org": 6, + "pagure.io": 6, + "cisa.gov": 6, + "weblog.rubyonrails.org": 6, + "markmail.org": 6, + "issues.liferay.com": 6, + "jspwiki-wiki.apache.org": 6, + "gitbox.apache.org": 6, + "bugs.gentoo.org": 5, + "docs.microsoft.com": 5, + "puppet.com": 5, + "incibe.es": 5, + "cxf.apache.org": 5, + "bugzilla.wikimedia.org": 5, + "mvnrepository.com": 5, + "usom.gov.tr": 5, + "bz.apache.org": 5, + "nodejs.org": 5, + "syzkaller.appspot.com": 5, + "vuln.ryotak.me": 5, + "docs.google.com": 5, + "spinics.net": 5, + "auth0.com": 5, + "basercms.net": 5, + "next-auth.js.org": 5, + "plugins.jenkins.io": 5, + "libreoffice.org": 5, + "tiny.cloud": 5, + "hg.openjdk.java.net": 5, + "owncloud.org": 5, + "community.otrs.com": 5, + "git.openstack.org": 5, + "osgeo-org.atlassian.net": 5, + "bugzilla.gnome.org": 5, + "forum.xpdfreader.com": 5, + "istio.io": 5, + "developers.ibexa.co": 5, + "portswigger.net": 5, + "wpvulndb.com": 5, + "issues.chromium.org": 5, + "advisories.mageia.org": 5, + "doc.powerdns.com": 5, + "patchwork.ozlabs.org": 5, + "ec-cube.net": 5, + "phpbb.com": 5, + "rfc-editor.org": 5, + "synology.com": 5, + "ocert.org": 5, + "mail-archives.us.apache.org": 5, + "crates.io": 5, + "en.wikipedia.org": 5, + "issues.asterisk.org": 5, + "jira.spring.io": 5, + "secuniaresearch.flexerasoftware.com": 5, + "tools.ietf.org": 5, + "wiki.jenkins-ci.org": 5, + "youtu.be": 5, + "camel.apache.org": 5, + "0dd.zone": 5, + "blog.gitea.io": 5, + "bugs.ghostscript.com": 5, + "bugzilla.novell.com": 5, + "developer.joomla.org": 5, + "gitlab.torproject.org": 5, + "lists.wikimedia.org": 5, + "trac.torproject.org": 5, + "wpscan.com": 5, + "ibm.com": 5, + "trac.roundcube.net": 5, + "pidgin.im": 5, + "git.ghostscript.com": 5, + "gerrit.wikimedia.org": 5, + "gitlab.freedesktop.org": 5, + "hiddenlayer.com": 5, + "support.hpe.com": 5, + "thread.gmane.org": 5, + "cvcn.gov.it": 5, + "activemq.apache.org": 5, + "article.gmane.org": 5, + "bakery.cakephp.org": 5, + "bugs.python.org": 5, + "chromium.googlesource.com": 5, + "docs.github.com": 5, + "docs.rs": 5, + "grimthereaperteam.medium.com": 5, + "mailman.nginx.org": 5, + "psirt.global.sonicwall.com": 5, + "selenic.com": 5, + "esecforte.com": 5, + "mend.io": 5, + "netsparker.com": 5, + "playframework.com": 5, + "securityreason.com": 5, + "files.opcfoundation.org": 5, + "lgtm.com": 5, + "stackoverflow.com": 5, + "tools.cisco.com": 5, + "redmine.org": 5, + "vicarius.io": 5, + "w1.fi": 5, + "blog.phusion.nl": 5, + "codeigniter4.github.io": 5, + "csirt.divd.nl": 5, + "documentation.centreon.com": 5, + "eprint.iacr.org": 5, + "git.gnome.org": 5, + "joel-malwarebenchmark.github.io": 5, + "magento.com": 5, + "mantisbt.org": 5, + "research.loginsoft.com": 5, + "neos.io": 5, + "syss.de": 5, + "xforce.iss.net": 5, + "advisories.nats.io": 5, + "bugs.tryton.org": 5, + "lists.launchpad.net": 5, + "salsa.debian.org": 5, + "saltproject.io": 5, + "shopware.com": 5, + "shawroot.cc": 5, + "blog.ripstech.com": 5, + "blog.sonarsource.com": 5, + "cheatsheetseries.owasp.org": 5, + "docs.craftercms.org": 5, + "docs.umbraco.com": 5, + "go-review.googlesource.com": 5, + "kc.mcafee.com": 5, + "labs.integrity.pt": 5, + "pyup.io": 5, + "rubysec.com": 5, + "support.ntp.org": 5, + "wiki.openstack.org": 5, + "concretecms.org": 5, + "darkmatter.ae": 5, + "linkedin.com": 5, + "otrs.com": 5, + "privoxy.org": 5, + "sonarsource.com": 5, + "trustwave.com": 5, + "vmware.com": 5, + "blog.rubygems.org": 5, + "karmainsecurity.com": 5, + "cloud.google.com": 5, + "docs.python.org": 5, + "git-wip-us.apache.org": 5, + "nvidia.custhelp.com": 5, + "pastebin.com": 5, + "pypi.python.org": 5, + "python-security.readthedocs.io": 5, + "rumbling-slice-eb0.notion.site": 5, + "rapid7.com": 5, + "advisory.dw1.io": 5, + "akka.io": 5, + "android.googlesource.com": 5, + "argo-cd.readthedocs.io": 5, + "blog.fuzzing-project.org": 5, + "blog.laravel.com": 5, + "codeberg.org": 5, + "docs.cilium.io": 5, + "docs.docker.com": 5, + "docs.wagtail.org": 5, + "downloads.powerdns.com": 5, + "electronjs.org": 5, + "forums.rancher.com": 5, + "irssi.org": 5, + "kallithea-scm.org": 5, + "legalhackers.com": 5, + "lists.freedesktop.org": 5, + "mellium.im": 5, + "rubyonrails.org": 5, + "spark.apache.org": 5, + "storyboard.openstack.org": 5, + "subversion.apache.org": 5, + "tls.mbed.org": 5, + "sunsolve.sun.com": 5, + "wiki.ubuntu.com": 5, + "apollographql.com": 5, + "herodevs.com": 5, + "imagemagick.org": 5, + "kde.org": 5, + "qualys.com": 5, + "usenix.org": 5, + "wizlynxgroup.com": 5, + "x41-dsec.de": 5, + "gentoo.org": 5, + "git.videolan.org": 5, + "moinmo.in": 5, + "puppetlabs.com": 5, + "arxiv.org": 5, + "blog.getbootstrap.com": 5, + "review.gluster.org": 5, + "roy.marples.name": 5, + "stackblitz.com": 5, + "facebook.com": 5, + "git.savannah.gnu.org": 5, + "blog.torproject.org": 5, + "codepen.io": 5, + "codeql.github.com": 5, + "commons.apache.org": 5, + "confluence.atlassian.com": 5, + "cs.opensource.google": 5, + "doc.akka.io": 5, + "docs.opennms.com": 5, + "docs.openstack.org": 5, + "docs.silverstripe.org": 5, + "doc.traefik.io": 5, + "doyensec.com": 5, + "foss.heptapod.net": 5, + "gcc.gnu.org": 5, + "git.sheetjs.com": 5, + "hackmd.io": 5, + "ics-cert.kaspersky.com": 5, + "kiwitcms.org": 5, + "kjur.github.io": 5, + "lists.gnupg.org": 5, + "logback.qos.ch": 5, + "logging.apache.org": 5, + "mail.python.org": 5, + "metacpan.org": 5, + "neo4j.com": 5, + "opcfoundation.org": 5, + "ostorlab.co": 5, + "palletsprojects.com": 5, + "portal.liferay.dev": 5, + "research.securitum.com": 5, + "sites.google.com": 5, + "source.codeaurora.org": 5, + "support.sonatype.com": 5, + "twistedmatrix.com": 5, + "edoardoottavianelli.it": 5, + "sec-consult.com": 5, + "sudo.ws": 5, + "vulnerability-lab.com": 5, + "yiiframework.com": 5, + "www-1.ibm.com": 5, + "cherrypy.org": 5, + "vapidlabs.com": 5, + "archiva.apache.org": 5, + "bugs.cacti.net": 5, + "docs.moodle.org": 5, + "freeradius.org": 5, + "git.php.net": 5, + "hmarco.org": 5, + "hyp3rlinx.altervista.org": 5, + "lists.bestpractical.com": 5, + "php.net": 5, + "qpid.apache.org": 5, + "blog.clamav.net": 5, + "codex.wordpress.org": 5, + "cxsecurity.com": 5, + "security.libvirt.org": 5, + "extensions.typo3.org": 5, + "gnunet.org": 5, + "pretix.eu": 5, + "software.intel.com": 5, + "syncope.apache.org": 5, + "apache.org": 5, + "blackhat.com": 5, + "bleepingcomputer.com": 5, + "cyrusimap.org": 5, + "openssh.com": 5, + "arubanetworks.com": 5, + "code610.blogspot.com": 5, + "code.djangoproject.com": 5, + "karaf.apache.org": 5, + "lists.openwall.net": 5, + "netty.io": 5, + "packetstormsecurity.org": 5, + "arstechnica.com": 5, + "aws.amazon.com": 5, + "benjamin-bouchet.com": 5, + "blog.jquery.com": 5, + "bugs.jquery.com": 5, + "bugs.php.net": 5, + "bugs.torproject.org": 5, + "cert.pl": 5, + "community.grafana.com": 5, + "community.shopware.com": 5, + "cves.at": 5, + "datatracker.ietf.org": 5, + "docs.geoserver.org": 5, + "docs.sentry.io": 5, + "edugit.org": 5, + "fedorahosted.org": 5, + "fetch.spec.whatwg.org": 5, + "froala.com": 5, + "gerrit.libreoffice.org": 5, + "git1-us-west.apache.org": 5, + "grimhacker.com": 5, + "hg.reportlab.com": 5, + "kb.pulsesecure.net": 5, + "laravel.com": 5, + "lwn.net": 5, + "mail.openvswitch.org": 5, + "matrix-org.github.io": 5, + "nukeviet.vn": 5, + "portal.microfocus.com": 5, + "research.nccgroup.com": 5, + "rt.cpan.org": 5, + "s.apache.org": 5, + "search.maven.org": 5, + "security.docs.wso2.com": 5, + "share.ez.no": 5, + "shibboleth.net": 5, + "solr.apache.org": 5, + "starlabs.sg": 5, + "support.microsoft.com": 5, + "support.zabbix.com": 5, + "sysdream.com": 5, + "tech.feedyourhead.at": 5, + "dolibarr.org": 5, + "hashicorp.com": 5, + "opennms.com": 5, + "passbolt.com": 5, + "reddit.com": 5, + "sqlite.org": 5, + "strongswan.org": 5, + "hawktesters.com": 5, + "topquadrant.com": 5, + "blog.mindedsecurity.com": 5, + "dev.eclipse.org": 5, + "downloads.digium.com": 5, + "permalink.gmane.org": 5, + "0xacab.org": 5, + "about.gitlab.com": 5, + "alephsecurity.com": 5, + "blog.cloudflare.com": 5, + "bugs.freedesktop.org": 5, + "bugs.kde.org": 5, + "community.openvpn.net": 5, + "core.spip.net": 5, + "docs.oracle.com": 5, + "dumpco.re": 5, + "edk2-docs.gitbooks.io": 5, + "forum.cosmos.network": 5, + "git.ffmpeg.org": 5, + "golang.org": 5, + "hex.pm": 5, + "lists.x.org": 5, + "me.sap.com": 5, + "pdfium.googlesource.com": 5, + "peps.python.org": 5, + "portal.perforce.com": 5, + "roundcube.net": 5, + "trac.ffmpeg.org": 5, + "trac.pjsip.org": 5, + "svn.cacti.net": 5, + "couchbase.com": 5, + "nlnetlabs.nl": 5, + "prevanders.net": 5, + "sourceclear.com": 5, + "suse.com": 5, + "novell.com": 5, + "openafs.org": 5, + "tryton.org": 5, + "zope.org": 5, + "blog.plataformatec.com.br": 5, + "blog.trendmicro.com": 5, + "geronimo.apache.org": 5, + "hg.tryton.org": 5, + "issues.roundup-tracker.org": 5, + "akerva.com": 5, + "api.slack.com": 5, + "backdropcms.org": 5, + "blog.certimetergroup.com": 5, + "blog.jetbrains.com": 5, + "codereview.qt-project.org": 5, + "codewhitesec.blogspot.com": 5, + "community.librenms.org": 5, + "craftcms.com": 5, + "discord.com": 5, + "discuss.flarum.org": 5, + "discuss.tryton.org": 5, + "dl.acm.org": 5, + "docs.aws.amazon.com": 5, + "docs.pinot.apache.org": 5, + "docs.projectdiscovery.io": 5, + "documentation.bonitasoft.com": 5, + "duo.com": 5, + "elixirforum.com": 5, + "extensions.xwiki.org": 5, + "ezplatform.com": 5, + "git.launchpad.net": 5, + "gitlord.com": 5, + "git.shibboleth.net": 5, + "gnutls.org": 5, + "gxx777.github.io": 5, + "ibb.co": 5, + "i.ibb.co": 5, + "internetcomputer.org": 5, + "issues.opennms.org": 5, + "jira.qos.ch": 5, + "jsoup.org": 5, + "kiali.io": 5, + "lab.louiz.org": 5, + "labs.bishopfox.com": 5, + "linkis.apache.org": 5, + "mattermost.com": 5, + "mycvee.blogspot.com": 5, + "notes.sjtu.edu.cn": 5, + "opcfoundation-onlineapplications.org": 5, + "openid.net": 5, + "patch-diff.githubusercontent.com": 5, + "pho3n1x-web.github.io": 5, + "podalirius.net": 5, + "portals.apache.org": 5, + "projects.eclipse.org": 5, + "pulsesecurity.co.nz": 5, + "quarkus.io": 5, + "reactjs.org": 5, + "review.typo3.org": 5, + "salvatoresecurity.com": 5, + "securelayer7.net": 5, + "secure.ucc.asn.au": 5, + "shrouded-trowel-50c.notion.site": 5, + "ssd-disclosure.com": 5, + "strapi.io": 5, + "surrealdb.com": 5, + "tailscale.com": 5, + "tufangungor.github.io": 5, + "support.avaya.com": 5, + "webargs.readthedocs.io": 5, + "whitehub.net": 5, + "ait.ac.at": 5, + "checkmarx.com": 5, + "cloudbees.com": 5, + "cloudera.com": 5, + "codeigniter.com": 5, + "digital.security": 5, + "dulwich.io": 5, + "freeipa.org": 5, + "gerritcodereview.com": 5, + "ghostccamm.com": 5, + "horizon3.ai": 5, + "imperva.com": 5, + "mediawiki.org": 5, + "openmicroscopy.org": 5, + "schedmd.com": 5, + "wwws.nightwatchcybersecurity.com": 5, + "synacktiv.com": 5, + "synopsys.com": 5, + "tigera.io": 5, + "twistlock.com": 5, + "w3.org": 5, + "verneet.com": 5, + "dnnsoftware.com": 5, + "halfdog.net": 5, + "hitachi.co.jp": 5, + "reviewboard.org": 5, + "us-cert.gov": 5, + "depot.galaxyproject.org": 5, + "codecanyon.net": 5, + "marketplace.visualstudio.com": 5, + "docs.telerik.com": 5, + "support.hcltechsw.com": 5, + "watchguard.com": 5, + "bazaar.launchpad.net": 5, + "comments.gmane.org": 5, + "db.apache.org": 5, + "debbugs.gnu.org": 5, + "emberjs.com": 5, + "git.ganeti.org": 5, + "lists.thekelleys.org.uk": 5, + "phpmyadmin.git.sourceforge.net": 5, + "projects.puppetlabs.com": 5, + "anonscm.debian.org": 5, + "apereo.github.io": 5, + "blogs.apache.org": 5, + "bugzilla.opensuse.org": 5, + "build.prestashop.com": 5, + "community.rapid7.com": 5, + "cryptography.io": 5, + "cybersecurityworks.com": 5, + "dev.mysql.com": 5, + "docs.opencast.org": 5, + "docs.pagure.org": 5, + "docs.powerdns.com": 5, + "ftp.openbsd.org": 5, + "gitlab.labs.nic.cz": 5, + "git.linuxtv.org": 5, + "git.spip.net": 5, + "googleprojectzero.blogspot.com": 5, + "helm.sh": 5, + "hermes.opensuse.org": 5, + "hg.prosody.im": 5, + "isc.sans.edu": 5, + "issues.jenkins-ci.org": 5, + "jfrog.com": 5, + "jira.hdfgroup.org": 5, + "jira.springsource.org": 5, + "lists.01.org": 5, + "lists.lysator.liu.se": 5, + "lists.torproject.org": 5, + "nealpoole.com": 5, + "nokogiri.org": 5, + "novysodope.github.io": 5, + "somevulnsofadlab.blogspot.jp": 5, + "source.ikiwiki.branchable.com": 5, + "patchwork.linuxtv.org": 5, + "prosody.im": 5, + "pub.freerdp.com": 5, + "research.checkpoint.com": 5, + "rt-solutions.de": 5, + "tracker.debian.org": 5, + "user-images.githubusercontent.com": 5, + "ambionics.io": 5, + "bouncycastle.org": 5, + "freebsd.org": 5, + "mautic.org": 5, + "theregister.com": 5, + "trac.edgewall.org": 5, + "fujitsu.com": 5, + "gopivotal.com": 5, + "huawei.com": 5, + "ush.it": 5, + "wiki.shikangsi.com": 5, + "xenbits.xenproject.org": 5, + "bytium.com": 5, + "enpass.io": 5, + "archives.seul.org": 5, + "blog.npmjs.org": 5, + "blog.talosintel.com": 5, + "bugs.jqueryui.com": 5, + "dev.rubyonrails.org": 5, + "download.lighttpd.net": 5, + "freeimage.cvs.sourceforge.net": 5, + "googlechromereleases.blogspot.ca": 5, + "ijbswa.cvs.sourceforge.net": 5, + "issues.umbraco.org": 5, + "jira.codehaus.org": 5, + "jruby.org": 5, + "lcamtuf.coredump.cx": 5, + "lists.gnutls.org": 5, + "lists.qt-project.org": 5, + "lists.xen.org": 5, + "mx.gw.com": 5, + "octobercms.com": 5, + "openmeetings.apache.org": 5, + "pear.php.net": 5, + "projects.theforeman.org": 5, + "research.cs.wisc.edu": 5, + "airflow.apache.org": 5, + "santuario.apache.org": 5, + "appcheck-ng.com": 5, + "app.safebase.io": 5, + "app.snyk.io": 5, + "argoproj.github.io": 5, + "artifacthub.io": 5, + "authjs.dev": 5, + "savannah.nongnu.org": 5, + "bbpress.org": 5, + "belong2yourself.github.io": 5, + "blocksecteam.medium.com": 5, + "blog.cal1.cn": 5, + "blog.detectify.com": 5, + "blog.doyensec.com": 5, + "blog.gitea.com": 5, + "blog.jcoglan.com": 5, + "blog.jitendrapatro.me": 5, + "blog.jqueryui.com": 5, + "blog.jupyter.org": 5, + "blog.moku.fr": 5, + "blog.payara.fish": 5, + "blogs.securiteam.com": 5, + "blog.trailofbits.com": 5, + "breakingthe3ma.app": 5, + "brooklyn.apache.org": 5, + "buddypress.org": 5, + "bugs.oxid-esales.com": 5, + "bz.mercurial-scm.org": 5, + "cgit.kde.org": 5, + "channels.readthedocs.io": 5, + "chromium-review.googlesource.com": 5, + "ckan.org": 5, + "clojure.atlassian.net": 5, + "clusterfuzz-external.appspot.com": 5, + "community.snowflake.com": 5, + "community.sonarsource.com": 5, + "community.synopsys.com": 5, + "cupc4k3.medium.com": 5, + "developer.arm.com": 5, + "dev.liferay.com": 5, + "discuss.lightbend.com": 5, + "docs.getdbt.com": 5, + "docs.gradle.com": 5, + "docs.jboss.org": 5, + "docs.magnolia-cms.com": 5, + "docs.mattermost.com": 5, + "docs.opendaylight.org": 5, + "docs.rundeck.com": 5, + "docs.sixlabors.com": 5, + "docs.strapi.io": 5, + "docs.typo3.org": 5, + "docs.wso2.com": 5, + "duartecsantos.github.io": 5, + "eclipse.dev": 5, + "eslint.org": 5, + "excellium-services.com": 5, + "exment.net": 5, + "fatihhcelik.blogspot.com": 5, + "fortiguard.com": 5, + "forum.ghost.org": 5, + "forum.mautic.org": 5, + "gerrit.onosproject.org": 5, + "geth.ethereum.org": 5, + "ghost.org": 5, + "gist.githubusercontent.com": 5, + "git.eclipse.org": 5, + "git.haproxy.org": 5, + "github.dev": 5, + "github.openssl.org": 5, + "gitlab.kitware.com": 5, + "gitlab.ow2.org": 5, + "gitlab.redox-os.org": 5, + "git.opendaylight.org": 5, + "git-scm.com": 5, + "gitweb.gentoo.org": 5, + "gitweb.torproject.org": 5, + "glassfish.org": 5, + "go-vela.github.io": 5, + "grails.org": 5, + "gstreamer.freedesktop.org": 5, + "guidovranken.wordpress.com": 5, + "hadoop.apache.org": 5, + "hexdocs.pm": 5, + "hg.python.org": 5, + "htcondor-wiki.cs.wisc.edu": 5, + "ihacktoprotect.com": 5, + "ikiwiki.info": 5, + "infosecwriteups.com": 5, + "inspector.pypi.io": 5, + "invdos.net": 5, + "iotaa.cn": 5, + "ipsilon-project.org": 5, + "issues.shibboleth.net": 5, + "james.apache.org": 5, + "javaee.github.io": 5, + "jdbc.postgresql.org": 5, + "jekyllrb.com": 5, + "jena.apache.org": 5, + "jira.duraspace.org": 5, + "jira.hyperledger.org": 5, + "johnjhacking.com": 5, + "jolokia.org": 5, + "jupyterhub.readthedocs.io": 5, + "knightlab.northwestern.edu": 5, + "know.bishopfox.com": 5, + "kramdown.gettalong.org": 5, + "kubernetes.io": 5, + "kyverno.io": 5, + "laravel-news.com": 5, + "launchpadlibrarian.net": 5, + "learn.microsoft.com": 5, + "lf-opendaylight.atlassian.net": 5, + "lists.exim.org": 5, + "lists.quagga.net": 5, + "lists.samba.org": 5, + "lists.schedmd.com": 5, + "loopback.io": 5, + "mail.zope.org": 5, + "makandracards.com": 5, + "mariadb.com": 5, + "mosquitto.org": 5, + "mp.weixin.qq.com": 5, + "openfga.dev": 5, + "oxidforge.org": 5, + "pentest.com.tr": 5, + "phab.bots.miraheze.wiki": 5, + "plugins.craftcms.com": 5, + "polarssl.org": 5, + "spreecommerce.com": 5, + "public-inbox.org": 5, + "rancher.com": 5, + "redmine.openinfosecfoundation.org": 5, + "repo.mercurial-scm.org": 5, + "rhinosecuritylabs.com": 5, + "runkit.com": 5, + "sansec.io": 5, + "savannah.gnu.org": 5, + "seanmonstar.com": 5, + "security.FreeBSD.org": 5, + "seemann.io": 5, + "sl1nki.page": 5, + "sources.debian.org": 5, + "stackdiary.com": 5, + "support.hazelcast.com": 5, + "supportportal.juniper.net": 5, + "support.servicenow.com": 5, + "swarm.ptsecurity.com": 5, + "tauri.app": 5, + "thecybergeek.co.uk": 5, + "tib36.github.io": 5, + "storm.apache.org": 5, + "trovent.github.io": 5, + "trovent.io": 5, + "uima.apache.org": 5, + "unit42.paloaltonetworks.com": 5, + "unparalleled.eu": 5, + "support.blackberry.com": 5, + "uwsgi-docs.readthedocs.io": 5, + "vufind.org": 5, + "vulmon.com": 5, + "vuln.go.dev": 5, + "vuln.ryotak.net": 5, + "websecnerd.blogspot.in": 5, + "wha13.github.io": 5, + "whitehatck01.blogspot.com": 5, + "wiki.mageia.org": 5, + "wiki.phpbb.com": 5, + "wp-crontrol.com": 5, + "ag-grid.com": 5, + "apiman.io": 5, + "bentley.com": 5, + "bitvise.com": 5, + "bluetooth.com": 5, + "bookstackapp.com": 5, + "cnblogs.com": 5, + "cnvd.org.cn": 5, + "code-intelligence.com": 5, + "compass-security.com": 5, + "django-cms.org": 5, + "doctrine-project.org": 5, + "eclipse.org": 5, + "elastic.co": 5, + "envoyproxy.io": 5, + "exploitalert.com": 5, + "fastly.com": 5, + "fzi.de": 5, + "getastra.com": 5, + "graylog.org": 5, + "htbridge.com": 5, + "kingkk.com": 5, + "libssh.org": 5, + "nccgroup.trust": 5, + "nginx.com": 5, + "openpolicyagent.org": 5, + "pac4j.org": 5, + "pgadmin.org": 5, + "phpmyfaq.de": 5, + "roundup-tracker.org": 5, + "rubydoc.info": 5, + "saltstack.com": 5, + "secpod.com": 5, + "shielder.it": 5, + "smarty.net": 5, + "smilecdr.com": 5, + "soluble.ai": 5, + "star123.top": 5, + "sumor.top": 5, + "swascan.com": 5, + "teeworlds.com": 5, + "verot.net": 5, + "videolan.org": 5, + "xwiki.org": 5, + "yuque.com": 5, + "yarnpkg.com": 5, + "zeppelin.apache.org": 5, + "unomi.apache.org": 5, + "web2py.com": 5, + "web.mit.edu": 5, + "chiark.greenend.org.uk": 5, + "conostix.com": 5, + "daimacn.com": 5, + "dojotoolkit.org": 5, + "igniterealtime.org": 5, + "logilab.org": 5, + "swi-prolog.org": 5, + "yuiblog.com": 5, + "xerces.apache.org": 5, + "yuilibrary.com": 5, + "zaranshaikh.blogspot.com": 5, + "checkmk.com": 5, + "support.eset.com": 5, + "git.canopsis.net": 5, + "forescout.com": 5, + "mitel.com": 5, + "cert-in.org.in": 5, + "schneider-elektronik.de": 5, + "security-advisory.acronis.com": 5, + "activemq.2283324.n4.nabble.com": 5, + "apache-ignite-developers.2346864.n4.nabble.com": 5, + "baraktawily.blogspot.co.il": 5, + "blog.diniscruz.com": 5, + "blog.kotowicz.net": 5, + "blog.neargle.com": 5, + "blog.nodejs.org": 5, + "blog.talosintelligence.com": 5, + "borgbackup.readthedocs.io": 5, + "bugs.proftpd.org": 5, + "cgit.drupalcode.org": 5, + "erlang.org": 5, + "extplorer.net": 5, + "foxglovesecurity.com": 5, + "freshmeat.net": 5, + "googlechromereleases.blogspot.de": 5, + "h20000.www2.hp.com": 5, + "htmlpurifier.org": 5, + "itrc.hp.com": 5, + "kalilinux.co": 5, + "lists.horde.org": 5, + "lists.suse.com": 5, + "lists.ucc.gu.uwa.edu.au": 5, + "lkml.iu.edu": 5, + "mercurial.selenic.com": 5, + "miki.it": 5, + "openocd.zylin.com": 5, + "phpmyadmin.svn.sourceforge.net": 5, + "poi.apache.org": 5, + "redmine.lighttpd.net": 5, + "repo.or.cz": 5, + "rubyforge.org": 5, + "aka.ms": 5, + "sakurity.com": 5, + "blog.filippo.io": 5, + "blog.ircmaxell.com": 5, + "blog.justinbull.ca": 5, + "blogs.akamai.com": 5, + "blog.sonatype.com": 5, + "blog.spip.net": 5, + "blog.srcclr.com": 5, + "brakemanscanner.org": 5, + "bridge.grumpy-troll.org": 5, + "buer.haus": 5, + "bugs.dojotoolkit.org": 5, + "bugs.openjdk.java.net": 5, + "bugs.otr.im": 5, + "bugzilla.proxmox.com": 5, + "collectiveidea.com": 5, + "connect2id.com": 5, + "coredns.io": 5, + "cosmosofcyberspace.github.io": 5, + "delvingbitcoin.org": 5, + "devco.re": 5, + "dev.icinga.org": 5, + "docs.cometbft.com": 5, + "docs.couchbase.com": 5, + "docs.getindico.io": 5, + "dovecot.org": 5, + "download.samba.org": 5, + "forge.glpi-project.org": 5, + "ftp.suse.com": 5, + "git.blender.org": 5, + "git.enlightenment.org": 5, + "git.musl-libc.org": 5, + "git.samba.org": 5, + "git.zx2c4.com": 5, + "grsecurity.net": 5, + "hackdefense.com": 5, + "hg.mozilla.org": 5, + "hibernate.atlassian.net": 5, + "ics-cert.us-cert.gov": 5, + "int21.de": 5, + "issues.igniterealtime.org": 5, + "issues.rpath.com": 5, + "kb.juniper.net": 5, + "kyberslash.cr.yp.to": 5, + "labs.twistedmatrix.com": 5, + "ledgersmb.org": 5, + "lib.openmpt.org": 5, + "lists.dns-oarc.net": 5, + "lists.fedorahosted.org": 5, + "lists.linuxfoundation.org": 5, + "lists.xapian.org": 5, + "minerva.crocs.fi.muni.cz": 5, + "mta.openssl.org": 5, + "somevulnsofadlab.blogspot.com": 5, + "oss-fuzz.com": 5, + "oss.oracle.com": 5, + "source.git-annex.branchable.com": 5, + "pub.dev": 5, + "quickview.cloudapps.cisco.com": 5, + "robotattack.org": 5, + "security.googleblog.com": 5, + "shells.systems": 5, + "sintonen.fi": 5, + "skia.googlesource.com": 5, + "staaldraad.github.io": 5, + "store.shopware.com": 5, + "sumofpwn.nl": 5, + "support.lenovo.com": 5, + "trac.osgeo.org": 5, + "trac.webkit.org": 5, + "trac.xiph.org": 5, + "udiniya.wordpress.com": 5, + "support.novell.com": 5, + "weechat.org": 5, + "withatwist.dev": 5, + "codeaurora.org": 5, + "exim.org": 5, + "foxmole.com": 5, + "genivia.com": 5, + "gosecure.net": 5, + "home-assistant.io": 5, + "quagga.net": 5, + "tarlogic.com": 5, + "varnish-cache.org": 5, + "xorl.wordpress.com": 5, + "zenar.io": 5, + "zone.spip.net": 5, + "zyan.scripts.mit.edu": 5, + "theforeman.org": 5, + "tracker.firebirdsql.org": 5, + "trubka.network.cz": 5, + "wiki.mahara.org": 5, + "cacti.net": 5, + "eterna.com.au": 5, + "eweek.com": 5, + "ghostscript.com": 5, + "h5l.org": 5, + "ietf.org": 5, + "information-security.fr": 5, + "iwantacve.cn": 5, + "links.org": 5, + "mutt.org": 5, + "slackware.com": 5, + "thewildbeast.co.uk": 5, + "tornadoweb.org": 5, + "znuny.com": 5, + "apps.apple.com": 5, + "dontvacuum.me": 5, + "wiz.io": 5, + "tsc-soft.co.jp": 5, + "2016.hack.lu": 5, + "agrrrdog.blogspot.com": 5, + "alexcrack.com": 5, + "apache-spark-developers-list.1001551.n3.nabble.com": 5, + "appwrite.com": 5, + "archive.hack.lu": 5, + "argo.com": 5, + "bandoche.com": 5, + "bilishim.com": 5, + "blackboxexporter.com": 5, + "blog.amossys.fr": 5, + "blog.angularjs.org": 5, + "blog.apps.npr.org": 5, + "blog.bestpractical.com": 5, + "blog.checkpoint.com": 5, + "blog.csdn.net": 5, + "blog.datomic.com": 5, + "blog.emaze.net": 5, + "blog.intothesymmetry.com": 5, + "blog.portswigger.net": 5, + "blog.recurity-labs.com": 5, + "blog.securelayer7.net": 5, + "boltcms.com": 5, + "breaktoprotect.blogspot.com": 5, + "bugs.jython.org": 5, + "bugs.ledger-cli.org": 5, + "buildbot.net": 5, + "bzr.linuxfoundation.org": 5, + "ca17.com": 5, + "census-labs.com": 5, + "centreon.com": 5, + "ceriksen.com": 5, + "chargen.matasano.com": 5, + "chrony.tuxfamily.org": 5, + "code.call-cc.org": 5, + "community.ca.com": 5, + "community.impresscms.org": 5, + "cpansearch.perl.org": 5, + "craft.com": 5, + "crafter.com": 5, + "cvs.moodle.org": 5, + "cvsweb.netbsd.org": 5, + "cyberworldmirror.com": 5, + "davidsopaslabs.blogspot.com": 5, + "dev.deluge-torrent.org": 5, + "dfn.dl.sourceforge.net": 5, + "directory.apache.org": 5, + "doc.scrapy.org": 5, + "doc.silverstripe.org": 5, + "docs.info.apple.com": 5, + "docs.python-requests.org": 5, + "docs.withknown.com": 5, + "dolibarr.com": 5, + "dotclear.org": 5, + "dx.doi.org": 5, + "elixir.ematia.de": 5, + "embed.plnkr.co": 5, + "en.0day.today": 5, + "encode.com": 5, + "etcd.com": 5, + "ethereum.com": 5, + "evilpacket.net": 5, + "exfiltrated.com": 5, + "extendedsubset.com": 5, + "fitnesse.org": 5, + "flyingmana.de": 5, + "freecode.com": 5, + "ghost.com": 5, + "gilacms.com": 5, + "git-blame.blogspot.com": 5, + "git.deluge-torrent.org": 5, + "githubcommherflower.com": 5, + "git.postgresql.org": 5, + "go-ethereum.com": 5, + "gogs.io": 5, + "gollum.com": 5, + "grailsblog.objectcomputing.com": 5, + "groovy-lang.org": 5, + "ha.cker.info": 5, + "haxx.ml": 5, + "heartex.com": 5, + "help.plot.ly": 5, + "hidden-one.co.in": 5, + "holisticinfosec.org": 5, + "hsqldb.org": 5, + "icecoder.com": 5, + "info.tiki.org": 5, + "jakarta.apache.org": 5, + "jeecg-boot.com": 5, + "jinja.pocoo.org": 5, + "joplin.com": 5, + "juddi.apache.org": 5, + "labelstud.io": 5, + "labs.mwrinfosecurity.com": 5, + "lambdaops.com": 5, + "langchain.com": 5, + "lavalite.com": 5, + "lcamtuf.blogspot.ca": 5, + "lesscss.org": 5, + "libvirt.org": 5, + "liferay.com": 5, + "linux.oracle.com": 5, + "lists.live555.com": 5, + "lists.mutt.org": 5, + "lists.roaringpenguin.com": 5, + "lists.typo3.org": 5, + "lua.2524044.n2.nabble.com": 5, + "lxml.de": 5, + "maven.apache.org": 5, + "mina.apache.org": 5, + "mumble.info": 5, + "n8.tumblr.com": 5, + "nevado.skyscreamer.org": 5, + "ngenuity-is.com": 5, + "nginx.org": 5, + "nodejs.com": 5, + "openapi-generator.com": 5, + "openbsd.org": 5, + "opennlp.apache.org": 5, + "ph0rse.me": 5, + "processwire.com": 5, + "prometheus.com": 5, + "pyload.com": 5, + "pypi.doubanio.com": 5, + "pypinksign.com": 5, + "python.6.x6.nabble.com": 5, + "qcubed.com": 5, + "rails.lighthouseapp.com": 5, + "raneto.com": 5, + "request-baskets.com": 5, + "resque.com": 5, + "roundup.svn.sourceforge.net": 5, + "ruffsecurity.blogspot.com": 5, + "0ang3el.blogspot.ru": 5, + "2018.zeronights.ru": 5, + "98587329.github.io": 5, + "aaltodoc.aalto.fi": 5, + "accumulo.apache.org": 5, + "acloudtree.com": 5, + "adamcaudill.com": 5, + "advisories.gitlab.com": 5, + "advisories.octopus.com": 5, + "aetsu.github.io": 5, + "agent-js.icp.xyz": 5, + "sailsjs.org": 5, + "aisec.today": 5, + "aleksis.org": 5, + "alesandroortiz.com": 5, + "alexsecurity.rocks": 5, + "alicangonullu.org": 5, + "anisiosantos.me": 5, + "anongit.mindrot.org": 5, + "ant.apache.org": 5, + "antmedia.io": 5, + "apidoc.gitee.com": 5, + "apidock.com": 5, + "apiiro.com": 5, + "app.intigriti.com": 5, + "appwrite.io": 5, + "atlaskit.atlassian.com": 5, + "attackerkb.com": 5, + "at-trustit.tuv.at": 5, + "autobahn.readthedocs.io": 5, + "avideo.tube": 5, + "backbonejs.org": 5, + "bad.code.blog": 5, + "badpackets.net": 5, + "bandit.readthedocs.io": 5, + "baomidou.com": 5, + "beanvalidation.org": 5, + "becomepentester.blogspot.ae": 5, + "bishopfox.com": 5, + "bitbucket.hdfgroup.org": 5, + "bittherapy.net": 5, + "blakeembrey.com": 5, + "blitiri.com.ar": 5, + "blog.0xzon.dev": 5, + "blog.appsecco.com": 5, + "blog.bentkowski.info": 5, + "blog.bssi.fr": 5, + "blog.caller.xyz": 5, + "blog.chebuya.com": 5, + "blog.compass-security.com": 5, + "blog.daniel-ruf.de": 5, + "blog.effectrenan.com": 5, + "blog.ethereum.org": 5, + "blog.exodusintel.com": 5, + "blog.gdssecurity.com": 5, + "blog.gradle.org": 5, + "blog.hackeriet.no": 5, + "blog.hackingforce.com.br": 5, + "blog.ipfs.io": 5, + "blog.isosceles.com": 5, + "blog.jiguang.xyz": 5, + "blog.librenms.org": 5, + "blog.litespeedtech.com": 5, + "blog.mevsec.com": 5, + "blog.nettitude.com": 5, + "blog.orange.tw": 5, + "blog.ostorlab.co": 5, + "blog.phpbb.com": 5, + "blog.pusher.com": 5, + "blog.qualys.com": 5, + "blog.sentry.io": 5, + "blogs.gnome.org": 5, + "blog.slonser.info": 5, + "blog.smithsecurity.biz": 5, + "blogs.opera.com": 5, + "blogs.sap.com": 5, + "blog.truesec.com": 5, + "blog.xss.am": 5, + "boats.gitlab.io": 5, + "boho.or.kr": 5, + "book.hacktricks.xyz": 5, + "brave.com": 5, + "breakandpray.com": 5, + "bsg.tech": 5, + "bto.bluecoat.com": 5, + "bugs.bitlbee.org": 5, + "bugs.libssh.org": 5, + "bugs.limesurvey.org": 5, + "bugs.otrs.org": 5, + "bugs.ruby-lang.org": 5, + "bugzilla.libav.org": 5, + "bundler.io": 5, + "burninatorsec.blogspot.com": 5, + "buttercup.pw": 5, + "caddy.community": 5, + "caddyserver.com": 5, + "scala-lang.org": 5, + "cardaci.xyz": 5, + "carl1l.github.io": 5, + "scarybeastsecurity.blogspot.de": 5, + "casdoor.org": 5, + "c-c-a.org": 5, + "cdn2.hubspot.net": 5, + "cdn.datatables.net": 5, + "cdn.sheetjs.com": 5, + "cert.enea.pl": 5, + "cgit.freebsd.org": 5, + "cgit.freedesktop.org": 5, + "chaos.social": 5, + "chartkick.com": 5, + "checkstyle.org": 5, + "chmod744.super.site": 5, + "chocapikk.com": 5, + "clerk.com": 5, + "clojars.org": 5, + "cloud-trustit.spp.at": 5, + "codeburst.io": 5, + "code.jeremyevans.net": 5, + "codex.bbpress.org": 5, + "codex.buddypress.org": 5, + "com0t.github.io": 5, + "commonmark.thephpleague.com": 5, + "community.contao.org": 5, + "community.developer.atlassian.com": 5, + "community.gravitee.io": 5, + "community.traefik.io": 5, + "community.veracode.com": 5, + "concretecms.com": 5, + "contrastsecurity.com": 5, + "cortexmetrics.io": 5, + "cowtowncoder.medium.com": 5, + "cratedb.com": 5, + "cryptosense.com": 5, + "csrc.nist.gov": 5, + "cube01.io": 5, + "cupc4k3.lol": 5, + "cure53.de": 5, + "cve.anastasi.link": 5, + "cve.naver.com": 5, + "cve.nstsec.com": 5, + "cyllective.com": 5, + "danielfett.de": 5, + "dannewitz.ninja": 5, + "darkbit.io": 5, + "datasette.io": 5, + "datnlq.gitbook.io": 5, + "deadsh0t.medium.com": 5, + "demo.ripstech.com": 5, + "deps.dev": 5, + "devdocs.magento.com": 5, + "devel0pment.de": 5, + "developer.apple.com": 5, + "developer.hashicorp.com": 5, + "developer.jboss.org": 5, + "developers.cloudflare.com": 5, + "developers.google.com": 5, + "developer.shopware.com": 5, + "developers.yubico.com": 5, + "developer.woocommerce.com": 5, + "develop.sentry.dev": 5, + "dev.gajim.org": 5, + "dev.to": 5, + "diff.coditsu.io": 5, + "diff.hex.pm": 5, + "digi.ninja": 5, + "directus.io": 5, + "discourse.vtk.org": 5, + "discuss.istio.io": 5, + "dist.apache.org": 5, + "dist.plone.org": 5, + "dnsdist.org": 5, + "doc.clickup.com": 5, + "doc.rust-lang.org": 5, + "docs.centreon.com": 5, + "docs.ceph.com": 5, + "docs.ckan.org": 5, + "docs.dapr.io": 5, + "docs.dask.org": 5, + "docs.dependencytrack.org": 5, + "docs.directus.io": 5, + "docs.flyte.org": 5, + "docs.gitlab.com": 5, + "docs.gofiber.io": 5, + "docs.gravityforms.com": 5, + "docs.libp2p.io": 5, + "docs.locust.io": 5, + "docs.mulesoft.com": 5, + "docs.nautobot.com": 5, + "docs.nginx.com": 5, + "docs.parseplatform.org": 5, + "docs.pylonsproject.org": 5, + "docs.r3.com": 5, + "docs.saltproject.io": 5, + "docs.scrapy.org": 5, + "docs.securesauce.dev": 5, + "docs.sigstore.dev": 5, + "docs.snowflake.com": 5, + "docs.spring.io": 5, + "docs.sqlalchemy.org": 5, + "docs.sympy.org": 5, + "docs.tigergraph.com": 5, + "docs.totaljs.com": 5, + "docs.velociraptor.app": 5, + "docs.veracode.com": 5, + "docs.vyperlang.org": 5, + "docs.wagtail.io": 5, + "documentation.concrete5.org": 5, + "documentation.wazuh.com": 5, + "doi.org": 5, + "dotnetnuke.codeplex.com": 5, + "dubell.io": 5, + "dustri.org": 5, + "dwrensha.github.io": 5, + "security.360.cn": 5, + "security.openttd.org": 5, + "securitywarrior9.blogspot.com": 5, + "edg.io": 5, + "edhunter484.medium.com": 5, + "eldstal.se": 5, + "electron.atom.io": 5, + "elgg.org": 5, + "emily.id.au": 5, + "en.bandisoft.com": 5, + "en.bitcoin.it": 5, + "engindemirbilek.github.io": 5, + "en.osdn.jp": 5, + "epadillas.github.io": 5, + "erlef.github.io": 5, + "eslam.io": 5, + "exceptionfactory.com": 5, + "experienceleague.adobe.com": 5, + "expressjs.com": 5, + "fastapi-admin-pro.long2ice.io": 5, + "fbdhhhh47.github.io": 5, + "febin0x4e4a.blogspot.com": 5, + "febin0x4e4a.wordpress.com": 5, + "febinj.medium.com": 5, + "ferrous-systems.com": 5, + "fgsec.net": 5, + "filezilla-project.org": 5, + "fisheye6.atlassian.com": 5, + "flask-limiter.readthedocs.io": 5, + "floqast.com": 5, + "flowiseai.com": 5, + "flyd.uk": 5, + "fmyyy1.github.io": 5, + "forge.centreon.com": 5, + "forge.typo3.org": 5, + "forum.codeigniter.com": 5, + "forum.datomic.com": 5, + "forum.ksec.co.uk": 5, + "forum.netgate.com": 5, + "forum.openzeppelin.com": 5, + "forums.couchbase.com": 5, + "forum.silverstripe.org": 5, + "forums.swift.org": 5, + "fossies.org": 5, + "freemarker.apache.org": 5, + "frycos.github.io": 5, + "fuo.fi": 5, + "fusionauth.io": 5, + "gainsec.com": 5, + "gauravnarwani.com": 5, + "genix.me": 5, + "geoserver.org": 5, + "gerrit.asterisk.org": 5, + "gerrit.ovirt.org": 5, + "getcomposer.org": 5, + "getgrav.org": 5, + "getkirby.com": 5, + "getlaminas.org": 5, + "getlektor.com": 5, + "git-annex.branchable.com": 5, + "git.fedorahosted.org": 5, + "github.com.mattermost": 5, + "git.libssh.org": 5, + "git.oschina.net": 5, + "git.reviewboard.kde.org": 5, + "git.sdaoden.eu": 5, + "git.xiph.org": 5, + "gld.mcphail.uk": 5, + "glitch.com": 5, + "gluu.org": 5, + "go2docs.graylog.org": 5, + "godoc.org": 5, + "gohugo.io": 5, + "go-review.git.corp.google.com": 5, + "graz.pure.elsevier.com": 5, + "greysec.net": 5, + "groups.drupal.org": 5, + "grumpz.net": 5, + "guides.spreecommerce.org": 5, + "gultsch.de": 5, + "hackerdna.com": 5, + "hacker.soarescorp.com": 5, + "hackers.report": 5, + "hackinglab.cz": 5, + "hackmysystems.tumblr.com": 5, + "hackpuntes.com": 5, + "hansmi.ch": 5, + "hapifhir.io": 5, + "help.ecostruxureit.com": 5, + "help.egroupware.org": 5, + "help.mulesoft.com": 5, + "help.panic.com": 5, + "help.passbolt.com": 5, + "help.rapid7.com": 5, + "help.sonatype.com": 5, + "shindig.apache.org": 5, + "shiro.apache.org": 5, + "homakov.blogspot.ru": 5, + "html5lib.readthedocs.io": 5, + "httpwg.org": 5, + "hub.docker.com": 5, + "hugegraph.apache.org": 5, + "i.blackhat.com": 5, + "ibm.github.io": 5, + "igniterealtime.atlassian.net": 5, + "silverpeas.com": 5, + "imgur.com": 5, + "infosec.exchange": 5, + "infra.spec.whatwg.org": 5, + "inhann.top": 5, + "inlong.apache.org": 5, + "insinuator.net": 5, + "intrix.com.au": 5, + "ipython.org": 5, + "ipython.readthedocs.io": 5, + "irrd.readthedocs.io": 5, + "isec.pl": 5, + "issues.ibexa.co": 5, + "issues.jasig.org": 5, + "issues.jenkins.io": 5, + "issues.sonatype.org": 5, + "it-sec.de": 5, + "itsmeanonartist.tech": 5, + "jadaptive.com": 5, + "janino-compiler.github.io": 5, + "java.net": 5, + "jay-from-future.github.io": 5, + "jenkins-ci.org": 5, + "jetpack.com": 5, + "jinja.palletsprojects.com": 5, + "jinmu1108.github.io": 5, + "jira.onosproject.org": 5, + "jira.opendaylight.org": 5, + "jira.sonarsource.com": 5, + "jira.whamcloud.com": 5, + "jquery.com": 5, + "jqueryui.com": 5, + "jqueryvalidation.org": 5, + "jsfiddle.net": 5, + "jub0bs.com": 5, + "junit.org": 5, + "kafka.apache.org": 5, + "kb.hitcon.org": 5, + "kb.netapp.com": 5, + "skipper.com": 5, + "koji.fedoraproject.org": 5, + "korelogic.com": 5, + "koz.io": 5, + "labanskoller.se": 5, + "labs.detectify.com": 5, + "lab.wallarm.com": 5, + "landave.io": 5, + "laravel-admin.org": 5, + "latestpcsolution.wordpress.com": 5, + "launchpad.support.sap.com": 5, + "laworigin.github.io": 5, + "layui.dev": 5, + "learn.snyk.io": 5, + "lednerb.de": 5, + "lessonsec.com": 5, + "libgit2.org": 5, + "libnmap.readthedocs.io": 5, + "lib.rs": 5, + "lightning.network": 5, + "limpidsecurity.pl": 5, + "linkerd.io": 5, + "linotp.org": 5, + "lists.cncf.io": 5, + "lists.denx.de": 5, + "lists.ffmpeg.org": 5, + "lists.mindrot.org": 5, + "lists.osgeo.org": 5, + "lists.w3.org": 5, + "lists.zx2c4.com": 5, + "locutus.io": 5, + "lutrasecurity.com": 5, + "lycshub.github.io": 5, + "m3n0sd0n4ld.github.io": 5, + "mail.gnome.org": 5, + "mailman-mail5.webfaction.com": 5, + "smalruby.jp": 5, + "marketplace.atlassian.com": 5, + "martinthomson.github.io": 5, + "mat4mee.notion.site": 5, + "matt.ucc.asn.au": 5, + "mayaseven.com": 5, + "media.dedaub.com": 5, + "mensfeld.pl": 5, + "meshery.io": 5, + "meta.wikimedia.org": 5, + "meterpreter.org": 5, + "minhnq22.medium.com": 5, + "mitmproxy.org": 5, + "mitogen.networkgenomics.com": 5, + "mksec.tk": 5, + "mlflow.org": 5, + "modwsgi.readthedocs.io": 5, + "monicz.dev": 5, + "morehouse.github.io": 5, + "mostwanted002.cf": 5, + "mouha.be": 5, + "movermeyer.com": 5, + "msrc-blog.microsoft.com": 5, + "mthbernardes.github.io": 5, + "mulch.dev": 5, + "my.f5.com": 5, + "my.goanywhere.com": 5, + "nablarch.atlassian.net": 5, + "nakedsecurity.sophos.com": 5, + "nandynarwhals.org": 5, + "narrow-oatmeal-0c0.notion.site": 5, + "nasa.github.io": 5, + "nest.pijul.com": 5, + "nghttp2.org": 5, + "nmap.org": 5, + "nodemailer.com": 5, + "node-postgres.com": 5, + "nostarttls.secvuln.info": 5, + "notes.netbytesec.com": 5, + "note.youdao.com": 5, + "note.zhaoj.in": 5, + "nova.app": 5, + "nowotarski.info": 5, + "nozero.io": 5, + "nsfocusglobal.com": 5, + "numanozdemir.com": 5, + "objectcomputing.com": 5, + "octoprint.org": 5, + "okankurtulus.com.tr": 5, + "omespino.com": 5, + "onekey.com": 5, + "openai.com": 5, + "openbase.com": 5, + "opencast.jira.com": 5, + "opencirt.com": 5, + "openjdk.org": 5, + "opennms.atlassian.net": 5, + "opensearch.org": 5, + "opensource.fast-report.com": 5, + "openssf.org": 5, + "opentelemetry.io": 5, + "opsecx.com": 5, + "orc.apache.org": 5, + "oroinc.com": 5, + "oryx-embedded.com": 5, + "oss-security.openwall.narkive.com": 5, + "osv.dev": 5, + "owncloud.com": 5, + "packaging.python.org": 5, + "pagehelper.github.io": 5, + "panda002.hashnode.dev": 5, + "papers.mathyvanhoef.com": 5, + "sparkjava.com": 5, + "payatu.com": 5, + "peckshield.com": 5, + "pentesterlab.com": 5, + "people.kingsds.network": 5, + "petl.readthedocs.io": 5, + "piraeus.io": 5, + "plotly.com": 5, + "plugins.gradle.org": 5, + "plus.google.com": 5, + "pony7.fr": 5, + "popalltheshells.medium.com": 5, + "powerful-bulb-c36.notion.site": 5, + "prestosql.io": 5, + "pretalx.com": 5, + "prismjs.com": 5, + "privatebin.info": 5, + "programmer.help": 5, + "projects.duckcorp.org": 5, + "prophaze.com": 5, + "publicobject.com": 5, + "pubs.acs.org": 5, + "pugjs.org": 5, + "pulsar.apache.org": 5, + "pydio.com": 5, + "pythonhosted.org": 5, + "python-hyper.org": 5, + "quilljs.com": 5, + "quiltmc.org": 5, + "r0.haxors.org": 5, + "ranchermanager.docs.rancher.com": 5, + "rankmath.com": 5, + "rastating.github.io": 5, + "ratpack.io": 5, + "reportportal.io": 5, + "repo.saltproject.io": 5, + "researchgate.net": 5, + "research.hisolutions.com": 5, + "research.insecurelabs.org": 5, + "restsharp.dev": 5, + "review.whamcloud.com": 5, + "rhodecode.com": 5, + "rhynorater.github.io": 5, + "robertheaton.com": 5, + "rodelllemit.medium.com": 5, + "rootdaemon.com": 5, + "roumenpetrov.info": 5, + "rpyc.readthedocs.io": 5, + "ruby-doc.org": 5, + "rudnkh.me": 5, + "sahildhar.github.io": 5, + "samuzora.com": 5, + "saturncloud.io": 5, + "sca.analysiscenter.veracode.com": 5, + "scalyr-static.s3.amazonaws.com": 5, + "scikit-learn.org": 5, + "scnps.co": 5, + "scratch.mit.edu": 5, + "sec.stealthcopter.com": 5, + "secure1.securityspace.com": 5, + "security-garage.com": 5, + "security.gradle.com": 5, + "securitylabs.datadoghq.com": 5, + "security.paloaltonetworks.com": 5, + "securitypitfalls.wordpress.com": 5, + "securitytrails.com": 5, + "segment.com": 5, + "sektioneins.de": 5, + "sethmlarson.dev": 5, + "setuptools.pypa.io": 5, + "shattered.io": 5, + "sheetjs.com": 5, + "siebene.github.io": 5, + "silverpeas.org": 5, + "skalatan.de": 5, + "skerritt.blog": 5, + "skii.dev": 5, + "skywalking.apache.org": 5, + "sling.apache.org": 5, + "smshrimant.com": 5, + "snicco.io": 5, + "softwaresupport.hpe.com": 5, + "solidus.io": 5, + "sonarsource.atlassian.net": 5, + "sources.debian.net": 5, + "spacemesh.io": 5, + "specifications.freedesktop.org": 5, + "spec.matrix.org": 5, + "src.fedoraproject.org": 5, + "srcincite.io": 5, + "stackstorm.com": 5, + "staging-website.elastic.co": 5, + "stash.kopano.io": 5, + "subrion.org": 5, + "support.aerospike.com": 5, + "support.ca.com": 5, + "support.citrix.com": 5, + "support.confluent.io": 5, + "support.contrastsecurity.com": 5, + "support.delphix.com": 5, + "support.herodevs.com": 5, + "support.snyk.io": 5, + "sw.aveva.com": 5, + "sylabs.io": 5, + "sylius.com": 5, + "sympa-community.github.io": 5, + "sysdig.com": 5, + "systeminformation.io": 5, + "systemweakness.com": 5, + "tada.github.io": 5, + "tadayoshi-sato.medium.com": 5, + "tantosec.com": 5, + "tches.iacr.org": 5, + "techblog.wikimedia.org": 5, + "technet.microsoft.com": 5, + "tempered.works": 5, + "tf1t.gitbook.io": 5, + "thehackernews.com": 5, + "the-it-wonders.blogspot.com": 5, + "thelia.net": 5, + "threat.tevora.com": 5, + "threema.ch": 5, + "tickets.puppetlabs.com": 5, + "tiki.org": 5, + "stimulsoft.com": 5, + "tprynn.github.io": 5, + "tracker.die-offenbachs.homelinux.org": 5, + "tracker.moodle.org": 5, + "tracker.phpbb.com": 5, + "tracker.zkoss.org": 5, + "trac.xapian.org": 5, + "trafficcontrol.apache.org": 5, + "travis-ci.com": 5, + "trends.builtwith.com": 5, + "strimzi.com": 5, + "trino.io": 5, + "tristartom.github.io": 5, + "truedigitalsecurity.com": 5, + "trufflesecurity.com": 5, + "trungvm.gitbook.io": 5, + "trust.neo4j.com": 5, + "tryhexadecimal.com": 5, + "tttang.com": 5, + "unbound.net": 5, + "unpoly.com": 5, + "updates.playhive.com": 5, + "updates.snyk.io": 5, + "support.zeus.com": 5, + "suricata-ids.org": 5, + "us-cert.cisa.gov": 5, + "vapor.codes": 5, + "vega.github.io": 5, + "veracode.com": 5, + "verichains.io": 5, + "versprite.com": 5, + "vertx.io": 5, + "vimeo.com": 5, + "vitejs.dev": 5, + "svn.savannah.gnu.org": 5, + "vulners.com": 5, + "w3lib.readthedocs.io": 5, + "warehouse.python.org": 5, + "weakdh.org": 5, + "webassembly.github.io": 5, + "webiny.com": 5, + "webmasters.googleblog.com": 5, + "wger.de": 5, + "wicket.apache.org": 5, + "wiki.duraspace.org": 5, + "wiki.eclipse.org": 5, + "wiki.folio.org": 5, + "wiki.gentoo.org": 5, + "wiki.mercurial-scm.org": 5, + "wiki.opendaylight.org": 5, + "wiki.shibboleth.net": 5, + "winscp.net": 5, + "wintercms.com": 5, + "winterdragon.ca": 5, + "wiremock.org": 5, + "withknown.com": 5, + "wtfsec.org": 5, + "activecyber.us": 5, + "admidio.org": 5, + "adminer.org": 5, + "aleksey.com": 5, + "alevsk.com": 5, + "alluxio.io": 5, + "ansible.com": 5, + "anyscale.com": 5, + "apereo.org": 5, + "apolloconfig.com": 5, + "baeldung.com": 5, + "barracuda.com": 5, + "benthamsgaze.org": 5, + "bountysource.com": 5, + "brics.dk": 5, + "broadcom.com": 5, + "brzozowski.io": 5, + "bsi.bund.de": 5, + "ccsq8.com": 5, + "certik.com": 5, + "chtsecurity.com": 5, + "cigital.com": 5, + "ciphertechs.com": 5, + "cloudflare.com": 5, + "cloudfoundry.org": 5, + "coalfire.com": 5, + "concrete5.org": 5, + "coresecurity.com": 5, + "crowdstrike.com": 5, + "crushftp.com": 5, + "cvedetails.com": 5, + "cybereagle.io": 5, + "cyfirma.com": 5, + "darkreading.com": 5, + "ddosi.org": 5, + "descope.com": 5, + "dnspython.org": 5, + "docker.com": 5, + "doyler.net": 5, + "dsecbypass.com": 5, + "dubget.com": 5, + "economizzer.org": 5, + "edwardthomson.com": 5, + "elementary-data.com": 5, + "enhavo.com": 5, + "enterprisedb.com": 5, + "equalexperts.com": 5, + "eyecontrol.nl": 5, + "fastify.io": 5, + "ffmpeg.org": 5, + "fireeye.com": 5, + "fork-cms.com": 5, + "fossil-scm.org": 5, + "gitpod.io": 5, + "gradio.app": 5, + "gsma.com": 5, + "hacksecproject.com": 5, + "hackthebox.com": 5, + "hakaioffensivesecurity.com": 5, + "haproxy.com": 5, + "htmlunit.org": 5, + "ihteam.net": 5, + "imperialviolet.org": 5, + "impresscms.org": 5, + "intruder.io": 5, + "isc.org": 5, + "jetbrains.com": 5, + "jhipster.tech": 5, + "lancom-systems.de": 5, + "landaire.net": 5, + "laravel-enlightn.com": 5, + "limesurvey.org": 5, + "logicallysecure.com": 5, + "mageni.net": 5, + "makotemplates.org": 5, + "mandiant.com": 5, + "menlosecurity.com": 5, + "mgm-sp.com": 5, + "monolune.com": 5, + "mpg123.de": 5, + "nds.ruhr-uni-bochum.de": 5, + "netlify.com": 5, + "netsarang.com": 5, + "ni.com": 5, + "nomadproject.io": 5, + "nu11secur1ty.com": 5, + "obrela.com": 5, + "octobot.online": 5, + "okta.com": 5, + "on-x.com": 5, + "op-c.net": 5, + "opencart.com": 5, + "opencrx.org": 5, + "optiv.com": 5, + "oxeye.io": 5, + "papermerge.com": 5, + "paramiko.org": 5, + "pethuraj.com": 5, + "phoronix.com": 5, + "pizzapower.me": 5, + "portainer.io": 5, + "postfix.org": 5, + "www-prd-trops.events.ibm.com": 5, + "prestashop.com": 5, + "privacy-wise.com": 5, + "projectcalico.org": 5, + "purplemet.com": 5, + "qemu.org": 5, + "raifberkaydincel.com": 5, + "rejetto.com": 5, + "reportlab.com": 5, + "rsaconference.com": 5, + "rust-lang.org": 5, + "sap.com": 5, + "scottbrady91.com": 5, + "secureauth.com": 5, + "securitymetrics.com": 5, + "semantic-mediawiki.org": 5, + "shielder.com": 5, + "shorebreaksecurity.com": 5, + "sidertia.com": 5, + "slf4j.org": 5, + "sockjs.org": 5, + "splunk.com": 5, + "symantec.com": 5, + "sympa.org": 5, + "tcpdump.org": 5, + "telerik.com": 5, + "terrapin-attack.com": 5, + "trustmatta.com": 5, + "vandyke.com": 5, + "vdoo.com": 5, + "viralpatel.net": 5, + "virtualbox.org": 5, + "whitehats.nl": 5, + "winimage.com": 5, + "yubico.com": 5, + "zdnet.com": 5, + "zenml.io": 5, + "zofrex.com": 5, + "zsh.org": 5, + "xalan.apache.org": 5, + "xavibel.com": 5, + "xmit.xyz": 5, + "xmlgraphics.apache.org": 5, + "xmpp.org": 5, + "yondon.blog": 5, + "zerosecuritypenetrationtesting.com": 5, + "zh-cn.tenable.com": 5, + "zookeeper.apache.org": 5, + "zuso.ai": 5, + "zxsecurity.co.nz": 5, + "testh5shanglv.minshengec.com:1024": 5, + "thinkphp.com": 5, + "tiddlywiki5.com": 5, + "trac.dojotoolkit.org": 5, + "tracker.ceph.com": 5, + "tree-kit.com": 5, + "unomi.apache.org.": 5, + "ureport.com": 5, + "users.encs.concordia.ca": 5, + "vsintelli.com": 5, + "wiki.apache.org": 5, + "wiki.rpath.com": 5, + "andmp.com": 5, + "benjaminfleischer.com": 5, + "bigdiao.cc": 5, + "blcat.cn": 5, + "brocade.com": 5, + "conviso.com.br": 5, + "coreftp.com": 5, + "cs.utexas.edu": 5, + "dest-unreach.org": 5, + "egroupware.org": 5, + "elasticsearch.com": 5, + "fetchmail.info": 5, + "fomori.org": 5, + "getsymphony.com": 5, + "gevent.org": 5, + "graphicsmagick.org": 5, + "hackersb.cn": 5, + "inspircd.org": 5, + "isg.rhul.ac.uk": 5, + "ja-sig.org": 5, + "jcraft.com": 5, + "jianshu.com": 5, + "linuxgrill.com": 5, + "mbsd.jp": 5, + "mega-nerd.com": 5, + "movabletype.org": 5, + "mpxj.org": 5, + "nemux.org": 5, + "netbytesec.com": 5, + "opera.com": 5, + "pip-installer.org": 5, + "procheckup.com": 5, + "rabbitmq.com": 5, + "roothc.com.br": 5, + "rootlabs.com.br": 5, + "rsyslog.com": 5, + "securityweek.com": 5, + "slideshare.net": 5, + "sqlalchemy.org": 5, + "square16.org": 5, + "stack.nl": 5, + "tinc-vpn.org": 5, + "tripwire.com": 5, + "uzbl.org": 5, + "zeroscience.mk": 5, + "xmlsoft.org": 5, + "xxl-job.com": 5, + "yehg.net": 5, + "yourls.org": 5, + "zalando.com": 5, + "sony.com": 5, + "strawberry.rocks": 5, + "werewolves.world": 5, + "newegg.com": 5, + "consumer.huawei.com": 5, + "kb.parallels.com": 5, + "support.google.com": 5, + "community.targit.com": 5, + "unisoc.com": 5, + "support.broadcom.com": 5, + "security-portal.versa-networks.com": 5, + "sliderrevolution.com": 5, + "autodesk.com": 5, + "trust.mi.com": 5, + "git.proxmox.com": 5, + "docs.azul.com": 5, + "kibty.town": 5, + "flowus.cn": 5, + "wp-osm-plugin.hyumika.com": 5, + "nokia.com": 5, + "bdosecurity.de": 5, + "khronokernel.com": 5, + "wiki.securepoint.de": 5, + "bridewell.com": 5, + "arc.net": 5, + "bravurasecurity.com": 5, + "chiggerlor.substack.com": 5, + "csflabs.github.io": 5, + "manageengine.com": 5, + "zoho.com": 5, + "devolutions.net": 5, + "technitium.com": 5, + "dell.com": 5, + "eviden.com": 5, + "ptzoptics.com": 5, + "security.freebsd.org": 5, + "giflib.com": 5, + "support.hp.com": 5, + "icecms.com": 5, + "cyberdanube.com": 5, + "customers.codesys.com": 5, + "kaiten.ru": 5, + "keyence.com": 5, + "trustedcare.entrust.com": 5, + "codefactor.io": 5, + "codeastro.com": 5, + "exchange.checkmk.com": 5, + "cs-cart.com": 5, + "usvn.info": 5, + "takex-eng.co.jp": 5, + "ti.qianxin.com": 5, + "stuxxn.github.io": 5, + "trevorkems.com": 5, + "cve.mahi.be": 5, + "0xmupa.github.io": 5, + "forum.proxmox.com": 5, + "puredata.info": 5, + "docs.rocket.chat": 5, + "cirosec.de": 5, + "docs.adacore.com": 5, + "forums.ivanti.com": 5, + "thewatch.centreon.com": 5, + "papercut.com": 5, + "sophos.com": 5, + "issue-tracker.miraheze.org": 5, + "tiptel.com": 5, + "e-tax.nta.go.jp": 5, + "teamviewer.com": 5, + "answers.webroot.com": 5, + "cert.vde.com": 5, + "support.bull.com": 5, + "cnnvd.org.cn": 5, + "westerndigital.com": 5, + "closed-loop.biz": 5, + "entrust.com": 5, + "solvait.com": 5, + "woodwing.com": 5, + "blog.hawktesters.com": 5, + "zyxel.com": 5, + "0xahmed.ninja": 5, + "2012.appsec-forum.ch": 5, + "acme.com": 5, + "aliyundrive-webdav.com": 5, + "als.regnet.cz": 5, + "aluigi.altervista.org": 5, + "antirez.com": 5, + "apt.inguza.net": 5, + "arthurdejong.org": 5, + "asfws12.files.wordpress.com": 5, + "askubuntu.com": 5, + "aspell.net": 5, + "badlock.org": 5, + "baraktawily.blogspot.com": 5, + "benmmurphy.github.io": 5, + "bh.ht.vc": 5, + "bird.network.cz": 5, + "bk.ntp.org": 5, + "blog.addepar.com": 5, + "blog.c22.cc": 5, + "blog.chromium.org": 5, + "blog.dscpl.com.au": 5, + "blog.fortify.com": 5, + "blog.fuseyism.com": 5, + "blog.gmane.org": 5, + "blog.g-sec.lu": 5, + "blog.guya.net": 5, + "blog.infosectcbr.com.au": 5, + "blog.kazuhooku.com": 5, + "blog.koehntopp.de": 5, + "blog.nibblesec.org": 5, + "blog.noobroot.com": 5, + "blog.o0o.nu": 5, + "blog.opensecurityresearch.com": 5, + "blog.pear.php.net": 5, + "blog.pi3.com.pl": 5, + "blog.prosody.im": 5, + "blog.python.org": 5, + "blogs.adobe.com": 5, + "blog.scrt.ch": 5, + "blog.securitymouse.com": 5, + "blog.sendsafely.com": 5, + "blog.senr.io": 5, + "blogs.iss.net": 5, + "blogs.oracle.com": 5, + "blog.spiderlabs.com": 5, + "blogs.sun.com": 5, + "blog.swiecki.net": 5, + "blog.swiftmailer.org": 5, + "blog.topsec.com.cn": 5, + "boinc.berkeley.edu": 5, + "bugs.dokuwiki.org": 5, + "bugs.gw.com": 5, + "bugs.icu-project.org": 5, + "bugs.java.com": 5, + "bugs.ntp.org": 5, + "bugs.openttd.org": 5, + "bugzilla.maptools.org": 5, + "buzz.typo3.org": 5, + "cakeforge.org": 5, + "cat.eyalro.net": 5, + "checkpw.sourceforge.net": 5, + "cisofy.com": 5, + "clicky.me": 5, + "co3k.org": 5, + "code.fabfile.org": 5, + "code.qt.io": 5, + "community.igniterealtime.org": 5, + "crypto.junod.info": 5, + "csrf.htmlpurifier.org": 5, + "cve.circl.lu": 5, + "cvs.sourceforge.net": 5, + "cybersecurity.upv.es": 5, + "danlec.com": 5, + "dev.exiv2.org": 5, + "dev.mutt.org": 5, + "dev.plone.org": 5, + "dev.subrion.org": 5, + "distro.conectiva.com.br": 5, + "docs.sequelizejs.com": 5, + "download.opensuse.org": 5, + "download.osgeo.org": 5, + "download.savannah.gnu.org": 5, + "download.strongswan.org": 5, + "drosenbe.blogspot.com": 5, + "drupalcode.org": 5, + "dsecrg.com": 5, + "em386.blogspot.com": 5, + "everdox.net": 5, + "fastcompression.blogspot.fr": 5, + "flash.flowplayer.org": 5, + "forum.xda-developers.com": 5, + "freeciv.wikia.com": 5, + "ftp.gnu.org": 5, + "ftp.naist.jp": 5, + "ftp.netbsd.org": 5, + "ftp.NetBSD.org": 5, + "gaganpreet.in": 5, + "ganglia.info": 5, + "geojson.org": 5, + "getid3.sourceforge.net": 5, + "git-blame.blogspot.com.es": 5, + "git.cyrusimap.org": 5, + "git.gluster.org": 5, + "git.hylafax.org": 5, + "git.infradead.org": 5, + "git.kernel.dk": 5, + "git.libav.org": 5, + "git.lxde.org": 5, + "git.mathias-kettner.de": 5, + "git.nordu.net": 5, + "gitorious.org": 5, + "git.savannah.nongnu.org": 5, + "git.tuxfamily.org": 5, + "googlechromereleases.blogspot.co.uk": 5, + "gridscheduler.sourceforge.net": 5, + "guac-dev.org": 5, + "h30499.www3.hp.com": 5, + "habrahabr.ru": 5, + "hac425.unaux.com": 5, + "ha.xxor.se": 5, + "hg.libsdl.org": 5, + "hg.savannah.gnu.org": 5, + "hkpco.kr": 5, + "homakov.blogspot.com": 5, + "icedtea.classpath.org": 5, + "inertiawar.com": 5, + "invisible-island.net": 5, + "ircrash.com": 5, + "isecpartners.github.io": 5, + "jaanuskp.blogspot.com": 5, + "jgarber.lighthouseapp.com": 5, + "jira.opensymphony.com": 5, + "jira.ow2.org": 5, + "juliusdavies.ca": 5, + "k3research.outerhaven.de": 5, + "kbase.redhat.com": 5, + "kerneltrap.org": 5, + "klikki.fi": 5, + "knoxin.blogspot.co.uk": 5, + "krbdev.mit.edu": 5, + "lab.cs.ttu.ee": 5, + "lcamtuf.blogspot.com.au": 5, + "ledgersmbdev.blogspot.ca": 5, + "libcloud.apache.org": 5, + "libexif.sourceforge.net": 5, + "linuxtesting.org": 5, + "linuxtv.org": 5, + "listengine.tuxfamily.org": 5, + "lists.alioth.debian.org": 5, + "lists.busybox.net": 5, + "lists.clamav.net": 5, + "lists.fusionforge.org": 5, + "lists.infradead.org": 5, + "lists.llvm.org": 5, + "lists.ntp.org": 5, + "lists.openvz.org": 5, + "lists.owasp.org": 5, + "lists.unbit.it": 5, + "lists.vmware.com": 5, + "lists.xiph.org": 5, + "lists.xymon.com": 5, + "lucene.apache.org": 5, + "lustre.org": 5, + "mahara.org": 5, + "mail.globnix.net": 5, + "mail.jabber.org": 5, + "mail.kde.org": 5, + "mailman.alsa-project.org": 5, + "mailman.mit.edu": 5, + "mail.openjdk.java.net": 5, + "mails.dpdk.org": 5, + "malloc.im": 5, + "martin.swende.se": 5, + "mathias-kettner.de": 5, + "mayaa.seasar.org": 5, + "metadata.ftp-master.debian.org": 5, + "mirror.easyname.at": 5, + "mirror.linux.org.au": 5, + "mod-security.svn.sourceforge.net": 5, + "modwsgi.readthedocs.org": 5, + "moinmoin.wikiwikiweb.de": 5, + "mov.sx": 5, + "mstrokin.com": 5, + "mysqlblog.fivefarmers.com": 5, + "netatalk.sourceforge.net": 5, + "net-ninja-mr.me": 5, + "news.dieweltistgarnichtso.net": 5, + "news.tryton.org": 5, + "nmav.gnutls.org": 5, + "nongnu.askapache.com": 5, + "old.blog.phusion.nl": 5, + "openjpeg.googlecode.com": 5, + "openvswitch.org": 5, + "osandamalith.wordpress.com": 5, + "osdir.com": 5, + "oss.sgi.com": 5, + "otiose.dhs.org": 5, + "owasp-esapi-java.googlecode.com": 5, + "owasp-java-html-sanitizer.googlecode.com": 5, + "ozlabs.org": 5, + "packetstormsecurity.nl": 5, + "pastie.caboo.se": 5, + "penturalabs.wordpress.com": 5, + "people.debian.org": 5, + "perception-point.io": 5, + "perl5.git.perl.org": 5, + "phpunit.vulnbusters.com": 5, + "pillow.readthedocs.org": 5, + "pmt.sourceforge.net": 5, + "post-office.corp.redhat.com": 5, + "proftpd.org": 5, + "projects.edgewall.com": 5, + "pseudo-flaw.net": 5, + "pylonshq.com": 5, + "pyropus.ca": 5, + "pywebdav.googlecode.com": 5, + "quassel-irc.org": 5, + "quickgit.kde.org": 5, + "qwertwwwe.github.io": 5, + "r00tin.blogspot.com": 5, + "rabbit.dereferenced.org": 5, + "rack.github.com": 5, + "railspikes.com": 5, + "restlet.org": 5, + "riddle.link": 5, + "rosariosis.com": 5, + "roundup.cvs.sourceforge.net": 5, + "rubysec.github.io": 5, + "01.org": 5, + "0day.work": 5, + "s1m0n.dft-labs.eu": 5, + "admin.hostpoint.ch": 5, + "alioth-lists.debian.net": 5, + "android-review.googlesource.com": 5, + "anotepad.com": 5, + "antichat.com": 5, + "appgateresearch.blogspot.com": 5, + "archive.apache.org": 5, + "artifex.com": 5, + "attachments.samba.org": 5, + "attic.apache.org": 5, + "bertjwregeer.keybase.pub": 5, + "bitslog.com": 5, + "blade.tencent.com": 5, + "blog.cloudpassage.com": 5, + "blog.convisoappsec.com": 5, + "blog.cryptographyengineering.com": 5, + "blog.documentfoundation.org": 5, + "blog.hartwork.org": 5, + "blog.hboeck.de": 5, + "blog.mozilla.org": 5, + "blog.qt.io": 5, + "blog.quarkslab.com": 5, + "blog.smarttecs.com": 5, + "blog.sqreen.com": 5, + "blog.wpscan.org": 5, + "bnbdr.github.io": 5, + "botan.randombit.net": 5, + "bpmn.io": 5, + "bro-tracker.atlassian.net": 5, + "bugreports.qt.io": 5, + "bugs.exim.org": 5, + "bugs.gnunet.org": 5, + "bugs.horde.org": 5, + "bugs.linuxfoundation.org": 5, + "bugs.openvz.org": 5, + "bugs.webkit.org": 5, + "bugzilla.clamav.net": 5, + "bugzilla.mindrot.org": 5, + "bugzilla.samba.org": 5, + "bugzilla.tianocore.org": 5, + "busybox.net": 5, + "c-ares.haxx.se": 5, + "ceph.io": 5, + "cfreal.github.io": 5, + "chromiumcodereview.appspot.com": 5, + "civicrm.org": 5, + "codechecker-demo.eastus.cloudapp.azure.com": 5, + "code.launchpad.net": 5, + "codereview.appspot.com": 5, + "codesearch.debian.net": 5, + "community.jboss.org": 5, + "community.qualys.com": 5, + "community.sophos.com": 5, + "contrib.spip.net": 5, + "coreymhudson.github.io": 5, + "crashes.fuzzing-project.org": 5, + "crrev.com": 5, + "crypto.stanford.edu": 5, + "defuse.ca": 5, + "deshal3v.github.io": 5, + "developer.atlassian.com": 5, + "dev.gnupg.org": 5, + "dev.recurly.com": 5, + "discourse.aurelia.io": 5, + "discourse.gnome.org": 5, + "discuss.gradle.org": 5, + "discuss.neos.io": 5, + "docs.inspircd.org": 5, + "dom4j.github.io": 5, + "donncha.is": 5, + "download.igniterealtime.org": 5, + "dyntopia.com": 5, + "secondlookforensics.com": 5, + "secureappdev.blogspot.com": 5, + "secureyourit.co.uk": 5, + "security.cucumberlinux.com": 5, + "edk2-docs.gitbook.io": 5, + "edk2.groups.io": 5, + "edoverflow.com": 5, + "efail.de": 5, + "elixir.bootlin.com": 5, + "en.bitcoinwiki.org": 5, + "enigmail.net": 5, + "esnet-security.github.io": 5, + "exploitbox.io": 5, + "eyalitkin.wordpress.com": 5, + "fakhrizulkifli.github.io": 5, + "fastcompression.blogspot.ca": 5, + "fastd.readthedocs.io": 5, + "feh.finalrewind.org": 5, + "fisheye.codehaus.org": 5, + "forums.grsecurity.net": 5, + "forum.snapcraft.io": 5, + "forums.servicestack.net": 5, + "francozappa.github.io": 5, + "sf.snu.ac.kr": 5, + "fuelphp.com": 5, + "gaffer.ptitcanardnoir.org": 5, + "gerrit.googlesource.com": 5, + "git.busybox.net": 5, + "git.centos.org": 5, + "git.entrouvert.org": 5, + "git.gnunet.org": 5, + "git.gnupg.org": 5, + "github.blog": 5, + "githubengineering.com": 5, + "gitlab.xiph.org": 5, + "git.netfilter.org": 5, + "git.tartarus.org": 5, + "git.tt-rss.org": 5, + "git.whamcloud.com": 5, + "git.zabbix.com": 5, + "gnats.netbsd.org": 5, + "gnupg.org": 5, + "googleonlinesecurity.blogspot.com": 5, + "googleprojectzero.blogspot.co.uk": 5, + "googleprojectzero.blogspot.cz": 5, + "gruss.cc": 5, + "guidovranken.files.wordpress.com": 5, + "gusralph.info": 5, + "h20564.www2.hpe.com": 5, + "h20566.www2.hp.com": 5, + "heasarc.gsfc.nasa.gov": 5, + "hg.ucc.asn.au": 5, + "shibboleth.internet2.edu": 5, + "hosein-vita.medium.com": 5, + "httpoxy.org": 5, + "icepng.github.io": 5, + "icinga.com": 5, + "insert-script.blogspot.com": 5, + "insights.sei.cmu.edu": 5, + "insights.ubuntu.com": 5, + "invent.kde.org": 5, + "ioquake3.org": 5, + "issues.civicrm.org": 5, + "issues.gerritcodereview.com": 5, + "issuetracker.google.com": 5, + "site.icu-project.org": 5, + "site.pi3.com.pl": 5, + "jeffhacks.com": 5, + "kate.io": 5, + "kb.bluecoat.com": 5, + "kingcope.wordpress.com": 5, + "knobattack.com": 5, + "komodoplatform.com": 5, + "leastauthority.com": 5, + "libexpat.github.io": 5, + "libosinfo.org": 5, + "libreswan.org": 5, + "lists.ath9k.org": 5, + "lists.clusterlabs.org": 5, + "lists.cypherpunks.ca": 5, + "lists.iai.uni-bonn.de": 5, + "lists.immunityinc.com": 5, + "lists.isc.org": 5, + "lists.linuxcontainers.org": 5, + "lists.nic.cz": 5, + "lists.openldap.org": 5, + "lists.open-mesh.org": 5, + "lists.osuosl.org": 5, + "lists.sequoia-pgp.org": 5, + "lists.tartarus.org": 5, + "lists.ubuntu.com": 5, + "lock.cmpxchg8b.com": 5, + "mailarchives.bentasker.co.uk": 5, + "mail.gnu.org": 5, + "mailman-eng.corp.redhat.com": 5, + "make.wordpress.org": 5, + "mariadb.atlassian.net": 5, + "marlam.de": 5, + "smarty-php.googlecode.com": 5, + "maustin.net": 5, + "mcabber.com": 5, + "mega.nz": 5, + "meltdownattack.com": 5, + "micahflee.com": 5, + "mirror.fail": 5, + "nathandavison.com": 5, + "neomutt.org": 5, + "neopg.io": 5, + "nethack.org": 5, + "newrelic.com": 5, + "nohats.ca": 5, + "snoopy.cvs.sourceforge.net": 5, + "nvisium.com": 5, + "old.reddit.com": 5, + "open-docs.neuvector.com": 5, + "openmpt.org": 5, + "openssf.slack.com": 5, + "openssl-library.org": 5, + "opnsec.com": 5, + "orpheus-lyre.info": 5, + "osdn.net": 5, + "oss.clusterlabs.org": 5, + "os-s.net": 5, + "ostif.org": 5, + "source.jboss.org": 5, + "packages.qa.debian.org": 5, + "paste.pound-python.org": 5, + "patchwork.ffmpeg.org": 5, + "patchwork.freedesktop.org": 5, + "people.csail.mit.edu": 5, + "people.fedoraproject.org": 5, + "phabricator.kde.org": 5, + "philwantsfish.github.io": 5, + "spl0it.org": 5, + "platypusattack.com": 5, + "postlister.uninett.no": 5, + "projects.gnome.org": 5, + "projects.ow2.org": 5, + "pyyaml.docsforge.com": 5, + "qa.debian.org": 5, + "qtpass.org": 5, + "rachelbythebay.com": 5, + "raw.github.com": 5, + "raw.globalsecuritydatabase.org": 5, + "rdot.org": 5, + "relistan.com": 5, + "robots.thoughtbot.com": 5, + "rtpbleed.com": 5, + "rt.perl.org": 5, + "ruhr-uni-bochum.sciebo.de": 5, + "scannell.me": 5, + "scarybeastsecurity.blogspot.ch": 5, + "scarybeastsecurity.blogspot.com": 5, + "scarybeastsecurity.blogspot.dk": 5, + "scumjr.github.io": 5, + "secure.phabricator.com": 5, + "secure.php.net": 5, + "security-center.intel.com": 5, + "serenity.is": 5, + "sick.codes": 5, + "sitewat.ch": 5, + "smacktls.com": 5, + "source.openmpt.org": 5, + "spectreattack.com": 5, + "speirofr.appspot.com": 5, + "ssrg.nicta.com.au": 5, + "support.cloud.engineyard.com": 5, + "support.process-one.net": 5, + "svn.filezilla-project.org": 5, + "svn.resiprocate.org": 5, + "sweet32.info": 5, + "swiftstack.com": 5, + "sword.bladex.cn": 5, + "sympa.inria.fr": 5, + "t2.fi": 5, + "tapestry.apache.org": 5, + "thejh.net": 5, + "therecord.media": 5, + "threatpost.com": 5, + "tomforb.es": 5, + "trac.gajim.org": 5, + "tracker.ardour.org": 5, + "trac.mplayerhq.hu": 5, + "trac.nginx.org": 5, + "tt-rss.org": 5, + "unicode-org.atlassian.net": 5, + "unix.stackexchange.com": 5, + "utcc.utoronto.ca": 5, + "vishnudevtj.github.io": 5, + "vndh.net": 5, + "svn.ec-cube.net": 5, + "svn.haxx.se": 5, + "svn.ruby-lang.org": 5, + "svn.tartarus.org": 5, + "vulnhive.com": 5, + "web-in-security.blogspot.ca": 5, + "wid.cert-bund.de": 5, + "wiki.openvz.org": 5, + "wiki.samba.org": 5, + "worthdoingbadly.com": 5, + "wpa3.mathyvanhoef.com": 5, + "131002.net": 5, + "www3.sqlite.org": 5, + "www3.trustwave.com": 5, + "acunetix.com": 5, + "alphabot.com": 5, + "amd.com": 5, + "arista.com": 5, + "armis.com": 5, + "bamsoftware.com": 5, + "bitdefender.com": 5, + "cabextract.org.uk": 5, + "cazzulino.com": 5, + "cisecurity.org": 5, + "cl.cam.ac.uk": 5, + "cncf.io": 5, + "contextis.com": 5, + "cubeyond.net": 5, + "davical.org": 5, + "eclypsium.com": 5, + "evonide.com": 5, + "exiv2.org": 5, + "flexera.com": 5, + "freeplane.org": 5, + "futureweb.at": 5, + "gnu.org": 5, + "icinga.org": 5, + "inputzero.io": 5, + "intezer.com": 5, + "java.com": 5, + "jsof-tech.com": 5, + "jwz.org": 5, + "kernelmode.blog": 5, + "krackattacks.com": 5, + "kvakil.me": 5, + "ldap-account-manager.org": 5, + "legacysecuritygroup.com": 5, + "libssh2.org": 5, + "lsexperts.de": 5, + "mehmetince.net": 5, + "midnight-commander.org": 5, + "nikhef.nl": 5, + "npmjs.org": 5, + "openldap.org": 5, + "percona.com": 5, + "portcullis-security.com": 5, + "pureftpd.org": 5, + "python.org": 5, + "qbittorrent.org": 5, + "redteam-pentesting.de": 5, + "reversinglabs.com": 5, + "rootshellsecurity.net": 5, + "rorsecurity.info": 5, + "saddns.net": 5, + "secfu.net": 5, + "secura.com": 5, + "seebug.org": 5, + "sentinelone.com": 5, + "sigsac.org": 5, + "stunnel.org": 5, + "teskalabs.com": 5, + "theregister.co.uk": 5, + "thezdi.com": 5, + "timmclean.net": 5, + "trendmicro.com": 5, + "vice.com": 5, + "vidocsecurity.com": 5, + "viestintavirasto.fi": 5, + "virustotal.com": 5, + "vusec.net": 5, + "xairy.github.io": 5, + "sysoev.ru": 5, + "t3.dotgnu.info": 5, + "tartarus.org": 5, + "taviso.decsystem.org": 5, + "telia.dl.sourceforge.net": 5, + "telussecuritylabs.com": 5, + "thekelleys.org.uk": 5, + "tickets.opscode.com": 5, + "trac.kodi.tv": 5, + "trac.transmissionbt.com": 5, + "vcs.pcre.org": 5, + "venom.crowdstrike.com": 5, + "vladz.devzero.fr": 5, + "webservsec.blogspot.com": 5, + "wiki2.dovecot.org": 5, + "wiki.audacityteam.org": 5, + "wiki.lustre.org": 5, + "wiki.postgresql.org": 5, + "worldofpadman.com": 5, + "www13.itrc.hp.com": 5, + "agarri.fr": 5, + "agrs.tu-berlin.de": 5, + "apsis.ch": 5, + "argyllcms.com": 5, + "arm.linux.org.uk": 5, + "asmail.be": 5, + "attrition.org": 5, + "bacula.org": 5, + "betanews.com": 5, + "binarysniper.net": 5, + "bugzilla.org": 5, + "cc.gatech.edu": 5, + "cert.fi": 5, + "ciac.org": 5, + "cisco.com": 5, + "codenomicon.com": 5, + "collabtive.o-dyn.de": 5, + "communities.hp.com": 5, + "cs.bu.edu": 5, + "cs.tau.ac.IL": 5, + "cs.tau.ac.il": 5, + "cs.technion.ac.il": 5, + "dotnetnuke.com": 5, + "droid-life.com": 5, + "droidrzr.com": 5, + "droidsec.org": 5, + "educatedguesswork.org": 5, + "elasticsearch.org": 5, + "elsherei.com": 5, + "eucalyptus.com": 5, + "evernote.com": 5, + "exploringbinary.com": 5, + "firebirdsql.org": 5, + "floyd.ch": 5, + "freelists.org": 5, + "freerdp.com": 5, + "gdssecurity.com": 5, + "getchef.com": 5, + "gulftech.org": 5, + "gwtproject.org": 5, + "haproxy.org": 5, + "hdwsec.fr": 5, + "hitachi-support.com": 5, + "htbridge.ch": 5, + "ijg.org": 5, + "infoq.com": 5, + "infradead.org": 5, + "ingate.com": 5, + "intelsecurity.com": 5, + "jbkempf.com": 5, + "jplayer.org": 5, + "kernelhub.org": 5, + "konakart.com": 5, + "libraw.org": 5, + "live555.com": 5, + "lua.org": 5, + "mandrakesecure.net": 5, + "maradns.org": 5, + "mathyvanhoef.com": 5, + "metasploit.com": 5, + "mh-sec.de": 5, + "mindrot.org": 5, + "mitls.org": 5, + "mongodb.org": 5, + "mono-project.com": 5, + "nds.rub.de": 5, + "networkworld.com": 5, + "nntp.perl.org": 5, + "nruns.com": 5, + "nsfocus.net": 5, + "oliverkarow.de": 5, + "opencms.org": 5, + "openoffice.org": 5, + "palemoon.org": 5, + "paul-moore.com": 5, + "phenoelit.org": 5, + "pimcore.org": 5, + "pnigos.com": 5, + "process-one.net": 5, + "quantumleap.it": 5, + "rafayhackingarticles.net": 5, + "rajatswarup.com": 5, + "ratbox.org": 5, + "rul3z.de": 5, + "sbosnet.nl": 5, + "securation.com": 5, + "securegoose.org": 5, + "securiteam.com": 5, + "sendmail.com": 5, + "simplesystems.org": 5, + "slimframework.com": 5, + "sogo.nu": 5, + "splitbrain.org": 5, + "springsource.com": 5, + "squirrelmail.org": 5, + "supercluster.org": 5, + "synacktiv.ninja": 5, + "tedunangst.com": 5, + "tele-consulting.com": 5, + "tombom.co.uk": 5, + "tt-forums.net": 5, + "undeadly.org": 5, + "unixodbc.org": 5, + "x.org": 5, + "zweitag.de": 5, + "xstream.codehaus.org": 5, + "xteam.baidu.com": 5, + "xync.org": 5, + "zsh.sourceforge.net": 5, + "project-zero.issues.chromium.org": 5, + "anker.com": 5, + "visionspace.com": 5, + "eufy.com": 5, + "advisory-inbox.githubapp.com": 5, + "myoffice.ru": 5, + "support.myoffice.ru": 5, + "planex.co.jp": 5, + "docs.iredmail.org": 5, + "iredmail.org": 5, +} diff --git a/vulnerablecode-json-api.png b/vulnerablecode-json-api.png deleted file mode 100644 index 9712a9f93..000000000 Binary files a/vulnerablecode-json-api.png and /dev/null differ diff --git a/vulnerablecode-ui.png b/vulnerablecode-ui.png deleted file mode 100644 index 7a0a28329..000000000 Binary files a/vulnerablecode-ui.png and /dev/null differ diff --git a/vulnerablecode/__init__.py b/vulnerablecode/__init__.py index 797c0871b..ee339e883 100644 --- a/vulnerablecode/__init__.py +++ b/vulnerablecode/__init__.py @@ -3,16 +3,14 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import os import sys -import warnings -from pathlib import Path -__version__ = "34.0.2" +__version__ = "35.1.0" def command_line(): diff --git a/vulnerablecode/context_processors.py b/vulnerablecode/context_processors.py index e93bb7bd3..ee5885b81 100644 --- a/vulnerablecode/context_processors.py +++ b/vulnerablecode/context_processors.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index f2f612098..0e545e0f2 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -349,3 +349,9 @@ }, }, } + +if DEBUG: + LOGGING["django"] = { + "handlers": ["console"], + "level": "ERROR", + } diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 28954d3a8..10f7db13f 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -20,6 +20,8 @@ from vulnerabilities.api import CPEViewSet from vulnerabilities.api import PackageViewSet from vulnerabilities.api import VulnerabilityViewSet +from vulnerabilities.api_v2 import PackageV2ViewSet +from vulnerabilities.api_v2 import VulnerabilityV2ViewSet from vulnerabilities.views import ApiUserCreateView from vulnerabilities.views import HomePage from vulnerabilities.views import PackageDetails @@ -43,7 +45,12 @@ def __init__(self, *args, **kwargs): api_router.register("cpes", CPEViewSet, basename="cpe") api_router.register("aliases", AliasViewSet, basename="alias") +api_v2_router = OptionalSlashRouter() +api_v2_router.register("packages", PackageV2ViewSet, basename="package-v2") +api_v2_router.register("vulnerabilities", VulnerabilityV2ViewSet, basename="vulnerability-v2") + urlpatterns = [ + path("api/v2/", include(api_v2_router.urls)), path( "robots.txt", TemplateView.as_view(template_name="robots.txt", content_type="text/plain"), diff --git a/vulnerablecode/wsgi.py b/vulnerablecode/wsgi.py index c17f4abeb..aa06c937a 100644 --- a/vulnerablecode/wsgi.py +++ b/vulnerablecode/wsgi.py @@ -3,7 +3,7 @@ # VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/vulnerablecode for support or download. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. #