Improves requirements handling. Fixes #14

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributing
 
-Apologies but I'm yet to flesh this out.
+Apologies but I'm yet to flesh this out fully.
 
 ## Developing
 

clean.sh

@@ -1,14 +1,14 @@
-#!/bin/bash
+#!/bin/bash -xe
 
-rm -rf .nox/
-rm -rf .pytest_cache/
-rm -rf build/
-rm -rf .mypy_cache/
-rm -rf .pytype/
-rm -rf junit/
+rm -rfv .nox/
+rm -rfv .pytest_cache/
+rm -rfv build/
+rm -rfv .mypy_cache/
+rm -rfv .pytype/
+rm -rfv junit/
 
-find -name "__pycache__" -type d -exec rm -r {} \;
-find -name ".pytest_cache" -type d -exec rm -r {} \;
-find -name "*.egg-info" -type d -exec rm -r {} \;
+find -name "__pycache__" -type d -exec rm -rv {} \;
+find -name ".pytest_cache" -type d -exec rm -rv {} \;
+find -name "*.egg-info" -type d -exec rm -rv {} \;
 find -name ".coverage" -type f -delete
 find -name "*.py,cover" -type f -delete

mypy.ini

@@ -33,3 +33,6 @@ ignore_missing_imports = True
 
 [mypy-safety.*]
 ignore_missing_imports = True
+
+[mypy-parsley.*]
+ignore_missing_imports = True

noxfile.py

@@ -228,6 +228,25 @@ def xdoctest(session: Session) -> None:
     session.run("python", "-m", "xdoctest", package, *args)
 
 
+@nox.session(python=general_py_version)
+def audit(session: Session) -> None:
+    """Run a Valiant audit on Valiant."""
+
+    def generate_audit(session: Session, requirements_file: str) -> None:
+        """Generate a Valiant audit."""
+        from pathlib import Path
+
+        Path("./build").mkdir(exist_ok=True)
+        session.run(
+            "valiant", "audit", "-o", "json", f"{requirements_file}",
+        )
+
+    packages = ["./"]
+    install_with_constraints(
+        session, packages=packages, include_dev=False, callback=generate_audit
+    )
+
+
 def generate_bom(session: Session, requirements_file: str) -> None:
     """Generate a Software Bill of Materials."""
     from pathlib import Path

pyproject.toml

@@ -57,6 +57,7 @@ requests-cache = "^0.5.2"
 structlog = "^20.1.0"
 safety = "^1.8.7"
 marshmallow-dataclass = "^7.5.2"
+Parsley = "^1.3"
 [tool.poetry.dev-dependencies]
 pytest = "^4.6"
 black = "^19"

src/valiant/console/commands/audit.py

@@ -23,6 +23,7 @@
 from typing import List, NamedTuple, Optional
 
 from valiant.reports import Finding, ReportSet
+from valiant.util import RequirementEntry, parse_requirements_file
 
 from .base_command import BaseCommand
 from .package_command import Payload
@@ -57,28 +58,21 @@ def handle(self) -> Optional[int]:  # noqa: D102
         requirements = Path(self.argument("requirements-file"))
         report_list: List[str] = []
         format = self.option("out")
-        package_list: List[AuditCommand._RequirementsEntry] = []
+        package_list: List[RequirementEntry] = []
         payloads: List[Payload] = []
 
         if self.argument("reports"):
             report_list = self.argument("reports").split(",")
 
         try:
-            if not requirements.is_file():
-                raise ValueError(
-                    f"The requirements file ({requirements}) doesn't exist."
-                )
-
-            with open(requirements, "r") as reqs:
-                for line in reqs:
-                    package, version = line.strip().split("==")
-                    package_list.append(
-                        AuditCommand._RequirementsEntry(package, version)
-                    )
+            package_list = parse_requirements_file(requirements)
 
             for req in package_list:
+                if len(req.versions) < 1 and req.versions[0][0] != "==":
+                    raise ValueError(f"A pinned version is required for {req.package}.")
+
                 package_metadata = self.valiant.get_package_metadata(
-                    package_name=req.package, package_version=req.version,
+                    package_name=req.package, package_version=req.versions[0][1],
                 )
                 reports = self.valiant.get_package_reports(
                     package_metadata, reports=set(report_list)

src/valiant/repositories/pypi/model.py

@@ -68,7 +68,7 @@ class Info:
     description: str
     description_content_type: Optional[str]
     docs_url: Optional[str]
-    download_url: str
+    download_url: Optional[str]
     downloads: Downloads
     home_page: str
     keywords: Optional[str]

src/valiant/util/__init__.py

@@ -22,6 +22,11 @@
 from enum import Enum
 
 from .dictionizer import Dictionizer
+from .requirements import (
+    RequirementEntry,
+    parse_requirements_file,
+    parse_requirements_entry,
+)
 
 
 class NoValue(Enum):

src/valiant/util/requirements.py

@@ -0,0 +1,225 @@
+"""PEP 508-based requirements parser.
+
+Refer to https://www.python.org/dev/peps/pep-0508
+
+
+Copyright 2021 The Valiant Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Tuple
+
+from parsley import makeGrammar
+
+"""Parsley grammer for PEP 508.
+
+Slightly modified from the original provided at
+https://www.python.org/dev/peps/pep-0508/#complete-grammar
+
+I've added the ability to parse the hash check:
+https://pip.pypa.io/en/stable/reference/pip_install/#hash-checking-mode
+"""
+pep_508_grammar = """
+wsp           = ' ' | '\t'
+version_cmp   = wsp* <'<=' | '<' | '!=' | '==' | '>=' | '>' | '~=' | '==='>
+version       = wsp* <( letterOrDigit | '-' | '_' | '.' | '*' | '+' | '!' )+>
+version_one   = version_cmp:op version:v wsp* -> (op, v)
+version_many  = version_one:v1 (wsp* ',' version_one)*:v2 -> [v1] + v2
+versionspec   = ('(' version_many:v ')' ->v) | version_many
+
+urlspec       = '@' wsp* <URI_reference>
+marker_op     = version_cmp | (wsp* 'in') | (wsp* 'not' wsp+ 'in')
+python_str_c  = (wsp | letter | digit | '(' | ')' | '.' | '{' | '}' |
+                 '-' | '_' | '*' | '#' | ':' | ';' | ',' | '/' | '?' |
+                 '[' | ']' | '!' | '~' | '`' | '@' | '$' | '%' | '^' |
+                 '&' | '=' | '+' | '|' | '<' | '>' )
+dquote        = '"'
+squote        = '\\''
+python_str    = (squote <(python_str_c | dquote)*>:s squote |
+                 dquote <(python_str_c | squote)*>:s dquote) -> s
+
+env_var       = ('python_version' | 'python_full_version' |
+                 'os_name' | 'sys_platform' | 'platform_release' |
+                 'platform_system' | 'platform_version' |
+                 'platform_machine' | 'platform_python_implementation' |
+                 'implementation_name' | 'implementation_version' |
+                 'extra' # ONLY when defined by a containing layer
+                 )
+
+hashes        = hash:h ( wsp* hash )*:hlist -> [h] + hlist
+hash_prefix   = '--hash='
+hash          = ( hash_prefix hash_type:t ':' identifier:i -> t, i )
+hash_type     = ('sha1' | 'sha224' | 'sha256' | 'sha384' | 'sha512')
+
+marker_var    = wsp* (env_var | python_str)
+marker_expr   = marker_var:l marker_op:o marker_var:r -> (o, l, r)
+              | wsp* '(' marker:m wsp* ')' -> m
+marker_and    = marker_expr:l wsp* 'and' marker_expr:r -> ('and', l, r)
+              | marker_expr:m -> m
+marker_or     = marker_and:l wsp* 'or' marker_and:r -> ('or', l, r)
+                  | marker_and:m -> m
+marker        = marker_or
+quoted_marker = ';' wsp* marker
+identifier_end = letterOrDigit | (('-' | '_' | '.' )* letterOrDigit)
+identifier    = < letterOrDigit identifier_end* >
+name          = identifier
+extras_list   = identifier:i (wsp* ',' wsp* identifier)*:ids -> [i] + ids
+extras        = '[' wsp* extras_list?:e wsp* ']' -> e
+name_req      = (name:n wsp* extras?:e wsp* versionspec?:v wsp* quoted_marker?:m wsp* hashes?:h
+                 -> (n, e or [], v or [], m, h))
+url_req       = (name:n wsp* extras?:e wsp* urlspec:v (wsp+ | end)
+                 quoted_marker?:m wsp* hashes?:h
+                 -> (n, e or [], v, m, h))
+specification = wsp* ( url_req | name_req ):s wsp* -> s
+# The result is a tuple - name, list-of-extras,
+# list-of-version-constraints-or-a-url, marker-ast or None
+
+
+URI_reference = <URI | relative_ref>
+URI           = scheme ':' hier_part ('?' query )? ( '#' fragment)?
+hier_part     = ('//' authority path_abempty) | path_absolute | path_rootless | path_empty
+absolute_URI  = scheme ':' hier_part ( '?' query )?
+relative_ref  = relative_part ( '?' query )? ( '#' fragment )?
+relative_part = '//' authority path_abempty | path_absolute | path_noscheme | path_empty
+scheme        = letter ( letter | digit | '+' | '-' | '.')*
+authority     = ( userinfo '@' )? host ( ':' port )?
+userinfo      = ( unreserved | pct_encoded | sub_delims | ':')*
+host          = IP_literal | IPv4address | reg_name
+port          = digit*
+IP_literal    = '[' ( IPv6address | IPvFuture) ']'
+IPvFuture     = 'v' hexdig+ '.' ( unreserved | sub_delims | ':')+
+IPv6address   = (
+                  ( h16 ':'){6} ls32
+                  | '::' ( h16 ':'){5} ls32
+                  | ( h16 )?  '::' ( h16 ':'){4} ls32
+                  | ( ( h16 ':')? h16 )? '::' ( h16 ':'){3} ls32
+                  | ( ( h16 ':'){0,2} h16 )? '::' ( h16 ':'){2} ls32
+                  | ( ( h16 ':'){0,3} h16 )? '::' h16 ':' ls32
+                  | ( ( h16 ':'){0,4} h16 )? '::' ls32
+                  | ( ( h16 ':'){0,5} h16 )? '::' h16
+                  | ( ( h16 ':'){0,6} h16 )? '::' )
+h16           = hexdig{1,4}
+ls32          = ( h16 ':' h16) | IPv4address
+IPv4address   = dec_octet '.' dec_octet '.' dec_octet '.' dec_octet
+nz            = ~'0' digit
+dec_octet     = (
+                  digit # 0-9
+                  | nz digit # 10-99
+                  | '1' digit{2} # 100-199
+                  | '2' ('0' | '1' | '2' | '3' | '4') digit # 200-249
+                  | '25' ('0' | '1' | '2' | '3' | '4' | '5') )# %250-255
+reg_name = ( unreserved | pct_encoded | sub_delims)*
+path = (
+        path_abempty # begins with '/' or is empty
+        | path_absolute # begins with '/' but not '//'
+        | path_noscheme # begins with a non-colon segment
+        | path_rootless # begins with a segment
+        | path_empty ) # zero characters
+path_abempty  = ( '/' segment)*
+path_absolute = '/' ( segment_nz ( '/' segment)* )?
+path_noscheme = segment_nz_nc ( '/' segment)*
+path_rootless = segment_nz ( '/' segment)*
+path_empty    = pchar{0}
+segment       = pchar*
+segment_nz    = pchar+
+segment_nz_nc = ( unreserved | pct_encoded | sub_delims | '@')+
+                # non-zero-length segment without any colon ':'
+pchar         = unreserved | pct_encoded | sub_delims | ':' | '@'
+query         = ( pchar | '/' | '?')*
+fragment      = ( pchar | '/' | '?')*
+pct_encoded   = '%' hexdig
+unreserved    = letter | digit | '-' | '.' | '_' | '~'
+reserved      = gen_delims | sub_delims
+gen_delims    = ':' | '/' | '?' | '#' | '(' | ')?' | '@'
+sub_delims    = '!' | '$' | '&' | '\\'' | '(' | ')' | '*' | '+' | ',' | ';' | '='
+hexdig        = digit | 'a' | 'A' | 'b' | 'B' | 'c' | 'C' | 'd' | 'D' | 'e' | 'E' | 'f' | 'F'
+"""
+
+pep_508_grammar_compiled = makeGrammar(pep_508_grammar, {})
+
+
+@dataclass(frozen=True)
+class RequirementEntry:
+    """The primary components of a requirements entry."""
+
+    package: str
+    versions: List[Tuple[str, str]]
+    extras: List[str]
+    environment_markers: Optional[Tuple]
+    hashes: Optional[List[Tuple[str, str]]]
+
+
+def parse_requirements_entry(entry: str) -> RequirementEntry:
+    """Parse an individual requirement entry.
+
+    Args:
+        entry: the PEP508-compliant entry.
+
+    Returns:
+        A RequirementEntry.
+    """
+    package, extras, versions, environment_markers, hashes = pep_508_grammar_compiled(
+        " ".join(entry.strip().splitlines())
+    ).specification()
+    return RequirementEntry(
+        package=package,
+        versions=versions,
+        extras=extras,
+        environment_markers=environment_markers,
+        hashes=hashes,
+    )
+
+
+def parse_requirements_file(requirements: Path) -> List[RequirementEntry]:
+    """Parse a requirements file and return a list of entries.
+
+    Args:
+        requirements: A Path to the requirements file
+
+    Returns:
+        A list of requirement entries
+
+    Raises:
+        ValueError: if the requirements file does not exist
+    """
+    package_list: List[RequirementEntry] = []
+
+    if not requirements.is_file():
+        raise ValueError(f"The requirements file ({requirements}) doesn't exist.")
+
+    with open(requirements, "r") as reqs:
+        for line in reqs:
+            line = line.rstrip("\n")
+            while line.endswith("\\"):
+                line = line[:-1] + next(reqs).rstrip("\n")
+            package_list.append(parse_requirements_entry(line))
+
+    return package_list
+
+
+if __name__ == "__main__":
+    reqs = [
+        "appdirs==1.4.4",
+        "appdirs==1.4.4"
+        " --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128",
+        "appdirs==1.4.4"
+        "    --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
+        "    --hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
+        """appdirs==1.4.4
+            --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128
+            --hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41""",
+    ]
+    for r in reqs:
+        print(parse_requirements_entry(r))