diff --git a/MANIFEST.in b/MANIFEST.in index 4fe5352..e2a0a4b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ include LICENSE.txt include README.rst include requirements/base.in +include requirements/constraints.txt diff --git a/pylintrc b/pylintrc index 2712717..11bc215 100644 --- a/pylintrc +++ b/pylintrc @@ -64,7 +64,7 @@ # SERIOUSLY. # # ------------------------------ -# Generated by edx-lint version: 5.3.0 +# Generated by edx-lint version: 5.3.7 # ------------------------------ [MASTER] ignore = migrations @@ -259,6 +259,7 @@ enable = useless-suppression, disable = bad-indentation, + broad-exception-raised, consider-using-f-string, duplicate-code, file-ignored, @@ -383,6 +384,6 @@ ext-import-graph = int-import-graph = [EXCEPTIONS] -overgeneral-exceptions = Exception +overgeneral-exceptions = builtins.Exception -# 1e3dbff6aa778644c68d245ad972c7cf9926188a +# 99437a773a20fabf842bf2a5e67e4308b8de8534 diff --git a/setup.py b/setup.py index 2e0a64a..4b228ce 100755 --- a/setup.py +++ b/setup.py @@ -10,43 +10,73 @@ def load_requirements(*requirements_paths): """ Load all requirements from the specified requirements files. - Requirements will include any constraints from files specified with -c in the requirements files. + Requirements will include any constraints from files specified + with -c in the requirements files. Returns a list of requirement strings. """ + # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why. + + # e.g. {"django": "Django", "confluent-kafka": "confluent_kafka[avro]"} + by_canonical_name = {} + + def check_name_consistent(package): + """ + Raise exception if package is named different ways. + + This ensures that packages are named consistently so we can match + constraints to packages. It also ensures that if we require a package + with extras we don't constrain it without mentioning the extras (since + that too would interfere with matching constraints.) + """ + canonical = package.lower().replace('_', '-').split('[')[0] + seen_spelling = by_canonical_name.get(canonical) + if seen_spelling is None: + by_canonical_name[canonical] = package + elif seen_spelling != package: + raise Exception( + f'Encountered both "{seen_spelling}" and "{package}" in requirements ' + 'and constraints files; please use just one or the other.' + ) + requirements = {} constraint_files = set() - # groups "my-package-name<=x.y.z,..." into ("my-package-name", "<=x.y.z,...") - requirement_line_regex = re.compile(r"([a-zA-Z0-9-_.]+)([<>=][^#\s]+)?") + # groups "pkg<=x.y.z,..." into ("pkg", "<=x.y.z,...") + re_package_name_base_chars = r"a-zA-Z0-9\-_." # chars allowed in base package name + # Two groups: name[maybe,extras], and optionally a constraint + requirement_line_regex = re.compile( + r"([%s]+(?:\[[%s,\s]+\])?)([<>=][^#\s]+)?" + % (re_package_name_base_chars, re_package_name_base_chars) + ) def add_version_constraint_or_raise(current_line, current_requirements, add_if_not_present): regex_match = requirement_line_regex.match(current_line) if regex_match: package = regex_match.group(1) version_constraints = regex_match.group(2) + check_name_consistent(package) existing_version_constraints = current_requirements.get(package, None) - # it's fine to add constraints to an unconstrained package, but raise an error if there are already - # constraints in place + # It's fine to add constraints to an unconstrained package, + # but raise an error if there are already constraints in place. if existing_version_constraints and existing_version_constraints != version_constraints: - raise BaseException( - f"Multiple constraint definitions found for {package}:" - f' "{existing_version_constraints}" and "{version_constraints}".' - f"Combine constraints into one location with {package}" - f"{existing_version_constraints},{version_constraints}." - ) + raise BaseException(f'Multiple constraint definitions found for {package}:' + f' "{existing_version_constraints}" and "{version_constraints}".' + f'Combine constraints into one location with {package}' + f'{existing_version_constraints},{version_constraints}.') if add_if_not_present or package in current_requirements: current_requirements[package] = version_constraints - # process .in files and store the path to any constraint files that are pulled in + # Read requirements from .in files and store the path to any + # constraint files that are pulled in. for path in requirements_paths: with open(path) as reqs: for line in reqs: if is_requirement(line): add_version_constraint_or_raise(line, requirements, True) - if line and line.startswith("-c") and not line.startswith("-c http"): - constraint_files.add(os.path.dirname(path) + "/" + line.split("#")[0].replace("-c", "").strip()) + if line and line.startswith('-c') and not line.startswith('-c http'): + constraint_files.add(os.path.dirname(path) + '/' + line.split('#')[0].replace('-c', '').strip()) - # process constraint files and add any new constraints found to existing requirements + # process constraint files: add constraints to existing requirements for constraint_file in constraint_files: with open(constraint_file) as reader: for line in reader: @@ -60,10 +90,15 @@ def add_version_constraint_or_raise(current_line, current_requirements, add_if_n def is_requirement(line): """ - Return True if the requirement line is a package requirement; - that is, it is not blank, a comment, a URL, or an included file. + Return True if the requirement line is a package requirement. + + Returns: + bool: True if the line is not blank, a comment, + a URL, or an included file """ - return line and not line.startswith(("-r", "#", "-e", "git+", "-c")) + # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why + + return line and line.strip() and not line.startswith(('-r', '#', '-e', 'git+', '-c')) def get_version(*file_paths):