Merge pull request #152 from openedx/feanil/update_setup_and_manifest

feanil/update setup and manifest

MANIFEST.in

@@ -1,3 +1,4 @@
 include LICENSE.txt
 include README.rst
 include requirements/base.in
+include requirements/constraints.txt

pylintrc

@@ -64,7 +64,7 @@
 # SERIOUSLY.
 #
 # ------------------------------
-# Generated by edx-lint version: 5.3.0
+# Generated by edx-lint version: 5.3.7
 # ------------------------------
 [MASTER]
 ignore = migrations
@@ -259,6 +259,7 @@ enable =
 	useless-suppression,
 disable = 
 	bad-indentation,
+	broad-exception-raised,
 	consider-using-f-string,
 	duplicate-code,
 	file-ignored,
@@ -383,6 +384,6 @@ ext-import-graph =
 int-import-graph = 
 
 [EXCEPTIONS]
-overgeneral-exceptions = Exception
+overgeneral-exceptions = builtins.Exception
 
-# 1e3dbff6aa778644c68d245ad972c7cf9926188a
+# 99437a773a20fabf842bf2a5e67e4308b8de8534

setup.py

@@ -10,43 +10,73 @@ def load_requirements(*requirements_paths):
     """
     Load all requirements from the specified requirements files.
 
-    Requirements will include any constraints from files specified with -c in the requirements files.
+    Requirements will include any constraints from files specified
+    with -c in the requirements files.
     Returns a list of requirement strings.
     """
+    # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why.
+
+    # e.g. {"django": "Django", "confluent-kafka": "confluent_kafka[avro]"}
+    by_canonical_name = {}
+
+    def check_name_consistent(package):
+        """
+        Raise exception if package is named different ways.
+
+        This ensures that packages are named consistently so we can match
+        constraints to packages. It also ensures that if we require a package
+        with extras we don't constrain it without mentioning the extras (since
+        that too would interfere with matching constraints.)
+        """
+        canonical = package.lower().replace('_', '-').split('[')[0]
+        seen_spelling = by_canonical_name.get(canonical)
+        if seen_spelling is None:
+            by_canonical_name[canonical] = package
+        elif seen_spelling != package:
+            raise Exception(
+                f'Encountered both "{seen_spelling}" and "{package}" in requirements '
+                'and constraints files; please use just one or the other.'
+            )
+
     requirements = {}
     constraint_files = set()
 
-    # groups "my-package-name<=x.y.z,..." into ("my-package-name", "<=x.y.z,...")
-    requirement_line_regex = re.compile(r"([a-zA-Z0-9-_.]+)([<>=][^#\s]+)?")
+    # groups "pkg<=x.y.z,..." into ("pkg", "<=x.y.z,...")
+    re_package_name_base_chars = r"a-zA-Z0-9\-_."  # chars allowed in base package name
+    # Two groups: name[maybe,extras], and optionally a constraint
+    requirement_line_regex = re.compile(
+        r"([%s]+(?:\[[%s,\s]+\])?)([<>=][^#\s]+)?"
+        % (re_package_name_base_chars, re_package_name_base_chars)
+    )
 
     def add_version_constraint_or_raise(current_line, current_requirements, add_if_not_present):
         regex_match = requirement_line_regex.match(current_line)
         if regex_match:
             package = regex_match.group(1)
             version_constraints = regex_match.group(2)
+            check_name_consistent(package)
             existing_version_constraints = current_requirements.get(package, None)
-            # it's fine to add constraints to an unconstrained package, but raise an error if there are already
-            # constraints in place
+            # It's fine to add constraints to an unconstrained package,
+            # but raise an error if there are already constraints in place.
             if existing_version_constraints and existing_version_constraints != version_constraints:
-                raise BaseException(
-                    f"Multiple constraint definitions found for {package}:"
-                    f' "{existing_version_constraints}" and "{version_constraints}".'
-                    f"Combine constraints into one location with {package}"
-                    f"{existing_version_constraints},{version_constraints}."
-                )
+                raise BaseException(f'Multiple constraint definitions found for {package}:'
+                                    f' "{existing_version_constraints}" and "{version_constraints}".'
+                                    f'Combine constraints into one location with {package}'
+                                    f'{existing_version_constraints},{version_constraints}.')
             if add_if_not_present or package in current_requirements:
                 current_requirements[package] = version_constraints
 
-    # process .in files and store the path to any constraint files that are pulled in
+    # Read requirements from .in files and store the path to any
+    # constraint files that are pulled in.
     for path in requirements_paths:
         with open(path) as reqs:
             for line in reqs:
                 if is_requirement(line):
                     add_version_constraint_or_raise(line, requirements, True)
-                if line and line.startswith("-c") and not line.startswith("-c http"):
-                    constraint_files.add(os.path.dirname(path) + "/" + line.split("#")[0].replace("-c", "").strip())
+                if line and line.startswith('-c') and not line.startswith('-c http'):
+                    constraint_files.add(os.path.dirname(path) + '/' + line.split('#')[0].replace('-c', '').strip())
 
-    # process constraint files and add any new constraints found to existing requirements
+    # process constraint files: add constraints to existing requirements
     for constraint_file in constraint_files:
         with open(constraint_file) as reader:
             for line in reader:
@@ -60,10 +90,15 @@ def add_version_constraint_or_raise(current_line, current_requirements, add_if_n
 
 def is_requirement(line):
     """
-    Return True if the requirement line is a package requirement;
-    that is, it is not blank, a comment, a URL, or an included file.
+    Return True if the requirement line is a package requirement.
+
+    Returns:
+        bool: True if the line is not blank, a comment,
+        a URL, or an included file
     """
-    return line and not line.startswith(("-r", "#", "-e", "git+", "-c"))
+    # UPDATED VIA SEMGREP - if you need to remove/modify this method remove this line and add a comment specifying why
+
+    return line and line.strip() and not line.startswith(('-r', '#', '-e', 'git+', '-c'))
 
 
 def get_version(*file_paths):