Refactor unittests for compatibility with pytest

Resolves #1318

This commit prepares unittests to run on official python tools like pytest by:

1. moving mocking modules to package_control/tests/mock and load them
   whenever `tests` package is imported outside of sublime text

2. moving internal test runner code to package_control_tests_command.py
   Note: Not sure whether this is still needed.

3. renaming all test modules to `test_...` so pytest can easily find them.

4. add pytest config to tox.ini

.github/workflows/ci.yml

@@ -3,8 +3,8 @@ name: CI
 on: [push, pull_request]
 
 jobs:
-  build-38:
-    name: Python 3.8.7 on ${{ matrix.os }} ${{ matrix.arch }}
+  test-all-38:
+    name: Python 3.8 on ${{ matrix.os }} ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -21,23 +21,23 @@ jobs:
           - os: ubuntu-20.04
             arch: x86
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8.7'
+          python-version: '3.8'
           architecture: ${{ matrix.arch }}
       - name: Install dependencies
-        run: python dev/deps.py
+        run: pip install -U flake8 pytest
       - name: Run linter
-        run: python dev/lint.py
+        run: flake8
       - name: Run tests
-        run: python dev/tests.py
+        run: pytest
         env:
           GH_PASS: ${{ secrets.GH_PASS }}
           GL_PASS: ${{ secrets.GL_PASS }}
           BB_PASS: ${{ secrets.BB_PASS }}
 
-  # build-linux-33:
+  # test-linux-33:
   #   name: Python 3.3.7 on ubuntu-18.04 x64
   #   runs-on: ubuntu-18.04
   #   steps:
@@ -57,7 +57,7 @@ jobs:
   #         GL_PASS: ${{ secrets.GL_PASS }}
   #         BB_PASS: ${{ secrets.BB_PASS }}
 
-  # build-mac-33:
+  # test-mac-33:
   #   name: Python 3.3.7 on macos-11 x64
   #   runs-on: macos-11
   #   steps:
@@ -84,7 +84,7 @@ jobs:
   #         GL_PASS: ${{ secrets.GL_PASS }}
   #         BB_PASS: ${{ secrets.BB_PASS }}
 
-  build-windows-33:
+  test-windows-33:
     name: Python 3.3.5 on windows-2019 ${{ matrix.arch }}
     runs-on: windows-2019
     strategy:

dev/tests.py

@@ -1,9 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals, division, absolute_import, print_function
 
-import importlib.machinery
 import os
-import re
 import sys
 import unittest
 
@@ -12,64 +10,29 @@
 
 sys.path.append(PACKAGE_ROOT)
 
-# Mock the sublime module for CLI usage
-sublime = importlib.machinery.SourceFileLoader(
-    "sublime",
-    os.path.join(PACKAGE_ROOT, "dev/sublime.py")
-).load_module()
 
-# Mock the sublime_plugin module for CLI usage
-sublime_plugin = importlib.machinery.SourceFileLoader(
-    "sublime_plugin",
-    os.path.join(PACKAGE_ROOT, "dev/sublime_plugin.py")
-).load_module()
-
-
-import package_control.tests
-
-
-def run(matcher=None):
+def run():
     """
     Runs tests
 
-    :param matcher:
-        A unicode string containing a regular expression to use to filter test
-        names by. A value of None will cause no filtering.
-
     :return:
         A bool - if tests did not find any errors
     """
 
     print('Python %s' % sys.version)
     print('Running tests')
 
-    loader = unittest.TestLoader()
-    test_list = []
-    for test_class in package_control.tests.TEST_CLASSES:
-        if matcher:
-            names = loader.getTestCaseNames(test_class)
-            for name in names:
-                if re.search(matcher, name):
-                    test_list.append(test_class(name))
-        else:
-            test_list.append(loader.loadTestsFromTestCase(test_class))
-
-    stream = sys.stdout
-    verbosity = 1
-    if matcher:
-        verbosity = 2
+    suite = unittest.TestLoader().discover(
+        pattern="test_*.py",
+        start_dir="package_control/tests",
+        top_level_dir=PACKAGE_ROOT
+    )
 
-    suite = unittest.TestSuite()
-    for test in test_list:
-        suite.addTest(test)
-    result = unittest.TextTestRunner(stream=stream, verbosity=verbosity).run(suite)
+    result = unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(suite)
 
     return len(result.errors) == 0 and len(result.failures) == 0
 
 
 if __name__ == "__main__":
-    if len(sys.argv) == 2:
-        result = run(sys.argv[1])
-    else:
-        result = run()
+    result = run()
     sys.exit(int(not result))

package_control/commands/package_control_tests_command.py

@@ -1,22 +1,65 @@
+import io
+import os.path
+import threading
+import unittest
+
 import sublime
 import sublime_plugin
 
 from ..settings import pc_settings_filename
 
-try:
-    from ..tests import TestRunner, TEST_CLASSES
-except ImportError:
-    class PackageControlTestsCommand:
+
+class OutputPanel(io.TextIOWrapper):
+    """
+    A TextIO wrapper to output test results in an output panel.
+    """
+
+    def __init__(self, window):
+        self.panel = window.get_output_panel("package_control_tests")
+        self.panel.settings().set("word_wrap", True)
+        self.panel.settings().set("scroll_past_end", False)
+        window.run_command("show_panel", {"panel": "output.package_control_tests"})
+
+    def write(self, data):
+        self.panel.run_command("package_control_insert", {"string": data})
+
+    def get(self):
+        pass
+
+    def flush(self):
         pass
 
-else:
-    class PackageControlTestsCommand(sublime_plugin.ApplicationCommand):
-        """
-        A command to run the tests for Package Control
-        """
 
-        def run(self):
-            TestRunner(args=(sublime.active_window(), TEST_CLASSES)).start()
+class PackageControlTestsCommand(sublime_plugin.ApplicationCommand):
+    """
+    A command to run the tests for Package Control
+    """
+
+    HAVE_TESTS = None
+
+    def run(self):
+        def worker():
+            package_root = os.path.join(sublime.packages_path(), "Package Control")
+
+            # tests are excluded from production builds
+            # so it's ok to rely on filesystem traversal
+            suite = unittest.TestLoader().discover(
+                pattern="test_*.py",
+                start_dir=os.path.join(package_root, "package_control", "tests"),
+                top_level_dir=package_root,
+            )
+
+            output = OutputPanel(sublime.active_window())
+            output.write("Running Package Control Tests\n\n")
+
+            unittest.TextTestRunner(stream=output, verbosity=1).run(suite)
+
+        threading.Thread(target=worker).start()
+
+    def is_visible(self):
+        if self.HAVE_TESTS is None:
+            self.HAVE_TESTS = os.path.exists(
+                os.path.join(sublime.packages_path(), "Package Control", "package_control", "tests")
+            )
 
-        def is_visible(self):
-            return sublime.load_settings(pc_settings_filename()).get("enable_tests", False)
+        return self.HAVE_TESTS and sublime.load_settings(pc_settings_filename()).get("enable_tests", False)

package_control/tests/__init__.py

@@ -1,90 +1,19 @@
-import threading
-import unittest
+from sys import modules
 
-from . import (
-    clients,
-    distinfo,
-    downloaders,
-    package_version,
-    pep440_specifier,
-    pep440_version,
-    pep508_marker,
-    providers,
-    selectors,
-)
+if "sublime" not in modules:
+    import importlib.machinery
+    import os
 
+    PACKAGE_ROOT = os.path.dirname(__file__)
 
-TEST_CLASSES = [
-    clients.BitBucketClientTests,
-    clients.GitHubClientTests,
-    clients.GitLabClientTests,
-    distinfo.DistinfoTests,
-    downloaders.CurlDownloaderTests,
-    downloaders.OscryptoDownloaderTests,
-    downloaders.ResolveUrlTests,
-    downloaders.UrlLibDownloaderTests,
-    downloaders.WgetDownloaderTests,
-    downloaders.WinINetDownloaderTests,
-    package_version.PackageVersionTests,
-    pep440_specifier.PEP440VersionSpecifierTests,
-    pep440_version.PEP440VersionTests,
-    pep508_marker.PEP508MarkerTests,
-    providers.BitBucketRepositoryProviderTests,
-    providers.ChannelProviderTests,
-    providers.GitHubRepositoryProviderTests,
-    providers.GitHubUserProviderTests,
-    providers.GitLabRepositoryProviderTests,
-    providers.GitLabUserProviderTests,
-    providers.JsonRepositoryProviderTests,
-    selectors.PlatformSelectorTests,
-    selectors.VersionSelectorTests,
-]
+    # Mock the sublime module for CLI usage
+    sublime = importlib.machinery.SourceFileLoader(
+        "sublime",
+        os.path.join(PACKAGE_ROOT, "mock_sublime.py")
+    ).load_module()
 
-
-class OutputPanel:
-    def __init__(self, window):
-        self.panel = window.get_output_panel("package_control_tests")
-        self.panel.settings().set("word_wrap", True)
-        self.panel.settings().set("scroll_past_end", False)
-        window.run_command("show_panel", {"panel": "output.package_control_tests"})
-
-    def write(self, data):
-        self.panel.run_command("package_control_insert", {"string": data})
-
-    def get(self):
-        pass
-
-    def flush(self):
-        pass
-
-
-class TestRunner(threading.Thread):
-    """
-    Runs tests in a thread and outputs the results to an output panel
-
-    :example:
-        TestRunner(args=(window, test_classes)).start()
-
-    :param window:
-        A sublime.Window object to use to display the results
-
-    :param test_classes:
-        A unittest.TestCase class, or list of classes
-    """
-
-    def run(self):
-        window, test_classes = self._args
-
-        output = OutputPanel(window)
-        output.write("Running Package Control Tests\n\n")
-
-        if not isinstance(test_classes, list) and not isinstance(test_classes, tuple):
-            test_classes = [test_classes]
-
-        suite = unittest.TestSuite()
-
-        loader = unittest.TestLoader()
-        for test_class in test_classes:
-            suite.addTest(loader.loadTestsFromTestCase(test_class))
-
-        unittest.TextTestRunner(stream=output, verbosity=1).run(suite)
+    # Mock the sublime_plugin module for CLI usage
+    sublime_plugin = importlib.machinery.SourceFileLoader(
+        "sublime_plugin",
+        os.path.join(PACKAGE_ROOT, "mock_sublime_plugin.py")
+    ).load_module()

dev/sublime_plugin.py → package_control/tests/mock_sublime_plugin.py

@@ -1,3 +1,2 @@
-
 class ZipImporter:
-	pass
+    pass

tox.ini

@@ -19,4 +19,10 @@ ignore =
 	W503,
 	# leading operators after line continuation
 	W504
-max-line-length = 120
+max-line-length = 120
+
+[pytest]
+minversion = 6.0
+addopts = -ra -q
+python_files = test_*.py
+testpaths = package_control/tests