deprecating fake user agent and installing from github (#83)

* deprecating fake user agent and installing from github
Signed-off-by: vsoch <[email protected]>

CHANGELOG.md

@@ -12,6 +12,7 @@ and **Merged pull requests**. Critical items to know are:
 Referenced versions in headers are tagged on Github, in parentheses are for pypi.
 
 ## [vxx](https://github.com/urlstechie/urlschecker-python/tree/master) (master)
+ - preparing to install from git for fake-useragent (0.0.33)
  - serial option for debugging (0.0.32)
  - adding support for web driver for harder URLs (0.0.31)
  - use ANSI escape sequences for colors, fake-useragent for agents (0.0.30)

setup.py

@@ -86,7 +86,10 @@ def get_reqs(lookup=None, key="INSTALL_REQUIRES"):
         long_description_content_type="text/markdown",
         keywords=KEYWORDS,
         setup_requires=["pytest-runner"],
-        install_requires=INSTALL_REQUIRES,
+        install_requires=INSTALL_REQUIRES
+        + [
+            "fake-useragent @ git+https://github.com/danger89/fake-useragent@master#egg=fake-useragent"
+        ],
         tests_require=TESTS_REQUIRES,
         extras_require={
             "all": INSTALL_REQUIRES_ALL,

tests/test_core_check.py

@@ -46,10 +46,7 @@ def test_difficult_urls(file_paths):
     results = checker.run(file_paths, timeout=20)
 
     # This should be the only failing (503)
-    assert (
-        "https://thisurldoesnotexist-pancakes.whatever"
-        in results["failed"]
-    )
+    assert "https://thisurldoesnotexist-pancakes.whatever" in results["failed"]
     working = [
         "https://www.hpcwire.com/2019/01/17/pfizer-hpc-engineer-aims-to-automate-software-stack-testing/",
         "https://www.sciencedirect.com/science/article/pii/S0013468608005045",

tests/test_core_fileproc.py

@@ -88,20 +88,15 @@ def test_get_file_paths(base_path, file_types):
         list of file paths.
     """
     file_paths = get_file_paths(base_path, file_types)
+    file_paths = get_file_paths(base_path, file_types)
     expected_paths = [
-        [
-            "tests/test_files/sample_test_file.md",
-            "tests/test_files/sample_test_file.py",
-            "tests/test_files/hard_urls.md",
-        ],
-        [
-            "tests/test_files/sample_test_file.py",
-            "tests/test_files/sample_test_file.md",
-            "tests/test_files/hard_urls.md",
-        ],
+        "tests/test_files/sample_test_file.md",
+        "tests/test_files/sample_test_file.py",
+        "tests/test_files/hard_urls.md",
     ]
     # assert
-    assert file_paths in expected_paths
+    for found in file_paths:
+        assert found in expected_paths
 
 
 @pytest.mark.parametrize(

urlchecker/core/check.py

@@ -202,6 +202,10 @@ def run(
         results = {}
         for file_name in file_paths:
 
+            # Re-use ports if we run out
+            if not ports:
+                ports = list(range(8000, 9999))
+
             # Export parameters and functions, use the same check task for all
             kwargs = {
                 "file_name": file_name,
@@ -224,7 +228,7 @@ def run(
             results = workers.run(funcs, tasks)  # type: ignore
         if not results:
             print("\U0001F914 There were no URLs to check.")
-            sys.exit(0)
+            return self.results
 
         for file_name, result in results.items():
             self.checks[file_name] = result

urlchecker/version.py

@@ -7,7 +7,7 @@
 
 """
 
-__version__ = "0.0.32"
+__version__ = "0.0.33"
 AUTHOR = "Ayoub Malek, Vanessa Sochat"
 AUTHOR_EMAIL = "[email protected], [email protected]"
 NAME = "urlchecker"
@@ -23,11 +23,7 @@
 # Global requirements
 
 
-INSTALL_REQUIRES = (
-    ("requests", {"min_version": "2.18.4"}),
-    # Recommended: pip install git+https://github.com/danger89/fake-useragent.git
-    ("fake-useragent", {"min_version": None}),
-)
+INSTALL_REQUIRES = (("requests", {"min_version": "2.18.4"}),)
 
 SELENIUM_REQUIRES = (("selenium", {"min_version": None}),)