Net mod (#28)

* rename writer_func to writer_wrapper,
move the decorator outside the Writer class
file_handle is an instance member
close_file_handle is an instance method
file_handle initialized in __init__, __get_file_handle not needed anymore

* using destructor to close file

* calling __get_table content in __to_ascii_table, not in __get_table and __get_table_transpose
Change 'Argument' in ascii table to ' '

* Now summarizing data using Writer.display_rows when printing ascii table to stdout

* removing 'full_name' from repository.display_rows

* adding imports to __init__.py, refactor __main__.py

* working on a networking module, using connection pooling to speed up recurring requests

* set block to True

* decouple networking logic from gitcomp_core and user. Using net_mod

* remove comments

* fetching user and repo data asynchronously

* reformat

* change NetMod._pool to __pool
bugfix using correct template while fetching repo data
Including full_name when writing ascii table to STDOUT

* following redirects. Handling connection failure

* handling 4xy and possibly 5xy errors

* remove __fetch_one_and_decode

* dedup args

src/gitcomp/__main__.py

@@ -20,7 +20,7 @@ def __get_arg_parser() -> argparse.ArgumentParser:
     mutually_exclusive = parser.add_mutually_exclusive_group()
 
     mutually_exclusive.add_argument('-u', '--user', type=str, nargs='+',
-                                    metavar='user_name', default=None, dest='user_names',
+                                    metavar='user_name', default=[], dest='user_names',
                                     help='''
                                          -u, --user <username...>
                                          The GitHub username(s) to query against.
@@ -29,7 +29,7 @@ def __get_arg_parser() -> argparse.ArgumentParser:
                                          ''')
 
     mutually_exclusive.add_argument('-r', '--repo', type=str, nargs='+',
-                                    metavar='repo', default=None, dest='repo_names',
+                                    metavar='repo', default=[], dest='repo_names',
                                     help='''
                                          -r, --repo <repo>
                                          The public GitHub repository to query against where repo takes the form:
@@ -70,11 +70,13 @@ def main():
     args = arg_parser.parse_args()
     if args.user_names is None and args.repo_names is None:
         safe_exit(arg_parser)
-    g = GitComp(users=args.user_names, repos=args.repo_names)
+    users = list(set(args.user_names)) or None
+    repos = list((set(args.repo_names))) or None
+    g = GitComp(users=users, repos=repos)
     prop = None
-    if args.user_names is not None:
+    if users is not None:
         prop = PROP['users'].value
-    elif args.repo_names is not None:
+    elif repos is not None:
         prop = PROP['repos'].value
     out_type = args.out_type[0]
     out_file = args.output_file[0]

src/gitcomp/gitcomp_core.py

@@ -1,20 +1,11 @@
-import json
 import re
-import urllib.request
 from .user import User
 from .repository import Repository
 from typing import List, Dict
+from .net_mod import NetMod
 
 
 class GitComp:
-    __api_base: str = 'https://api.github.com/'
-    """
-    explicitly request v3 of the API
-    https://docs.github.com/en/rest/overview/resources-in-the-rest-api#current-version
-    """
-    __headers: Dict[str, str] = {
-        'Accept': 'application/vnd.github.v3+json'
-    }
     users: List[str]
     repos: List[str]
     user_data: Dict[str, User] = None
@@ -35,10 +26,9 @@ def __init__(self, users: List[str] = None, repos: List[str] = None):
 
     def __fetch_user_data(self):
         self.__validate_user_names()
-        api_route = 'users/'
-        for user in self.users:
-            req = urllib.request.Request(url=f'{self.__api_base}{api_route}{user}', headers=GitComp.__headers)
-            self.user_data[user] = User(GitComp.__make_request(request=req))
+        response = NetMod().fetch_users_data(self.users)
+        for user in response:
+            self.user_data[user] = User(response[user])
 
     def __validate_user_names(self):
         for user in self.users:
@@ -48,11 +38,10 @@ def __validate_user_names(self):
                 """)
 
     def __fetch_repo_data(self):
-        api_route = 'repos/'
         self.__validate_repo_string()
-        for repo in self.repos:
-            req = urllib.request.Request(url=f'{self.__api_base}{api_route}{repo}', headers=GitComp.__headers)
-            self.repo_data[repo] = Repository(GitComp.__make_request(request=req))
+        response = NetMod().fetch_repos_data(self.repos)
+        for repo in response:
+            self.repo_data[repo] = Repository(response[repo])
 
     def __validate_repo_string(self):
         for repo in self.repos:
@@ -61,9 +50,3 @@ def __validate_repo_string(self):
                 Improper repository format.
                 Provide the repository name as: <user-name>/<repository-name>
                 """)
-
-    @staticmethod
-    def __make_request(request: urllib.request.Request):
-        with urllib.request.urlopen(request) as req:
-            data = json.loads(req.read().decode())
-            return data

src/gitcomp/net_mod.py

@@ -0,0 +1,75 @@
+import json
+import sys
+from concurrent.futures import ThreadPoolExecutor, Future
+from urllib3.connectionpool import HTTPSConnectionPool, HTTPResponse
+from urllib3.exceptions import NewConnectionError, MaxRetryError, HTTPError
+from typing import Dict, List, Any
+from string import Template
+
+
+class NetMod:
+    _instance = None
+    __pool: HTTPSConnectionPool
+    __pool_size: int = 5
+    __api_base: str = 'api.github.com'
+    __port: int = 443
+    __timeout: float = 5.0
+    __repo_route: Template = Template('/repos/$repo')
+    __user_route: Template = Template('/users/$user')
+    __org_route: Template = Template('/users/$user/orgs')
+
+    """
+    explicitly request v3 of the API
+    https://docs.github.com/en/rest/overview/resources-in-the-rest-api#current-version
+    """
+    __headers: Dict[str, str] = {
+        'Accept': 'application/vnd.github.v3+json',
+        'User-Agent': 'Python-urllib/3',
+        'Authorization': ''
+    }
+
+    """
+    referenced from
+    https://python-patterns.guide/gang-of-four/singleton/
+    """
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance = super(NetMod, cls).__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        self.__pool = HTTPSConnectionPool(host=NetMod.__api_base, maxsize=NetMod.__pool_size, headers=NetMod.__headers,
+                                          timeout=NetMod.__timeout, port=NetMod.__port, block=True)
+
+    def __make_request(self, api_route: str, method: str = 'get') -> Dict[str, Any]:
+        try:
+            response: HTTPResponse = self.__pool.request(method, api_route, release_conn=True, redirect=True)
+            res_data = json.loads(response.data)
+            if response.status != 200:
+                raise HTTPError(response.status, res_data['message'])
+            return res_data
+        except (NewConnectionError, MaxRetryError):
+            sys.exit("""Failed to connect. Exiting...""")
+        except HTTPError as err:
+            sys.exit(err)
+
+    def fetch_repos_data(self, repos: List[str]) -> Dict[str, Any]:
+        api_routes = [self.__repo_route.substitute(repo=repo) for repo in repos]
+        return self.__fetch_all__concurrent(repos, api_routes)
+
+    def fetch_users_data(self, users: List[str]) -> Dict[str, Any]:
+        api_routes = [self.__user_route.substitute(user=user) for user in users]
+        return self.__fetch_all__concurrent(users, api_routes)
+
+    def fetch_org_data(self, user: str) -> Dict[str, Any]:
+        api_route = self.__org_route.substitute(user=user)
+        return self.__make_request(api_route)
+
+    def __fetch_all__concurrent(self, entries: List[str], api_routes: List[str]) -> Dict[str, Any]:
+        max_workers = max(len(entries), self.__pool_size)
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            res: Dict[str, Future[Dict[str, Any]]] = {entry: executor.submit(self.__make_request, route) for
+                                                      entry, route in
+                                                      zip(entries, api_routes)}
+        return {user: data.result() for user, data in res.items()}

src/gitcomp/repository.py

@@ -26,7 +26,7 @@ class Repository:
     subscribers_count: int
     git_score: int
     license: str = None
-    display_rows = ['forks', 'open_issues', 'watches', 'network_count', 'subscribers_count', 'git_score']
+    display_rows = ['full_name', 'forks', 'open_issues', 'watches', 'network_count', 'subscribers_count', 'git_score']
     __date_fmt = '%Y-%m-%dT%H:%M:%SZ'
     __total_weight = 100 / 16
 

src/gitcomp/ser_de.py

@@ -131,7 +131,6 @@ def __get_table_content(self, g: Dict[str, Union[User, Repository]]):
         dict_repr = Writer.__to_dict(g)
         if self.out_file is stdout and self.type == 'ascii':
             dict_repr = self.__summarize(dict_repr)
-        print(dict_repr)
         headers = Writer.__get_headers(dict_repr)
         rows = Writer.__get_entries_as_rows(dict_repr)
         return headers, rows

src/gitcomp/user.py

@@ -1,6 +1,5 @@
-import json
-import urllib.request
 from dataclasses import dataclass
+from .net_mod import NetMod
 import sys
 
 
@@ -37,15 +36,12 @@ def __init__(self, user_data: dict):
         self.location = user_data['location']
         self.public_repos = user_data['public_repos']
         self.public_gists = user_data['public_gists']
-        self.organizations = self.__get_orgs_len(user_data['organizations_url'])
+        self.organizations = self.__get_orgs_len()
         self.git_score = self.get_score()
 
-    @staticmethod
-    def __get_orgs_len(url: str):
-        api_end_point = url
-        with urllib.request.urlopen(api_end_point) as req:
-            data = json.loads(req.read().decode())
-            return len(data)
+    def __get_orgs_len(self):
+        response = NetMod().fetch_org_data(self.login)
+        return len(response)
 
     def feature_score(self, name, val, weight=1, metric={}):
         fscore = 0