From d4fec6ae45a38a3bc0aaacc3b46aa593c49bf36c Mon Sep 17 00:00:00 2001 From: Andrew Brain Date: Sat, 13 Jul 2024 08:44:59 -0500 Subject: [PATCH] Move repo_info to new graphql endpoint Signed-off-by: Andrew Brain --- augur/tasks/github/repo_info/core.py | 146 ++++++------------ .../github/util/github_graphql_data_access.py | 19 ++- 2 files changed, 62 insertions(+), 103 deletions(-) diff --git a/augur/tasks/github/repo_info/core.py b/augur/tasks/github/repo_info/core.py index 0cf6705fca..24e72355ea 100644 --- a/augur/tasks/github/repo_info/core.py +++ b/augur/tasks/github/repo_info/core.py @@ -2,6 +2,7 @@ import json import sqlalchemy as s from augur.tasks.github.util.github_paginator import GithubPaginator +from augur.tasks.github.util.github_graphql_data_access import GithubGraphQlDataAccess from augur.tasks.github.util.github_paginator import hit_api from augur.tasks.github.util.util import get_owner_repo from augur.tasks.github.util.gh_graphql_entities import request_graphql_dict @@ -98,123 +99,74 @@ def repo_info_model(key_auth, repo_orm_obj, logger): owner, repo = get_owner_repo(repo_orm_obj.repo_git) - url = 'https://api.github.com/graphql' - - query = """ - { - repository(owner:"%s", name:"%s"){ - updatedAt - hasIssuesEnabled - issues(states:OPEN) { + query = """query($repo: String!, $owner: String!) { + repository(name: $repo, owner: $owner) { + updatedAt + hasIssuesEnabled + issues(states: OPEN) { totalCount - } - hasWikiEnabled - forkCount - defaultBranchRef { + } + hasWikiEnabled + forkCount + defaultBranchRef { name - } - watchers { + } + watchers { totalCount - } - id - licenseInfo { + } + id + licenseInfo { name url - } - stargazers { + } + stargazers { totalCount - } - codeOfConduct { + } + codeOfConduct { name url - } - issue_count: issues { + } + issue_count: issues { totalCount - } - issues_closed: issues(states:CLOSED) { + } + issues_closed: issues(states: CLOSED) { totalCount - } - pr_count: pullRequests { + } + pr_count: pullRequests { totalCount - } - pr_open: pullRequests(states: OPEN) { + } + pr_open: pullRequests(states: OPEN) { totalCount - } - pr_closed: pullRequests(states: CLOSED) { + } + pr_closed: pullRequests(states: CLOSED) { totalCount - } - pr_merged: pullRequests(states: MERGED) { + } + pr_merged: pullRequests(states: MERGED) { totalCount - } - defaultBranchRef { + } + defaultBranchRef { target { ... on Commit { - history { - totalCount - } + history { + totalCount } + } + } } } - } - } - - """ % (owner, repo) - - ############################## - # { - # repository(owner: "chaoss", name: "augur") { - # updatedAt - # hasIssuesEnabled - # issues(states: OPEN) { - # totalCount - # } - # hasWikiEnabled - # forkCount - # defaultBranchRef { - # name - # } - # watchers { - # totalCount - # } - # id - # licenseInfo { - # name - # url - # } - # stargazers { - # totalCount - # } - # codeOfConduct { - # name - # url - # } - # issue_count: issues { - # totalCount - # } - # issues_closed: issues(states: CLOSED) { - # totalCount - # } - # pr_count: pullRequests { - # totalCount - # } - # pr_open: pullRequests(states: OPEN) { - # totalCount - # } - # pr_closed: pullRequests(states: CLOSED) { - # totalCount - # } - # pr_merged: pullRequests(states: MERGED) { - # totalCount - # } - # stargazerCount - # } - # } + } + """ + + github_graphql_data_access = GithubGraphQlDataAccess(key_auth, logger) - try: - data = grab_repo_info_from_graphql_endpoint(key_auth, logger, query) - except Exception as e: - logger.error(f"Could not grab info for repo {repo_orm_obj.repo_id}") - raise e + variables = { + "owner": owner, + "repo": repo + } + + result_keys = ("repository") + + data = github_graphql_data_access.get_resource(query, variables, result_keys) # Get committers count info that requires seperate endpoint committers_count = query_committers_count(key_auth, logger, owner, repo) diff --git a/augur/tasks/github/util/github_graphql_data_access.py b/augur/tasks/github/util/github_graphql_data_access.py index 485c1de3b3..99794b5554 100644 --- a/augur/tasks/github/util/github_graphql_data_access.py +++ b/augur/tasks/github/util/github_graphql_data_access.py @@ -17,16 +17,26 @@ class UrlNotFoundException(Exception): pass class GithubGraphQlDataAccess: + def __init__(self, key_manager, logger: logging.Logger): self.logger = logger self.key_manager = key_manager + def get_resource(self, query, variables, result_keys): + + result_json = self.make_request_with_retries(query, variables).json() + + data = self.__extract_data_section(result_keys, result_json) + + return self.__extract_raw_data_into_list(data) + + def paginate_resource(self, query, variables, result_keys): params = { - "numRecords" : self.per_page, + "numRecords" : 100, "cursor" : None } params.update(variables) @@ -36,7 +46,7 @@ def paginate_resource(self, query, variables, result_keys): data = self.__extract_data_section(result_keys, result_json) if self.__get_total_count(data) == 0: - return + return yield from self.__extract_raw_data_into_list(data) @@ -49,7 +59,7 @@ def paginate_resource(self, query, variables, result_keys): yield from self.__extract_raw_data_into_list(data) - def make_request(self, query, variables={}, timeout=40): + def make_request(self, query, variables, timeout=40): with httpx.Client() as client: @@ -185,9 +195,6 @@ def __get_total_count(data): if not data["totalCount"]: raise Exception(f"Error: totalCount is null. Data: {data}") - if not data["totalCount"]: - raise Exception(f"Error: totalCount is null. Data: {data}") - try: return int(data["totalCount"]) except ValueError: