Skip to content

Commit

Permalink
Move repo_info to new graphql endpoint
Browse files Browse the repository at this point in the history 
Signed-off-by: Andrew Brain <[email protected]>
  • Loading branch information
@ABrain7710
ABrain7710 committed Jul 13, 2024
1 parent 30350fc commit d4fec6a
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 103 deletions.
146 changes: 49 additions & 97 deletions augur/tasks/github/repo_info/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import json
import sqlalchemy as s
from augur.tasks.github.util.github_paginator import GithubPaginator
from augur.tasks.github.util.github_graphql_data_access import GithubGraphQlDataAccess
from augur.tasks.github.util.github_paginator import hit_api
from augur.tasks.github.util.util import get_owner_repo
from augur.tasks.github.util.gh_graphql_entities import request_graphql_dict
Expand Down Expand Up @@ -98,123 +99,74 @@ def repo_info_model(key_auth, repo_orm_obj, logger):

owner, repo = get_owner_repo(repo_orm_obj.repo_git)

url = 'https://api.github.com/graphql'

query = """
{
repository(owner:"%s", name:"%s"){
updatedAt
hasIssuesEnabled
issues(states:OPEN) {
query = """query($repo: String!, $owner: String!) {
repository(name: $repo, owner: $owner) {
updatedAt
hasIssuesEnabled
issues(states: OPEN) {
totalCount
}
hasWikiEnabled
forkCount
defaultBranchRef {
}
hasWikiEnabled
forkCount
defaultBranchRef {
name
}
watchers {
}
watchers {
totalCount
}
id
licenseInfo {
}
id
licenseInfo {
name
url
}
stargazers {
}
stargazers {
totalCount
}
codeOfConduct {
}
codeOfConduct {
name
url
}
issue_count: issues {
}
issue_count: issues {
totalCount
}
issues_closed: issues(states:CLOSED) {
}
issues_closed: issues(states: CLOSED) {
totalCount
}
pr_count: pullRequests {
}
pr_count: pullRequests {
totalCount
}
pr_open: pullRequests(states: OPEN) {
}
pr_open: pullRequests(states: OPEN) {
totalCount
}
pr_closed: pullRequests(states: CLOSED) {
}
pr_closed: pullRequests(states: CLOSED) {
totalCount
}
pr_merged: pullRequests(states: MERGED) {
}
pr_merged: pullRequests(states: MERGED) {
totalCount
}
defaultBranchRef {
}
defaultBranchRef {
target {
... on Commit {
history {
totalCount
}
history {
totalCount
}
}
}
}
}
}
}
""" % (owner, repo)

##############################
# {
# repository(owner: "chaoss", name: "augur") {
# updatedAt
# hasIssuesEnabled
# issues(states: OPEN) {
# totalCount
# }
# hasWikiEnabled
# forkCount
# defaultBranchRef {
# name
# }
# watchers {
# totalCount
# }
# id
# licenseInfo {
# name
# url
# }
# stargazers {
# totalCount
# }
# codeOfConduct {
# name
# url
# }
# issue_count: issues {
# totalCount
# }
# issues_closed: issues(states: CLOSED) {
# totalCount
# }
# pr_count: pullRequests {
# totalCount
# }
# pr_open: pullRequests(states: OPEN) {
# totalCount
# }
# pr_closed: pullRequests(states: CLOSED) {
# totalCount
# }
# pr_merged: pullRequests(states: MERGED) {
# totalCount
# }
# stargazerCount
# }
# }
}
"""

github_graphql_data_access = GithubGraphQlDataAccess(key_auth, logger)

try:
data = grab_repo_info_from_graphql_endpoint(key_auth, logger, query)
except Exception as e:
logger.error(f"Could not grab info for repo {repo_orm_obj.repo_id}")
raise e
variables = {
"owner": owner,
"repo": repo
}

result_keys = ("repository")

data = github_graphql_data_access.get_resource(query, variables, result_keys)

# Get committers count info that requires seperate endpoint
committers_count = query_committers_count(key_auth, logger, owner, repo)
Expand Down
19 changes: 13 additions & 6 deletions augur/tasks/github/util/github_graphql_data_access.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,26 @@ class UrlNotFoundException(Exception):
pass

class GithubGraphQlDataAccess:


def __init__(self, key_manager, logger: logging.Logger):

self.logger = logger
self.key_manager = key_manager

def get_resource(self, query, variables, result_keys):

result_json = self.make_request_with_retries(query, variables).json()

data = self.__extract_data_section(result_keys, result_json)

return self.__extract_raw_data_into_list(data)


def paginate_resource(self, query, variables, result_keys):

params = {
"numRecords" : self.per_page,
"numRecords" : 100,
"cursor" : None
}
params.update(variables)
Expand All @@ -36,7 +46,7 @@ def paginate_resource(self, query, variables, result_keys):
data = self.__extract_data_section(result_keys, result_json)

if self.__get_total_count(data) == 0:
return
return

yield from self.__extract_raw_data_into_list(data)

Expand All @@ -49,7 +59,7 @@ def paginate_resource(self, query, variables, result_keys):

yield from self.__extract_raw_data_into_list(data)

def make_request(self, query, variables={}, timeout=40):
def make_request(self, query, variables, timeout=40):

with httpx.Client() as client:

Expand Down Expand Up @@ -185,9 +195,6 @@ def __get_total_count(data):
if not data["totalCount"]:
raise Exception(f"Error: totalCount is null. Data: {data}")

if not data["totalCount"]:
raise Exception(f"Error: totalCount is null. Data: {data}")

try:
return int(data["totalCount"])
except ValueError:
Expand Down

0 comments on commit d4fec6a

Please sign in to comment.