Debugging of dev branch looks complete
#2810
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,9 +3,11 @@ on: | |
| push: | ||
| branches: | ||
| - main | ||
| - dev | ||
| pull_request: | ||
| branches: | ||
| - main | ||
| - dev | ||
| release: | ||
| types: | ||
| - published | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,11 @@ | ||
| import requests | ||
| import logging | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
||
| def get_NPM_data(package): | ||
| url = "https://registry.npmjs.org/%s" % package | ||
|
There was a problem hiding this comment. [pylint] reported by reviewdog 🐶 |
||
| r = requests.get(url) | ||
|
Check warning on line 8 in augur/tasks/git/dependency_libyear_tasks/libyear_util/npm_libyear_utils.py
|
||
| if r.status_code < 400: | ||
| return r.json() | ||
| return {} | ||
|
|
@@ -42,10 +45,16 @@ | |
|
|
||
|
|
||
| def get_lastest_minor(version, data): | ||
| try: | ||
| versions = data['versions'] | ||
| except Exception as e: | ||
| logger.info(f'error is {e} on the NPM. Hey, its NODEJS, of course it does not work :D ') | ||
| raise e | ||
|
|
||
| try: | ||
| index = list(versions.keys()).index(version) | ||
| except ValueError as e: | ||
| logger.info(f'error is {e} on the NPM. Some kind of value error. Probably a VALUES error for Node, #AmIRight?') | ||
| raise e | ||
|
|
||
| major,minor,patch = split_version(version) | ||
|
There was a problem hiding this comment. [pylint] reported by reviewdog 🐶 |
||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,12 +5,12 @@ | |
| from augur.tasks.init.celery_app import AugurCoreRepoCollectionTask | ||
| from augur.application.db.data_parse import * | ||
|
There was a problem hiding this comment. [pylint] reported by reviewdog 🐶 |
||
| from augur.tasks.github.util.github_paginator import GithubPaginator | ||
| from augur.tasks.github.util.github_task_session import GithubTaskManifest | ||
| from augur.tasks.util.worker_util import remove_duplicate_dicts | ||
| from augur.tasks.github.util.util import get_owner_repo | ||
| from augur.application.db.models import PullRequest, Message, Issue, PullRequestMessageRef, IssueMessageRef, Contributor, Repo, CollectionStatus | ||
| from augur.application.db import get_engine, get_session | ||
| from sqlalchemy.sql import text | ||
|
|
||
| platform_id = 1 | ||
|
|
||
|
|
@@ -27,8 +27,8 @@ def collect_github_messages(repo_git: str) -> None: | |
| Repo.repo_git == repo_git).one().repo_id | ||
|
|
||
| owner, repo = get_owner_repo(repo_git) | ||
| task_name = f"{owner}/{repo}: Message Task" | ||
|
|
||
|
|
||
| if is_repo_small(repo_id): | ||
| message_data = fast_retrieve_all_pr_and_issue_messages(repo_git, logger, manifest.key_auth, task_name) | ||
|
|
@@ -133,7 +133,7 @@ def process_large_issue_and_pr_message_collection(repo_id, repo_git: str, logger | |
| process_messages(all_data, task_name, repo_id, logger, augur_db) | ||
|
|
||
|
|
||
| def process_messages(messages, task_name, repo_id, logger, augur_db): | ||
|
There was a problem hiding this comment. [pylint] reported by reviewdog 🐶 There was a problem hiding this comment. [pylint] reported by reviewdog 🐶 There was a problem hiding this comment. [pylint] reported by reviewdog 🐶 |
||
|
|
||
| tool_source = "Pr comment task" | ||
| tool_version = "2.0" | ||
|
|
@@ -152,13 +152,13 @@ def process_messages(messages, task_name, repo_id, logger): | |
|
|
||
| # create mapping from issue url to issue id of current issues | ||
| issue_url_to_id_map = {} | ||
| issues = augur_db.session.query(Issue).filter(Issue.repo_id == repo_id).all() | ||
| for issue in issues: | ||
| issue_url_to_id_map[issue.issue_url] = issue.issue_id | ||
|
|
||
| # create mapping from pr url to pr id of current pull requests | ||
| pr_issue_url_to_id_map = {} | ||
| prs = augur_db.session.query(PullRequest).filter(PullRequest.repo_id == repo_id).all() | ||
| for pr in prs: | ||
| pr_issue_url_to_id_map[pr.pr_issue_url] = pr.pull_request_id | ||
|
|
||
|
|
@@ -229,13 +229,13 @@ def process_messages(messages, task_name, repo_id, logger): | |
| contributors = remove_duplicate_dicts(contributors) | ||
|
|
||
| logger.info(f"{task_name}: Inserting {len(contributors)} contributors") | ||
| augur_db.insert_data(contributors, Contributor, ["cntrb_id"]) | ||
|
|
||
| logger.info(f"{task_name}: Inserting {len(message_dicts)} messages") | ||
| message_natural_keys = ["platform_msg_id", "pltfrm_id"] | ||
| message_return_columns = ["msg_id", "platform_msg_id"] | ||
| message_string_fields = ["msg_text"] | ||
| message_return_data = augur_db.insert_data(message_dicts, Message, message_natural_keys, | ||
| return_columns=message_return_columns, string_fields=message_string_fields) | ||
| if message_return_data is None: | ||
| return | ||
|
|
@@ -258,11 +258,11 @@ def process_messages(messages, task_name, repo_id, logger): | |
|
|
||
| logger.info(f"{task_name}: Inserting {len(pr_message_ref_dicts)} pr messages ref rows") | ||
| pr_message_ref_natural_keys = ["pull_request_id", "pr_message_ref_src_comment_id"] | ||
| augur_db.insert_data(pr_message_ref_dicts, PullRequestMessageRef, pr_message_ref_natural_keys) | ||
|
|
||
| logger.info(f"{task_name}: Inserting {len(issue_message_ref_dicts)} issue messages ref rows") | ||
| issue_message_ref_natural_keys = ["issue_id", "issue_msg_ref_src_comment_id"] | ||
| augur_db.insert_data(issue_message_ref_dicts, IssueMessageRef, issue_message_ref_natural_keys) | ||
|
|
||
| logger.info(f"{task_name}: Inserted {len(message_dicts)} messages. {len(issue_message_ref_dicts)} from issues and {len(pr_message_ref_dicts)} from prs") | ||
|
|
||
|
|
@@ -287,4 +287,4 @@ def process_github_comment_contributors(message, tool_source, tool_version, data | |
| # This is done by searching all the dicts for the given key that has the specified value | ||
| def find_dict_in_list_of_dicts(data, key, value): | ||
|
|
||
| return next((item for item in data if item[key] == value), None) | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,22 +2,26 @@ | |
| from augur.tasks.github.util.gh_graphql_entities import GraphQlPageCollection | ||
| from augur.application.db.models import * | ||
|
There was a problem hiding this comment. [pylint] reported by reviewdog 🐶 |
||
| from augur.tasks.github.util.util import get_owner_repo | ||
| from augur.application.db.util import execute_session_query | ||
| import traceback | ||
|
|
||
| def pull_request_files_model(repo_id,logger, augur_db, key_auth): | ||
|
|
||
| # query existing PRs and the respective url we will append the commits url to | ||
| pr_number_sql = s.sql.text(""" | ||
| SELECT DISTINCT pr_src_number as pr_src_number, pull_requests.pull_request_id | ||
| FROM pull_requests--, pull_request_meta | ||
| WHERE repo_id = :repo_id | ||
| """).bindparams(repo_id=repo_id) | ||
| pr_numbers = [] | ||
| #pd.read_sql(pr_number_sql, self.db, params={}) | ||
|
|
||
| result = augur_db.execute_sql(pr_number_sql)#.fetchall() | ||
| pr_numbers = [dict(row) for row in result.mappings()] | ||
|
|
||
| query = augur_db.session.query(Repo).filter(Repo.repo_id == repo_id) | ||
| repo = execute_session_query(query, 'one') | ||
|
|
||
| owner, name = get_owner_repo(repo.repo_git) | ||
|
|
||
| pr_file_rows = [] | ||
|
|
@@ -59,20 +63,31 @@ def pull_request_files_model(repo,logger, key_auth): | |
| 'values' : values | ||
| } | ||
|
|
||
| logger.debug(f"query: {query}; key_auth: {key_auth}; params: {params}") | ||
| file_collection = GraphQlPageCollection(query, key_auth, logger,bind=params) | ||
|
|
||
| logger.debug(f"Results of file_collection: {file_collection}") | ||
|
|
||
| for pr_file in file_collection: | ||
| logger.debug(f"CHECK: {repr(file_collection)}") | ||
| if pr_file and 'path' in pr_file: | ||
| logger.debug(f"Checks out for {repr(pr_file)} and {repr(file_collection)}") | ||
|
|
||
| try: | ||
| pr_file_rows += [{ | ||
| 'pull_request_id': pr_info['pull_request_id'], | ||
| 'pr_file_additions': pr_file['additions'] if 'additions' in pr_file else None, | ||
| 'pr_file_deletions': pr_file['deletions'] if 'deletions' in pr_file else None, | ||
| 'pr_file_path': pr_file['path'], | ||
| 'data_source': 'GitHub API', | ||
| 'repo_id': repo_id, | ||
| } for pr_file in file_collection if pr_file and 'path' in pr_file] | ||
| except Exception as e: | ||
| logger.error(f"PR Files Traceback: {''.join(traceback.format_exception(None, e, e.__traceback__))}") | ||
|
|
||
|
|
||
|
|
||
| if len(pr_file_rows) > 0: | ||
| #Execute a bulk upsert with sqlalchemy | ||
| pr_file_natural_keys = ["pull_request_id", "repo_id", "pr_file_path"] | ||
| augur_db.insert_data(pr_file_rows, PullRequestFile, pr_file_natural_keys) | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[pylint] reported by reviewdog 🐶
C0114: Missing module docstring (missing-module-docstring)