Dev Release to Main

.github/workflows/checks.yml

@@ -0,0 +1,31 @@
+name: "run-linting-checks"
+on:
+  pull_request:
+    branches: [main, dev]
+
+jobs:
+  run-pylint:
+    name: runner / pylint
+    permissions: write-all
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: dciborow/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          reporter: github-pr-review
+          level: warning
+          glob_pattern: "**/*.py"
+          filter_mode: "file"
+
+  misspell:
+    name: runner / misspell
+    runs-on: ubuntu-latest
+    steps:
+      - name: Highlight any misspellings in changes.
+        uses: actions/checkout@v4
+      - name: misspell
+        uses: reviewdog/action-misspell@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          locale: "US"

.pylintrc

@@ -12,7 +12,7 @@
 #refactoring checker
 #enable=R
 
-disable=E0611,E1101,W1203,R0801,W0614,W0611,C0411,C0103,C0301,C0303,C0304,C0305,W0311
+disable=E0611,E1101,W1203,R0801,W0614,W0611,C0411,C0103,C0301,C0303,C0304,C0305,W0311,E0401
 
 
 # Analyse import fallback blocks. This can be used to support both Python 2 and

README.md

@@ -1,4 +1,6 @@
-# Augur NEW Release v0.60.2
+# Augur NEW Release v0.62.0
+
+CHECK out Augur's Public Instance at https://ai.chaoss.io 
 
 [![first-timers-only](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) We follow the [First Timers Only](https://www.firsttimersonly.com/) philosophy of tagging issues for first timers only, and walking one newcomer through the resolution process weekly. [You can find these issues tagged with "first timers only" on our issues list.](https://github.com/chaoss/augur/labels/first-timers-only).
 
@@ -7,7 +9,7 @@
 ## NEW RELEASE ALERT!
 ### [If you want to jump right in, updated docker build/compose and bare metal installation instructions are available here](docs/new-install.md)
 
-Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.60.2
+Augur is now releasing a dramatically improved new version to the main branch. It is also available here: https://github.com/chaoss/augur/releases/tag/v0.62.0
 - The `main` branch is a stable version of our new architecture, which features:
   - Dramatic improvement in the speed of large scale data collection (100,000+ repos). All data is obtained for 100k+ repos within 2 weeks.
   - A new job management architecture that uses Celery and Redis to manage queues, and enables users to run a Flower job monitoring dashboard

augur/api/routes/dei.py

@@ -52,7 +52,7 @@ def dei_track_repo(application: ClientApplication):
         return jsonify({"status": "Repo already exists"})
 
     frontend_repo_group: RepoGroup = session.query(RepoGroup).filter(RepoGroup.rg_name == FRONTEND_REPO_GROUP_NAME).first()
-    repo_id = Repo.insert(session, repo_url, frontend_repo_group.repo_group_id, "API.DEI", repo_type="")
+    repo_id = Repo.insert_github_repo(session, repo_url, frontend_repo_group.repo_group_id, "API.DEI", repo_type="")
     if not repo_id:
         return jsonify({"status": "Error adding repo"})
 

augur/api/routes/user.py

@@ -227,7 +227,7 @@ def add_user_repo():
     repo = request.args.get("repo_url")
     group_name = request.args.get("group_name")
 
-    result = current_user.add_repo(group_name, repo)
+    result = current_user.add_github_repo(group_name, repo)
 
     return jsonify(result[1])
 
@@ -260,7 +260,7 @@ def add_user_org():
     org = request.args.get("org_url")
     group_name = request.args.get("group_name")
 
-    result = current_user.add_org(group_name, org)
+    result = current_user.add_github_org(group_name, org)
 
     return jsonify(result[1])
 

augur/api/routes/util.py

@@ -1,21 +1,18 @@
 #SPDX-License-Identifier: MIT
+from augur.api.routes import AUGUR_API_VERSION
+from ..server import app, engine
 import base64
 import sqlalchemy as s
 import pandas as pd
 import json
 from flask import Response
-import logging
 
 from augur.application.db.session import DatabaseSession
 from augur.application.logs import AugurLogger
 from augur.application.config import AugurConfig
 
 logger = AugurLogger("augur").get_logger()
 
-from augur.api.routes import AUGUR_API_VERSION
-from ..server import app, engine
-
-
 @app.route('/{}/repo-groups'.format(AUGUR_API_VERSION))
 def get_all_repo_groups(): #TODO: make this name automatic - wrapper?
     repoGroupsSQL = s.sql.text("""
@@ -54,9 +51,9 @@ def get_all_repos():
             (select * from api_get_all_repos_issues) b
             on
             repo.repo_id = b.repo_id
-            left outer join 
-            (select * from api_get_all_repo_prs) c 
-            on repo.repo_id=c.repo_id 
+            left outer join
+            (select * from api_get_all_repo_prs) c
+            on repo.repo_id=c.repo_id
             JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
         order by repo_name
     """)
@@ -95,9 +92,9 @@ def get_repos_in_repo_group(repo_group_id):
             (select * from api_get_all_repos_issues) b
             on
             repo.repo_id = b.repo_id
-            left outer join 
-            (select * from api_get_all_repo_prs) c 
-            on repo.repo_id=c.repo_id                 
+            left outer join
+            (select * from api_get_all_repo_prs) c
+            on repo.repo_id=c.repo_id
             JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
         WHERE
             repo_groups.repo_group_id = :repo_group_id
@@ -111,6 +108,49 @@ def get_repos_in_repo_group(repo_group_id):
                     status=200,
                     mimetype="application/json")
 
+@app.route('/{}/repos/<repo_id>'.format(AUGUR_API_VERSION))
+def get_repo_by_id(repo_id: int) -> Response:
+    repo_by_id_SQL = s.sql.text("""
+        SELECT
+            repo.repo_id,
+            repo.repo_name,
+            repo.description,
+            repo.repo_git AS url,
+            a.commits_all_time,
+            b.issues_all_time,
+            c.pull_requests_all_time,
+            rg_name,
+            repo.repo_group_id
+        FROM
+            repo
+            LEFT OUTER JOIN
+                (SELECT * FROM api_get_all_repos_commits) a
+            ON repo.repo_id = a.repo_id
+            LEFT OUTER JOIN
+                (SELECT * FROM api_get_all_repos_issues) b
+            ON repo.repo_id = b.repo_id
+            LEFT OUTER JOIN
+                (SELECT * FROM api_get_all_repo_prs) c
+            ON repo.repo_id = c.repo_id
+            JOIN repo_groups ON repo_groups.repo_group_id = repo.repo_group_id
+        WHERE
+            repo.repo_id = :id
+    """)
+
+    results = pd.read_sql(repo_by_id_SQL, engine, params={"id": repo_id})
+    results["url"] = results["url"].apply(lambda datum: datum.split("//")[1])  # cut "https://" off the URL
+    results["base64_url"] = [base64.b64encode(results.at[i, "url"].encode()) for i in results.index]
+    data = results.to_json(orient="records", date_format="iso", date_unit="ms")
+
+    if not data or data == "[]":
+        return Response(response='{"status": "Repository ' + str(repo_id) + ' does not exist"}',
+                        status=400,
+                        mimetype="application/json")
+
+    return Response(response=data[1:-1],  # cut off brackets at each end, turns list of length 1 into single value
+                    status=200,
+                    mimetype="application/json")
+
 @app.route('/{}/owner/<owner>/repo/<repo>'.format(AUGUR_API_VERSION))
 def get_repo_by_git_name(owner, repo):
 

augur/api/view/api.py

@@ -1,23 +1,23 @@
 from flask import Flask, render_template, render_template_string, request, abort, jsonify, redirect, url_for, session, flash
 import re
 from flask_login import current_user, login_required
 from augur.application.db.models import Repo, RepoGroup, UserGroup, UserRepo
 from augur.tasks.frontend import add_org_repo_list, parse_org_and_repo_name, parse_org_name
 from .utils import *
 from ..server import app, engine
 from augur.application.db.session import DatabaseSession
 
 @app.route('/cache/file/')
 @app.route('/cache/file/<path:file>')
 def cache(file=None):
     if file is None:
         return redirect(url_for('static', filename="cache"))
     return redirect(url_for('static', filename="cache/" + toCacheFilename(file, False)))
 
 
 def add_existing_repo_to_group(session, user_id, group_name, repo_id):
 
     logger.info("Adding existing repo to group")
 
     group_id = UserGroup.convert_group_name_to_id(session, user_id, group_name)
     if group_id is None:
@@ -102,7 +102,18 @@
                 if rg_obj:
                     # add the orgs repos to the group
                     add_existing_org_to_group(session, current_user.user_id, group, rg_obj.repo_group_id)
-
+
+            # matches https://gitlab.com/{org}/{repo}/ or http://gitlab.com/{org}/{repo}
+            elif Repo.parse_gitlab_repo_url(url)[0]:
+
+                org_name, repo_name = Repo.parse_github_repo_url(url)
+                repo_git = f"https://gitlab.com/{org_name}/{repo_name}"
+
+                # TODO: gitlab ensure the whole repo git is inserted so it can be found here
+                repo_obj = Repo.get_by_repo_git(session, repo_git)
+                if repo_obj:
+                    add_existing_repo_to_group(session, current_user.user_id, group, repo_obj.repo_id)
+
             else:
                 invalid_urls.append(url)
 

augur/api/view/routes.py

@@ -1,4 +1,8 @@
+"""
+Defines the api routes for the augur views
+"""
 import logging
+import math
 from flask import Flask, render_template, render_template_string, request, abort, jsonify, redirect, url_for, session, flash
 from sqlalchemy.orm.exc import NoResultFound
 from .utils import *
@@ -37,9 +41,9 @@ def root(path=""):
 def logo(brand=None):
     if brand is None:
         return redirect(url_for('static', filename='img/augur_logo.png'))
-    elif "augur" in brand:
+    if "augur" in brand:
         return logo(None)
-    elif "chaoss" in brand:
+    if "chaoss" in brand:
         return redirect(url_for('static', filename='img/Chaoss_Logo_white.png'))
     return ""
 
@@ -74,17 +78,15 @@ def repo_table_view():
     pagination_offset = config.get_value("frontend", "pagination_offset")
 
     if current_user.is_authenticated:
-        data = load_repos_test(user = current_user, search = query, page = page, sort = sorting, direction = direction, source = "user")
-        page_count = load_repos_test(user = current_user, search = query, count = True, source = "user")
-        # data = current_user.get_repos(page = page, sort = sorting, direction = direction, search=query)[0]
-        # page_count = (current_user.get_repo_count(search = query)[0] or 0) // pagination_offset
+        data = current_user.get_repos(page = page, sort = sorting, direction = direction, search=query)[0]
+        repos_count = (current_user.get_repo_count(search = query)[0] or 0)
     else:
-        data = load_repos_test(search = query, page = page, sort = sorting, direction = direction)
-        page_count = load_repos_test(search = query, count = True)
-        # data = get_all_repos(page = page, sort = sorting, direction = direction, search=query)[0]
-        # page_count = (get_all_repos_count(search = query)[0] or 0) // pagination_offset
+        data = get_all_repos(page = page, sort = sorting, direction = direction, search=query)[0]
+        repos_count = (get_all_repos_count(search = query)[0] or 0)
+
+    page_count = math.ceil(repos_count / pagination_offset) - 1
 
-    if not data.count():
+    if not data:
         data = None
 
 

augur/api/view/utils.py

@@ -1,6 +1,10 @@
+"""
+Defines utility functions used by the augur api views
+"""
 from pathlib import Path
 from concurrent.futures import ThreadPoolExecutor
 from flask import render_template, flash, url_for, Flask
+from .init import init_logging
 from .init import *
 from ..server import app, db_session
 from augur.application.config import AugurConfig
@@ -9,11 +13,13 @@
 from augur.application.db.session import DatabaseSession
 from augur.application.db.engine import DatabaseEngine
 from augur.application.db.models import User, Repo, RepoGroup, UserGroup, UserRepo
-from sqlalchemy import Column, Table, Integer, MetaData, or_, Label
+from sqlalchemy import Column, Table, Integer, MetaData, or_
 from sqlalchemy.sql.operators import ilike_op, distinct_op
 from sqlalchemy.sql.functions import coalesce
 from augur.application.db.models.base import Base
 
+from sqlalchemy.orm import Query
+
 init_logging()
 
 from .init import logger
@@ -309,67 +315,5 @@ def render_module(module, **args):
     args.setdefault("body", module)
     return render_template('index.j2', **args)
 
-""" ----------------------------------------------------------------
-    No longer used
-"""
-# My attempt at a loading page
-def renderLoading(dest, query, request):
-    cache_files_requested.append(request)
-    return render_template('index.j2', body="loading", title="Loading", d=dest, query_key=query, api_url=getSetting('serving'))
-
-with DatabaseEngine() as engine:
-    augur_data_schema = MetaData(schema = "augur_data")
-    augur_data_schema.reflect(bind = engine, views = True)
-
-    commits_materialized_view: Table = augur_data_schema.tables["augur_data.api_get_all_repos_commits"]
-    issues_materialized_view: Table = augur_data_schema.tables["augur_data.api_get_all_repos_issues"]
-
 """ ----------------------------------------------------------------
 """
-def load_repos_test(count = False, source = None, **kwargs):
-    columns: list[Label] = [
-        Repo.repo_id.distinct().label("repo_id"),
-        Repo.description.label("description"),
-        Repo.repo_git.label("url"),
-        coalesce(commits_materialized_view.columns.commits_all_time, 0).label("commits_all_time"),
-        coalesce(issues_materialized_view.columns.issues_all_time, 0).label("issues_all_time"),
-        RepoGroup.rg_name.label("rg_name"),
-        Repo.repo_git.regexp_replace('.*github\.com\/[A-Za-z0-9 \- _]+\/([A-Za-z0-9 \- _ .]+)$', "\\1").label("repo_name"),
-        Repo.repo_git.regexp_replace('.*github\.com\/([A-Za-z0-9 \- _]+)\/[A-Za-z0-9 \- _ .]+$', "\\1").label("repo_owner"),
-        RepoGroup.repo_group_id.label("repo_group_id")
-    ]
-
-    def get_colum_by_label(label: str)-> Label:
-        for column in columns:
-            if column.name == label:
-                return column
-
-    repos = db_session.query(*columns)\
-        .outerjoin(commits_materialized_view, Repo.repo_id == commits_materialized_view.columns.repo_id)\
-        .outerjoin(issues_materialized_view, Repo.repo_id == issues_materialized_view.columns.repo_id)\
-        .join(RepoGroup, Repo.repo_group_id == RepoGroup.repo_group_id)
-
-    user: User = kwargs.get("user")
-    if user:
-        repos = repos.join(UserRepo, Repo.repo_id == UserRepo.repo_id)\
-            .join(UserGroup, UserGroup.group_id == UserRepo.group_id)\
-            .filter(UserGroup.user_id == user.user_id)
-
-    search = kwargs.get("search")
-    qkey = kwargs.get("query_key") or ["repo_name", "repo_owner"]
-    if search:
-        if isinstance(qkey, list) and len(qkey) > 0:
-            repos = repos.filter(or_(ilike_op(get_colum_by_label(filter_column), f"%{search}%") for filter_column in qkey))
-        else:
-            repos = repos.filter(ilike_op(get_colum_by_label(qkey), f"%{search}%"))
-
-    page_size: int = kwargs.get("page_size") or 25
-    if count:
-        c = repos.count()
-        return math.ceil(c / page_size) - 1
-
-    page: int = kwargs.get("page") or 0
-    offset = page * page_size
-
-    return repos.slice(offset, offset + page_size)
-

augur/application/cli/backend.py

@@ -91,9 +91,12 @@ def start(disable_collection, development, port):
             logger.info("Deleting old task schedule")
             os.remove("celerybeat-schedule.db")
 
-    celery_beat_process = None
-    celery_command = "celery -A augur.tasks.init.celery_app.celery_app beat -l debug"
-    celery_beat_process = subprocess.Popen(celery_command.split(" "))    
+    with DatabaseSession(logger) as db_session:
+        config = AugurConfig(logger, db_session)
+        log_level = config.get_value("Logging", "log_level")
+        celery_beat_process = None
+        celery_command = f"celery -A augur.tasks.init.celery_app.celery_app beat -l {log_level.lower()}"
+        celery_beat_process = subprocess.Popen(celery_command.split(" "))    
 
     if not disable_collection: