Merge pull request #274 from r-world-devs/258-let-whole-internal-git-…

…platform-be-scanned 258 let whole internal git platform be scanned
r-world-devs · Aug 25, 2023 · c157491 · c157491
2 parents 00854e9 + 2019be2
commit c157491
Show file tree

Hide file tree

Showing 30 changed files with 682 additions and 365 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -1,7 +1,8 @@
 GitStats 0.1.0.9000
 
 - added setting tokens by default - if a user does have all the PATs set up in environment variables (as e.g. `GITHUB_PAT` or `GITLAB_PAT`), there is no need to pass them as an arugment to `set_connection` (I: #120 PR: #268),
-- added `get_users()` function to pull information on users (I: #199 PR: #238)
+- added `get_users()` function to pull information on users (I: #199 PR: #238),
+- added possibility of scanning whole internal git platforms if no `orgs` are passed (I: #258),
 - added switching to REST engine in case GraphQL fails with 502 error (I: #225 PRs: #227 (for repos) #261 (for commits))
 - added GraphQL engine for getting GitLab repos by organization (I: #218 PR: #233)
 - removed `contributors` as basic stat when pulling `repos` by `org` and by `phrase` to improve speed of pulling repositories data. Added `add_repos_contributors()` user function and `add_contributors` parameter to `get_repos()` function to add conditionally information on contributors to repositories table (I: #235 PRs: #243 #264)

diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R
@@ -15,10 +15,13 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL",
      #' @description Create `EngineGraphQL` object.
      #' @param gql_api_url GraphQL API url.
      #' @param token A token.
-     initialize = function(gql_api_url,
-                           token) {
+     #' @param scan_all A boolean.
+     initialize = function(gql_api_url = NA,
+                           token = NA,
+                           scan_all = FALSE) {
        self$gql_api_url <- gql_api_url
        private$token <- token
+       private$scan_all <- scan_all
      },
 
      #' @description Wrapper of GraphQL API request and response.
@@ -49,6 +52,9 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL",
      # @field token A token authorizing access to API.
      token = NULL,
 
+     # @field A boolean.
+     scan_all = FALSE,
+
      # @description A method to pull information on user.
      # @param username A login.
      # @return A user response.

diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R
@@ -10,13 +10,36 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
     #' @description Create `EngineGraphQLGitHub` object.
     #' @param gql_api_url GraphQL API url.
     #' @param token A token.
+    #' @param scan_all A boolean.
     initialize = function(gql_api_url,
-                          token) {
+                          token,
+                          scan_all = FALSE) {
       super$initialize(gql_api_url = gql_api_url,
-                       token = token)
+                       token = token,
+                       scan_all = scan_all)
       self$gql_query <- GQLQueryGitHub$new()
     },
 
+    #' @description Get all groups from GitLab.
+    get_orgs = function() {
+      end_cursor <- NULL
+      has_next_page <- TRUE
+      full_orgs_list <- list()
+      while(has_next_page) {
+        response <- self$gql_response(
+          gql_query = self$gql_query$orgs(
+            end_cursor = end_cursor
+          )
+        )
+        orgs_list <- purrr::map(response$data$search$edges, ~stringr::str_match(.$node$url, "[^\\/]*$"))
+        full_orgs_list <- append(full_orgs_list, orgs_list)
+        has_next_page <- response$data$search$pageInfo$hasNextPage
+        end_cursor <- response$data$search$pageInfo$endCursor
+      }
+      all_orgs <- unlist(full_orgs_list)
+      return(all_orgs)
+    },
+
     #' @description A method to retrieve all repositories for an organization in
     #'   a table format.
     #' @param org An organization.
@@ -26,14 +49,18 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
                          settings) {
       if (settings$search_param %in% c("org", "team")) {
         if (settings$search_param == "org") {
-          cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...")
+          if (!private$scan_all) {
+            cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...")
+          }
           repos_table <- private$pull_repos(
             from = "org",
             org = org
           ) %>%
             private$prepare_repos_table()
         } else {
-          cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling repositories...")
+          if (!private$scan_all) {
+            cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling repositories...")
+          }
           repos_table <- private$pull_repos_from_team(
             team = settings$team
           ) %>%
@@ -79,7 +106,9 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
       repos_names <- repos_table$name
 
       if (settings$search_param == "org") {
-        cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling commits...")
+        if (!private$scan_all) {
+          cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling commits...")
+        }
         repos_list_with_commits <- private$pull_commits_from_repos(
           org = org,
           repos = repos_names,
@@ -88,7 +117,9 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
         )
       }
       if (settings$search_param == "team") {
-        cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling commits...")
+        if (!private$scan_all) {
+          cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling commits...")
+        }
         repos_list_with_commits <- private$pull_commits_from_repos(
           org = org,
           repos = repos_names,
@@ -275,7 +306,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
           }
           return(full_commits_list)
         }
-      }, .progress = TRUE)
+      }, .progress = !private$scan_all)
       return(repos_list_with_commits)
     },
 

diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R
@@ -9,13 +9,35 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab",
      #' @description Create `EngineGraphQLGitLab` object.
      #' @param gql_api_url GraphQL API url.
      #' @param token A token.
+     #' @param scan_all A boolean.
      initialize = function(gql_api_url,
-                           token) {
+                           token,
+                           scan_all = FALSE) {
        super$initialize(gql_api_url = gql_api_url,
-                        token = token)
+                        token = token,
+                        scan_all = scan_all)
        self$gql_query <- GQLQueryGitLab$new()
      },
 
+     #' @description Get all groups from GitLab.
+     get_orgs = function() {
+       group_cursor <- ""
+       has_next_page <- TRUE
+       full_orgs_list <- list()
+       while(has_next_page) {
+         response <- self$gql_response(
+           gql_query = self$gql_query$groups(),
+           vars = list("groupCursor" = group_cursor)
+         )
+         orgs_list <- purrr::map(response$data$groups$edges, ~.$node$fullPath)
+         full_orgs_list <- append(full_orgs_list, orgs_list)
+         has_next_page <- response$data$groups$pageInfo$hasNextPage
+         group_cursor <- response$data$groups$pageInfo$endCursor
+       }
+       all_orgs <- unlist(full_orgs_list)
+       return(all_orgs)
+     },
+
      #' @description A method to retrieve all repositories for an organization in
      #'   a table format.
      #' @param org An organization.
@@ -24,7 +46,9 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab",
      get_repos = function(org,
                           settings) {
        if (settings$search_param == "org") {
-         cli::cli_alert_info("[GitLab][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...")
+         if (!private$scan_all) {
+           cli::cli_alert_info("[GitLab][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...")
+         }
          repos_table <- private$pull_repos(
            from = "org",
            org = org

diff --git a/R/EngineRest.R b/R/EngineRest.R
@@ -13,11 +13,14 @@ EngineRest <- R6::R6Class("EngineRest",
     #' @description Create a new `Rest` object
     #' @param rest_api_url A character, url of Rest API.
     #' @param token A token.
+    #' @param scan_all A boolean.
     #' @return A `Rest` object.
     initialize = function(rest_api_url = NA,
-                          token = NA) {
+                          token = NA,
+                          scan_all = FALSE) {
       self$rest_api_url <- rest_api_url
-      private$token <- token
+      private$token <- private$check_token(token)
+      private$scan_all <- scan_all
     },
 
     #' @description A wrapper for httr2 functions to perform get request to REST API endpoints.
@@ -34,13 +37,47 @@ EngineRest <- R6::R6Class("EngineRest",
       }
 
       return(result)
+    },
+
+    #' @description Check if an organization exists
+    #' @param orgs A character vector of organizations
+    #' @return orgs or NULL.
+    check_organizations = function(orgs) {
+      orgs <- purrr::map(orgs, function(org) {
+        org_endpoint <- if(grepl("github", self$rest_api_url)) "/orgs/" else "/groups/"
+        withCallingHandlers(
+          {
+            self$response(endpoint = paste0(self$rest_api_url, org_endpoint, org))
+          },
+          message = function(m) {
+            if (grepl("404", m)) {
+              cli::cli_alert_danger("Organization you provided does not exist or its name was passed in a wrong way: {org}")
+              cli::cli_alert_warning("Please type your organization name as you see it in `url`.")
+              cli::cli_alert_info("E.g. do not use spaces. Organization names as you see on the page may differ from their 'address' name.")
+              org <<- NULL
+            }
+          }
+        )
+        return(org)
+      }) %>%
+        purrr::keep(~ length(.) > 0) %>%
+        unlist()
+
+      if (length(orgs) == 0) {
+        return(NULL)
+      }
+      orgs
     }
+
   ),
   private = list(
 
     # @field token A token authorizing access to API.
     token = NULL,
 
+    # @field A boolean.
+    scan_all = FALSE,
+
     # @description Check whether the token exists.
     # @param token A token.
     # @return A token.
@@ -91,14 +128,16 @@ EngineRest <- R6::R6Class("EngineRest",
         },
         error = function(e) {
           if (!is.null(e$status)) {
-            if (e$status == 400) {
-              message("HTTP 400 Bad Request.")
-            } else if (e$status == 401) {
-              message("HTTP 401 Unauthorized.")
-            } else if (e$status == 403) {
-              message("HTTP 403 API limit reached.")
-            } else if (e$status == 404) {
-              message("HTTP 404 No such address")
+            if (!private$scan_all) {
+              if (e$status == 400) {
+                message("HTTP 400 Bad Request.")
+              } else if (e$status == 401) {
+                message("HTTP 401 Unauthorized.")
+              } else if (e$status == 403) {
+                message("HTTP 403 API limit reached.")
+              } else if (e$status == 404) {
+                message("HTTP 404 No such address")
+              }
             }
           } else if (grepl("Could not resolve host", e)) {
             cli::cli_abort(c(