diff --git a/NEWS.md b/NEWS.md index dda98ccb..aca0a0fc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,8 @@ GitStats 0.1.0.9000 - added setting tokens by default - if a user does have all the PATs set up in environment variables (as e.g. `GITHUB_PAT` or `GITLAB_PAT`), there is no need to pass them as an arugment to `set_connection` (I: #120 PR: #268), -- added `get_users()` function to pull information on users (I: #199 PR: #238) +- added `get_users()` function to pull information on users (I: #199 PR: #238), +- added possibility of scanning whole internal git platforms if no `orgs` are passed (I: #258), - added switching to REST engine in case GraphQL fails with 502 error (I: #225 PRs: #227 (for repos) #261 (for commits)) - added GraphQL engine for getting GitLab repos by organization (I: #218 PR: #233) - removed `contributors` as basic stat when pulling `repos` by `org` and by `phrase` to improve speed of pulling repositories data. Added `add_repos_contributors()` user function and `add_contributors` parameter to `get_repos()` function to add conditionally information on contributors to repositories table (I: #235 PRs: #243 #264) diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R index 12c7664e..56fc46e7 100644 --- a/R/EngineGraphQL.R +++ b/R/EngineGraphQL.R @@ -15,10 +15,13 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL", #' @description Create `EngineGraphQL` object. #' @param gql_api_url GraphQL API url. #' @param token A token. - initialize = function(gql_api_url, - token) { + #' @param scan_all A boolean. + initialize = function(gql_api_url = NA, + token = NA, + scan_all = FALSE) { self$gql_api_url <- gql_api_url private$token <- token + private$scan_all <- scan_all }, #' @description Wrapper of GraphQL API request and response. @@ -49,6 +52,9 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL", # @field token A token authorizing access to API. token = NULL, + # @field A boolean. + scan_all = FALSE, + # @description A method to pull information on user. # @param username A login. # @return A user response. diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 000b9521..276625b1 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -10,13 +10,36 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", #' @description Create `EngineGraphQLGitHub` object. #' @param gql_api_url GraphQL API url. #' @param token A token. + #' @param scan_all A boolean. initialize = function(gql_api_url, - token) { + token, + scan_all = FALSE) { super$initialize(gql_api_url = gql_api_url, - token = token) + token = token, + scan_all = scan_all) self$gql_query <- GQLQueryGitHub$new() }, + #' @description Get all groups from GitLab. + get_orgs = function() { + end_cursor <- NULL + has_next_page <- TRUE + full_orgs_list <- list() + while(has_next_page) { + response <- self$gql_response( + gql_query = self$gql_query$orgs( + end_cursor = end_cursor + ) + ) + orgs_list <- purrr::map(response$data$search$edges, ~stringr::str_match(.$node$url, "[^\\/]*$")) + full_orgs_list <- append(full_orgs_list, orgs_list) + has_next_page <- response$data$search$pageInfo$hasNextPage + end_cursor <- response$data$search$pageInfo$endCursor + } + all_orgs <- unlist(full_orgs_list) + return(all_orgs) + }, + #' @description A method to retrieve all repositories for an organization in #' a table format. #' @param org An organization. @@ -26,14 +49,18 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", settings) { if (settings$search_param %in% c("org", "team")) { if (settings$search_param == "org") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...") + } repos_table <- private$pull_repos( from = "org", org = org ) %>% private$prepare_repos_table() } else { - cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling repositories...") + } repos_table <- private$pull_repos_from_team( team = settings$team ) %>% @@ -79,7 +106,9 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", repos_names <- repos_table$name if (settings$search_param == "org") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling commits...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling commits...") + } repos_list_with_commits <- private$pull_commits_from_repos( org = org, repos = repos_names, @@ -88,7 +117,9 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", ) } if (settings$search_param == "team") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling commits...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling commits...") + } repos_list_with_commits <- private$pull_commits_from_repos( org = org, repos = repos_names, @@ -275,7 +306,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", } return(full_commits_list) } - }, .progress = TRUE) + }, .progress = !private$scan_all) return(repos_list_with_commits) }, diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 5b0fb764..fa43417f 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -9,13 +9,35 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab", #' @description Create `EngineGraphQLGitLab` object. #' @param gql_api_url GraphQL API url. #' @param token A token. + #' @param scan_all A boolean. initialize = function(gql_api_url, - token) { + token, + scan_all = FALSE) { super$initialize(gql_api_url = gql_api_url, - token = token) + token = token, + scan_all = scan_all) self$gql_query <- GQLQueryGitLab$new() }, + #' @description Get all groups from GitLab. + get_orgs = function() { + group_cursor <- "" + has_next_page <- TRUE + full_orgs_list <- list() + while(has_next_page) { + response <- self$gql_response( + gql_query = self$gql_query$groups(), + vars = list("groupCursor" = group_cursor) + ) + orgs_list <- purrr::map(response$data$groups$edges, ~.$node$fullPath) + full_orgs_list <- append(full_orgs_list, orgs_list) + has_next_page <- response$data$groups$pageInfo$hasNextPage + group_cursor <- response$data$groups$pageInfo$endCursor + } + all_orgs <- unlist(full_orgs_list) + return(all_orgs) + }, + #' @description A method to retrieve all repositories for an organization in #' a table format. #' @param org An organization. @@ -24,7 +46,9 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab", get_repos = function(org, settings) { if (settings$search_param == "org") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...") + } repos_table <- private$pull_repos( from = "org", org = org diff --git a/R/EngineRest.R b/R/EngineRest.R index e0476129..70d4f37d 100644 --- a/R/EngineRest.R +++ b/R/EngineRest.R @@ -13,11 +13,14 @@ EngineRest <- R6::R6Class("EngineRest", #' @description Create a new `Rest` object #' @param rest_api_url A character, url of Rest API. #' @param token A token. + #' @param scan_all A boolean. #' @return A `Rest` object. initialize = function(rest_api_url = NA, - token = NA) { + token = NA, + scan_all = FALSE) { self$rest_api_url <- rest_api_url - private$token <- token + private$token <- private$check_token(token) + private$scan_all <- scan_all }, #' @description A wrapper for httr2 functions to perform get request to REST API endpoints. @@ -34,13 +37,47 @@ EngineRest <- R6::R6Class("EngineRest", } return(result) + }, + + #' @description Check if an organization exists + #' @param orgs A character vector of organizations + #' @return orgs or NULL. + check_organizations = function(orgs) { + orgs <- purrr::map(orgs, function(org) { + org_endpoint <- if(grepl("github", self$rest_api_url)) "/orgs/" else "/groups/" + withCallingHandlers( + { + self$response(endpoint = paste0(self$rest_api_url, org_endpoint, org)) + }, + message = function(m) { + if (grepl("404", m)) { + cli::cli_alert_danger("Organization you provided does not exist or its name was passed in a wrong way: {org}") + cli::cli_alert_warning("Please type your organization name as you see it in `url`.") + cli::cli_alert_info("E.g. do not use spaces. Organization names as you see on the page may differ from their 'address' name.") + org <<- NULL + } + } + ) + return(org) + }) %>% + purrr::keep(~ length(.) > 0) %>% + unlist() + + if (length(orgs) == 0) { + return(NULL) + } + orgs } + ), private = list( # @field token A token authorizing access to API. token = NULL, + # @field A boolean. + scan_all = FALSE, + # @description Check whether the token exists. # @param token A token. # @return A token. @@ -91,14 +128,16 @@ EngineRest <- R6::R6Class("EngineRest", }, error = function(e) { if (!is.null(e$status)) { - if (e$status == 400) { - message("HTTP 400 Bad Request.") - } else if (e$status == 401) { - message("HTTP 401 Unauthorized.") - } else if (e$status == 403) { - message("HTTP 403 API limit reached.") - } else if (e$status == 404) { - message("HTTP 404 No such address") + if (!private$scan_all) { + if (e$status == 400) { + message("HTTP 400 Bad Request.") + } else if (e$status == 401) { + message("HTTP 401 Unauthorized.") + } else if (e$status == 403) { + message("HTTP 403 API limit reached.") + } else if (e$status == 404) { + message("HTTP 404 No such address") + } } } else if (grepl("Could not resolve host", e)) { cli::cli_abort(c( diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index 1a1d2cdc..7b0cc351 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -4,45 +4,6 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", inherit = EngineRest, public = list( - #' @description Create new `EngineRestGitHub` object. - #' @param rest_api_url A REST API url. - #' @param token A token. - initialize = function(rest_api_url, - token) { - super$initialize( - rest_api_url = rest_api_url, - token = private$check_token(token) - ) - }, - - #' @description Check if an organization exists - #' @param orgs A character vector of organizations - #' @return orgs or NULL. - check_organizations = function(orgs) { - orgs <- purrr::map(orgs, function(org) { - org_endpoint <- "/orgs/" - withCallingHandlers( - { - self$response(endpoint = paste0(self$rest_api_url, org_endpoint, org)) - }, - message = function(m) { - if (grepl("404", m)) { - cli::cli_alert_danger("Organization you provided does not exist. Check spelling in: {org}") - org <<- NULL - } - } - ) - return(org) - }) %>% - purrr::keep(~ length(.) > 0) %>% - unlist() - - if (length(orgs) == 0) { - return(NULL) - } - orgs - }, - #' @description Method to get repositories with phrase in code blobs. #' @param org An organization #' @param settings A list of `GitStats` settings. @@ -50,7 +11,9 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", get_repos = function(org, settings) { if (settings$search_param == "phrase") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][phrase:{settings$phrase}][org:{org}] Searching repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][phrase:{settings$phrase}][org:{org}] Searching repositories...") + } repos_table <- private$search_repos_by_phrase( org = org, phrase = settings$phrase, @@ -72,8 +35,11 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", #' @return A table of repositories. get_repos_supportive = function(org, settings) { + repos_table <- NULL if (settings$search_param %in% c("org")) { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") + } repos_table <- private$pull_repos_from_org( org = org ) %>% @@ -114,10 +80,12 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", org = org, settings = list(search_param = "org") ) - if (settings$search_param == "org") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}] Pulling commits...") - } else if (settings$search_param == "team") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling commits...") + if (!private$scan_all) { + if (settings$search_param == "org") { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}] Pulling commits...") + } else if (settings$search_param == "team") { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling commits...") + } } repos_list_with_commits <- private$pull_commits_from_org( repos_table = repos_table, @@ -144,7 +112,9 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", #' @return A table of repositories with added information on contributors. add_repos_contributors = function(repos_table) { if (nrow(repos_table) > 0) { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}] Pulling contributors...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}] Pulling contributors...") + } repo_iterator <- paste0(repos_table$organization, "/", repos_table$name) user_name <- rlang::expr(.$login) repos_table$contributors <- purrr::map_chr(repo_iterator, function(repos_id) { @@ -224,10 +194,15 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", org, language, byte_max = "384000") { + org_url <- if (!private$scan_all) { + paste0("'+user:", org) + } else { + "" + } search_endpoint <- if (language != "All") { - paste0(self$rest_api_url, "/search/code?q='", phrase, "'+user:", org, "+language:", language) + paste0(self$rest_api_url, "/search/code?q='", phrase, org_url, "+language:", language) } else { - paste0(self$rest_api_url, "/search/code?q='", phrase, "'+user:", org) + paste0(self$rest_api_url, "/search/code?q='", phrase, org_url) } total_n <- self$response(search_endpoint)[["total_count"]] @@ -269,14 +244,14 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", resp_list <- list() index <- c(0, 50) - pb <- progress::progress_bar$new( - format = "GitHub search limit (1000 results) exceeded. Results will be divided. :elapsedfull" + spinner <- cli::make_spinner( + template = cli::col_grey("GitHub search limit (1000 results) exceeded. Results will be divided. {spin}") ) while (index[2] < as.numeric(byte_max)) { size_formula <- paste0("+size:", as.character(index[1]), "..", as.character(index[2])) - pb$tick(0) + spinner$spin() n_count <- tryCatch( { @@ -313,7 +288,7 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", index[2] <- index[2] + 10000 } } - + spinner$finish() resp_list } }, @@ -328,8 +303,8 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", "name" = repo$name, "stars" = repo$stargazers_count, "forks" = repo$forks_count, - "created_at" = repo$created_at, - "last_activity_at" = repo$pushed_at, + "created_at" = gts_to_posixt(repo$created_at), + "last_activity_at" = if (!is.null(repo$pushed_at)) gts_to_posixt(repo$pushed_at) else gts_to_posixt(repo$created_at), "languages" = repo$language, "issues_open" = repo$issues_open, "issues_closed" = repo$issues_closed, diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 85ac17f9..fd60257a 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -4,47 +4,6 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", inherit = EngineRest, public = list( - #' @description Create new `EngineRestGitLab` object. - #' @param rest_api_url A REST API url. - #' @param token A token. - initialize = function(rest_api_url, - token) { - super$initialize( - rest_api_url = rest_api_url, - token = private$check_token(token) - ) - }, - - #' @description Check if an organization exists - #' @param orgs A character vector of organizations - #' @return orgs or NULL. - check_organizations = function(orgs) { - orgs <- purrr::map(orgs, function(org) { - org_endpoint <- "/groups/" - withCallingHandlers( - { - self$response(endpoint = paste0(self$rest_api_url, org_endpoint, org)) - }, - message = function(m) { - if (grepl("404", m)) { - cli::cli_alert_danger("Group name passed in a wrong way: {org}") - cli::cli_alert_warning("If you are using `GitLab`, please type your group name as you see it in `url`.") - cli::cli_alert_info("E.g. do not use spaces. Group names as you see on the page may differ from their 'address' name.") - org <<- NULL - } - } - ) - return(org) - }) %>% - purrr::keep(~ length(.) > 0) %>% - unlist() - - if (length(orgs) == 0) { - return(NULL) - } - orgs - }, - #' @description A method to retrieve all repositories for an organization in #' a table format. #' @param org A character, a group of projects. @@ -53,7 +12,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", get_repos = function(org, settings) { if (settings$search_param == "phrase") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][phrase:{settings$phrase}][org:{org}] Searching repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][phrase:{settings$phrase}][org:{org}] Searching repositories...") + } repos_table <- private$search_repos_by_phrase( org = org, phrase = settings$phrase, @@ -63,7 +24,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", private$prepare_repos_table() %>% private$add_repos_issues() } else if (settings$search_param == "team") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling repositories...") + } org <- private$get_group_id(org) repos_table <- private$pull_repos_from_org(org) %>% private$tailor_repos_info() %>% @@ -87,8 +50,11 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", #' @return Nothing. get_repos_supportive = function(org, settings) { + repos_table <- NULL if (settings$search_param == "org") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") + } org <- private$get_group_id(org) repos_table <- private$pull_repos_from_org(org) %>% private$tailor_repos_info() %>% @@ -103,7 +69,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", #' @return A table of repositories with added information on contributors. add_repos_contributors = function(repos_table) { if (nrow(repos_table) > 0) { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}] Pulling contributors...") + if (!private$scan_all) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}] Pulling contributors...") + } repo_iterator <- repos_table$id user_name <- rlang::expr(.$name) repos_table$contributors <- purrr::map_chr(repo_iterator, function(repos_id) { @@ -141,13 +109,13 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", org = org, settings = list(search_param = "org") ) - - if (settings$search_param == "org") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}] Pulling commits...") - } else if (settings$search_param == "team") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling commits...") + if (!private$scan_all) { + if (settings$search_param == "org") { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}] Pulling commits...") + } else if (settings$search_param == "team") { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling commits...") + } } - repos_list_with_commits <- private$pull_commits_from_org( repos_table = repos_table, date_from = date_from, @@ -230,12 +198,15 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", page <- 1 still_more_hits <- TRUE resp_list <- list() - groups_id <- private$get_group_id(org) - + groups_url <- if (!private$scan_all) { + paste0("/groups/", private$get_group_id(org)) + } else { + "" + } while (still_more_hits | page < page_max) { resp <- self$response( paste0( - self$rest_api_url, "/groups/", groups_id, + self$rest_api_url, groups_url, "/search?scope=blobs&search=", phrase, "&per_page=100&page=", page ) ) @@ -364,7 +335,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", date_until = date_until ) return(commits_from_repo) - }, .progress = TRUE) + }, .progress = !private$scan_all) names(repos_list_with_commits) <- repos_names return(repos_list_with_commits) }, diff --git a/R/GQLQueryGitHub.R b/R/GQLQueryGitHub.R index e8e0e9f3..0ccc10ce 100644 --- a/R/GQLQueryGitHub.R +++ b/R/GQLQueryGitHub.R @@ -4,6 +4,35 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub", public = list( + #' @description Prepare query to list organizations from GitHub. + #' @param end_cursor An end cursor to paginate. + #' @return A query. + orgs = function(end_cursor) { + if (is.null(end_cursor)) { + pagination_phrase <- '' + } else { + pagination_phrase <- paste0('after: "', end_cursor, '"') + } + + paste0( + 'query { + search(first: 100, type: USER, query: "type:org" ', pagination_phrase, ') { + pageInfo { + hasNextPage + endCursor + } + edges { + node{ + ... on Organization { + name + url + } + } + } + } + }') + }, + #' @description Prepare query to get repositories from GitHub. #' @param org An organization of repositories. #' @param repo_cursor An end cursor for repositories page. diff --git a/R/GQLQueryGitLab.R b/R/GQLQueryGitLab.R index 31b5ae92..2f2a3a33 100644 --- a/R/GQLQueryGitLab.R +++ b/R/GQLQueryGitLab.R @@ -4,6 +4,24 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab", public = list( + #' @description Prepare query to list groups from GitLab. + #' @return A query. + groups = function() { + 'query GetGroups($groupCursor: String!) { + groups (after: $groupCursor) { + pageInfo { + endCursor + hasNextPage + } + edges { + node { + fullPath + } + } + } + }' + }, + #' @description Prepare query to get repositories from GitLab. #' @param repo_cursor An end cursor for repositories page. #' @return A query. diff --git a/R/GitHost.R b/R/GitHost.R index 31aeb0ec..e737ab32 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -18,31 +18,36 @@ GitHost <- R6::R6Class("GitHost", token = NA, api_url = NA) { private$api_url <- api_url + private$is_public <- private$check_if_public() + private$host <- private$set_host() if (is.null(token)){ - token <- private$set_default_token() + private$token <- private$set_default_token() + } else { + private$token <- token } - if (grepl("https://", api_url) && grepl("github", api_url)) { - private$engines$rest <- EngineRestGitHub$new( - token = token, - rest_api_url = api_url - ) - private$engines$graphql <- EngineGraphQLGitHub$new( - token = token, - gql_api_url = private$set_gql_url(api_url) - ) - } else if (grepl("https://", api_url) && grepl("gitlab|code", api_url)) { - private$engines$rest <- EngineRestGitLab$new( - token = token, - rest_api_url = api_url - ) - private$engines$graphql <- EngineGraphQLGitLab$new( - token = token, - gql_api_url = private$set_gql_url(api_url) - ) + if (is.null(orgs)) { + if (private$is_public) { + cli::cli_abort(c( + "You need to specify `orgs` for public Git Host.", + "x" = "Host will not be added.", + "i" = "Add organizations to your `orgs` parameter." + ), + call = NULL) + } else { + cli::cli_alert_warning(cli::col_yellow( + "No `orgs` specified. I will pull all organizations from the Git Host." + )) + private$scan_all <- TRUE + } + } + private$engines$rest <- private$setup_engine(type = "rest") + private$engines$graphql <- private$setup_engine(type = "graphql") + if (private$scan_all) { + cli::cli_alert_info("[{private$host}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") + private$orgs <- private$engines$graphql$get_orgs() } else { - stop("This connection is not supported by GitStats class object.") + private$orgs <- private$engines$rest$check_organizations(orgs) } - private$orgs <- private$engines$rest$check_organizations(orgs) }, #' @description A method to list all repositories for an organization, a @@ -52,32 +57,9 @@ GitHost <- R6::R6Class("GitHost", #' column to repositories table. #' @return A data.frame of repositories. get_repos = function(settings, add_contributors = FALSE) { - repos_table <- purrr::map(private$orgs, function(org) { - tryCatch({ - repos_list <- purrr::map(private$engines, function (engine) { - engine$get_repos( - org = org, - settings = settings - ) - }) - }, - error = function(e) { - if (grepl("502", e)) { - cli::cli_alert_warning(cli::col_yellow("HTTP 502 Bad Gateway Error. Switch to another Engine.")) - repos_list <<- purrr::map(private$engines, function (engine) { - engine$get_repos_supportive( - org = org, - settings = settings - ) - }) - } else { - e - } - }) - repos_table_org <- purrr::list_rbind(repos_list) - return(repos_table_org) - }) %>% - purrr::list_rbind() + repos_table <- private$pull_repos_from_orgs( + settings = settings + ) if (settings$search_param == "team") { add_contributors <- TRUE @@ -128,7 +110,9 @@ GitHost <- R6::R6Class("GitHost", "i" = "Please change your `search_param` either to 'org' or 'team' with `setup()`." )) } - + if (private$scan_all) { + cli::cli_alert_info("[Host:{private$host}] {cli::col_yellow('Pulling commits from all organizations...')}") + } commits_table <- purrr::map(private$orgs, function(org) { tryCatch({ commits_table_org <- purrr::map(private$engines, ~ .$get_commits( @@ -161,7 +145,7 @@ GitHost <- R6::R6Class("GitHost", }) return(commits_table_org) - }) %>% + }, .progress = private$scan_all) %>% purrr::list_rbind() return(commits_table) @@ -187,12 +171,49 @@ GitHost <- R6::R6Class("GitHost", # @field A REST API url. api_url = NULL, + # @field A token. + token = NULL, + + # @field public A boolean. + is_public = NULL, + + # @field Host name. + host = NULL, + # @field orgs A character vector of repo organizations. orgs = NULL, + # @field A boolean. + scan_all = FALSE, + # @field engines A placeholder for REST and GraphQL Engine classes. engines = list(), + # @description Check whether Git platform is public or internal. + check_if_public = function() { + if (grepl("api.github.com|gitlab.com/api", private$api_url)) { + TRUE + } else { + FALSE + } + }, + + # @description Set name of a Git Host. + set_host = function() { + if (grepl("https://", private$api_url) && grepl("github", private$api_url)) { + "GitHub" + } else if (grepl("https://", private$api_url) && grepl("gitlab|code", private$api_url)) { + "GitLab" + } else { + cli::cli_abort(c( + "This connection is not supported by GitStats class object.", + "x" = "Host will not be added." + ), + call = NULL + ) + } + }, + # @description Set default token if none exists. set_default_token = function() { if (grepl("github", private$api_url)) { @@ -217,6 +238,40 @@ GitHost <- R6::R6Class("GitHost", return(token) }, + # Setup REST and GraphQL engines + setup_engine = function(type) { + engine <- if (private$host == "GitHub") { + if (type == "rest") { + EngineRestGitHub$new( + rest_api_url = private$api_url, + token = private$token, + scan_all = private$scan_all + ) + } else { + EngineGraphQLGitHub$new( + gql_api_url = private$set_gql_url(private$api_url), + token = private$token, + scan_all = private$scan_all + ) + } + } else { + if (type == "rest") { + EngineRestGitLab$new( + rest_api_url = private$api_url, + token = private$token, + scan_all = private$scan_all + ) + } else { + EngineGraphQLGitLab$new( + gql_api_url = private$set_gql_url(private$api_url), + token = private$token, + scan_all = private$scan_all + ) + } + } + return(engine) + }, + # @description Helper to test if a token works test_token = function(token) { response <- NULL @@ -242,6 +297,45 @@ GitHost <- R6::R6Class("GitHost", paste0(gsub("/v+.*", "", rest_api_url), "/graphql") }, + # @description Pull repositories from organisations. + pull_repos_from_orgs = function(settings) { + orgs <- private$orgs + if (private$scan_all) { + cli::cli_alert_info("[Host:{private$host}] {cli::col_yellow('Pulling repositories from all organizations...')}") + if (settings$search_param == "phrase") { + orgs <- "no_orgs" + } + } + repos_table <- purrr::map(orgs, function(org) { + tryCatch({ + repos_list <- purrr::map(private$engines, function (engine) { + engine$get_repos( + org = org, + settings = settings + ) + }) + }, + error = function(e) { + if (!private$scan_all) { + if (grepl("502", e)) { + cli::cli_alert_warning(cli::col_yellow("HTTP 502 Bad Gateway Error. Switch to another Engine.")) + } else { + cli::cli_alert_warning(cli::col_yellow("Error. Switch to another Engine.")) + } + } + repos_list <<- purrr::map(private$engines, function (engine) { + engine$get_repos_supportive( + org = org, + settings = settings + ) + }) + }) + repos_table_org <- purrr::list_rbind(repos_list) + return(repos_table_org) + }, .progress = private$scan_all) %>% + purrr::list_rbind() + }, + # @description Filter repositories by contributors. # @details If at least one member of a team is a contributor than a project # passes through the filter. diff --git a/R/GitStats.R b/R/GitStats.R index ab55174d..7459b945 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -82,27 +82,23 @@ GitStats <- R6::R6Class("GitStats", token, orgs) { new_host <- NULL - tryCatch({ - new_host <- GitHost$new( - orgs = orgs, - token = token, - api_url = api_url - ) - if (grepl("https://", api_url) && grepl("github", api_url)) { - cli::cli_alert_success("Set connection to GitHub.") - } else if (grepl("https://", api_url) && grepl("gitlab|code", api_url)) { - cli::cli_alert_success("Set connection to GitLab.") - } - }, - error = function(e){ - cli::cli_alert_warning(e$message) - cli::cli_alert_danger("Host will not be passed.") - }) - if (!is.null(new_host)) { - private$hosts <- new_host %>% - private$check_for_duplicate_hosts() %>% - append(private$hosts, .) + + new_host <- GitHost$new( + orgs = orgs, + token = token, + api_url = api_url + ) + if (grepl("https://", api_url) && grepl("github", api_url)) { + cli::cli_alert_success("Set connection to GitHub.") + } else if (grepl("https://", api_url) && grepl("gitlab|code", api_url)) { + cli::cli_alert_success("Set connection to GitLab.") } + + if (!is.null(new_host)) { + private$hosts <- new_host %>% + private$check_for_duplicate_hosts() %>% + append(private$hosts, .) + } }, #' @description A method to add a team member. @@ -242,8 +238,7 @@ GitStats <- R6::R6Class("GitStats", orgs <- purrr::map(private$hosts, function(host) { host_priv <- environment(host$initialize)$private orgs <- host_priv$orgs - paste0(orgs, collapse = ", ") - }) %>% paste0(collapse = ", ") + }) private$print_item("Organisations", orgs) private$print_item("Search preference", private$settings$search_param) private$print_item("Team", private$settings$team_name, paste0(private$settings$team_name, " (", length(private$settings$team), " members)")) @@ -329,6 +324,18 @@ GitStats <- R6::R6Class("GitStats", print_item = function(item_name, item_to_check, item_to_print = item_to_check) { + if (item_name == "Organisations") { + item_to_print <- unlist(item_to_print) + if (length(item_to_print) < 10) { + list_items <- paste0(item_to_print, collapse = ", ") + + } else { + item_to_print_cut <- item_to_print[1:10] + list_items <- paste0(item_to_print_cut, collapse = ", ") %>% + paste0("... and ", length(item_to_print) - 10, " more") + } + item_to_print <- paste0("[", cli::col_green(length(item_to_print)), "] ", list_items) + } cat(paste0( cli::col_blue(paste0(item_name, ": ")), ifelse(is.null(item_to_check), diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index c13523c4..1b0af12d 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -13,8 +13,12 @@ create_gitstats <- function() { #' @param gitstats_obj A GitStats object. #' @param api_url A character, url address of API. #' @param token A token. -#' @param orgs A character vector of organisations (owners of repositories -#' in case of GitHub and groups of projects in case of GitLab). +#' @param orgs A character vector of organisations (owners of repositories in +#' case of GitHub and groups of projects in case of GitLab). You do not need +#' to define `orgs` if you wish to scan whole internal Git platform (such as +#' enterprise version of GitHub or GitLab). In case of a public one (like +#' GitHub) you need to define `orgs` as scanning through all organizations +#' would be an overkill. #' @return A `GitStats` class object with added information on connection #' (`$hosts` field). #' @examples @@ -34,7 +38,7 @@ create_gitstats <- function() { set_connection <- function(gitstats_obj, api_url, token = NULL, - orgs) { + orgs = NULL) { gitstats_obj$add_host( api_url = api_url, token = token, diff --git a/R/test_helpers.R b/R/test_helpers.R index bb9cdd2d..65067212 100644 --- a/R/test_helpers.R +++ b/R/test_helpers.R @@ -29,8 +29,9 @@ TestHost <- R6::R6Class("TestHost", token = NA, api_url = NA) { private$api_url <- api_url + private$host <- private$set_host() if (grepl("https://", api_url) && grepl("github", api_url)) { - private$engines$rest <- EngineRestGitHub$new( + private$engines$rest <- TestEngineRestGitHub$new( token = token, rest_api_url = api_url ) @@ -39,7 +40,7 @@ TestHost <- R6::R6Class("TestHost", gql_api_url = private$set_gql_url(api_url) ) } else if (grepl("https://", api_url) && grepl("gitlab|code", api_url)) { - private$engines$rest <- EngineRestGitLab$new( + private$engines$rest <- TestEngineRest$new( token = token, rest_api_url = api_url ) @@ -83,6 +84,32 @@ TestEngineRest <- R6::R6Class("TestEngineRest", ) ) +#' @noRd +#' @description A helper class to use in tests. +TestEngineRestGitHub <- R6::R6Class("TestEngineRestGitHub", + inherit = EngineRestGitHub, + public = list( + initialize = function(token, + rest_api_url) { + private$token <- token + self$rest_api_url <- rest_api_url + } + ) +) + +#' @noRd +#' @description A helper class to use in tests. +TestEngineRestGitLab <- R6::R6Class("TestEngineRestGitLab", + inherit = EngineRestGitLab, + public = list( + initialize = function(token, + rest_api_url) { + private$token <- token + self$rest_api_url <- rest_api_url + } + ) +) + #' @noRd create_testrest <- function(rest_api_url = "https://api.github.com", token, diff --git a/man/EngineGraphQL.Rd b/man/EngineGraphQL.Rd index ac593d1d..da8c6b09 100644 --- a/man/EngineGraphQL.Rd +++ b/man/EngineGraphQL.Rd @@ -30,7 +30,7 @@ A class for methods wrapping GitHub's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQL} object. \subsection{Usage}{ -\if{html}{\out{