From 714e1cdf8cb953b82a8119f5969cdf0b029c76ec Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Tue, 22 Aug 2023 13:01:03 +0000 Subject: [PATCH 1/9] Develop pulling all groups for GitLab. --- R/EngineGraphQL.R | 8 +- R/EngineGraphQLGitHub.R | 7 +- R/EngineGraphQLGitLab.R | 31 +++++- R/EngineRest.R | 26 +++-- R/EngineRestGitHub.R | 11 ++- R/EngineRestGitLab.R | 19 +++- R/GQLQueryGitLab.R | 18 ++++ R/GitHost.R | 125 ++++++++++++++++++------ R/GitStats.R | 15 ++- R/gitstats_functions.R | 10 +- man/EngineGraphQL.Rd | 4 +- man/EngineGraphQLGitHub.Rd | 4 +- man/EngineGraphQLGitLab.Rd | 15 ++- man/EngineRest.Rd | 4 +- man/EngineRestGitHub.Rd | 4 +- man/EngineRestGitLab.Rd | 4 +- man/GQLQueryGitLab.Rd | 14 +++ man/set_connection.Rd | 10 +- tests/testthat/_snaps/set_connection.md | 2 +- tests/testthat/test-GitHost.R | 34 +++++++ 20 files changed, 296 insertions(+), 69 deletions(-) diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R index 12c7664e..b1333a5a 100644 --- a/R/EngineGraphQL.R +++ b/R/EngineGraphQL.R @@ -15,10 +15,13 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL", #' @description Create `EngineGraphQL` object. #' @param gql_api_url GraphQL API url. #' @param token A token. + #' @param scan_whole_host A boolean. initialize = function(gql_api_url, - token) { + token, + scan_whole_host) { self$gql_api_url <- gql_api_url private$token <- token + private$scan_whole_host <- scan_whole_host }, #' @description Wrapper of GraphQL API request and response. @@ -49,6 +52,9 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL", # @field token A token authorizing access to API. token = NULL, + # @field A boolean. + scan_whole_host = FALSE, + # @description A method to pull information on user. # @param username A login. # @return A user response. diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 000b9521..e162cc96 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -10,10 +10,13 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", #' @description Create `EngineGraphQLGitHub` object. #' @param gql_api_url GraphQL API url. #' @param token A token. + #' @param scan_whole_host A boolean. initialize = function(gql_api_url, - token) { + token, + scan_whole_host) { super$initialize(gql_api_url = gql_api_url, - token = token) + token = token, + scan_whole_host = scan_whole_host) self$gql_query <- GQLQueryGitHub$new() }, diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 5b0fb764..25c59cb0 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -9,13 +9,36 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab", #' @description Create `EngineGraphQLGitLab` object. #' @param gql_api_url GraphQL API url. #' @param token A token. + #' @param scan_whole_host A boolean. initialize = function(gql_api_url, - token) { + token, + scan_whole_host) { super$initialize(gql_api_url = gql_api_url, - token = token) + token = token, + scan_whole_host = scan_whole_host) self$gql_query <- GQLQueryGitLab$new() }, + #' @description Get all groups from GitLab. + get_orgs = function() { + group_cursor <- "" + has_next_page <- TRUE + full_orgs_list <- list() + i <- 1 + while(has_next_page) { + response <- self$gql_response( + gql_query = self$gql_query$groups(), + vars = list("groupCursor" = group_cursor) + ) + orgs_list <- purrr::map(response$data$groups$edges, ~.$node$fullPath) + full_orgs_list <- append(full_orgs_list, orgs_list) + has_next_page <- response$data$groups$pageInfo$hasNextPage + group_cursor <- response$data$groups$pageInfo$endCursor + } + all_orgs <- unlist(full_orgs_list) + return(all_orgs) + }, + #' @description A method to retrieve all repositories for an organization in #' a table format. #' @param org An organization. @@ -24,7 +47,9 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab", get_repos = function(org, settings) { if (settings$search_param == "org") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...") + if (!private$scan_whole_host) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...") + } repos_table <- private$pull_repos( from = "org", org = org diff --git a/R/EngineRest.R b/R/EngineRest.R index e0476129..56b28b17 100644 --- a/R/EngineRest.R +++ b/R/EngineRest.R @@ -13,11 +13,14 @@ EngineRest <- R6::R6Class("EngineRest", #' @description Create a new `Rest` object #' @param rest_api_url A character, url of Rest API. #' @param token A token. + #' @param scan_whole_host A boolean. #' @return A `Rest` object. initialize = function(rest_api_url = NA, - token = NA) { + token = NA, + scan_whole_host) { self$rest_api_url <- rest_api_url private$token <- token + private$scan_whole_host <- scan_whole_host }, #' @description A wrapper for httr2 functions to perform get request to REST API endpoints. @@ -41,6 +44,9 @@ EngineRest <- R6::R6Class("EngineRest", # @field token A token authorizing access to API. token = NULL, + # @field A boolean. + scan_whole_host = FALSE, + # @description Check whether the token exists. # @param token A token. # @return A token. @@ -91,14 +97,16 @@ EngineRest <- R6::R6Class("EngineRest", }, error = function(e) { if (!is.null(e$status)) { - if (e$status == 400) { - message("HTTP 400 Bad Request.") - } else if (e$status == 401) { - message("HTTP 401 Unauthorized.") - } else if (e$status == 403) { - message("HTTP 403 API limit reached.") - } else if (e$status == 404) { - message("HTTP 404 No such address") + if (!private$scan_whole_host) { + if (e$status == 400) { + message("HTTP 400 Bad Request.") + } else if (e$status == 401) { + message("HTTP 401 Unauthorized.") + } else if (e$status == 403) { + message("HTTP 403 API limit reached.") + } else if (e$status == 404) { + message("HTTP 404 No such address") + } } } else if (grepl("Could not resolve host", e)) { cli::cli_abort(c( diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index 1a1d2cdc..ac3202e8 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -7,11 +7,14 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", #' @description Create new `EngineRestGitHub` object. #' @param rest_api_url A REST API url. #' @param token A token. + #' @param scan_whole_host A boolean. initialize = function(rest_api_url, - token) { + token, + scan_whole_host) { super$initialize( rest_api_url = rest_api_url, - token = private$check_token(token) + token = private$check_token(token), + scan_whole_host = scan_whole_host ) }, @@ -73,7 +76,9 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", get_repos_supportive = function(org, settings) { if (settings$search_param %in% c("org")) { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") + if (!private$scan_whole_host) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") + } repos_table <- private$pull_repos_from_org( org = org ) %>% diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 85ac17f9..aaae14f2 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -7,11 +7,14 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", #' @description Create new `EngineRestGitLab` object. #' @param rest_api_url A REST API url. #' @param token A token. + #' @param scan_whole_host A boolean. initialize = function(rest_api_url, - token) { + token, + scan_whole_host) { super$initialize( rest_api_url = rest_api_url, - token = private$check_token(token) + token = private$check_token(token), + scan_whole_host = scan_whole_host ) }, @@ -53,7 +56,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", get_repos = function(org, settings) { if (settings$search_param == "phrase") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][phrase:{settings$phrase}][org:{org}] Searching repositories...") + if (!private$scan_whole_host) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][phrase:{settings$phrase}][org:{org}] Searching repositories...") + } repos_table <- private$search_repos_by_phrase( org = org, phrase = settings$phrase, @@ -63,7 +68,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", private$prepare_repos_table() %>% private$add_repos_issues() } else if (settings$search_param == "team") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling repositories...") + if (!private$scan_whole_host) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling repositories...") + } org <- private$get_group_id(org) repos_table <- private$pull_repos_from_org(org) %>% private$tailor_repos_info() %>% @@ -88,7 +95,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", get_repos_supportive = function(org, settings) { if (settings$search_param == "org") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") + if (!private$scan_whole_host) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") + } org <- private$get_group_id(org) repos_table <- private$pull_repos_from_org(org) %>% private$tailor_repos_info() %>% diff --git a/R/GQLQueryGitLab.R b/R/GQLQueryGitLab.R index 31b5ae92..2f2a3a33 100644 --- a/R/GQLQueryGitLab.R +++ b/R/GQLQueryGitLab.R @@ -4,6 +4,24 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab", public = list( + #' @description Prepare query to list groups from GitLab. + #' @return A query. + groups = function() { + 'query GetGroups($groupCursor: String!) { + groups (after: $groupCursor) { + pageInfo { + endCursor + hasNextPage + } + edges { + node { + fullPath + } + } + } + }' + }, + #' @description Prepare query to get repositories from GitLab. #' @param repo_cursor An end cursor for repositories page. #' @return A query. diff --git a/R/GitHost.R b/R/GitHost.R index 31aeb0ec..11f7c547 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -18,31 +18,58 @@ GitHost <- R6::R6Class("GitHost", token = NA, api_url = NA) { private$api_url <- api_url + private$is_public <- private$check_if_public(api_url) + private$host <- private$set_host_name(api_url) if (is.null(token)){ token <- private$set_default_token() } + if (is.null(orgs)) { + if (private$is_public) { + cli::cli_abort( + "You need to specify `orgs` for public Git platform." + ) + } else { + cli::cli_alert_warning(cli::col_yellow( + "No `orgs` specified. I will pull all organizations from the Git Host." + )) + private$scan_whole_host <- TRUE + } + } else { + private$orgs <- private$engines$rest$check_organizations(orgs) + } if (grepl("https://", api_url) && grepl("github", api_url)) { private$engines$rest <- EngineRestGitHub$new( token = token, - rest_api_url = api_url + rest_api_url = api_url, + scan_whole_host = private$scan_whole_host ) private$engines$graphql <- EngineGraphQLGitHub$new( token = token, - gql_api_url = private$set_gql_url(api_url) + gql_api_url = private$set_gql_url(api_url), + scan_whole_host = private$scan_whole_host ) } else if (grepl("https://", api_url) && grepl("gitlab|code", api_url)) { private$engines$rest <- EngineRestGitLab$new( token = token, - rest_api_url = api_url + rest_api_url = api_url, + scan_whole_host = private$scan_whole_host ) private$engines$graphql <- EngineGraphQLGitLab$new( token = token, - gql_api_url = private$set_gql_url(api_url) + gql_api_url = private$set_gql_url(api_url), + scan_whole_host = private$scan_whole_host ) } else { stop("This connection is not supported by GitStats class object.") } - private$orgs <- private$engines$rest$check_organizations(orgs) + if (private$scan_whole_host) { + cli::cli_alert_info("[{private$host}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") + if (private$host == "GitLab") { + private$orgs <- private$engines$graphql$get_orgs() + } else { + # private$orgs <- private$engines$rest$get_orgs() + } + } }, #' @description A method to list all repositories for an organization, a @@ -52,32 +79,9 @@ GitHost <- R6::R6Class("GitHost", #' column to repositories table. #' @return A data.frame of repositories. get_repos = function(settings, add_contributors = FALSE) { - repos_table <- purrr::map(private$orgs, function(org) { - tryCatch({ - repos_list <- purrr::map(private$engines, function (engine) { - engine$get_repos( - org = org, - settings = settings - ) - }) - }, - error = function(e) { - if (grepl("502", e)) { - cli::cli_alert_warning(cli::col_yellow("HTTP 502 Bad Gateway Error. Switch to another Engine.")) - repos_list <<- purrr::map(private$engines, function (engine) { - engine$get_repos_supportive( - org = org, - settings = settings - ) - }) - } else { - e - } - }) - repos_table_org <- purrr::list_rbind(repos_list) - return(repos_table_org) - }) %>% - purrr::list_rbind() + repos_table <- private$pull_repos_from_orgs( + settings = settings + ) if (settings$search_param == "team") { add_contributors <- TRUE @@ -187,12 +191,39 @@ GitHost <- R6::R6Class("GitHost", # @field A REST API url. api_url = NULL, + # @field public A boolean. + is_public = NULL, + + # @field Host name. + host = NULL, + # @field orgs A character vector of repo organizations. orgs = NULL, + # @field A boolean. + scan_whole_host = FALSE, + # @field engines A placeholder for REST and GraphQL Engine classes. engines = list(), + # @description Check whether Git platform is public or internal. + check_if_public = function(api_url) { + if (grepl("api.github.com|gitlab.com/api", api_url)) { + TRUE + } else { + FALSE + } + }, + + # @description Set name of a Git Host. + set_host_name = function(api_url) { + if (grepl("github", api_url)) { + "GitHub" + } else { + "GitLab" + } + }, + # @description Set default token if none exists. set_default_token = function() { if (grepl("github", private$api_url)) { @@ -242,6 +273,38 @@ GitHost <- R6::R6Class("GitHost", paste0(gsub("/v+.*", "", rest_api_url), "/graphql") }, + # @description Pull repositories from organisations. + pull_repos_from_orgs = function(settings) { + repos_table <- purrr::map(private$orgs, function(org) { + tryCatch({ + repos_list <- purrr::map(private$engines, function (engine) { + engine$get_repos( + org = org, + settings = settings + ) + }) + }, + error = function(e) { + if (!private$scan_whole_host) { + if (grepl("502", e)) { + cli::cli_alert_warning(cli::col_yellow("HTTP 502 Bad Gateway Error. Switch to another Engine.")) + } else { + cli::cli_alert_warning(cli::col_yellow("Error. Switch to another Engine.")) + } + } + repos_list <<- purrr::map(private$engines, function (engine) { + engine$get_repos_supportive( + org = org, + settings = settings + ) + }) + }) + repos_table_org <- purrr::list_rbind(repos_list) + return(repos_table_org) + }, .progress = private$scan_whole_host) %>% + purrr::list_rbind() + }, + # @description Filter repositories by contributors. # @details If at least one member of a team is a contributor than a project # passes through the filter. diff --git a/R/GitStats.R b/R/GitStats.R index ab55174d..331ee0a0 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -242,8 +242,7 @@ GitStats <- R6::R6Class("GitStats", orgs <- purrr::map(private$hosts, function(host) { host_priv <- environment(host$initialize)$private orgs <- host_priv$orgs - paste0(orgs, collapse = ", ") - }) %>% paste0(collapse = ", ") + }) private$print_item("Organisations", orgs) private$print_item("Search preference", private$settings$search_param) private$print_item("Team", private$settings$team_name, paste0(private$settings$team_name, " (", length(private$settings$team), " members)")) @@ -329,6 +328,18 @@ GitStats <- R6::R6Class("GitStats", print_item = function(item_name, item_to_check, item_to_print = item_to_check) { + if (item_name == "Organisations") { + item_to_print <- unlist(item_to_print) + if (length(item_to_print) < 10) { + list_items <- paste0(item_to_print, collapse = ", ") + + } else { + item_to_print_cut <- item_to_print[1:10] + list_items <- paste0(item_to_print_cut, collapse = ", ") %>% + paste0("... and ", length(item_to_print) - 10, " more") + } + item_to_print <- paste0("[", cli::col_green(length(item_to_print)), "] ", list_items) + } cat(paste0( cli::col_blue(paste0(item_name, ": ")), ifelse(is.null(item_to_check), diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index c13523c4..1b0af12d 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -13,8 +13,12 @@ create_gitstats <- function() { #' @param gitstats_obj A GitStats object. #' @param api_url A character, url address of API. #' @param token A token. -#' @param orgs A character vector of organisations (owners of repositories -#' in case of GitHub and groups of projects in case of GitLab). +#' @param orgs A character vector of organisations (owners of repositories in +#' case of GitHub and groups of projects in case of GitLab). You do not need +#' to define `orgs` if you wish to scan whole internal Git platform (such as +#' enterprise version of GitHub or GitLab). In case of a public one (like +#' GitHub) you need to define `orgs` as scanning through all organizations +#' would be an overkill. #' @return A `GitStats` class object with added information on connection #' (`$hosts` field). #' @examples @@ -34,7 +38,7 @@ create_gitstats <- function() { set_connection <- function(gitstats_obj, api_url, token = NULL, - orgs) { + orgs = NULL) { gitstats_obj$add_host( api_url = api_url, token = token, diff --git a/man/EngineGraphQL.Rd b/man/EngineGraphQL.Rd index ac593d1d..55869a42 100644 --- a/man/EngineGraphQL.Rd +++ b/man/EngineGraphQL.Rd @@ -30,7 +30,7 @@ A class for methods wrapping GitHub's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQL} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineGraphQL$new(gql_api_url, token)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineGraphQL$new(gql_api_url, token, scan_whole_host)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -39,6 +39,8 @@ Create \code{EngineGraphQL} object. \item{\code{gql_api_url}}{GraphQL API url.} \item{\code{token}}{A token.} + +\item{\code{scan_whole_host}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/EngineGraphQLGitHub.Rd b/man/EngineGraphQLGitHub.Rd index 1b23ccd2..daede726 100644 --- a/man/EngineGraphQLGitHub.Rd +++ b/man/EngineGraphQLGitHub.Rd @@ -34,7 +34,7 @@ A class for methods wrapping GitHub's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQLGitHub} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineGraphQLGitHub$new(gql_api_url, token)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineGraphQLGitHub$new(gql_api_url, token, scan_whole_host)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -43,6 +43,8 @@ Create \code{EngineGraphQLGitHub} object. \item{\code{gql_api_url}}{GraphQL API url.} \item{\code{token}}{A token.} + +\item{\code{scan_whole_host}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/EngineGraphQLGitLab.Rd b/man/EngineGraphQLGitLab.Rd index 70de5164..bb9ac35a 100644 --- a/man/EngineGraphQLGitLab.Rd +++ b/man/EngineGraphQLGitLab.Rd @@ -13,6 +13,7 @@ A class for methods wrapping GitLab's GraphQL API responses. \subsection{Public methods}{ \itemize{ \item \href{#method-new}{\code{EngineGraphQLGitLab$new()}} +\item \href{#method-get_orgs}{\code{EngineGraphQLGitLab$get_orgs()}} \item \href{#method-get_repos}{\code{EngineGraphQLGitLab$get_repos()}} \item \href{#method-get_repos_supportive}{\code{EngineGraphQLGitLab$get_repos_supportive()}} \item \href{#method-get_commits}{\code{EngineGraphQLGitLab$get_commits()}} @@ -33,7 +34,7 @@ A class for methods wrapping GitLab's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQLGitLab} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineGraphQLGitLab$new(gql_api_url, token)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineGraphQLGitLab$new(gql_api_url, token, scan_whole_host)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -42,9 +43,21 @@ Create \code{EngineGraphQLGitLab} object. \item{\code{gql_api_url}}{GraphQL API url.} \item{\code{token}}{A token.} + +\item{\code{scan_whole_host}}{A boolean.} } \if{html}{\out{}} } +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-get_orgs}{}}} +\subsection{Method \code{get_orgs()}}{ +Get all groups from GitLab. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{EngineGraphQLGitLab$get_orgs()}\if{html}{\out{
}} +} + } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/EngineRest.Rd b/man/EngineRest.Rd index c94af738..80e6e97c 100644 --- a/man/EngineRest.Rd +++ b/man/EngineRest.Rd @@ -27,7 +27,7 @@ A superclass for methods wrapping Rest API responses. \subsection{Method \code{new()}}{ Create a new \code{Rest} object \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRest$new(rest_api_url = NA, token = NA)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineRest$new(rest_api_url = NA, token = NA, scan_whole_host)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -36,6 +36,8 @@ Create a new \code{Rest} object \item{\code{rest_api_url}}{A character, url of Rest API.} \item{\code{token}}{A token.} + +\item{\code{scan_whole_host}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/EngineRestGitHub.Rd b/man/EngineRestGitHub.Rd index 5c2a0d4f..b5917f8d 100644 --- a/man/EngineRestGitHub.Rd +++ b/man/EngineRestGitHub.Rd @@ -35,7 +35,7 @@ A class for methods wrapping GitHub's REST API responses. \subsection{Method \code{new()}}{ Create new \code{EngineRestGitHub} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRestGitHub$new(rest_api_url, token)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineRestGitHub$new(rest_api_url, token, scan_whole_host)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -44,6 +44,8 @@ Create new \code{EngineRestGitHub} object. \item{\code{rest_api_url}}{A REST API url.} \item{\code{token}}{A token.} + +\item{\code{scan_whole_host}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/EngineRestGitLab.Rd b/man/EngineRestGitLab.Rd index feec80a5..9b514839 100644 --- a/man/EngineRestGitLab.Rd +++ b/man/EngineRestGitLab.Rd @@ -34,7 +34,7 @@ A class for methods wrapping GitLab's REST API responses. \subsection{Method \code{new()}}{ Create new \code{EngineRestGitLab} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRestGitLab$new(rest_api_url, token)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineRestGitLab$new(rest_api_url, token, scan_whole_host)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -43,6 +43,8 @@ Create new \code{EngineRestGitLab} object. \item{\code{rest_api_url}}{A REST API url.} \item{\code{token}}{A token.} + +\item{\code{scan_whole_host}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/GQLQueryGitLab.Rd b/man/GQLQueryGitLab.Rd index 5443603f..f3087e53 100644 --- a/man/GQLQueryGitLab.Rd +++ b/man/GQLQueryGitLab.Rd @@ -9,12 +9,26 @@ A class with methods to build GraphQL Queries for GitLab. \section{Methods}{ \subsection{Public methods}{ \itemize{ +\item \href{#method-groups}{\code{GQLQueryGitLab$groups()}} \item \href{#method-repos_by_org}{\code{GQLQueryGitLab$repos_by_org()}} \item \href{#method-user}{\code{GQLQueryGitLab$user()}} \item \href{#method-clone}{\code{GQLQueryGitLab$clone()}} } } \if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-groups}{}}} +\subsection{Method \code{groups()}}{ +Prepare query to list groups from GitLab. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{GQLQueryGitLab$groups()}\if{html}{\out{
}} +} + +\subsection{Returns}{ +A query. +} +} +\if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-repos_by_org}{}}} \subsection{Method \code{repos_by_org()}}{ diff --git a/man/set_connection.Rd b/man/set_connection.Rd index a5863594..b0982d9c 100644 --- a/man/set_connection.Rd +++ b/man/set_connection.Rd @@ -4,7 +4,7 @@ \alias{set_connection} \title{Setting connections} \usage{ -set_connection(gitstats_obj, api_url, token = NULL, orgs) +set_connection(gitstats_obj, api_url, token = NULL, orgs = NULL) } \arguments{ \item{gitstats_obj}{A GitStats object.} @@ -13,8 +13,12 @@ set_connection(gitstats_obj, api_url, token = NULL, orgs) \item{token}{A token.} -\item{orgs}{A character vector of organisations (owners of repositories -in case of GitHub and groups of projects in case of GitLab).} +\item{orgs}{A character vector of organisations (owners of repositories in +case of GitHub and groups of projects in case of GitLab). You do not need +to define \code{orgs} if you wish to scan whole internal Git platform (such as +enterprise version of GitHub or GitLab). In case of a public one (like +GitHub) you need to define \code{orgs} as scanning through all organizations +would be an overkill.} } \value{ A \code{GitStats} class object with added information on connection diff --git a/tests/testthat/_snaps/set_connection.md b/tests/testthat/_snaps/set_connection.md index eec10bba..88d1189f 100644 --- a/tests/testthat/_snaps/set_connection.md +++ b/tests/testthat/_snaps/set_connection.md @@ -40,7 +40,7 @@ test_gitstats %>% set_connection(api_url = "https://api.github.com", token = Sys.getenv( "GITHUB_PAT")) Message - ! argument "orgs" is missing, with no default + ! You need to specify `orgs` for public Git platform. x Host will not be passed. # Warning shows, when wrong input is passed when setting connection and host is not passed diff --git a/tests/testthat/test-GitHost.R b/tests/testthat/test-GitHost.R index 3247d782..ebb1d3df 100644 --- a/tests/testthat/test-GitHost.R +++ b/tests/testthat/test-GitHost.R @@ -22,6 +22,40 @@ test_host <- create_testhost( mode = "private" ) +test_that("`check_if_public` checks correctly if Git Platform is public or not", { + expect_true( + test_host$check_if_public("https://api.github.com") + ) + expect_true( + test_host$check_if_public("https://gitlab.com/api/v4") + ) + expect_false( + test_host$check_if_public("https://code.internal.com/api/v4") + ) + expect_false( + test_host$check_if_public("https://github.internal.com/api/v4") + ) +}) + +test_that("`set_host_name` checks correctly if Git Platform is public or not", { + expect_equal( + test_host$set_host_name("https://api.github.com"), + "GitHub" + ) + expect_equal( + test_host$set_host_name("https://gitlab.com/api/v4"), + "GitLab" + ) + expect_equal( + test_host$set_host_name("https://code.internal.com/api/v4"), + "GitLab" + ) + expect_equal( + test_host$set_host_name("https://github.internal.com/api/v4"), + "GitHub" + ) +}) + test_that("`set_default_token` sets default token for public GitHub", { expect_snapshot( default_token <- test_host$set_default_token() From 7beb934a10b5e08ebb4dff577d977ca0c492b966 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 23 Aug 2023 09:35:06 +0000 Subject: [PATCH 2/9] Add pulling all orgs from internal GitHub. --- R/EngineGraphQL.R | 8 ++-- R/EngineGraphQLGitHub.R | 34 ++++++++++++-- R/EngineGraphQLGitLab.R | 9 ++-- R/EngineRest.R | 10 ++-- R/EngineRestGitHub.R | 17 +++---- R/EngineRestGitLab.R | 13 +++--- R/GQLQueryGitHub.R | 29 ++++++++++++ R/GitHost.R | 96 +++++++++++++++++++++----------------- R/GitStats.R | 2 +- man/EngineGraphQL.Rd | 4 +- man/EngineGraphQLGitHub.Rd | 15 +++++- man/EngineGraphQLGitLab.Rd | 4 +- man/EngineRest.Rd | 4 +- man/EngineRestGitHub.Rd | 4 +- man/EngineRestGitLab.Rd | 4 +- man/GQLQueryGitHub.Rd | 21 +++++++++ 16 files changed, 186 insertions(+), 88 deletions(-) diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R index b1333a5a..34c59eab 100644 --- a/R/EngineGraphQL.R +++ b/R/EngineGraphQL.R @@ -15,13 +15,13 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL", #' @description Create `EngineGraphQL` object. #' @param gql_api_url GraphQL API url. #' @param token A token. - #' @param scan_whole_host A boolean. + #' @param scan_all A boolean. initialize = function(gql_api_url, token, - scan_whole_host) { + scan_all) { self$gql_api_url <- gql_api_url private$token <- token - private$scan_whole_host <- scan_whole_host + private$scan_all <- scan_all }, #' @description Wrapper of GraphQL API request and response. @@ -53,7 +53,7 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL", token = NULL, # @field A boolean. - scan_whole_host = FALSE, + scan_all = FALSE, # @description A method to pull information on user. # @param username A login. diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index e162cc96..5ec25ae0 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -10,16 +10,36 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", #' @description Create `EngineGraphQLGitHub` object. #' @param gql_api_url GraphQL API url. #' @param token A token. - #' @param scan_whole_host A boolean. + #' @param scan_all A boolean. initialize = function(gql_api_url, token, - scan_whole_host) { + scan_all) { super$initialize(gql_api_url = gql_api_url, token = token, - scan_whole_host = scan_whole_host) + scan_all = scan_all) self$gql_query <- GQLQueryGitHub$new() }, + #' @description Get all groups from GitLab. + get_orgs = function() { + end_cursor <- NULL + has_next_page <- TRUE + full_orgs_list <- list() + while(has_next_page) { + response <- self$gql_response( + gql_query = self$gql_query$orgs( + end_cursor = end_cursor + ) + ) + orgs_list <- purrr::map(response$data$search$edges, ~stringr::str_match(.$node$url, "[^\\/]*$")) + full_orgs_list <- append(full_orgs_list, orgs_list) + has_next_page <- response$data$search$pageInfo$hasNextPage + end_cursor <- response$data$search$pageInfo$endCursor + } + all_orgs <- unlist(full_orgs_list) + return(all_orgs) + }, + #' @description A method to retrieve all repositories for an organization in #' a table format. #' @param org An organization. @@ -29,14 +49,18 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", settings) { if (settings$search_param %in% c("org", "team")) { if (settings$search_param == "org") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...") + } repos_table <- private$pull_repos( from = "org", org = org ) %>% private$prepare_repos_table() } else { - cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling repositories...") + } repos_table <- private$pull_repos_from_team( team = settings$team ) %>% diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 25c59cb0..a719ca2e 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -9,13 +9,13 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab", #' @description Create `EngineGraphQLGitLab` object. #' @param gql_api_url GraphQL API url. #' @param token A token. - #' @param scan_whole_host A boolean. + #' @param scan_all A boolean. initialize = function(gql_api_url, token, - scan_whole_host) { + scan_all) { super$initialize(gql_api_url = gql_api_url, token = token, - scan_whole_host = scan_whole_host) + scan_all = scan_all) self$gql_query <- GQLQueryGitLab$new() }, @@ -24,7 +24,6 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab", group_cursor <- "" has_next_page <- TRUE full_orgs_list <- list() - i <- 1 while(has_next_page) { response <- self$gql_response( gql_query = self$gql_query$groups(), @@ -47,7 +46,7 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab", get_repos = function(org, settings) { if (settings$search_param == "org") { - if (!private$scan_whole_host) { + if (!private$scan_all) { cli::cli_alert_info("[GitLab][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling repositories...") } repos_table <- private$pull_repos( diff --git a/R/EngineRest.R b/R/EngineRest.R index 56b28b17..41a5f087 100644 --- a/R/EngineRest.R +++ b/R/EngineRest.R @@ -13,14 +13,14 @@ EngineRest <- R6::R6Class("EngineRest", #' @description Create a new `Rest` object #' @param rest_api_url A character, url of Rest API. #' @param token A token. - #' @param scan_whole_host A boolean. + #' @param scan_all A boolean. #' @return A `Rest` object. initialize = function(rest_api_url = NA, token = NA, - scan_whole_host) { + scan_all) { self$rest_api_url <- rest_api_url private$token <- token - private$scan_whole_host <- scan_whole_host + private$scan_all <- scan_all }, #' @description A wrapper for httr2 functions to perform get request to REST API endpoints. @@ -45,7 +45,7 @@ EngineRest <- R6::R6Class("EngineRest", token = NULL, # @field A boolean. - scan_whole_host = FALSE, + scan_all = FALSE, # @description Check whether the token exists. # @param token A token. @@ -97,7 +97,7 @@ EngineRest <- R6::R6Class("EngineRest", }, error = function(e) { if (!is.null(e$status)) { - if (!private$scan_whole_host) { + if (!private$scan_all) { if (e$status == 400) { message("HTTP 400 Bad Request.") } else if (e$status == 401) { diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index ac3202e8..62045178 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -7,14 +7,14 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", #' @description Create new `EngineRestGitHub` object. #' @param rest_api_url A REST API url. #' @param token A token. - #' @param scan_whole_host A boolean. + #' @param scan_all A boolean. initialize = function(rest_api_url, token, - scan_whole_host) { + scan_all) { super$initialize( rest_api_url = rest_api_url, token = private$check_token(token), - scan_whole_host = scan_whole_host + scan_all = scan_all ) }, @@ -29,8 +29,8 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", self$response(endpoint = paste0(self$rest_api_url, org_endpoint, org)) }, message = function(m) { - if (grepl("404", m)) { - cli::cli_alert_danger("Organization you provided does not exist. Check spelling in: {org}") + if (grepl("40", m)) { + cli::cli_abort("Improper name of organization.") org <<- NULL } } @@ -75,8 +75,9 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", #' @return A table of repositories. get_repos_supportive = function(org, settings) { + repos_table <- NULL if (settings$search_param %in% c("org")) { - if (!private$scan_whole_host) { + if (!private$scan_all) { cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") } repos_table <- private$pull_repos_from_org( @@ -333,8 +334,8 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", "name" = repo$name, "stars" = repo$stargazers_count, "forks" = repo$forks_count, - "created_at" = repo$created_at, - "last_activity_at" = repo$pushed_at, + "created_at" = gts_to_posixt(repo$created_at), + "last_activity_at" = if (!is.null(repo$pushed_at)) gts_to_posixt(repo$pushed_at) else gts_to_posixt(repo$created_at), "languages" = repo$language, "issues_open" = repo$issues_open, "issues_closed" = repo$issues_closed, diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index aaae14f2..499da67f 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -7,14 +7,14 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", #' @description Create new `EngineRestGitLab` object. #' @param rest_api_url A REST API url. #' @param token A token. - #' @param scan_whole_host A boolean. + #' @param scan_all A boolean. initialize = function(rest_api_url, token, - scan_whole_host) { + scan_all) { super$initialize( rest_api_url = rest_api_url, token = private$check_token(token), - scan_whole_host = scan_whole_host + scan_all = scan_all ) }, @@ -56,7 +56,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", get_repos = function(org, settings) { if (settings$search_param == "phrase") { - if (!private$scan_whole_host) { + if (!private$scan_all) { cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][phrase:{settings$phrase}][org:{org}] Searching repositories...") } repos_table <- private$search_repos_by_phrase( @@ -68,7 +68,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", private$prepare_repos_table() %>% private$add_repos_issues() } else if (settings$search_param == "team") { - if (!private$scan_whole_host) { + if (!private$scan_all) { cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling repositories...") } org <- private$get_group_id(org) @@ -94,8 +94,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", #' @return Nothing. get_repos_supportive = function(org, settings) { + repos_table <- NULL if (settings$search_param == "org") { - if (!private$scan_whole_host) { + if (!private$scan_all) { cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}] Pulling repositories...") } org <- private$get_group_id(org) diff --git a/R/GQLQueryGitHub.R b/R/GQLQueryGitHub.R index e8e0e9f3..0ccc10ce 100644 --- a/R/GQLQueryGitHub.R +++ b/R/GQLQueryGitHub.R @@ -4,6 +4,35 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub", public = list( + #' @description Prepare query to list organizations from GitHub. + #' @param end_cursor An end cursor to paginate. + #' @return A query. + orgs = function(end_cursor) { + if (is.null(end_cursor)) { + pagination_phrase <- '' + } else { + pagination_phrase <- paste0('after: "', end_cursor, '"') + } + + paste0( + 'query { + search(first: 100, type: USER, query: "type:org" ', pagination_phrase, ') { + pageInfo { + hasNextPage + endCursor + } + edges { + node{ + ... on Organization { + name + url + } + } + } + } + }') + }, + #' @description Prepare query to get repositories from GitHub. #' @param org An organization of repositories. #' @param repo_cursor An end cursor for repositories page. diff --git a/R/GitHost.R b/R/GitHost.R index 11f7c547..57faeeda 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -19,9 +19,9 @@ GitHost <- R6::R6Class("GitHost", api_url = NA) { private$api_url <- api_url private$is_public <- private$check_if_public(api_url) - private$host <- private$set_host_name(api_url) + private$host <- private$set_host(api_url) if (is.null(token)){ - token <- private$set_default_token() + private$token <- private$set_default_token() } if (is.null(orgs)) { if (private$is_public) { @@ -32,43 +32,16 @@ GitHost <- R6::R6Class("GitHost", cli::cli_alert_warning(cli::col_yellow( "No `orgs` specified. I will pull all organizations from the Git Host." )) - private$scan_whole_host <- TRUE + private$scan_all <- TRUE } - } else { - private$orgs <- private$engines$rest$check_organizations(orgs) } - if (grepl("https://", api_url) && grepl("github", api_url)) { - private$engines$rest <- EngineRestGitHub$new( - token = token, - rest_api_url = api_url, - scan_whole_host = private$scan_whole_host - ) - private$engines$graphql <- EngineGraphQLGitHub$new( - token = token, - gql_api_url = private$set_gql_url(api_url), - scan_whole_host = private$scan_whole_host - ) - } else if (grepl("https://", api_url) && grepl("gitlab|code", api_url)) { - private$engines$rest <- EngineRestGitLab$new( - token = token, - rest_api_url = api_url, - scan_whole_host = private$scan_whole_host - ) - private$engines$graphql <- EngineGraphQLGitLab$new( - token = token, - gql_api_url = private$set_gql_url(api_url), - scan_whole_host = private$scan_whole_host - ) - } else { - stop("This connection is not supported by GitStats class object.") - } - if (private$scan_whole_host) { + private$engines$rest <- private$setup_engine(type = "rest") + private$engines$graphql <- private$setup_engine(type = "graphql") + if (private$scan_all) { cli::cli_alert_info("[{private$host}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") - if (private$host == "GitLab") { - private$orgs <- private$engines$graphql$get_orgs() - } else { - # private$orgs <- private$engines$rest$get_orgs() - } + private$orgs <- private$engines$graphql$get_orgs() + } else { + private$orgs <- private$engines$rest$check_organizations(orgs) } }, @@ -191,6 +164,9 @@ GitHost <- R6::R6Class("GitHost", # @field A REST API url. api_url = NULL, + # @field A token. + token = NULL, + # @field public A boolean. is_public = NULL, @@ -201,7 +177,7 @@ GitHost <- R6::R6Class("GitHost", orgs = NULL, # @field A boolean. - scan_whole_host = FALSE, + scan_all = FALSE, # @field engines A placeholder for REST and GraphQL Engine classes. engines = list(), @@ -216,11 +192,13 @@ GitHost <- R6::R6Class("GitHost", }, # @description Set name of a Git Host. - set_host_name = function(api_url) { - if (grepl("github", api_url)) { + set_host = function(api_url) { + if (grepl("https://", private$api_url) && grepl("github", private$api_url)) { "GitHub" - } else { + } else if (grepl("https://", private$api_url) && grepl("gitlab|code", private$api_url)) { "GitLab" + } else { + stop("This connection is not supported by GitStats class object.") } }, @@ -248,6 +226,40 @@ GitHost <- R6::R6Class("GitHost", return(token) }, + # Setup REST and GraphQL engines + setup_engine = function(type) { + engine <- if (private$host == "GitHub") { + if (type == "rest") { + EngineRestGitHub$new( + rest_api_url = private$api_url, + token = private$token, + scan_all = private$scan_all + ) + } else { + EngineGraphQLGitHub$new( + gql_api_url = private$set_gql_url(private$api_url), + token = private$token, + scan_all = private$scan_all + ) + } + } else { + if (type == "rest") { + EngineRestGitLab$new( + rest_api_url = private$api_url, + token = private$token, + scan_all = private$scan_all + ) + } else { + EngineGraphQLGitLab$new( + gql_api_url = private$set_gql_url(private$api_url), + token = private$token, + scan_all = private$scan_all + ) + } + } + return(engine) + }, + # @description Helper to test if a token works test_token = function(token) { response <- NULL @@ -285,7 +297,7 @@ GitHost <- R6::R6Class("GitHost", }) }, error = function(e) { - if (!private$scan_whole_host) { + if (!private$scan_all) { if (grepl("502", e)) { cli::cli_alert_warning(cli::col_yellow("HTTP 502 Bad Gateway Error. Switch to another Engine.")) } else { @@ -301,7 +313,7 @@ GitHost <- R6::R6Class("GitHost", }) repos_table_org <- purrr::list_rbind(repos_list) return(repos_table_org) - }, .progress = private$scan_whole_host) %>% + }, .progress = private$scan_all) %>% purrr::list_rbind() }, diff --git a/R/GitStats.R b/R/GitStats.R index 331ee0a0..11b203ec 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -95,7 +95,7 @@ GitStats <- R6::R6Class("GitStats", } }, error = function(e){ - cli::cli_alert_warning(e$message) + print(e) cli::cli_alert_danger("Host will not be passed.") }) if (!is.null(new_host)) { diff --git a/man/EngineGraphQL.Rd b/man/EngineGraphQL.Rd index 55869a42..8607af04 100644 --- a/man/EngineGraphQL.Rd +++ b/man/EngineGraphQL.Rd @@ -30,7 +30,7 @@ A class for methods wrapping GitHub's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQL} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineGraphQL$new(gql_api_url, token, scan_whole_host)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineGraphQL$new(gql_api_url, token, scan_all)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -40,7 +40,7 @@ Create \code{EngineGraphQL} object. \item{\code{token}}{A token.} -\item{\code{scan_whole_host}}{A boolean.} +\item{\code{scan_all}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/EngineGraphQLGitHub.Rd b/man/EngineGraphQLGitHub.Rd index daede726..2eb96977 100644 --- a/man/EngineGraphQLGitHub.Rd +++ b/man/EngineGraphQLGitHub.Rd @@ -13,6 +13,7 @@ A class for methods wrapping GitHub's GraphQL API responses. \subsection{Public methods}{ \itemize{ \item \href{#method-new}{\code{EngineGraphQLGitHub$new()}} +\item \href{#method-get_orgs}{\code{EngineGraphQLGitHub$get_orgs()}} \item \href{#method-get_repos}{\code{EngineGraphQLGitHub$get_repos()}} \item \href{#method-get_repos_supportive}{\code{EngineGraphQLGitHub$get_repos_supportive()}} \item \href{#method-get_commits}{\code{EngineGraphQLGitHub$get_commits()}} @@ -34,7 +35,7 @@ A class for methods wrapping GitHub's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQLGitHub} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineGraphQLGitHub$new(gql_api_url, token, scan_whole_host)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineGraphQLGitHub$new(gql_api_url, token, scan_all)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -44,10 +45,20 @@ Create \code{EngineGraphQLGitHub} object. \item{\code{token}}{A token.} -\item{\code{scan_whole_host}}{A boolean.} +\item{\code{scan_all}}{A boolean.} } \if{html}{\out{}} } +} +\if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-get_orgs}{}}} +\subsection{Method \code{get_orgs()}}{ +Get all groups from GitLab. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{EngineGraphQLGitHub$get_orgs()}\if{html}{\out{
}} +} + } \if{html}{\out{
}} \if{html}{\out{}} diff --git a/man/EngineGraphQLGitLab.Rd b/man/EngineGraphQLGitLab.Rd index bb9ac35a..e12f010b 100644 --- a/man/EngineGraphQLGitLab.Rd +++ b/man/EngineGraphQLGitLab.Rd @@ -34,7 +34,7 @@ A class for methods wrapping GitLab's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQLGitLab} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineGraphQLGitLab$new(gql_api_url, token, scan_whole_host)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineGraphQLGitLab$new(gql_api_url, token, scan_all)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -44,7 +44,7 @@ Create \code{EngineGraphQLGitLab} object. \item{\code{token}}{A token.} -\item{\code{scan_whole_host}}{A boolean.} +\item{\code{scan_all}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/EngineRest.Rd b/man/EngineRest.Rd index 80e6e97c..d3bf96cf 100644 --- a/man/EngineRest.Rd +++ b/man/EngineRest.Rd @@ -27,7 +27,7 @@ A superclass for methods wrapping Rest API responses. \subsection{Method \code{new()}}{ Create a new \code{Rest} object \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRest$new(rest_api_url = NA, token = NA, scan_whole_host)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineRest$new(rest_api_url = NA, token = NA, scan_all)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -37,7 +37,7 @@ Create a new \code{Rest} object \item{\code{token}}{A token.} -\item{\code{scan_whole_host}}{A boolean.} +\item{\code{scan_all}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/EngineRestGitHub.Rd b/man/EngineRestGitHub.Rd index b5917f8d..df7a87a8 100644 --- a/man/EngineRestGitHub.Rd +++ b/man/EngineRestGitHub.Rd @@ -35,7 +35,7 @@ A class for methods wrapping GitHub's REST API responses. \subsection{Method \code{new()}}{ Create new \code{EngineRestGitHub} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRestGitHub$new(rest_api_url, token, scan_whole_host)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineRestGitHub$new(rest_api_url, token, scan_all)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -45,7 +45,7 @@ Create new \code{EngineRestGitHub} object. \item{\code{token}}{A token.} -\item{\code{scan_whole_host}}{A boolean.} +\item{\code{scan_all}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/EngineRestGitLab.Rd b/man/EngineRestGitLab.Rd index 9b514839..adc03397 100644 --- a/man/EngineRestGitLab.Rd +++ b/man/EngineRestGitLab.Rd @@ -34,7 +34,7 @@ A class for methods wrapping GitLab's REST API responses. \subsection{Method \code{new()}}{ Create new \code{EngineRestGitLab} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRestGitLab$new(rest_api_url, token, scan_whole_host)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineRestGitLab$new(rest_api_url, token, scan_all)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -44,7 +44,7 @@ Create new \code{EngineRestGitLab} object. \item{\code{token}}{A token.} -\item{\code{scan_whole_host}}{A boolean.} +\item{\code{scan_all}}{A boolean.} } \if{html}{\out{}} } diff --git a/man/GQLQueryGitHub.Rd b/man/GQLQueryGitHub.Rd index dd19b1f6..47fc5d94 100644 --- a/man/GQLQueryGitHub.Rd +++ b/man/GQLQueryGitHub.Rd @@ -9,6 +9,7 @@ A class with methods to build GraphQL Queries for GitHub. \section{Methods}{ \subsection{Public methods}{ \itemize{ +\item \href{#method-orgs}{\code{GQLQueryGitHub$orgs()}} \item \href{#method-repos_by_org}{\code{GQLQueryGitHub$repos_by_org()}} \item \href{#method-repos_by_user}{\code{GQLQueryGitHub$repos_by_user()}} \item \href{#method-user}{\code{GQLQueryGitHub$user()}} @@ -17,6 +18,26 @@ A class with methods to build GraphQL Queries for GitHub. } } \if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-orgs}{}}} +\subsection{Method \code{orgs()}}{ +Prepare query to list organizations from GitHub. +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{GQLQueryGitHub$orgs(end_cursor)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{end_cursor}}{An end cursor to paginate.} +} +\if{html}{\out{
}} +} +\subsection{Returns}{ +A query. +} +} +\if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-repos_by_org}{}}} \subsection{Method \code{repos_by_org()}}{ From 9c7791bcedcad3903620eee6c97f294db0992374 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 23 Aug 2023 10:47:22 +0000 Subject: [PATCH 3/9] Silence message when searching through whole platform. --- R/EngineRestGitHub.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index 62045178..a06997de 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -53,7 +53,9 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", get_repos = function(org, settings) { if (settings$search_param == "phrase") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][phrase:{settings$phrase}][org:{org}] Searching repositories...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][phrase:{settings$phrase}][org:{org}] Searching repositories...") + } repos_table <- private$search_repos_by_phrase( org = org, phrase = settings$phrase, From 1204183e5d6e9ced91aa72691dabfdd5a02e76a6 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Wed, 23 Aug 2023 11:31:21 +0000 Subject: [PATCH 4/9] Improve informing user. --- R/EngineGraphQLGitHub.R | 10 +++++++--- R/EngineRestGitHub.R | 14 +++++++++----- R/EngineRestGitLab.R | 18 ++++++++++-------- R/GitHost.R | 9 +++++++-- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 5ec25ae0..9d5545ea 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -106,7 +106,9 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", repos_names <- repos_table$name if (settings$search_param == "org") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling commits...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}] Pulling commits...") + } repos_list_with_commits <- private$pull_commits_from_repos( org = org, repos = repos_names, @@ -115,7 +117,9 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", ) } if (settings$search_param == "team") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling commits...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_yellow('GraphQL')}][org:{org}][team:{settings$team_name}] Pulling commits...") + } repos_list_with_commits <- private$pull_commits_from_repos( org = org, repos = repos_names, @@ -302,7 +306,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", } return(full_commits_list) } - }, .progress = TRUE) + }, .progress = !private$scan_all) return(repos_list_with_commits) }, diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index a06997de..dbb06867 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -122,10 +122,12 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", org = org, settings = list(search_param = "org") ) - if (settings$search_param == "org") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}] Pulling commits...") - } else if (settings$search_param == "team") { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling commits...") + if (!private$scan_all) { + if (settings$search_param == "org") { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}] Pulling commits...") + } else if (settings$search_param == "team") { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling commits...") + } } repos_list_with_commits <- private$pull_commits_from_org( repos_table = repos_table, @@ -152,7 +154,9 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", #' @return A table of repositories with added information on contributors. add_repos_contributors = function(repos_table) { if (nrow(repos_table) > 0) { - cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}] Pulling contributors...") + if (!private$scan_all) { + cli::cli_alert_info("[GitHub][Engine:{cli::col_green('REST')}] Pulling contributors...") + } repo_iterator <- paste0(repos_table$organization, "/", repos_table$name) user_name <- rlang::expr(.$login) repos_table$contributors <- purrr::map_chr(repo_iterator, function(repos_id) { diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 499da67f..a07f2cdc 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -113,7 +113,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", #' @return A table of repositories with added information on contributors. add_repos_contributors = function(repos_table) { if (nrow(repos_table) > 0) { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}] Pulling contributors...") + if (!private$scan_all) { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}] Pulling contributors...") + } repo_iterator <- repos_table$id user_name <- rlang::expr(.$name) repos_table$contributors <- purrr::map_chr(repo_iterator, function(repos_id) { @@ -151,13 +153,13 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", org = org, settings = list(search_param = "org") ) - - if (settings$search_param == "org") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}] Pulling commits...") - } else if (settings$search_param == "team") { - cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling commits...") + if (!private$scan_all) { + if (settings$search_param == "org") { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}] Pulling commits...") + } else if (settings$search_param == "team") { + cli::cli_alert_info("[GitLab][Engine:{cli::col_green('REST')}][org:{org}][team:{settings$team_name}] Pulling commits...") + } } - repos_list_with_commits <- private$pull_commits_from_org( repos_table = repos_table, date_from = date_from, @@ -374,7 +376,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", date_until = date_until ) return(commits_from_repo) - }, .progress = TRUE) + }, .progress = !private$scan_all) names(repos_list_with_commits) <- repos_names return(repos_list_with_commits) }, diff --git a/R/GitHost.R b/R/GitHost.R index 57faeeda..bea3579a 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -105,7 +105,9 @@ GitHost <- R6::R6Class("GitHost", "i" = "Please change your `search_param` either to 'org' or 'team' with `setup()`." )) } - + if (private$scan_all) { + cli::cli_alert_info("[Host:{private$host}] {cli::col_yellow('Pulling commits from all organizations...')}") + } commits_table <- purrr::map(private$orgs, function(org) { tryCatch({ commits_table_org <- purrr::map(private$engines, ~ .$get_commits( @@ -138,7 +140,7 @@ GitHost <- R6::R6Class("GitHost", }) return(commits_table_org) - }) %>% + }, .progress = private$scan_all) %>% purrr::list_rbind() return(commits_table) @@ -287,6 +289,9 @@ GitHost <- R6::R6Class("GitHost", # @description Pull repositories from organisations. pull_repos_from_orgs = function(settings) { + if (private$scan_all) { + cli::cli_alert_info("[Host:{private$host}] {cli::col_yellow('Pulling repositories from all organizations...')}") + } repos_table <- purrr::map(private$orgs, function(org) { tryCatch({ repos_list <- purrr::map(private$engines, function (engine) { From 46baf332ceb8981e5fd7683843be4d04fdb853f9 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 24 Aug 2023 07:49:30 +0000 Subject: [PATCH 5/9] Update News. --- NEWS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index dda98ccb..dcac041e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,8 @@ GitStats 0.1.0.9000 - added setting tokens by default - if a user does have all the PATs set up in environment variables (as e.g. `GITHUB_PAT` or `GITLAB_PAT`), there is no need to pass them as an arugment to `set_connection` (I: #120 PR: #268), -- added `get_users()` function to pull information on users (I: #199 PR: #238) +- added `get_users()` function to pull information on users (I: #199 PR: #238), +- added scanning whole internal git platforms (I: #258), - added switching to REST engine in case GraphQL fails with 502 error (I: #225 PRs: #227 (for repos) #261 (for commits)) - added GraphQL engine for getting GitLab repos by organization (I: #218 PR: #233) - removed `contributors` as basic stat when pulling `repos` by `org` and by `phrase` to improve speed of pulling repositories data. Added `add_repos_contributors()` user function and `add_contributors` parameter to `get_repos()` function to add conditionally information on contributors to repositories table (I: #235 PRs: #243 #264) From f4b55064107b4760e95e9bf599f575084609c596 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 24 Aug 2023 07:49:54 +0000 Subject: [PATCH 6/9] Update NEWS --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index dcac041e..aca0a0fc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,7 +2,7 @@ GitStats 0.1.0.9000 - added setting tokens by default - if a user does have all the PATs set up in environment variables (as e.g. `GITHUB_PAT` or `GITLAB_PAT`), there is no need to pass them as an arugment to `set_connection` (I: #120 PR: #268), - added `get_users()` function to pull information on users (I: #199 PR: #238), -- added scanning whole internal git platforms (I: #258), +- added possibility of scanning whole internal git platforms if no `orgs` are passed (I: #258), - added switching to REST engine in case GraphQL fails with 502 error (I: #225 PRs: #227 (for repos) #261 (for commits)) - added GraphQL engine for getting GitLab repos by organization (I: #218 PR: #233) - removed `contributors` as basic stat when pulling `repos` by `org` and by `phrase` to improve speed of pulling repositories data. Added `add_repos_contributors()` user function and `add_contributors` parameter to `get_repos()` function to add conditionally information on contributors to repositories table (I: #235 PRs: #243 #264) From 9e065ef30039b415f011ac5e7fe620172f1ccca2 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 24 Aug 2023 07:54:10 +0000 Subject: [PATCH 7/9] Fix tests and adjust code. One of major adjustments was moving `check_organizations` and initialize methods to EngineRest superclass. Also error handling of wrong inputs to `set_connection` on `GitStats` level was removed for the sake of simplicity and making neat messages to user. --- R/EngineGraphQL.R | 6 +-- R/EngineGraphQLGitHub.R | 2 +- R/EngineGraphQLGitLab.R | 2 +- R/EngineRest.R | 35 ++++++++++++++- R/EngineRestGitHub.R | 42 ------------------ R/EngineRestGitLab.R | 44 ------------------ R/GitHost.R | 28 ++++++++---- R/GitStats.R | 36 +++++++-------- R/test_helpers.R | 30 ++++++++++++- man/EngineGraphQL.Rd | 2 +- man/EngineGraphQLGitHub.Rd | 2 +- man/EngineGraphQLGitLab.Rd | 2 +- man/EngineRest.Rd | 23 +++++++++- man/EngineRestGitHub.Rd | 45 +------------------ man/EngineRestGitLab.Rd | 45 +------------------ tests/testthat/_snaps/GitStats.md | 6 +-- tests/testthat/_snaps/reset_language.md | 2 +- tests/testthat/_snaps/set_connection.md | 57 ++++++++---------------- tests/testthat/_snaps/setup.md | 6 +-- tests/testthat/test-02-EngineRest.R | 2 +- tests/testthat/test-GitHost.R | 59 +++++++++++++++---------- tests/testthat/test-set_connection.R | 31 +++++-------- 22 files changed, 203 insertions(+), 304 deletions(-) diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R index 34c59eab..56fc46e7 100644 --- a/R/EngineGraphQL.R +++ b/R/EngineGraphQL.R @@ -16,9 +16,9 @@ EngineGraphQL <- R6::R6Class("EngineGraphQL", #' @param gql_api_url GraphQL API url. #' @param token A token. #' @param scan_all A boolean. - initialize = function(gql_api_url, - token, - scan_all) { + initialize = function(gql_api_url = NA, + token = NA, + scan_all = FALSE) { self$gql_api_url <- gql_api_url private$token <- token private$scan_all <- scan_all diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 9d5545ea..276625b1 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -13,7 +13,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", #' @param scan_all A boolean. initialize = function(gql_api_url, token, - scan_all) { + scan_all = FALSE) { super$initialize(gql_api_url = gql_api_url, token = token, scan_all = scan_all) diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index a719ca2e..fa43417f 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -12,7 +12,7 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab", #' @param scan_all A boolean. initialize = function(gql_api_url, token, - scan_all) { + scan_all = FALSE) { super$initialize(gql_api_url = gql_api_url, token = token, scan_all = scan_all) diff --git a/R/EngineRest.R b/R/EngineRest.R index 41a5f087..70d4f37d 100644 --- a/R/EngineRest.R +++ b/R/EngineRest.R @@ -17,9 +17,9 @@ EngineRest <- R6::R6Class("EngineRest", #' @return A `Rest` object. initialize = function(rest_api_url = NA, token = NA, - scan_all) { + scan_all = FALSE) { self$rest_api_url <- rest_api_url - private$token <- token + private$token <- private$check_token(token) private$scan_all <- scan_all }, @@ -37,7 +37,38 @@ EngineRest <- R6::R6Class("EngineRest", } return(result) + }, + + #' @description Check if an organization exists + #' @param orgs A character vector of organizations + #' @return orgs or NULL. + check_organizations = function(orgs) { + orgs <- purrr::map(orgs, function(org) { + org_endpoint <- if(grepl("github", self$rest_api_url)) "/orgs/" else "/groups/" + withCallingHandlers( + { + self$response(endpoint = paste0(self$rest_api_url, org_endpoint, org)) + }, + message = function(m) { + if (grepl("404", m)) { + cli::cli_alert_danger("Organization you provided does not exist or its name was passed in a wrong way: {org}") + cli::cli_alert_warning("Please type your organization name as you see it in `url`.") + cli::cli_alert_info("E.g. do not use spaces. Organization names as you see on the page may differ from their 'address' name.") + org <<- NULL + } + } + ) + return(org) + }) %>% + purrr::keep(~ length(.) > 0) %>% + unlist() + + if (length(orgs) == 0) { + return(NULL) + } + orgs } + ), private = list( diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index dbb06867..ee899b50 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -4,48 +4,6 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", inherit = EngineRest, public = list( - #' @description Create new `EngineRestGitHub` object. - #' @param rest_api_url A REST API url. - #' @param token A token. - #' @param scan_all A boolean. - initialize = function(rest_api_url, - token, - scan_all) { - super$initialize( - rest_api_url = rest_api_url, - token = private$check_token(token), - scan_all = scan_all - ) - }, - - #' @description Check if an organization exists - #' @param orgs A character vector of organizations - #' @return orgs or NULL. - check_organizations = function(orgs) { - orgs <- purrr::map(orgs, function(org) { - org_endpoint <- "/orgs/" - withCallingHandlers( - { - self$response(endpoint = paste0(self$rest_api_url, org_endpoint, org)) - }, - message = function(m) { - if (grepl("40", m)) { - cli::cli_abort("Improper name of organization.") - org <<- NULL - } - } - ) - return(org) - }) %>% - purrr::keep(~ length(.) > 0) %>% - unlist() - - if (length(orgs) == 0) { - return(NULL) - } - orgs - }, - #' @description Method to get repositories with phrase in code blobs. #' @param org An organization #' @param settings A list of `GitStats` settings. diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index a07f2cdc..c128a0c2 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -4,50 +4,6 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", inherit = EngineRest, public = list( - #' @description Create new `EngineRestGitLab` object. - #' @param rest_api_url A REST API url. - #' @param token A token. - #' @param scan_all A boolean. - initialize = function(rest_api_url, - token, - scan_all) { - super$initialize( - rest_api_url = rest_api_url, - token = private$check_token(token), - scan_all = scan_all - ) - }, - - #' @description Check if an organization exists - #' @param orgs A character vector of organizations - #' @return orgs or NULL. - check_organizations = function(orgs) { - orgs <- purrr::map(orgs, function(org) { - org_endpoint <- "/groups/" - withCallingHandlers( - { - self$response(endpoint = paste0(self$rest_api_url, org_endpoint, org)) - }, - message = function(m) { - if (grepl("404", m)) { - cli::cli_alert_danger("Group name passed in a wrong way: {org}") - cli::cli_alert_warning("If you are using `GitLab`, please type your group name as you see it in `url`.") - cli::cli_alert_info("E.g. do not use spaces. Group names as you see on the page may differ from their 'address' name.") - org <<- NULL - } - } - ) - return(org) - }) %>% - purrr::keep(~ length(.) > 0) %>% - unlist() - - if (length(orgs) == 0) { - return(NULL) - } - orgs - }, - #' @description A method to retrieve all repositories for an organization in #' a table format. #' @param org A character, a group of projects. diff --git a/R/GitHost.R b/R/GitHost.R index bea3579a..18ac837b 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -18,16 +18,21 @@ GitHost <- R6::R6Class("GitHost", token = NA, api_url = NA) { private$api_url <- api_url - private$is_public <- private$check_if_public(api_url) - private$host <- private$set_host(api_url) + private$is_public <- private$check_if_public() + private$host <- private$set_host() if (is.null(token)){ private$token <- private$set_default_token() + } else { + private$token <- token } if (is.null(orgs)) { if (private$is_public) { - cli::cli_abort( - "You need to specify `orgs` for public Git platform." - ) + cli::cli_abort(c( + "You need to specify `orgs` for public Git Host.", + "x" = "Host will not be added.", + "i" = "Add organizations to your `orgs` parameter." + ), + call = NULL) } else { cli::cli_alert_warning(cli::col_yellow( "No `orgs` specified. I will pull all organizations from the Git Host." @@ -185,8 +190,8 @@ GitHost <- R6::R6Class("GitHost", engines = list(), # @description Check whether Git platform is public or internal. - check_if_public = function(api_url) { - if (grepl("api.github.com|gitlab.com/api", api_url)) { + check_if_public = function() { + if (grepl("api.github.com|gitlab.com/api", private$api_url)) { TRUE } else { FALSE @@ -194,13 +199,18 @@ GitHost <- R6::R6Class("GitHost", }, # @description Set name of a Git Host. - set_host = function(api_url) { + set_host = function() { if (grepl("https://", private$api_url) && grepl("github", private$api_url)) { "GitHub" } else if (grepl("https://", private$api_url) && grepl("gitlab|code", private$api_url)) { "GitLab" } else { - stop("This connection is not supported by GitStats class object.") + cli::cli_abort(c( + "This connection is not supported by GitStats class object.", + "x" = "Host will not be added." + ), + call = NULL + ) } }, diff --git a/R/GitStats.R b/R/GitStats.R index 11b203ec..7459b945 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -82,27 +82,23 @@ GitStats <- R6::R6Class("GitStats", token, orgs) { new_host <- NULL - tryCatch({ - new_host <- GitHost$new( - orgs = orgs, - token = token, - api_url = api_url - ) - if (grepl("https://", api_url) && grepl("github", api_url)) { - cli::cli_alert_success("Set connection to GitHub.") - } else if (grepl("https://", api_url) && grepl("gitlab|code", api_url)) { - cli::cli_alert_success("Set connection to GitLab.") - } - }, - error = function(e){ - print(e) - cli::cli_alert_danger("Host will not be passed.") - }) - if (!is.null(new_host)) { - private$hosts <- new_host %>% - private$check_for_duplicate_hosts() %>% - append(private$hosts, .) + + new_host <- GitHost$new( + orgs = orgs, + token = token, + api_url = api_url + ) + if (grepl("https://", api_url) && grepl("github", api_url)) { + cli::cli_alert_success("Set connection to GitHub.") + } else if (grepl("https://", api_url) && grepl("gitlab|code", api_url)) { + cli::cli_alert_success("Set connection to GitLab.") } + + if (!is.null(new_host)) { + private$hosts <- new_host %>% + private$check_for_duplicate_hosts() %>% + append(private$hosts, .) + } }, #' @description A method to add a team member. diff --git a/R/test_helpers.R b/R/test_helpers.R index bb9cdd2d..5bc65239 100644 --- a/R/test_helpers.R +++ b/R/test_helpers.R @@ -30,7 +30,7 @@ TestHost <- R6::R6Class("TestHost", api_url = NA) { private$api_url <- api_url if (grepl("https://", api_url) && grepl("github", api_url)) { - private$engines$rest <- EngineRestGitHub$new( + private$engines$rest <- TestEngineRestGitHub$new( token = token, rest_api_url = api_url ) @@ -39,7 +39,7 @@ TestHost <- R6::R6Class("TestHost", gql_api_url = private$set_gql_url(api_url) ) } else if (grepl("https://", api_url) && grepl("gitlab|code", api_url)) { - private$engines$rest <- EngineRestGitLab$new( + private$engines$rest <- TestEngineRest$new( token = token, rest_api_url = api_url ) @@ -83,6 +83,32 @@ TestEngineRest <- R6::R6Class("TestEngineRest", ) ) +#' @noRd +#' @description A helper class to use in tests. +TestEngineRestGitHub <- R6::R6Class("TestEngineRestGitHub", + inherit = EngineRestGitHub, + public = list( + initialize = function(token, + rest_api_url) { + private$token <- token + self$rest_api_url <- rest_api_url + } + ) +) + +#' @noRd +#' @description A helper class to use in tests. +TestEngineRestGitLab <- R6::R6Class("TestEngineRestGitLab", + inherit = EngineRestGitLab, + public = list( + initialize = function(token, + rest_api_url) { + private$token <- token + self$rest_api_url <- rest_api_url + } + ) +) + #' @noRd create_testrest <- function(rest_api_url = "https://api.github.com", token, diff --git a/man/EngineGraphQL.Rd b/man/EngineGraphQL.Rd index 8607af04..da8c6b09 100644 --- a/man/EngineGraphQL.Rd +++ b/man/EngineGraphQL.Rd @@ -30,7 +30,7 @@ A class for methods wrapping GitHub's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQL} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineGraphQL$new(gql_api_url, token, scan_all)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineGraphQL$new(gql_api_url = NA, token = NA, scan_all = FALSE)}\if{html}{\out{
}} } \subsection{Arguments}{ diff --git a/man/EngineGraphQLGitHub.Rd b/man/EngineGraphQLGitHub.Rd index 2eb96977..7d401621 100644 --- a/man/EngineGraphQLGitHub.Rd +++ b/man/EngineGraphQLGitHub.Rd @@ -35,7 +35,7 @@ A class for methods wrapping GitHub's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQLGitHub} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineGraphQLGitHub$new(gql_api_url, token, scan_all)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineGraphQLGitHub$new(gql_api_url, token, scan_all = FALSE)}\if{html}{\out{
}} } \subsection{Arguments}{ diff --git a/man/EngineGraphQLGitLab.Rd b/man/EngineGraphQLGitLab.Rd index e12f010b..482ed5ed 100644 --- a/man/EngineGraphQLGitLab.Rd +++ b/man/EngineGraphQLGitLab.Rd @@ -34,7 +34,7 @@ A class for methods wrapping GitLab's GraphQL API responses. \subsection{Method \code{new()}}{ Create \code{EngineGraphQLGitLab} object. \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineGraphQLGitLab$new(gql_api_url, token, scan_all)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineGraphQLGitLab$new(gql_api_url, token, scan_all = FALSE)}\if{html}{\out{
}} } \subsection{Arguments}{ diff --git a/man/EngineRest.Rd b/man/EngineRest.Rd index d3bf96cf..2ef035e2 100644 --- a/man/EngineRest.Rd +++ b/man/EngineRest.Rd @@ -18,6 +18,7 @@ A superclass for methods wrapping Rest API responses. \itemize{ \item \href{#method-new}{\code{EngineRest$new()}} \item \href{#method-response}{\code{EngineRest$response()}} +\item \href{#method-check_organizations}{\code{EngineRest$check_organizations()}} \item \href{#method-clone}{\code{EngineRest$clone()}} } } @@ -27,7 +28,7 @@ A superclass for methods wrapping Rest API responses. \subsection{Method \code{new()}}{ Create a new \code{Rest} object \subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRest$new(rest_api_url = NA, token = NA, scan_all)}\if{html}{\out{
}} +\if{html}{\out{
}}\preformatted{EngineRest$new(rest_api_url = NA, token = NA, scan_all = FALSE)}\if{html}{\out{
}} } \subsection{Arguments}{ @@ -68,6 +69,26 @@ A content of response formatted to list. } } \if{html}{\out{
}} +\if{html}{\out{}} +\if{latex}{\out{\hypertarget{method-check_organizations}{}}} +\subsection{Method \code{check_organizations()}}{ +Check if an organization exists +\subsection{Usage}{ +\if{html}{\out{
}}\preformatted{EngineRest$check_organizations(orgs)}\if{html}{\out{
}} +} + +\subsection{Arguments}{ +\if{html}{\out{
}} +\describe{ +\item{\code{orgs}}{A character vector of organizations} +} +\if{html}{\out{
}} +} +\subsection{Returns}{ +orgs or NULL. +} +} +\if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-clone}{}}} \subsection{Method \code{clone()}}{ diff --git a/man/EngineRestGitHub.Rd b/man/EngineRestGitHub.Rd index df7a87a8..9e838d7f 100644 --- a/man/EngineRestGitHub.Rd +++ b/man/EngineRestGitHub.Rd @@ -12,8 +12,6 @@ A class for methods wrapping GitHub's REST API responses. \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{EngineRestGitHub$new()}} -\item \href{#method-check_organizations}{\code{EngineRestGitHub$check_organizations()}} \item \href{#method-get_repos}{\code{EngineRestGitHub$get_repos()}} \item \href{#method-get_repos_supportive}{\code{EngineRestGitHub$get_repos_supportive()}} \item \href{#method-get_commits}{\code{EngineRestGitHub$get_commits()}} @@ -25,52 +23,13 @@ A class for methods wrapping GitHub's REST API responses. \if{html}{ \out{
Inherited methods} \itemize{ +\item \out{}\href{../../GitStats/html/EngineRest.html#method-check_organizations}{\code{GitStats::EngineRest$check_organizations()}}\out{} +\item \out{}\href{../../GitStats/html/EngineRest.html#method-initialize}{\code{GitStats::EngineRest$initialize()}}\out{} \item \out{}\href{../../GitStats/html/EngineRest.html#method-response}{\code{GitStats::EngineRest$response()}}\out{} } \out{
} } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} -\subsection{Method \code{new()}}{ -Create new \code{EngineRestGitHub} object. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRestGitHub$new(rest_api_url, token, scan_all)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{rest_api_url}}{A REST API url.} - -\item{\code{token}}{A token.} - -\item{\code{scan_all}}{A boolean.} -} -\if{html}{\out{
}} -} -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-check_organizations}{}}} -\subsection{Method \code{check_organizations()}}{ -Check if an organization exists -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRestGitHub$check_organizations(orgs)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{orgs}}{A character vector of organizations} -} -\if{html}{\out{
}} -} -\subsection{Returns}{ -orgs or NULL. -} -} -\if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-get_repos}{}}} \subsection{Method \code{get_repos()}}{ diff --git a/man/EngineRestGitLab.Rd b/man/EngineRestGitLab.Rd index adc03397..e040d8be 100644 --- a/man/EngineRestGitLab.Rd +++ b/man/EngineRestGitLab.Rd @@ -12,8 +12,6 @@ A class for methods wrapping GitLab's REST API responses. \section{Methods}{ \subsection{Public methods}{ \itemize{ -\item \href{#method-new}{\code{EngineRestGitLab$new()}} -\item \href{#method-check_organizations}{\code{EngineRestGitLab$check_organizations()}} \item \href{#method-get_repos}{\code{EngineRestGitLab$get_repos()}} \item \href{#method-get_repos_supportive}{\code{EngineRestGitLab$get_repos_supportive()}} \item \href{#method-add_repos_contributors}{\code{EngineRestGitLab$add_repos_contributors()}} @@ -24,52 +22,13 @@ A class for methods wrapping GitLab's REST API responses. \if{html}{ \out{
Inherited methods} \itemize{ +\item \out{}\href{../../GitStats/html/EngineRest.html#method-check_organizations}{\code{GitStats::EngineRest$check_organizations()}}\out{} +\item \out{}\href{../../GitStats/html/EngineRest.html#method-initialize}{\code{GitStats::EngineRest$initialize()}}\out{} \item \out{}\href{../../GitStats/html/EngineRest.html#method-response}{\code{GitStats::EngineRest$response()}}\out{} } \out{
} } \if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-new}{}}} -\subsection{Method \code{new()}}{ -Create new \code{EngineRestGitLab} object. -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRestGitLab$new(rest_api_url, token, scan_all)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{rest_api_url}}{A REST API url.} - -\item{\code{token}}{A token.} - -\item{\code{scan_all}}{A boolean.} -} -\if{html}{\out{
}} -} -} -\if{html}{\out{
}} -\if{html}{\out{}} -\if{latex}{\out{\hypertarget{method-check_organizations}{}}} -\subsection{Method \code{check_organizations()}}{ -Check if an organization exists -\subsection{Usage}{ -\if{html}{\out{
}}\preformatted{EngineRestGitLab$check_organizations(orgs)}\if{html}{\out{
}} -} - -\subsection{Arguments}{ -\if{html}{\out{
}} -\describe{ -\item{\code{orgs}}{A character vector of organizations} -} -\if{html}{\out{
}} -} -\subsection{Returns}{ -orgs or NULL. -} -} -\if{html}{\out{
}} \if{html}{\out{}} \if{latex}{\out{\hypertarget{method-get_repos}{}}} \subsection{Method \code{get_repos()}}{ diff --git a/tests/testthat/_snaps/GitStats.md b/tests/testthat/_snaps/GitStats.md index 3592be43..5ed25219 100644 --- a/tests/testthat/_snaps/GitStats.md +++ b/tests/testthat/_snaps/GitStats.md @@ -5,7 +5,7 @@ Output A object for 0 hosts: Hosts: - Organisations: + Organisations: [0] Search preference: org Team: Phrase: @@ -20,7 +20,7 @@ Output A object for 2 hosts: Hosts: https://api.github.com, https://gitlab.com/api/v4 - Organisations: r-world-devs, openpharma, mbtests + Organisations: [3] r-world-devs, openpharma, mbtests Search preference: org Team: Phrase: @@ -35,7 +35,7 @@ Output A object for 2 hosts: Hosts: https://api.github.com, https://gitlab.com/api/v4 - Organisations: r-world-devs, openpharma, mbtests + Organisations: [3] r-world-devs, openpharma, mbtests Search preference: team Team: RWD-IE (0 members) Phrase: diff --git a/tests/testthat/_snaps/reset_language.md b/tests/testthat/_snaps/reset_language.md index eff39d52..72acec3d 100644 --- a/tests/testthat/_snaps/reset_language.md +++ b/tests/testthat/_snaps/reset_language.md @@ -7,7 +7,7 @@ Output A object for 2 hosts: Hosts: https://api.github.com, https://gitlab.com/api/v4 - Organisations: r-world-devs, openpharma, mbtests + Organisations: [3] r-world-devs, openpharma, mbtests Search preference: phrase Team: Phrase: test-phrase diff --git a/tests/testthat/_snaps/set_connection.md b/tests/testthat/_snaps/set_connection.md index 88d1189f..fc4a1eae 100644 --- a/tests/testthat/_snaps/set_connection.md +++ b/tests/testthat/_snaps/set_connection.md @@ -34,23 +34,16 @@ i Using PAT from GITLAB_PAT envar. v Set connection to GitLab. -# Warning shows if organizations are not specified and host is not passed +# Error shows if organizations are not specified and host is not passed - Code - test_gitstats %>% set_connection(api_url = "https://api.github.com", token = Sys.getenv( - "GITHUB_PAT")) - Message - ! You need to specify `orgs` for public Git platform. - x Host will not be passed. + You need to specify `orgs` for public Git Host. + x Host will not be added. + i Add organizations to your `orgs` parameter. -# Warning shows, when wrong input is passed when setting connection and host is not passed +# Error shows, when wrong input is passed when setting connection and host is not passed - Code - set_connection(gitstats_obj = test_gitstats, api_url = "https://avengers.com", - token = Sys.getenv("GITLAB_PAT_PUBLIC")) - Message - ! This connection is not supported by GitStats class object. - x Host will not be passed. + This connection is not supported by GitStats class object. + x Host will not be added. # Error pops out, when two clients of the same url api are passed as input @@ -71,7 +64,9 @@ test_gitstats <- create_gitstats() %>% set_connection(api_url = "https://api.github.com", token = Sys.getenv("GITHUB_PAT"), orgs = c("openparma")) Message - x Organization you provided does not exist. Check spelling in: openparma + x Organization you provided does not exist or its name was passed in a wrong way: openparma + ! Please type your organization name as you see it in `url`. + i E.g. do not use spaces. Organization names as you see on the page may differ from their 'address' name. Message HTTP 404 No such address Message @@ -83,23 +78,9 @@ test_gitstats <- create_gitstats() %>% set_connection(api_url = "https://gitlab.com/api/v4", token = Sys.getenv("GITLAB_PAT_PUBLIC"), orgs = c("openparma", "mbtests")) Message - x Group name passed in a wrong way: openparma - ! If you are using `GitLab`, please type your group name as you see it in `url`. - i E.g. do not use spaces. Group names as you see on the page may differ from their 'address' name. - Message - HTTP 404 No such address - Message - v Set connection to GitLab. - -# Error with message pops out, when you pass to your `GitLab` connection group name as you see it on the page (not from url) - - Code - test_gitstats <- create_gitstats() %>% set_connection(api_url = "https://gitlab.com/api/v4", - token = Sys.getenv("GITLAB_PAT_PUBLIC"), orgs = "MB Tests") - Message - x Group name passed in a wrong way: MB Tests - ! If you are using `GitLab`, please type your group name as you see it in `url`. - i E.g. do not use spaces. Group names as you see on the page may differ from their 'address' name. + x Organization you provided does not exist or its name was passed in a wrong way: openparma + ! Please type your organization name as you see it in `url`. + i E.g. do not use spaces. Organization names as you see on the page may differ from their 'address' name. Message HTTP 404 No such address Message @@ -108,14 +89,14 @@ --- Code - test_gitstats <- create_gitstats() %>% set_connection(api_url = "https://gitlab.com/api/v4", - token = Sys.getenv("GITLAB_PAT_PUBLIC"), orgs = c("mbtests", "MB Tests")) + test_gitstats <- create_gitstats() %>% set_connection(api_url = "https://api.github.com", + token = Sys.getenv("GITHUB_PAT"), orgs = c("openpharma", "r_world_devs")) Message - x Group name passed in a wrong way: MB Tests - ! If you are using `GitLab`, please type your group name as you see it in `url`. - i E.g. do not use spaces. Group names as you see on the page may differ from their 'address' name. + x Organization you provided does not exist or its name was passed in a wrong way: r_world_devs + ! Please type your organization name as you see it in `url`. + i E.g. do not use spaces. Organization names as you see on the page may differ from their 'address' name. Message HTTP 404 No such address Message - v Set connection to GitLab. + v Set connection to GitHub. diff --git a/tests/testthat/_snaps/setup.md b/tests/testthat/_snaps/setup.md index 51ad27b0..464753a5 100644 --- a/tests/testthat/_snaps/setup.md +++ b/tests/testthat/_snaps/setup.md @@ -11,7 +11,7 @@ Output A object for 0 hosts: Hosts: - Organisations: + Organisations: [0] Search preference: phrase Team: RWD-IE (0 members) Phrase: covid @@ -28,7 +28,7 @@ Output A object for 0 hosts: Hosts: - Organisations: + Organisations: [0] Search preference: org Team: RWD-IE (0 members) Phrase: covid @@ -43,7 +43,7 @@ Output A object for 0 hosts: Hosts: - Organisations: + Organisations: [0] Search preference: org Team: RWD-IE (0 members) Phrase: covid diff --git a/tests/testthat/test-02-EngineRest.R b/tests/testthat/test-02-EngineRest.R index 9aea0ee6..73df8a76 100644 --- a/tests/testthat/test-02-EngineRest.R +++ b/tests/testthat/test-02-EngineRest.R @@ -68,7 +68,7 @@ test_that("`response()` returns search response from GitHub's REST API", { test_mocker$cache(gh_search_response) }) -test_rest <- EngineRest$new( +test_rest <- create_testrest( rest_api_url = "https://gitlab.com/api/v4", token = Sys.getenv("GITLAB_PAT_PUBLIC") ) diff --git a/tests/testthat/test-GitHost.R b/tests/testthat/test-GitHost.R index ebb1d3df..868e9135 100644 --- a/tests/testthat/test-GitHost.R +++ b/tests/testthat/test-GitHost.R @@ -15,47 +15,58 @@ test_that("GitHost gets users tables", { # private methods -test_host <- create_testhost( - api_url = "https://api.github.com", - token = Sys.getenv("GITHUB_PAT"), - orgs = c("openpharma", "r-world-devs"), - mode = "private" -) - -test_that("`check_if_public` checks correctly if Git Platform is public or not", { - expect_true( - test_host$check_if_public("https://api.github.com") +test_that("`check_if_public` and `set_host` work correctly", { + test_host <- create_testhost( + api_url = "https://api.github.com", + mode = "private" ) expect_true( - test_host$check_if_public("https://gitlab.com/api/v4") + test_host$check_if_public() ) - expect_false( - test_host$check_if_public("https://code.internal.com/api/v4") - ) - expect_false( - test_host$check_if_public("https://github.internal.com/api/v4") - ) -}) - -test_that("`set_host_name` checks correctly if Git Platform is public or not", { expect_equal( - test_host$set_host_name("https://api.github.com"), + test_host$set_host(), "GitHub" ) + test_host <- create_testhost( + api_url = "https://gitlab.com/api/v4", + mode = "private" + ) + expect_true( + test_host$check_if_public() + ) expect_equal( - test_host$set_host_name("https://gitlab.com/api/v4"), + test_host$set_host(), "GitLab" ) + test_host <- create_testhost( + api_url = "https://code.internal.com/api/v4", + mode = "private" + ) + expect_false( + test_host$check_if_public() + ) expect_equal( - test_host$set_host_name("https://code.internal.com/api/v4"), + test_host$set_host(), "GitLab" ) + test_host <- create_testhost( + api_url = "https://github.internal.com/api/v4", + mode = "private" + ) + expect_false( + test_host$check_if_public() + ) expect_equal( - test_host$set_host_name("https://github.internal.com/api/v4"), + test_host$set_host(), "GitHub" ) }) +test_host <- create_testhost( + api_url = "https://api.github.com", + mode = "private" +) + test_that("`set_default_token` sets default token for public GitHub", { expect_snapshot( default_token <- test_host$set_default_token() diff --git a/tests/testthat/test-set_connection.R b/tests/testthat/test-set_connection.R index ca8690fc..aea90b76 100644 --- a/tests/testthat/test-set_connection.R +++ b/tests/testthat/test-set_connection.R @@ -40,21 +40,25 @@ test_that("When empty token for GitLab, GitStats pulls default token", { ) }) -test_that("Warning shows if organizations are not specified and host is not passed", { +test_that("Error shows if organizations are not specified and host is not passed", { test_gitstats <- create_gitstats() - expect_snapshot( + expect_snapshot_error( test_gitstats %>% set_connection( api_url = "https://api.github.com", token = Sys.getenv("GITHUB_PAT") ) ) + expect_length( + test_gitstats$.__enclos_env__$private$hosts, + 0 + ) }) -test_that("Warning shows, when wrong input is passed when setting connection and host is not passed", { +test_that("Error shows, when wrong input is passed when setting connection and host is not passed", { test_gitstats <- create_gitstats() - expect_snapshot( + expect_snapshot_error( set_connection( gitstats_obj = test_gitstats, api_url = "https://avengers.com", @@ -100,26 +104,13 @@ test_that("`Org` name is not passed to the object if it does not exist", { orgs = c("openparma", "mbtests") ) ) -}) - -test_that("Error with message pops out, when you pass to your `GitLab` connection group name as you see it on the page (not from url)", { - testthat::skip_on_ci() expect_snapshot( test_gitstats <- create_gitstats() %>% set_connection( - api_url = "https://gitlab.com/api/v4", - token = Sys.getenv("GITLAB_PAT_PUBLIC"), - orgs = "MB Tests" - ) - ) - - expect_snapshot( - test_gitstats <- create_gitstats() %>% - set_connection( - api_url = "https://gitlab.com/api/v4", - token = Sys.getenv("GITLAB_PAT_PUBLIC"), - orgs = c("mbtests", "MB Tests") + api_url = "https://api.github.com", + token = Sys.getenv("GITHUB_PAT"), + orgs = c("openpharma", "r_world_devs") ) ) }) From a134a28c808e3fc64920dcfcf30053383334f153 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Thu, 24 Aug 2023 10:35:59 +0000 Subject: [PATCH 8/9] Cover code with tests. --- R/test_helpers.R | 1 + tests/testthat/_snaps/GitHost.md | 16 +++++++++++ tests/testthat/test-GitHost.R | 46 ++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/R/test_helpers.R b/R/test_helpers.R index 5bc65239..65067212 100644 --- a/R/test_helpers.R +++ b/R/test_helpers.R @@ -29,6 +29,7 @@ TestHost <- R6::R6Class("TestHost", token = NA, api_url = NA) { private$api_url <- api_url + private$host <- private$set_host() if (grepl("https://", api_url) && grepl("github", api_url)) { private$engines$rest <- TestEngineRestGitHub$new( token = token, diff --git a/tests/testthat/_snaps/GitHost.md b/tests/testthat/_snaps/GitHost.md index 61279b21..1be0b14e 100644 --- a/tests/testthat/_snaps/GitHost.md +++ b/tests/testthat/_snaps/GitHost.md @@ -14,6 +14,13 @@ Message i Using PAT from GITLAB_PAT envar. +# GitHost pulls repos from orgs + + Code + gh_repos_table <- test_host$pull_repos_from_orgs(settings) + Message + i [GitHub][Engine:GraphQL][org:r-world-devs] Pulling repositories... + # GitHost filters GitHub repositories' (pulled by org) table by languages Code @@ -56,6 +63,15 @@ Message i Filtering by language. +# get_repos returns table of repositories + + Code + repos_table <- test_host$get_repos(settings = list(search_param = "org", + language = "All")) + Message + i [GitHub][Engine:GraphQL][org:openpharma] Pulling repositories... + i [GitHub][Engine:GraphQL][org:r-world-devs] Pulling repositories... + # add_repos_contributors returns table with contributors Code diff --git a/tests/testthat/test-GitHost.R b/tests/testthat/test-GitHost.R index 868e9135..70cd83e8 100644 --- a/tests/testthat/test-GitHost.R +++ b/tests/testthat/test-GitHost.R @@ -113,6 +113,25 @@ test_that("`test_token` works properly", { test_host$test_token("false_token") ) }) +suppressMessages( + test_host <- GitHost$new( + api_url = "https://api.github.com", + token = Sys.getenv("GITHUB_PAT"), + orgs = "r-world-devs" + ) +) +test_host <- test_host$.__enclos_env__$private + +test_that("`setup_engine` adds engines to host", { + expect_s3_class( + test_host$setup_engine(type = "rest"), + "EngineRest" + ) + expect_s3_class( + test_host$setup_engine(type = "graphql"), + "EngineGraphQL" + ) +}) test_that("`set_gql_url()` correctly sets gql api url - for public GitHub", { expect_equal( @@ -128,6 +147,16 @@ test_that("`set_gql_url()` correctly sets gql api url - for public GitLab", { ) }) +test_that("GitHost pulls repos from orgs", { + settings <- list(search_param = "org") + expect_snapshot( + gh_repos_table <- test_host$pull_repos_from_orgs(settings) + ) + expect_repos_table( + gh_repos_table + ) +}) + test_that("GitHost filters GitHub repositories' (pulled by org) table by languages", { repos_table <- test_mocker$use("gh_repos_table") expect_snapshot( @@ -262,6 +291,23 @@ test_host <- create_testhost( orgs = c("openpharma", "r-world-devs") ) +test_that("get_repos returns table of repositories", { + mockery::stub( + test_host$get_repos, + "private$pull_repos_from_org", + test_mocker$use("gh_repos_table") + ) + expect_snapshot( + repos_table <- test_host$get_repos( + settings = list(search_param = "org", + language = "All") + ) + ) + expect_repos_table( + repos_table + ) +}) + test_that("add_repos_contributors returns table with contributors", { repos_table_1 <- test_mocker$use("gh_repos_table") From 2019be202c28e5761d30b3643cc393b457742560 Mon Sep 17 00:00:00 2001 From: Maciej Banas Date: Fri, 25 Aug 2023 07:03:20 +0000 Subject: [PATCH 9/9] Improve searching when `private$scan_all` set to `TRUE`: do not iterate over all orgs but use search endpoint on whole git platform (override orgs). --- R/EngineRestGitHub.R | 17 +++++++++++------ R/EngineRestGitLab.R | 9 ++++++--- R/GitHost.R | 6 +++++- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index ee899b50..7b0cc351 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -194,10 +194,15 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", org, language, byte_max = "384000") { + org_url <- if (!private$scan_all) { + paste0("'+user:", org) + } else { + "" + } search_endpoint <- if (language != "All") { - paste0(self$rest_api_url, "/search/code?q='", phrase, "'+user:", org, "+language:", language) + paste0(self$rest_api_url, "/search/code?q='", phrase, org_url, "+language:", language) } else { - paste0(self$rest_api_url, "/search/code?q='", phrase, "'+user:", org) + paste0(self$rest_api_url, "/search/code?q='", phrase, org_url) } total_n <- self$response(search_endpoint)[["total_count"]] @@ -239,14 +244,14 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", resp_list <- list() index <- c(0, 50) - pb <- progress::progress_bar$new( - format = "GitHub search limit (1000 results) exceeded. Results will be divided. :elapsedfull" + spinner <- cli::make_spinner( + template = cli::col_grey("GitHub search limit (1000 results) exceeded. Results will be divided. {spin}") ) while (index[2] < as.numeric(byte_max)) { size_formula <- paste0("+size:", as.character(index[1]), "..", as.character(index[2])) - pb$tick(0) + spinner$spin() n_count <- tryCatch( { @@ -283,7 +288,7 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", index[2] <- index[2] + 10000 } } - + spinner$finish() resp_list } }, diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index c128a0c2..fd60257a 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -198,12 +198,15 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", page <- 1 still_more_hits <- TRUE resp_list <- list() - groups_id <- private$get_group_id(org) - + groups_url <- if (!private$scan_all) { + paste0("/groups/", private$get_group_id(org)) + } else { + "" + } while (still_more_hits | page < page_max) { resp <- self$response( paste0( - self$rest_api_url, "/groups/", groups_id, + self$rest_api_url, groups_url, "/search?scope=blobs&search=", phrase, "&per_page=100&page=", page ) ) diff --git a/R/GitHost.R b/R/GitHost.R index 18ac837b..e737ab32 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -299,10 +299,14 @@ GitHost <- R6::R6Class("GitHost", # @description Pull repositories from organisations. pull_repos_from_orgs = function(settings) { + orgs <- private$orgs if (private$scan_all) { cli::cli_alert_info("[Host:{private$host}] {cli::col_yellow('Pulling repositories from all organizations...')}") + if (settings$search_param == "phrase") { + orgs <- "no_orgs" + } } - repos_table <- purrr::map(private$orgs, function(org) { + repos_table <- purrr::map(orgs, function(org) { tryCatch({ repos_list <- purrr::map(private$engines, function (engine) { engine$get_repos(