diff --git a/DESCRIPTION b/DESCRIPTION index cdfba9aa..b6e5fb16 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Standardized Git Repository Data -Version: 2.1.2.9003 +Version: 2.1.2.9004 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), diff --git a/NEWS.md b/NEWS.md index 64459297..62178505 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,7 @@ ## Features: +- From now on it is possible to pass `orgs` and `repos` in `set_*_host()` functions ([#400](https://github.com/r-world-devs/GitStats/issues/400)). - Improved `get_commits_stats()` function ([#556](https://github.com/r-world-devs/GitStats/issues/556), [#557](https://github.com/r-world-devs/GitStats/issues/557)) with: - giving possibility to customize grouping variable by passing it with the `group_var` parameter, - changing name of the `time_interval` parameter to `time_aggregation`, @@ -13,6 +14,7 @@ - Fixed pulling commits for GitLab subgroups when repositories are set as scope to scan ([#551](https://github.com/r-world-devs/GitStats/issues/551)). - Filled more information on `author_name` and `author_login` if it was missing in `commits_table` ([#550](https://github.com/r-world-devs/GitStats/issues/550)). - Handled a `GraphQL` response error when pulling repositories with R error. Earlier, `GitStats` just returned empty table with no clue on what has happened, as errors from `GraphQL` are returned as list outputs (they do not break code). +- Fixed getting R package usage when repositories are set ([#548](https://github.com/r-world-devs/GitStats/issues/548)). # GitStats 2.1.2 diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R index 7036b23a..99d7f7aa 100644 --- a/R/EngineGraphQL.R +++ b/R/EngineGraphQL.R @@ -75,7 +75,6 @@ EngineGraphQL <- R6::R6Class( }, get_path_from_files_structure = function(host_files_structure, - only_text_files, org, repo = NULL) { if (is.null(repo)) { @@ -85,9 +84,7 @@ EngineGraphQL <- R6::R6Class( } else { file_path <- host_files_structure[[org]][[repo]] } - if (only_text_files) { - file_path <- file_path[!grepl(non_text_files_pattern, file_path)] - } + file_path <- file_path[grepl(text_files_pattern, file_path)] return(file_path) } ) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 5ce4ddae..64a52c1b 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -49,7 +49,7 @@ EngineGraphQLGitHub <- R6::R6Class( while (next_page) { repos_response <- private$get_repos_page( login = org, - type = type, + type = type, repo_cursor = repo_cursor ) repositories <- if (type == "organization") { @@ -175,9 +175,8 @@ EngineGraphQLGitHub <- R6::R6Class( get_files_from_org = function(org, type, repos, - file_paths, - host_files_structure, - only_text_files, + file_paths = NULL, + host_files_structure = NULL, verbose = TRUE, progress = TRUE) { repo_data <- private$get_repos_data( @@ -193,7 +192,6 @@ EngineGraphQLGitHub <- R6::R6Class( org = org, file_paths = file_paths, host_files_structure = host_files_structure, - only_text_files = only_text_files, progress = progress ) names(org_files_list) <- repositories @@ -204,7 +202,7 @@ EngineGraphQLGitHub <- R6::R6Class( }, # Prepare files table. - prepare_files_table = function(files_response, org, file_path) { + prepare_files_table = function(files_response, org) { if (!is.null(files_response)) { files_table <- purrr::map(files_response, function(repository) { purrr::imap(repository, function(file_data, file_name) { @@ -230,10 +228,10 @@ EngineGraphQLGitHub <- R6::R6Class( # Pull all files from all repositories of an organization. get_files_structure_from_org = function(org, type, - repos, - pattern = NULL, - depth = Inf, - verbose = FALSE, + repos = NULL, + pattern = NULL, + depth = Inf, + verbose = FALSE, progress = TRUE) { repo_data <- private$get_repos_data( org = org, @@ -297,8 +295,8 @@ EngineGraphQLGitHub <- R6::R6Class( }, # Prepare releases table. - prepare_releases_table = function(releases_response, org, date_from, date_until) { - if (!is.null(releases_response)) { + prepare_releases_table = function(releases_response, org, since, until) { + if (length(releases_response) > 0) { releases_table <- purrr::map(releases_response, function(release) { release_table <- purrr::map(release$data$repository$releases$nodes, function(node) { @@ -310,7 +308,7 @@ EngineGraphQLGitHub <- R6::R6Class( release_log = node$description ) }) %>% - purrr::list_rbind() %>% + purrr::list_rbind() |> dplyr::mutate( repo_name = release$data$repository$name, repo_url = release$data$repository$url @@ -321,14 +319,14 @@ EngineGraphQLGitHub <- R6::R6Class( ) return(release_table) }) %>% - purrr::list_rbind() %>% + purrr::list_rbind() |> dplyr::filter( - published_at <= as.POSIXct(date_until) + published_at <= as.POSIXct(until) ) - if (!is.null(date_from)) { + if (!is.null(since)) { releases_table <- releases_table %>% dplyr::filter( - published_at >= as.POSIXct(date_from) + published_at >= as.POSIXct(since) ) } } else { @@ -453,19 +451,17 @@ EngineGraphQLGitHub <- R6::R6Class( def_branches, org, host_files_structure, - only_text_files, file_paths, progress) { purrr::map2(repositories, def_branches, function(repo, def_branch) { if (!is.null(host_files_structure)) { file_paths <- private$get_path_from_files_structure( host_files_structure = host_files_structure, - only_text_files = only_text_files, org = org, repo = repo ) - } else if (is.null(host_files_structure) && only_text_files) { - file_paths <- file_paths[!grepl(non_text_files_pattern, file_paths)] + } else if (is.null(host_files_structure)) { + file_paths <- file_paths[grepl(text_files_pattern, file_paths)] } repo_files_list <- purrr::map(file_paths, function(file_path) { private$get_file_response( diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 3f75ef19..5251b0f8 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -82,23 +82,24 @@ EngineGraphQLGitLab <- R6::R6Class( if (length(repos_list) > 0) { repos_table <- purrr::map(repos_list, function(repo) { repo <- repo$node - repo$default_branch <- repo$repository$rootRef %||% "" + repo[["repo_id"]] <- sub(".*/(\\d+)$", "\\1", repo$repo_id) + repo[["default_branch"]] <- repo$repository$rootRef %||% "" repo$repository <- NULL - repo$languages <- if (length(repo$languages) > 0) { + repo[["languages"]] <- if (length(repo$languages) > 0) { purrr::map_chr(repo$languages, ~ .$name) %>% paste0(collapse = ", ") } else { "" } - repo$created_at <- gts_to_posixt(repo$created_at) - repo$issues_open <- repo$issues$opened - repo$issues_closed <- repo$issues$closed + repo[["created_at"]] <- gts_to_posixt(repo$created_at) + repo[["issues_open"]] <- repo$issues$opened + repo[["issues_closed"]] <- repo$issues$closed repo$issues <- NULL - repo$last_activity_at <- as.POSIXct(repo$last_activity_at) - repo$organization <- repo$namespace$path + repo[["last_activity_at"]] <- as.POSIXct(repo$last_activity_at) + repo[["organization"]] <- repo$namespace$path repo$namespace <- NULL repo$repo_path <- NULL # temporary to close issue 338 - data.frame(repo) + return(data.frame(repo)) }) %>% purrr::list_rbind() %>% dplyr::relocate( @@ -124,9 +125,8 @@ EngineGraphQLGitLab <- R6::R6Class( get_files_from_org = function(org, type, repos, - file_paths, - host_files_structure, - only_text_files, + file_paths = NULL, + host_files_structure = NULL, verbose = FALSE, progress = FALSE) { org <- URLdecode(org) @@ -136,11 +136,10 @@ EngineGraphQLGitLab <- R6::R6Class( if (!is.null(host_files_structure)) { file_paths <- private$get_path_from_files_structure( host_files_structure = host_files_structure, - only_text_files = only_text_files, org = org ) - } else if (is.null(host_files_structure) && only_text_files) { - file_paths <- file_paths[!grepl(non_text_files_pattern, file_paths)] + } else { + file_paths <- file_paths[grepl(text_files_pattern, file_paths)] } if (type == "organization") { while (next_page) { @@ -177,7 +176,6 @@ EngineGraphQLGitLab <- R6::R6Class( repos = repos, file_paths = file_paths, host_files_structure = host_files_structure, - only_text_files = only_text_files, verbose = verbose, progress = progress ) @@ -194,13 +192,13 @@ EngineGraphQLGitLab <- R6::R6Class( purrr::discard(~ length(.$repository$blobs$nodes) == 0) if (is.null(files_list)) files_list <- list() if (length(files_list) > 0) { - next_page <- files_response$pageInfo$hasNextPage + next_page <- projects$pageInfo$hasNextPage } else { next_page <- FALSE } if (is.null(next_page)) next_page <- FALSE if (next_page) { - end_cursor <- files_response$pageInfo$endCursor + end_cursor <- projects$pageInfo$endCursor } else { end_cursor <- "" } @@ -219,7 +217,6 @@ EngineGraphQLGitLab <- R6::R6Class( repos = repos, file_paths = file_paths, host_files_structure = host_files_structure, - only_text_files = only_text_files, verbose = verbose, progress = progress ) @@ -235,7 +232,6 @@ EngineGraphQLGitLab <- R6::R6Class( repos, file_paths = NULL, host_files_structure = NULL, - only_text_files = TRUE, verbose = FALSE, progress = FALSE) { if (is.null(repos)) { @@ -250,16 +246,15 @@ EngineGraphQLGitLab <- R6::R6Class( if (!is.null(host_files_structure)) { file_paths <- private$get_path_from_files_structure( host_files_structure = host_files_structure, - only_text_files = only_text_files, - org = org, - repo = repo + org = org, + repo = repo ) } files_response <- tryCatch( { private$get_file_blobs_response( - org = org, - repo = repo, + org = org, + repo = repo, file_paths = file_paths ) }, @@ -272,7 +267,7 @@ EngineGraphQLGitLab <- R6::R6Class( }, # Prepare files table. - prepare_files_table = function(files_response, org, file_path) { + prepare_files_table = function(files_response, org) { if (!is.null(files_response)) { if (private$response_prepared_by_iteration(files_response)) { files_table <- purrr::map(files_response, function(response_data) { @@ -315,7 +310,7 @@ EngineGraphQLGitLab <- R6::R6Class( get_files_structure_from_org = function(org, type, - repos, + repos = NULL, pattern = NULL, depth = Inf, verbose = TRUE, @@ -368,7 +363,7 @@ EngineGraphQLGitLab <- R6::R6Class( response <- self$gql_response( gql_query = releases_from_repo_query, vars = list( - "project_path" = utils::URLdecode(repository) + "project_path" = paste0(org, "/", utils::URLdecode(repository)) ) ) return(response) @@ -378,7 +373,7 @@ EngineGraphQLGitLab <- R6::R6Class( }, # Prepare releases table. - prepare_releases_table = function(releases_response, org, date_from, date_until) { + prepare_releases_table = function(releases_response, org, since, until) { if (length(releases_response) > 0) { releases_table <- purrr::map(releases_response, function(release) { @@ -404,12 +399,12 @@ EngineGraphQLGitLab <- R6::R6Class( }) %>% purrr::list_rbind() %>% dplyr::filter( - published_at <= as.POSIXct(date_until) + published_at <= as.POSIXct(until) ) - if (!is.null(date_from)) { + if (!is.null(since)) { releases_table <- releases_table %>% dplyr::filter( - published_at >= as.POSIXct(date_from) + published_at >= as.POSIXct(since) ) } } else { diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index 86ec5e91..0add30e4 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -24,35 +24,58 @@ EngineRestGitHub <- R6::R6Class( return(files_list) }, + # Prepare files table from REST API. + prepare_files_table = function(files_list) { + files_table <- NULL + if (!is.null(files_list)) { + files_table <- purrr::map(files_list, function(file_data) { + repo_fullname <- private$get_repo_fullname(file_data$url) + org_repo <- stringr::str_split_1(repo_fullname, "/") + data.frame( + "repo_name" = org_repo[2], + "repo_id" = NA_character_, + "organization" = org_repo[1], + "file_path" = file_data$path, + "file_content" = file_data$content, + "file_size" = file_data$size, + "repo_url" = private$set_repo_url(file_data$url) + ) + }) %>% + purrr::list_rbind() + } + return(files_table) + }, + # Pulling repositories where code appears get_repos_by_code = function(code, - org = NULL, - filename = NULL, - in_path = FALSE, - output = "table_full", - verbose = TRUE, - progress = TRUE) { - user_query <- if (!is.null(org)) { - paste0('+user:', org) - } else { - '' - } - query <- if (!in_path) { - paste0('"', code, '"', user_query) - } else { - paste0('"', code, '"+in:path', user_query) - } - if (!is.null(filename)) { - query <- paste0(query, '+in:file+filename:', filename) + org = NULL, + repos = NULL, + filename = NULL, + in_path = FALSE, + output = "table_full", + verbose = TRUE, + progress = TRUE) { + if (!is.null(org)) { + search_result <- private$search_for_code( + code = code, + org = org, + filename = filename, + in_path = in_path, + verbose = verbose, + progress = progress + ) } - search_endpoint <- paste0(private$endpoints[["search"]], query) - if (verbose) cli::cli_alert_info("Searching for code [{code}]...") - total_n <- self$response(search_endpoint)[["total_count"]] - if (length(total_n) > 0) { - search_result <- private$search_response( - search_endpoint = search_endpoint, - total_n = total_n + if (!is.null(repos)) { + search_result <- private$search_repos_for_code( + code = code, + repos = repos, + filename = filename, + in_path = in_path, + verbose = verbose, + progress = progress ) + } + if (length(search_result) > 0) { if (output == "table_full" || output == "table_min") { search_output <- private$map_search_into_repos( search_response = search_result, @@ -107,8 +130,14 @@ EngineRestGitHub <- R6::R6Class( #' Pull all repositories URLS from organization get_repos_urls = function(type, org, repos) { - repos_response <- self$response( - endpoint = paste0(private$endpoints[["organizations"]], org, "/repos") + owner_type <- attr(org, "type") %||% "organization" + if (owner_type == "user") { + repo_endpoint <- paste0(private$endpoints[["users"]], org, "/repos") + } else { + repo_endpoint <- paste0(private$endpoints[["organizations"]], org, "/repos") + } + repos_response <- private$paginate_results( + endpoint = repo_endpoint ) if (!is.null(repos)) { repos_response <- repos_response %>% @@ -194,12 +223,82 @@ EngineRestGitHub <- R6::R6Class( self$rest_api_url, "/orgs/" ) + private$endpoints[["users"]] <- paste0( + self$rest_api_url, + "/users/" + ) private$endpoints[["repositories"]] <- paste0( self$rest_api_url, "/repos/" ) }, + search_for_code = function(code, + org, + filename, + in_path, + verbose, + progress) { + user_query <- if (!is.null(org)) { + paste0('+user:', org) + } else { + '' + } + query <- if (!in_path) { + paste0('"', code, '"', user_query) + } else { + paste0('"', code, '"+in:path', user_query) + } + if (!is.null(filename)) { + query <- paste0(query, '+in:file+filename:', filename) + } + search_endpoint <- paste0(private$endpoints[["search"]], query) + if (verbose) cli::cli_alert_info("Searching for code [{code}]...") + total_n <- self$response(search_endpoint)[["total_count"]] + search_result <- if (length(total_n) > 0) { + private$search_response( + search_endpoint = search_endpoint, + total_n = total_n + ) + } else { + list() + } + return(search_result) + }, + + search_repos_for_code = function(code, + repos, + filename, + in_path, + verbose, + progress) { + if (verbose) cli::cli_alert_info("Searching for code [{code}]...") + search_result <- purrr::map(repos, function(repo) { + repo_query <- paste0('+repo:', repo) + query <- if (!in_path) { + paste0('"', code, '"', repo_query) + } else { + paste0('"', code, '"+in:path', repo_query) + } + if (!is.null(filename)) { + query <- paste0(query, '+in:file+filename:', filename) + } + search_endpoint <- paste0(private$endpoints[["search"]], query) + total_n <- self$response(search_endpoint)[["total_count"]] + result <- if (length(total_n) > 0) { + private$search_response( + search_endpoint = search_endpoint, + total_n = total_n + ) + } else { + list() + } + return(result) + }) |> + purrr::list_flatten() + return(search_result) + }, + # A wrapper for proper pagination of GitHub search REST API # @param search_endpoint A character, a search endpoint # @param total_n Number of results @@ -302,6 +401,13 @@ EngineRestGitHub <- R6::R6Class( purrr::map(search_result, ~ self$response(.$url), .progress = glue::glue("Adding file [{filename}] info...")) %>% unique() + }, + + # Get repository full name + get_repo_fullname = function(file_url) { + stringr::str_remove_all(file_url, + paste0(private$endpoints$repositories, "/")) %>% + stringr::str_replace_all("/contents.*", "") } ) ) diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 2ce2455d..791e7b1e 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -62,19 +62,31 @@ EngineRestGitLab <- R6::R6Class( # filtering by language. For more information look here: # https://gitlab.com/gitlab-org/gitlab/-/issues/340333 get_repos_by_code = function(code, - org = NULL, + org = NULL, + repos = NULL, filename = NULL, - in_path = FALSE, - output = "table_full", - verbose = TRUE, + in_path = FALSE, + output = "table_full", + verbose = TRUE, progress = TRUE) { - search_response <- private$search_for_code( - code = code, - filename = filename, - in_path = in_path, - org = org, - verbose = verbose - ) + if (!is.null(org)) { + search_response <- private$search_for_code( + code = code, + filename = filename, + in_path = in_path, + org = utils::URLencode(org, reserved = TRUE), + verbose = verbose + ) + } + if (!is.null(repos)) { + search_response <- private$search_repos_for_code( + code = code, + filename = filename, + in_path = in_path, + repos = repos, + verbose = verbose + ) + } if (output == "raw") { search_output <- search_response } else if (output == "table_full" || output == "table_min") { @@ -168,8 +180,10 @@ EngineRestGitLab <- R6::R6Class( # Pull all repositories URLs from organization get_repos_urls = function(type, org, repos) { - repos_response <- self$response( - endpoint = paste0(private$endpoints[["organizations"]], utils::URLencode(org, reserved = TRUE), "/projects") + repos_response <- private$paginate_results( + endpoint = paste0(private$endpoints[["organizations"]], + utils::URLencode(org, reserved = TRUE), + "/projects") ) if (!is.null(repos)) { repos_response <- repos_response %>% @@ -317,21 +331,25 @@ EngineRestGitLab <- R6::R6Class( # Set search endpoint set_search_endpoint = function(org = NULL) { - groups_search <- if (!private$scan_all) { - private$set_groups_search_endpoint(org) + scope_endpoint <- if (!is.null(org)) { + paste0("/groups/", private$get_group_id(org)) } else { "" } - private$endpoints[["search"]] <- paste0( + paste0( self$rest_api_url, - groups_search, + scope_endpoint, "/search?scope=blobs&search=" ) }, - # set groups search endpoint - set_groups_search_endpoint = function(org) { - paste0("/groups/", private$get_group_id(org)) + set_projects_search_endpoint = function(repo) { + paste0( + self$rest_api_url, + "/projects/", + utils::URLencode(repo, reserved = TRUE), + "/search?scope=blobs&search=" + ) }, # Iterator over pulling pages of repositories. @@ -357,7 +375,7 @@ EngineRestGitLab <- R6::R6Class( page <- 1 still_more_hits <- TRUE full_repos_list <- list() - private$set_search_endpoint(org) + search_endpoint <- private$set_search_endpoint(org) if (verbose) cli::cli_alert_info("Searching for code [{code}]...") if (!in_path) { query <- paste0("%22", code, "%22") @@ -370,7 +388,7 @@ EngineRestGitLab <- R6::R6Class( while (still_more_hits | page < page_max) { search_result <- self$response( paste0( - private$endpoints[["search"]], + search_endpoint, query, "&per_page=100&page=", page @@ -387,11 +405,53 @@ EngineRestGitLab <- R6::R6Class( return(full_repos_list) }, + search_repos_for_code = function(code, + repos, + filename = NULL, + in_path = FALSE, + page_max = 1e6, + verbose = TRUE) { + if (verbose) cli::cli_alert_info("Searching for code [{code}]...") + if (!in_path) { + query <- paste0("%22", code, "%22") + } else { + query <- paste0("path:", code) + } + if (!is.null(filename)) { + query <- paste0(query, "%20filename:", filename) + } + search_response <- purrr::map(repos, function(repo) { + page <- 1 + still_more_hits <- TRUE + full_repos_list <- list() + search_endpoint <- private$set_projects_search_endpoint(repo) + while (still_more_hits | page < page_max) { + search_result <- self$response( + paste0( + search_endpoint, + query, + "&per_page=100&page=", + page + ) + ) + if (length(search_result) == 0) { + still_more_hits <- FALSE + break() + } else { + full_repos_list <- append(full_repos_list, search_result) + page <- page + 1 + } + } + return(full_repos_list) + }) |> + purrr::list_flatten() + return(search_response) + }, + # Parse search response into repositories output map_search_into_repos = function(search_response, progress) { repos_ids <- purrr::map_chr(search_response, ~ as.character(.$project_id)) %>% unique() - repos_list <- purrr::map(repos_ids, function(repo_id) { content <- self$response( endpoint = paste0(private$endpoints[["projects"]], repo_id) diff --git a/R/GQLQueryGitLab.R b/R/GQLQueryGitLab.R index 6f6270bf..fa3def62 100644 --- a/R/GQLQueryGitLab.R +++ b/R/GQLQueryGitLab.R @@ -238,7 +238,7 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab", opened } namespace { - path + path: fullPath } repo_url: webUrl } diff --git a/R/GitHost.R b/R/GitHost.R index 0fb6981c..d373f1c4 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -86,38 +86,32 @@ GitHost <- R6::R6Class( }, # Get repositories URLS from the Git host - get_repos_urls = function(type = "web", + get_repos_urls = function(type = "web", with_code = NULL, - in_files = NULL, + in_files = NULL, with_file = NULL, - verbose = TRUE, - progress = TRUE) { + verbose = TRUE, + progress = TRUE) { if (!is.null(with_code)) { - repo_urls <- private$get_repos_with_code( - code = with_code, + repo_urls <- private$get_repos_urls_with_code( + type = type, + code = with_code, in_files = in_files, - output = "raw", - verbose = verbose - ) %>% - private$get_repo_url_from_response( - type = type, - progress = progress - ) + verbose = verbose, + progress = progress + ) } else if (!is.null(with_file)) { - repo_urls <- private$get_repos_with_code( - code = with_file, + repo_urls <- private$get_repos_urls_with_code( + type = type, + code = with_file, in_path = TRUE, - output = "raw", - verbose = verbose - ) %>% - private$get_repo_url_from_response( - type = type, - progress = progress - ) + verbose = verbose, + progress = progress + ) } else { repo_urls <- private$get_all_repos_urls( - type = type, - verbose = verbose, + type = type, + verbose = verbose, progress = progress ) } @@ -131,14 +125,26 @@ GitHost <- R6::R6Class( progress = TRUE) { if (private$scan_all && is.null(private$orgs) && verbose) { cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") - private$orgs <- private$engines$graphql$get_orgs() + graphql_engine <- private$engines$graphql + private$orgs <- graphql_engine$get_orgs() } - commits_table <- private$get_commits_from_orgs( + commits_from_orgs <- private$get_commits_from_orgs( since = since, until = until, verbose = verbose, progress = progress ) + commits_from_repos <- private$get_commits_from_repos( + since = since, + until = until, + verbose = verbose, + progress = progress + ) + commits_table <- list( + commits_from_orgs, + commits_from_repos + ) |> + purrr::list_rbind() return(commits_table) }, @@ -157,20 +163,35 @@ GitHost <- R6::R6Class( #' a table format. get_files_content = function(file_path, host_files_structure = NULL, - only_text_files = TRUE, - verbose = TRUE, - progress = TRUE) { - files_table <- if (!private$scan_all) { - private$get_files_content_from_orgs( - file_path = file_path, + verbose = TRUE, + progress = TRUE) { + if (is.null(host_files_structure)) { + if (!private$scan_all) { + files_content_from_orgs <- private$get_files_content_from_orgs( + file_path = file_path, + verbose = verbose, + progress = progress + ) + files_content_from_repos <- private$get_files_content_from_repos( + file_path = file_path, + verbose = verbose, + progress = progress + ) + files_table <- rbind( + files_content_from_orgs, + files_content_from_repos + ) + } else { + files_table <- private$get_files_content_from_host( + file_path = file_path, + verbose = verbose, + progress = progress + ) + } + } + if (!is.null(host_files_structure)) { + files_table <- private$get_files_content_from_files_structure( host_files_structure = host_files_structure, - only_text_files = only_text_files, - verbose = verbose, - progress = progress - ) - } else { - private$get_files_content_from_host( - file_path = file_path, verbose = verbose, progress = progress ) @@ -189,12 +210,22 @@ GitHost <- R6::R6Class( "i" = "Set `orgs` or `repos` arguments in `set_*_host()` if you wish to run this function." ), call = NULL) } - files_structure <- private$get_files_structure_from_orgs( - pattern = pattern, - depth = depth, - verbose = verbose, + files_structure_from_orgs <- private$get_files_structure_from_orgs( + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress + ) + files_structure_from_repos <- private$get_files_structure_from_repos( + pattern = pattern, + depth = depth, + verbose = verbose, progress = progress ) + files_structure <- append( + files_structure_from_orgs %||% list(), + files_structure_from_repos %||% list() + ) return(files_structure) }, @@ -204,39 +235,26 @@ GitHost <- R6::R6Class( if (verbose) { cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") } - private$orgs <- private$engines$graphql$get_orgs() + graphql_engine <- private$engines$graphql + private$orgs <- graphql_engine$get_orgs() } until <- until %||% Sys.time() - release_logs_table <- purrr::map(private$orgs, function(org) { - org <- utils::URLdecode(org) - release_logs_table_org <- NULL - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = "Pulling release logs" - ) - } - repos_names <- private$set_repositories( - org = org - ) - graphql_engine <- private$engines$graphql - if (length(repos_names) > 0) { - release_logs_table_org <- graphql_engine$get_release_logs_from_org( - org = org, - repos_names = repos_names - ) %>% - graphql_engine$prepare_releases_table(org, since, until) - } else { - releases_logs_table_org <- NULL - } - return(release_logs_table_org) - }, .progress = if (progress) { - glue::glue("[GitHost:{private$host_name}] Pulling release logs...") - } else { - FALSE - }) %>% + release_logs_from_orgs <- private$get_release_logs_from_orgs( + since = since, + until = until, + verbose = verbose, + progress = progress + ) + release_logs_from_repos <- private$get_release_logs_from_repos( + since = since, + until = until, + verbose = verbose, + progress = progress + ) + release_logs_table <- list( + release_logs_from_orgs, + release_logs_from_repos + ) |> purrr::list_rbind() return(release_logs_table) } @@ -367,9 +385,10 @@ GitHost <- R6::R6Class( if (is.null(repos) && is.null(orgs)) { if (private$is_public) { cli::cli_abort(c( - "You need to specify `orgs` for public Git Host.", + "You need to specify `orgs` or/and `repos` for public Git Host.", "x" = "Host will not be added.", - "i" = "Add organizations to your `orgs` parameter." + "i" = "Add organizations to your `orgs` and/or repositories to + `repos` parameter." ), call = NULL) } else { @@ -385,25 +404,11 @@ GitHost <- R6::R6Class( private$scan_all <- TRUE } } - if (!is.null(repos) && is.null(orgs)) { - if (verbose) { - cli::cli_alert_info(cli::col_grey("Searching scope set to [repo].")) - } - private$searching_scope <- "repo" - } - if (is.null(repos) && !is.null(orgs)) { - if (verbose) { - cli::cli_alert_info(cli::col_grey("Searching scope set to [org].")) - } - private$searching_scope <- "org" + if (!is.null(repos)) { + private$searching_scope <- c(private$searching_scope, "repo") } - if (!is.null(repos) && !is.null(orgs)) { - cli::cli_abort(c( - "Do not specify `orgs` while specifing `repos`.", - "x" = "Host will not be added.", - "i" = "Specify `orgs` or `repos`." - ), - call = NULL) + if (!is.null(orgs)) { + private$searching_scope <- c(private$searching_scope, "org") } }, @@ -422,10 +427,7 @@ GitHost <- R6::R6Class( verbose = verbose ) private$repos_fullnames <- repos - orgs_repos <- private$extract_repos_and_orgs(repos) - private$orgs <- private$set_owner_type( - owners = names(orgs_repos) - ) + orgs_repos <- private$extract_repos_and_orgs(private$repos_fullnames) private$repos <- unname(unlist(orgs_repos)) private$orgs_repos <- orgs_repos } @@ -447,7 +449,7 @@ GitHost <- R6::R6Class( repo <- NULL } return(repo) - }) %>% + }, .progress = verbose) %>% purrr::keep(~ length(.) > 0) %>% unlist() if (length(repos) == 0) { @@ -492,7 +494,7 @@ GitHost <- R6::R6Class( cli::cli_abort( c( "x" = "{type} you provided does not exist or its name was passed - in a wrong way: {cli::col_red({endpoint})}", + in a wrong way: {cli::col_red({utils::URLdecode(endpoint)})}", "!" = "Please type your {tolower(type)} name as you see it in web URL.", "i" = "E.g. do not use spaces. {type} names as you see on the @@ -587,16 +589,6 @@ GitHost <- R6::R6Class( return(orgs_repo_list) }, - # Set repositories - set_repos = function(org) { - if (private$searching_scope == "repo") { - repos <- private$orgs_repos[[org]] - } else { - repos <- NULL - } - return(repos) - }, - # Filter repositories table by host filter_repos_by_host = function(repos_table) { dplyr::filter( @@ -605,7 +597,6 @@ GitHost <- R6::R6Class( ) }, - #' Retrieve all repositories for an organization in a table format. get_all_repos = function(verbose = TRUE, progress = TRUE) { if (private$scan_all && is.null(private$orgs)) { if (verbose) { @@ -615,43 +606,79 @@ GitHost <- R6::R6Class( information = "Pulling all organizations" ) } - private$orgs <- private$engines$graphql$get_orgs() + graphql_engine <- private$engines$graphql + private$orgs <- graphql_engine$get_orgs() } - graphql_engine <- private$engines$graphql - repos_table <- purrr::map(private$orgs, function(org) { - type <- attr(org, "type") %||% "organization" - org <- utils::URLdecode(org) - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = "Pulling repositories" - ) - } - repos <- private$set_repos(org) - repos_table <- graphql_engine$get_repos_from_org( - org = org, - type = type - ) %>% - graphql_engine$prepare_repos_table() - if (!is.null(repos)) { - repos_table <- repos_table %>% - dplyr::filter(repo_name %in% repos) - } - return(repos_table) - }, .progress = progress) %>% - purrr::list_rbind() + repos_table <- purrr::list_rbind( + list( + private$get_repos_from_orgs(verbose, progress), + private$get_repos_from_repos(verbose, progress) + ) + ) return(repos_table) }, + get_repos_from_orgs = function(verbose, progress) { + if ("org" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + purrr::map(private$orgs, function(org) { + type <- attr(org, "type") %||% "organization" + org <- utils::URLdecode(org) + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling repositories" + ) + } + repos_table <- graphql_engine$get_repos_from_org( + org = org, + type = type + ) |> + graphql_engine$prepare_repos_table() + return(repos_table) + }, .progress = progress) |> + purrr::list_rbind() + } + }, + + get_repos_from_repos = function(verbose, progress) { + if ("repo" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) + purrr::map(orgs, function(org) { + type <- attr(org, "type") %||% "organization" + org <- utils::URLdecode(org) + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling repositories" + ) + } + repos_table <- graphql_engine$get_repos_from_org( + org = org, + type = type + ) |> + graphql_engine$prepare_repos_table() |> + dplyr::filter(repo_name %in% private$orgs_repos[[org]]) + return(repos_table) + }, .progress = progress) |> + purrr::list_rbind() + } + }, + # Pull repositories with specific code get_repos_with_code = function(code, - in_files = NULL, - in_path = FALSE, - output = "table_full", - verbose = TRUE, - progress = TRUE) { + in_files = NULL, + in_path = FALSE, + output = "table_full", + verbose = TRUE, + progress = TRUE) { if (private$scan_all) { repos_table <- private$get_repos_with_code_from_host( code = code, @@ -663,7 +690,7 @@ GitHost <- R6::R6Class( ) } if (!private$scan_all) { - repos_table <- private$get_repos_with_code_from_orgs( + repos_from_org <- private$get_repos_with_code_from_orgs( code = code, in_files = in_files, in_path = in_path, @@ -671,6 +698,15 @@ GitHost <- R6::R6Class( verbose = verbose, progress = progress ) + repos_from_repos <- private$get_repos_with_code_from_repos( + code = code, + in_files = in_files, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + repos_table <- rbind(repos_from_org, repos_from_repos) } return(repos_table) }, @@ -680,33 +716,79 @@ GitHost <- R6::R6Class( if (private$scan_all && is.null(private$orgs)) { if (verbose) { show_message( - host = private$host_name, - engine = "graphql", + host = private$host_name, + engine = "graphql", information = "Pulling all organizations" ) } - private$orgs <- private$engines$graphql$get_orgs() + graphql_engine <- private$engines$graphql + private$orgs <- graphql_engine$get_orgs() } - rest_engine <- private$engines$rest - repos_vector <- purrr::map(private$orgs, function(org) { - org <- utils::URLdecode(org) - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "rest", - scope = org, - information = "Pulling repositories (URLS)" + repos_urls_from_orgs <- private$get_repos_urls_from_orgs( + type = type, + verbose = verbose, + progress = progress + ) + repos_urls_from_repos <- private$get_repos_urls_from_repos( + type = type, + verbose = verbose, + progress = progress + ) + repos_vector <- c( + repos_urls_from_orgs, + repos_urls_from_repos + ) + return(repos_vector) + }, + + get_repos_urls_from_orgs = function(type, verbose, progress) { + if ("org" %in% private$searching_scope) { + rest_engine <- private$engines$rest + repos_vector <- purrr::map(private$orgs, function(org) { + org <- utils::URLdecode(org) + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = org, + information = "Pulling repositories (URLs)" + ) + } + repos_urls <- rest_engine$get_repos_urls( + type = type, + org = org, + repos = NULL ) - } - repos_urls <- rest_engine$get_repos_urls( - type = type, - org = org, - repos = private$set_repos(org) + return(repos_urls) + }, .progress = progress) %>% + unlist() + } + }, + + get_repos_urls_from_repos = function(type, verbose, progress) { + if ("repo" %in% private$searching_scope) { + rest_engine <- private$engines$rest + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) ) - return(repos_urls) - }, .progress = progress) %>% - unlist() - return(repos_vector) + repos_vector <- purrr::map(orgs, function(org) { + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = org, + information = "Pulling repositories (URLs)" + ) + } + repos_urls <- rest_engine$get_repos_urls( + type = type, + org = org, + repos = private$orgs_repos[[org]] + ) + return(repos_urls) + }, .progress = progress) %>% + unlist() + } }, # Pull repositories with code from whole Git Host @@ -742,14 +824,14 @@ GitHost <- R6::R6Class( verbose = verbose, progress = progress ) - }) %>% + }) |> purrr::list_flatten() } if (output != "raw") { repos_table <- repos_response %>% rest_engine$tailor_repos_response( output = output - ) %>% + ) |> rest_engine$prepare_repos_table( output = output, verbose = verbose @@ -773,66 +855,185 @@ GitHost <- R6::R6Class( output = "table_full", verbose = TRUE, progress = TRUE) { - repos_list <- purrr::map(private$orgs, function(org) { + if ("org" %in% private$searching_scope) { + repos_list <- purrr::map(private$orgs, function(org) { + if (verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = utils::URLdecode(org), + code = code, + information = "Pulling repositories" + ) + } + rest_engine <- private$engines$rest + if (is.null(in_files)) { + repos_response <- rest_engine$get_repos_by_code( + org = org, + code = code, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + } else { + repos_response <- purrr::map(in_files, function(filename) { + rest_engine$get_repos_by_code( + org = org, + code = code, + filename = filename, + in_path = in_path, + output = output, + verbose = verbose, + progress = progress + ) + }) %>% + purrr::list_flatten() + } + if (output != "raw") { + repos_table <- repos_response %>% + rest_engine$tailor_repos_response( + output = output + ) %>% + rest_engine$prepare_repos_table( + output = output, + verbose = verbose + ) + if (output == "table_full") { + repos_table <- repos_table %>% + rest_engine$get_repos_issues( + progress = progress + ) + } + return(repos_table) + } else { + return(repos_response) + } + }, .progress = progress) + if (output != "raw") { + repos_output <- purrr::list_rbind(repos_list) + } else { + repos_output <- purrr::list_flatten(repos_list) + } + return(repos_output) + } + }, + + # Pull repositories with code from given organizations + get_repos_with_code_from_repos = function(code, + in_files = NULL, + in_path = FALSE, + output = "table_full", + verbose = TRUE, + progress = TRUE) { + orgs <- names(private$orgs_repos) + if ("repo" %in% private$searching_scope) { if (verbose) { show_message( host = private$host_name, engine = "rest", - scope = utils::URLdecode(org), + scope = utils::URLdecode(paste0(orgs, collapse = "|")), code = code, information = "Pulling repositories" ) } rest_engine <- private$engines$rest if (is.null(in_files)) { - repos_response <- rest_engine$get_repos_by_code( - org = org, - code = code, - in_path = in_path, - output = output, - verbose = verbose, + repos_output <- rest_engine$get_repos_by_code( + repos = private$repos_fullnames, + code = code, + in_path = in_path, + output = output, + verbose = verbose, progress = progress ) } else { - repos_response <- purrr::map(in_files, function(filename) { + repos_output <- purrr::map(in_files, function(filename) { rest_engine$get_repos_by_code( - org = org, - code = code, + repos = private$repos_fullnames, + code = code, filename = filename, - in_path = in_path, - output = output, - verbose = verbose, + in_path = in_path, + output = output, + verbose = verbose, progress = progress ) }) %>% purrr::list_flatten() } if (output != "raw") { - repos_table <- repos_response %>% + repos_output <- repos_output %>% rest_engine$tailor_repos_response( output = output ) %>% rest_engine$prepare_repos_table( - output = output, + output = output, verbose = verbose ) if (output == "table_full") { - repos_table <- repos_table %>% + repos_output <- repos_output %>% rest_engine$get_repos_issues( progress = progress ) } - return(repos_table) - } else { - return(repos_response) } - }, .progress = progress) - if (output != "raw") { - repos_output <- purrr::list_rbind(repos_list) - } else { - repos_output <- purrr::list_flatten(repos_list) + return(repos_output) } - return(repos_output) + }, + + get_repos_urls_with_code = function(type, + code, + in_files = NULL, + in_path = FALSE, + verbose, + progress) { + repos_urls_from_orgs <- private$get_repos_urls_with_code_from_orgs( + type = type, + code = code, + in_files = in_files, + in_path = in_path, + verbose = verbose, + progress = progress + ) + repos_urls_from_repos <- private$get_repos_urls_with_code_from_repos( + type = type, + code = code, + in_files = in_files, + in_path = in_path, + verbose = verbose, + progress = progress + ) + repos_urls <- c(repos_urls_from_orgs, repos_urls_from_repos) + return(repos_urls) + }, + + get_repos_urls_with_code_from_orgs = function(type, code, in_files, in_path, verbose, progress) { + private$get_repos_with_code_from_orgs( + code = code, + in_files = in_files, + in_path = in_path, + output = "raw", + verbose = verbose + ) |> + private$get_repo_url_from_response( + type = type, + progress = progress + ) + }, + + get_repos_urls_with_code_from_repos = function(type, code, in_files, in_path, verbose, progress) { + private$get_repos_with_code_from_repos( + code = code, + in_files = in_files, + in_path = in_path, + output = "raw", + verbose = verbose + ) |> + private$get_repo_url_from_response( + type = type, + repos_fullnames = private$repos_fullnames, + progress = progress + ) }, add_platform = function(repos_table) { @@ -866,55 +1067,112 @@ GitHost <- R6::R6Class( # Pull files content from organizations get_files_content_from_orgs = function(file_path, - host_files_structure = NULL, - only_text_files = TRUE, - verbose = TRUE, - progress = TRUE) { + verbose = TRUE, + progress = TRUE) { + if ("org" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + files_table <- purrr::map(private$orgs, function(org) { + if (verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") + ) + } + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_from_org( + org = org, + type = type, + repos = NULL, + file_paths = file_path, + verbose = verbose, + progress = progress + ) |> + graphql_engine$prepare_files_table( + org = org + ) + }) |> + purrr::list_rbind() |> + private$add_repo_api_url() + return(files_table) + } + }, + + # Pull files content from organizations + get_files_content_from_repos = function(file_path, + verbose = TRUE, + progress = TRUE) { + if ("repo" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) + files_table <- purrr::map(orgs, function(org) { + if (verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") + ) + } + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_from_org( + org = org, + type = type, + repos = private$orgs_repos[[org]], + file_paths = file_path, + verbose = verbose, + progress = progress + ) |> + graphql_engine$prepare_files_table( + org = org + ) + }) |> + purrr::list_rbind() |> + private$add_repo_api_url() + return(files_table) + } + }, + + get_files_content_from_files_structure = function(host_files_structure, + verbose = TRUE, + progress = TRUE) { graphql_engine <- private$engines$graphql - if (!is.null(host_files_structure)) { - if (verbose) { - cli::cli_alert_info(cli::col_green("I will make use of files structure stored in GitStats.")) - } - result <- private$get_orgs_and_repos_from_files_structure( - host_files_structure = host_files_structure + if (verbose) { + cli::cli_alert_info( + cli::col_green("I will make use of files structure stored in GitStats.") ) - orgs <- result$orgs - repos <- result$repos - } else { - orgs <- private$orgs - repos <- private$repos } + result <- private$get_orgs_and_repos_from_files_structure( + host_files_structure = host_files_structure + ) + orgs <- result$orgs + repos <- result$repos files_table <- purrr::map(orgs, function(org) { if (verbose) { - user_msg <- if (!is.null(host_files_structure)) { - "Pulling files from files structure" - } else { - glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") - } show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = user_msg + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling files from files structure" ) } type <- attr(org, "type") %||% "organization" graphql_engine$get_files_from_org( - org = org, - type = type, - repos = repos, - file_paths = file_path, + org = org, + type = type, + repos = repos, host_files_structure = host_files_structure, - only_text_files = only_text_files, - verbose = verbose, - progress = progress - ) %>% + verbose = verbose, + progress = progress + ) |> graphql_engine$prepare_files_table( - org = org, - file_path = file_path + org = org ) - }) %>% - purrr::list_rbind() %>% + }) |> + purrr::list_rbind() |> private$add_repo_api_url() return(files_table) }, @@ -931,43 +1189,92 @@ GitHost <- R6::R6Class( depth, verbose = TRUE, progress = TRUE) { - graphql_engine <- private$engines$graphql - files_structure_list <- purrr::map(private$orgs, function(org) { - if (verbose) { - user_info <- if (!is.null(pattern)) { - glue::glue("Pulling files structure...[files matching pattern: '{pattern}']") - } else { - glue::glue("Pulling files structure...") + if ("org" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + files_structure_list <- purrr::map(private$orgs, function(org) { + if (verbose) { + user_info <- if (!is.null(pattern)) { + glue::glue("Pulling files structure...[files matching pattern: '{pattern}']") + } else { + glue::glue("Pulling files structure...") + } + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = user_info + ) } - show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = user_info + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_structure_from_org( + org = org, + type = type, + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress + ) + }) + names(files_structure_list) <- private$orgs + files_structure_list <- files_structure_list %>% + purrr::discard(~ length(.) == 0) + if (length(files_structure_list) == 0 && verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "For {private$host_name} no files structure found." + ) ) } - type <- attr(org, "type") %||% "organization" - graphql_engine$get_files_structure_from_org( - org = org, - type = type, - repos = private$repos, - pattern = pattern, - depth = depth, - verbose = verbose, - progress = progress + return(files_structure_list) + } + }, + + get_files_structure_from_repos = function(pattern, + depth, + verbose = TRUE, + progress = TRUE) { + if ("repo" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) ) - }) - names(files_structure_list) <- private$orgs - files_structure_list <- files_structure_list %>% - purrr::discard(~ length(.) == 0) - if (length(files_structure_list) == 0 && verbose) { - cli::cli_alert_warning( - cli::col_yellow( - "For {private$host_name} no files structure found." + files_structure_list <- purrr::map(orgs, function(org) { + if (verbose) { + user_info <- if (!is.null(pattern)) { + glue::glue("Pulling files structure...[files matching pattern: '{pattern}']") + } else { + glue::glue("Pulling files structure...") + } + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = user_info + ) + } + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_structure_from_org( + org = org, + type = type, + repos = private$repos, + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress ) - ) + }) + names(files_structure_list) <- orgs + files_structure_list <- files_structure_list %>% + purrr::discard(~ length(.) == 0) + if (length(files_structure_list) == 0 && verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "For {private$host_name} no files structure found." + ) + ) + } + return(files_structure_list) } - return(files_structure_list) }, # Pull files from host @@ -987,9 +1294,87 @@ GitHost <- R6::R6Class( verbose = verbose, progress = progress ) %>% - private$prepare_files_table_from_rest() %>% + rest_engine$prepare_files_table() %>% private$add_repo_api_url() return(files_table) + }, + + get_release_logs_from_orgs = function(since, until, verbose, progress) { + if ("org" %in% private$searching_scope) { + release_logs_table <- purrr::map(private$orgs, function(org) { + org <- utils::URLdecode(org) + release_logs_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling release logs" + ) + } + repos_names <- private$get_repos_names( + org = org + ) + graphql_engine <- private$engines$graphql + if (length(repos_names) > 0) { + release_logs_table_org <- graphql_engine$get_release_logs_from_org( + repos_names = repos_names, + org = org + ) %>% + graphql_engine$prepare_releases_table( + org = org, + since = since, + until = until + ) + } else { + releases_logs_table_org <- NULL + } + return(release_logs_table_org) + }, .progress = if (progress) { + glue::glue("[GitHost:{private$host_name}] Pulling release logs...") + } else { + FALSE + }) %>% + purrr::list_rbind() + return(release_logs_table) + } + }, + + get_release_logs_from_repos = function(since, until, verbose, progress) { + if ("repo" %in% private$searching_scope) { + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) + release_logs_table <- purrr::map(orgs, function(org) { + org <- utils::URLdecode(org) + release_logs_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = paste0(org, "/", private$orgs_repos[[org]]), + information = "Pulling release logs" + ) + } + graphql_engine <- private$engines$graphql + release_logs_table_org <- graphql_engine$get_release_logs_from_org( + repos_names = private$orgs_repos[[org]], + org = org + ) %>% + graphql_engine$prepare_releases_table( + org = org, + since = since, + until = until + ) + return(release_logs_table_org) + }, .progress = if (progress) { + glue::glue("[GitHost:{private$host_name}] Pulling release logs...") + } else { + FALSE + }) %>% + purrr::list_rbind() + return(release_logs_table) + } } ) ) diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index 4b0089c3..f0ac14fc 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -155,7 +155,11 @@ GitHostGitHub <- R6::R6Class( }, # Get projects URL from search response - get_repo_url_from_response = function(search_response, type, progress = TRUE) { + get_repo_url_from_response = function(search_response, repos_fullnames = NULL, type, progress = TRUE) { + if (!is.null(repos_fullnames)) { + search_response <- search_response |> + purrr::keep(~ .$repository$full_name %in% repos_fullnames) + } purrr::map_vec(search_response, function(project) { if (type == "api") { project$repository$url @@ -167,80 +171,89 @@ GitHostGitHub <- R6::R6Class( # Pull commits from GitHub get_commits_from_orgs = function(since, until, verbose, progress) { - graphql_engine <- private$engines$graphql - commits_table <- purrr::map(private$orgs, function(org) { - commits_table_org <- NULL - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = "Pulling commits" - ) - } - repos_names <- private$set_repositories( - org = org - ) - commits_table_org <- graphql_engine$get_commits_from_repos( - org = org, - repos_names = repos_names, - since = since, - until = until, - progress = progress - ) %>% - graphql_engine$prepare_commits_table( - org = org - ) - return(commits_table_org) - }, .progress = if (private$scan_all && progress) { - "[GitHost:GitHub] Pulling commits..." - } else { - FALSE - }) %>% - purrr::list_rbind() - return(commits_table) - }, - - # Use repositories either from parameter or, if not set, pull them from API - set_repositories = function(org) { - if (private$searching_scope == "repo") { - repos_names <- private$orgs_repos[[org]] - } else { - repos_table <- private$get_all_repos( - verbose = FALSE - ) - repos_names <- repos_table$repo_name + if ("org" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + commits_table <- purrr::map(private$orgs, function(org) { + commits_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling commits" + ) + } + repos_names <- private$get_repos_names(org) + commits_table_org <- graphql_engine$get_commits_from_repos( + org = org, + repos_names = repos_names, + since = since, + until = until, + progress = progress + ) %>% + graphql_engine$prepare_commits_table( + org = org + ) + return(commits_table_org) + }, .progress = if (private$scan_all && progress) { + "[GitHost:GitHub] Pulling commits..." + } else { + FALSE + }) %>% + purrr::list_rbind() + return(commits_table) } - return(repos_names) }, - # Prepare files table from REST API. - prepare_files_table_from_rest = function(files_list) { - files_table <- NULL - if (!is.null(files_list)) { - files_table <- purrr::map(files_list, function(file_data) { - repo_fullname <- private$get_repo_fullname(file_data$url) - org_repo <- stringr::str_split_1(repo_fullname, "/") - data.frame( - "repo_name" = org_repo[2], - "repo_id" = NA_character_, - "organization" = org_repo[1], - "file_path" = file_data$path, - "file_content" = file_data$content, - "file_size" = file_data$size, - "repo_url" = private$set_repo_url(file_data$url) - ) + # Pull commits from GitHub + get_commits_from_repos = function(since, until, verbose, progress) { + if ("repo" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) + ) + commits_table <- purrr::map(orgs, function(org) { + commits_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = paste0(org, "/", private$orgs_repos[[org]], collapse = ", "), + information = "Pulling commits" + ) + } + commits_table_org <- graphql_engine$get_commits_from_repos( + org = org, + repos_names = private$orgs_repos[[org]], + since = since, + until = until, + progress = progress + ) %>% + graphql_engine$prepare_commits_table( + org = org + ) + return(commits_table_org) + }, .progress = if (private$scan_all && progress) { + "[GitHost:GitHub] Pulling commits..." + } else { + FALSE }) %>% purrr::list_rbind() + return(commits_table) } - return(files_table) }, - # Get repository full name - get_repo_fullname = function(file_url) { - stringr::str_remove_all(file_url, - paste0(private$endpoints$repositories, "/")) %>% - stringr::str_replace_all("/contents.*", "") + # Use repositories either from parameter or, if not set, pull them from API + get_repos_names = function(org) { + type <- attr(org, "type") %||% "organization" + org <- utils::URLdecode(org) + graphql_engine <- private$engines$graphql + repos_names <- graphql_engine$get_repos_from_org( + org = org, + type = type + ) |> + purrr::map_vec(~ .$repo_name) + return(repos_names) }, # Get repository url diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 61e40006..4f2f0957 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -21,56 +21,6 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", if (verbose) { cli::cli_alert_success("Set connection to GitLab.") } - }, - - # Retrieve content of given text files from all repositories for a host in - # a table format. - get_files_content = function(file_path, - host_files_structure = NULL, - only_text_files = TRUE, - verbose = TRUE, - progress = TRUE) { - if (!private$scan_all && private$are_non_text_files(file_path, host_files_structure)) { - if (only_text_files) { - files_table <- private$get_files_content_from_orgs( - file_path = file_path, - host_files_structure = host_files_structure, - only_text_files = only_text_files, - verbose = verbose, - progress = progress - ) - } else { - text_files_table <- private$get_files_content_from_orgs( - file_path = file_path, - host_files_structure = host_files_structure, - only_text_files = TRUE, - verbose = verbose, - progress = progress - ) - non_text_files_table <- private$get_files_content_from_orgs_via_rest( - file_path = file_path, - host_files_structure = host_files_structure, - clean_files_content = FALSE, - only_non_text_files = TRUE, - verbose = verbose, - progress = progress - ) - files_table <- purrr::list_rbind( - list( - text_files_table, - non_text_files_table - ) - ) - } - } else { - files_table <- super$get_files_content( - file_path = file_path, - host_files_structure = host_files_structure, - verbose = verbose, - progress = progress - ) - } - return(files_table) } ), private = list( @@ -203,8 +153,8 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", }, # Get projects API URL from search response - get_repo_url_from_response = function(search_response, type, progress = TRUE) { - purrr::map_vec(search_response, function(response) { + get_repo_url_from_response = function(search_response, type, repos_fullnames = NULL, progress = TRUE) { + repo_urls <- purrr::map_vec(search_response, function(response) { api_url <- paste0(private$api_url, "/projects/", response$project_id) if (type == "api") { return(api_url) @@ -221,62 +171,104 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", } else { FALSE }) + return(repo_urls) }, - # Pull commits from GitHub get_commits_from_orgs = function(since, until, verbose = TRUE, progress = verbose) { - rest_engine <- private$engines$rest - commits_table <- purrr::map(private$orgs, function(org) { - commits_table_org <- NULL - if (!private$scan_all && verbose) { - show_message( - host = private$host_name, - engine = "rest", - scope = utils::URLdecode(org), - information = "Pulling commits" + if ("org" %in% private$searching_scope) { + rest_engine <- private$engines$rest + commits_table <- purrr::map(private$orgs, function(org) { + commits_table_org <- NULL + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = utils::URLdecode(org), + information = "Pulling commits" + ) + } + repos_names <- private$get_repos_names( + org = org ) - } - repos_names <- private$set_repositories( - org = org - ) - commits_table_org <- rest_engine$get_commits_from_repos( - repos_names = repos_names, - since = since, - until = until, - progress = progress - ) %>% - rest_engine$tailor_commits_info(org = org) %>% - rest_engine$prepare_commits_table() %>% - rest_engine$get_commits_authors_handles_and_names( - verbose = verbose, + commits_table_org <- rest_engine$get_commits_from_repos( + repos_names = paste0(org, "%2f", repos_names), + since = since, + until = until, progress = progress - ) - return(commits_table_org) - }, .progress = if (private$scan_all && progress) { - "[GitHost:GitLab] Pulling commits..." - } else { - FALSE - }) %>% - purrr::list_rbind() - return(commits_table) + ) %>% + rest_engine$tailor_commits_info(org = org) %>% + rest_engine$prepare_commits_table() %>% + rest_engine$get_commits_authors_handles_and_names( + verbose = verbose, + progress = progress + ) + return(commits_table_org) + }, .progress = if (private$scan_all && progress) { + "[GitHost:GitLab] Pulling commits..." + } else { + FALSE + }) %>% + purrr::list_rbind() + return(commits_table) + } }, - # Use repositories either from parameter or, if not set, pull them from API - set_repositories = function(org, settings) { - if (private$searching_scope == "repo") { - repos <- private$orgs_repos[[org]] - repos_names <- paste0(utils::URLencode(org, reserved = TRUE), "%2f", repos) - } else { - repos_table <- private$get_all_repos( - verbose = FALSE + get_commits_from_repos = function(since, + until, + verbose = TRUE, + progress = verbose) { + if ("repo" %in% private$searching_scope) { + rest_engine <- private$engines$rest + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) ) - gitlab_web_url <- stringr::str_extract(private$api_url, "^.*?(?=api)") - repos <- stringr::str_remove(repos_table$repo_url, gitlab_web_url) - repos_names <- utils::URLencode(repos, reserved = TRUE) + commits_table <- purrr::map(orgs, function(org) { + commits_table_org <- NULL + repos <- private$orgs_repos[[org]] + repos_names <- paste0(utils::URLencode(org, reserved = TRUE), "%2f", repos) + if (!private$scan_all && verbose) { + show_message( + host = private$host_name, + engine = "rest", + scope = utils::URLdecode(paste0(repos_names, collapse = "|")), + information = "Pulling commits" + ) + } + commits_table_org <- rest_engine$get_commits_from_repos( + repos_names = repos_names, + since = since, + until = until, + progress = progress + ) %>% + rest_engine$tailor_commits_info(org = org) %>% + rest_engine$prepare_commits_table() %>% + rest_engine$get_commits_authors_handles_and_names( + verbose = verbose, + progress = progress + ) + return(commits_table_org) + }, .progress = if (private$scan_all && progress) { + "[GitHost:GitLab] Pulling commits..." + } else { + FALSE + }) %>% + purrr::list_rbind() + return(commits_table) } + }, + + # Use repositories either from parameter or, if not set, pull them from API + get_repos_names = function(org) { + graphql_engine <- private$engines$graphql + type <- attr(org, "type") %||% "organization" + repos_names <- graphql_engine$get_repos_from_org( + org = utils::URLdecode(org), + type = type + ) |> + purrr::map_vec(~ .$node$repo_path) return(repos_names) }, @@ -290,56 +282,80 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", } }, - # Pull files from orgs via rest - get_files_content_from_orgs_via_rest = function(file_path, - host_files_structure, - only_non_text_files, - clean_files_content, - verbose, - progress) { - rest_engine <- private$engines$rest - if (!is.null(host_files_structure)) { - if (verbose) { - cli::cli_alert_info(cli::col_green("I will make use of files structure stored in GitStats.")) - } - result <- private$get_orgs_and_repos_from_files_structure( - host_files_structure = host_files_structure + # Pull files content from organizations + get_files_content_from_repos = function(file_path, + verbose = TRUE, + progress = TRUE) { + if ("repo" %in% private$searching_scope) { + graphql_engine <- private$engines$graphql + orgs <- private$set_owner_type( + owners = names(private$orgs_repos) ) - orgs <- result$orgs - repos <- result$repos - } else { - orgs <- private$orgs - repos <- private$repos + files_table <- purrr::map(orgs, function(org) { + if (verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") + ) + } + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_from_org_per_repo( + org = org, + type = type, + repos = private$orgs_repos[[org]], + file_paths = file_path, + verbose = verbose, + progress = progress + ) |> + graphql_engine$prepare_files_table( + org = org + ) + }) |> + purrr::list_rbind() |> + private$add_repo_api_url() + return(files_table) } + }, + + get_files_content_from_files_structure = function(host_files_structure, + verbose = TRUE, + progress = TRUE) { + graphql_engine <- private$engines$graphql if (verbose) { - user_msg <- if (!is.null(host_files_structure)) { - "Pulling files from files structure" - } else { - glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") - } - show_message( - host = private$host_name, - engine = "rest", - information = user_msg + cli::cli_alert_info( + cli::col_green("I will make use of files structure stored in GitStats.") ) } + result <- private$get_orgs_and_repos_from_files_structure( + host_files_structure = host_files_structure + ) + orgs <- result$orgs + repos <- result$repos files_table <- purrr::map(orgs, function(org) { - if (!is.null(host_files_structure)) { - file_path <- host_files_structure[[org]] %>% unlist(use.names = FALSE) %>% unique() - } - if (only_non_text_files) { - file_path <- file_path[grepl(non_text_files_pattern, file_path)] + if (verbose) { + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = "Pulling files from files structure" + ) } - files_table <- rest_engine$get_files( - file_paths = file_path, - clean_files_content = clean_files_content, - org = org, - verbose = FALSE, - progress = progress - ) %>% - rest_engine$prepare_files_table() - }, .progress = progress) %>% - purrr::list_rbind() %>% + type <- attr(org, "type") %||% "organization" + graphql_engine$get_files_from_org_per_repo( + org = org, + type = type, + repos = repos, + host_files_structure = host_files_structure, + verbose = verbose, + progress = progress + ) |> + graphql_engine$prepare_files_table( + org = org + ) + }) |> + purrr::list_rbind() |> private$add_repo_api_url() return(files_table) } diff --git a/R/GitStats.R b/R/GitStats.R index a58518b8..b4fa288e 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -114,14 +114,21 @@ GitStats <- R6::R6Class( with_files = with_files, verbose = verbose, progress = progress - ) %>% - private$set_object_class( + ) + if (!is.null(repos_urls)) { + repos_urls <- private$set_object_class( + object = repos_urls, class = "repos_urls", attr_list = args_list ) - private$save_to_storage( - table = repos_urls - ) + private$save_to_storage( + table = repos_urls + ) + } else if (verbose) { + cli::cli_alert_warning( + cli::col_yellow("No findings.") + ) + } } else { repos_urls <- private$get_from_storage( table = "repos_urls", @@ -194,14 +201,12 @@ GitStats <- R6::R6Class( get_files_content = function(file_path = NULL, use_files_structure = TRUE, - only_text_files = TRUE, cache = TRUE, verbose = TRUE, progress = verbose) { private$check_for_host() args_list <- list("file_path" = file_path, - "use_files_structure" = use_files_structure, - "only_text_files" = only_text_files) + "use_files_structure" = use_files_structure) trigger <- private$trigger_pulling( cache = cache, storage = "files", @@ -212,7 +217,6 @@ GitStats <- R6::R6Class( files <- private$get_files_content_from_hosts( file_path = file_path, use_files_structure = use_files_structure, - only_text_files = only_text_files, verbose = verbose, progress = progress ) %>% @@ -707,7 +711,6 @@ GitStats <- R6::R6Class( # Pull content of a text file in a table form get_files_content_from_hosts = function(file_path, use_files_structure, - only_text_files, verbose, progress) { purrr::map(private$hosts, function(host) { @@ -729,11 +732,10 @@ GitStats <- R6::R6Class( NULL } else { host$get_files_content( - file_path = file_path, + file_path = file_path, host_files_structure = host_files_structure, - only_text_files = only_text_files, - verbose = verbose, - progress = progress + verbose = verbose, + progress = progress ) } }) %>% @@ -1021,13 +1023,13 @@ GitStats <- R6::R6Class( print_orgs_and_repos = function() { orgs <- purrr::map(private$hosts, function(host) { host_priv <- environment(host$initialize)$private - if (host_priv$searching_scope == "org") { + if ("org" %in% host_priv$searching_scope) { orgs <- host_priv$orgs } }) repos <- purrr::map(private$hosts, function(host) { host_priv <- environment(host$initialize)$private - if (host_priv$searching_scope == "repo") { + if ("repo" %in% host_priv$searching_scope) { repos <- host_priv$repos_fullnames } }) diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index fb95e945..e38fce10 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -283,16 +283,15 @@ get_users <- function(gitstats, cache = TRUE, verbose = is_verbose(gitstats)) { gitstats$get_users( - logins = logins, - cache = cache, + logins = logins, + cache = cache, verbose = verbose ) } #' @title Get content of files #' @name get_files_content -#' @description Pull text files content for a given scope (orgs, repos or whole -#' git hosts). +#' @description Pulls content of text files. #' @param gitstats A GitStats object. #' @param file_path Optional. A standardized path to file(s) in repositories. #' May be a character vector if multiple files are to be pulled. If set to @@ -303,9 +302,6 @@ get_users <- function(gitstats, #' `get_files_structure()` function and kept in storage. If there is no #' `files_structure` in storage, an error will be returned. If `file_path` is #' defined, it will override `use_files_structure` parameter. -#' @param only_text_files A logical, `TRUE` by default. If set to `FALSE`, apart -#' from files with text content shows in table output also non-text files with -#' `NA` value for text content. #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing @@ -324,37 +320,35 @@ get_users <- function(gitstats, #' orgs = "mbtests" #' ) #' get_files_content( -#' gitstats_obj = my_gitstats, +#' gitstats = my_gitstats, #' file_path = c("LICENSE", "DESCRIPTION") #' ) #' #' # example with files structure #' files_structure <- get_files_structure( -#' gitstats_obj = my_gitstats, +#' gitstats = my_gitstats, #' pattern = "\\.Rmd", #' depth = 2L #' ) #' # get_files_content() will make use of pulled earlier files structure #' files_content <- get_files_content( -#' gitstats_obj = my_gitstats +#' gitstats = my_gitstats #' ) #' } #' @return A data.frame. #' @export get_files_content <- function(gitstats, - file_path = NULL, + file_path = NULL, use_files_structure = TRUE, - only_text_files = TRUE, - cache = TRUE, - verbose = is_verbose(gitstats), - progress = verbose) { + cache = TRUE, + verbose = is_verbose(gitstats), + progress = verbose) { gitstats$get_files_content( - file_path = file_path, + file_path = file_path, use_files_structure = use_files_structure, - only_text_files = only_text_files, - cache = cache, - verbose = verbose, - progress = progress + cache = cache, + verbose = verbose, + progress = progress ) } @@ -386,23 +380,23 @@ get_files_content <- function(gitstats, #' orgs = "mbtests" #' ) #' get_files_structure( -#' gitstats_obj = my_gitstats, +#' gitstats = my_gitstats, #' pattern = "\\.md" #' ) #' } #' @return A list of vectors. #' @export get_files_structure <- function(gitstats, - pattern = NULL, - depth = Inf, - cache = TRUE, - verbose = is_verbose(gitstats), + pattern = NULL, + depth = Inf, + cache = TRUE, + verbose = is_verbose(gitstats), progress = verbose) { gitstats$get_files_structure( - pattern = pattern, - depth = depth, - cache = cache, - verbose = verbose, + pattern = pattern, + depth = depth, + cache = cache, + verbose = verbose, progress = progress ) } diff --git a/R/global.R b/R/global.R index df3db824..26726851 100644 --- a/R/global.R +++ b/R/global.R @@ -7,4 +7,7 @@ globalVariables(c( "contributors_n", "githost" )) -non_text_files_pattern <- "\\.(png||.jpg||.jpeg||.bmp||.gif||.tiff)$" +text_ext_files <- "\\.(txt|md|qmd|Rmd|markdown|yaml|yml|csv|json|xml|html|htm|css|js|r|py|sh|bat|ini|conf|log|sql|tsv|mdx)$" +no_ext_files <- "^[^\\.]+$" + +text_files_pattern <- paste0("(", text_ext_files, "|", no_ext_files, ")") diff --git a/R/test_helpers.R b/R/test_helpers.R index 056e2a86..18afac35 100644 --- a/R/test_helpers.R +++ b/R/test_helpers.R @@ -127,6 +127,28 @@ create_github_testhost <- function(host = NULL, return(test_host) } +#' @noRd +create_github_testhost_all <- function(host = NULL, + orgs = NULL, + repos = NULL, + token = NULL, + mode = "") { + suppressMessages( + test_host <- GitHostGitHubTest$new( + host = NULL, + token = token, + orgs = orgs, + repos = repos + ) + ) + test_host$.__enclos_env__$private$orgs <- NULL + test_host$.__enclos_env__$private$scan_all <- TRUE + if (mode == "private") { + test_host <- environment(test_host$initialize)$private + } + return(test_host) +} + #' @noRd create_gitlab_testhost <- function(host = NULL, orgs = NULL, diff --git a/inst/example_workflow.R b/inst/example_workflow.R new file mode 100644 index 00000000..2b17f83f --- /dev/null +++ b/inst/example_workflow.R @@ -0,0 +1,77 @@ +git_stats <- create_gitstats() |> + set_github_host( + orgs = "r-world-devs", + repos = "openpharma/DataFakeR" + ) + +get_repos(git_stats) + +get_repos_urls(git_stats, + with_files = "project_metadata.yaml", + progress = FALSE) + +get_repos_urls(git_stats, + with_files = "project_metadata.yaml", + cache = FALSE, + verbose = FALSE, + progress = TRUE) + +get_repos_urls(git_stats, + with_code = "Shiny", + in_files = "DESCRIPTION", + cache = FALSE, + verbose = FALSE) + +get_repos(git_stats) + +get_repos(git_stats, + cache = FALSE, + verbose = FALSE, + progress = TRUE) + +get_repos(git_stats, + cache = FALSE, + verbose = FALSE) + +get_repos(git_stats, + with_code = "Shiny") + +get_repos(git_stats, + with_code = "Shiny", + cache = FALSE, + verbose = FALSE) + +get_repos(git_stats, + with_code = "Shiny", + cache = FALSE, + verbose = FALSE, + progress = TRUE) + +get_repos(git_stats, + with_code = "Shiny", + in_files = "DESCRIPTION", + cache = FALSE) + +get_repos(git_stats, + with_code = c("shiny", "purrr"), + in_files = c("DESCRIPTION", "NAMESPACE"), + verbose = FALSE) + +get_commits(git_stats, since = "2024-06-01") + +get_commits(git_stats, + since = "2024-06-02", + verbose = FALSE, + progress = TRUE) + +get_release_logs( + gitstats = git_stats, + since = "2024-06-02", + verbose = FALSE +) + +get_release_logs( + gitstats = git_stats, + sinces = "2024-06-01", + verbose = TRUE +) diff --git a/inst/get_files_workflow.R b/inst/get_files_workflow.R index 960e7427..6aa09d4d 100644 --- a/inst/get_files_workflow.R +++ b/inst/get_files_workflow.R @@ -9,12 +9,38 @@ test_gitstats <- create_gitstats() |> ) get_files_content( - gitstats_obj = test_gitstats, + gitstats = test_gitstats, file_path = c("LICENSE", "DESCRIPTION") ) +github_stats <- create_gitstats() |> + set_github_host( + orgs = c("r-world-devs"), + repos = "openpharma/DataFakeR" + ) |> + set_gitlab_host( + repos = "mbtests/graphql_tests" + ) + +github_stats + +get_files_content( + gitstats = github_stats, + file_path = "DESCRIPTION" +) + +datafaker_stats <- create_gitstats() |> + set_github_host( + repos = "openpharma/DataFakeR" + ) + +get_files_content( + gitstats = datafaker_stats, + file_path = "DESCRIPTION" +) + md_files_structure <- get_files_structure( - gitstats_obj = test_gitstats, + gitstats = test_gitstats, pattern = "\\.md|.R", depth = 2L ) @@ -22,8 +48,8 @@ md_files_structure <- get_files_structure( get_files_content(test_gitstats) md_files_structure <- get_files_structure( - gitstats_obj = test_gitstats, - pattern = "\\.md|\\.qmd|\\.Rmd", + gitstats = test_gitstats, + pattern = "DESCRIPTION|\\.md|\\.qmd|\\.Rmd", depth = 2L, verbose = FALSE ) diff --git a/inst/get_storage_workflow.R b/inst/get_storage_workflow.R index 10bc2221..c8ebb4ce 100644 --- a/inst/get_storage_workflow.R +++ b/inst/get_storage_workflow.R @@ -7,17 +7,17 @@ git_stats <- create_gitstats() %>% ) release_logs <- get_release_logs( - gitstats_object = git_stats, + gitstats = git_stats, since = "2024-01-01" ) repos_urls <- get_repos_urls( - gitstats_object = git_stats, + gitstats = git_stats, with_code = "shiny" ) files_structure <- get_files_structure( - gitstats_object = git_stats, + gitstats = git_stats, pattern = "\\.md", depth = 1L ) diff --git a/inst/package_usage_workflow.R b/inst/package_usage_workflow.R index d7cf27e8..15db6f50 100644 --- a/inst/package_usage_workflow.R +++ b/inst/package_usage_workflow.R @@ -1,19 +1,36 @@ devtools::load_all(".") -test_gitstats <- create_gitstats() %>% +openpharma_stats <- create_gitstats() %>% set_github_host( orgs = "openpharma" ) -get_R_package_usage(test_gitstats, packages = "no_such_package") +get_R_package_usage(openpharma_stats, packages = "no_such_package") get_R_package_usage( - test_gitstats, + openpharma_stats, packages = c("purrr", "shiny") ) get_R_package_usage( - test_gitstats, + openpharma_stats, packages = c("dplyr", "shiny"), split_output = TRUE ) + +pharmaverse_stats <- create_gitstats() %>% + set_github_host( + orgs = "pharmaverse" + ) + +get_R_package_usage(pharmaverse_stats, + packages = c("purrr", "shiny")) + +rwd_stats <- create_gitstats() %>% + set_github_host( + repos = "openpharma/DataFakeR", + orgs = "r-world-devs" + ) + +get_R_package_usage(rwd_stats, + packages = c("purrr", "shiny")) diff --git a/man/get_files_content.Rd b/man/get_files_content.Rd index 1aa1a830..f2acc20e 100644 --- a/man/get_files_content.Rd +++ b/man/get_files_content.Rd @@ -8,7 +8,6 @@ get_files_content( gitstats, file_path = NULL, use_files_structure = TRUE, - only_text_files = TRUE, cache = TRUE, verbose = is_verbose(gitstats), progress = verbose @@ -28,10 +27,6 @@ will try to pull data from \code{files_structure} (see below).} \code{files_structure} in storage, an error will be returned. If \code{file_path} is defined, it will override \code{use_files_structure} parameter.} -\item{only_text_files}{A logical, \code{TRUE} by default. If set to \code{FALSE}, apart -from files with text content shows in table output also non-text files with -\code{NA} value for text content.} - \item{cache}{A logical, if set to \code{TRUE} GitStats will retrieve the last result from its storage.} @@ -45,8 +40,7 @@ output is switched off.} A data.frame. } \description{ -Pull text files content for a given scope (orgs, repos or whole -git hosts). +Pulls content of text files. } \examples{ \dontrun{ @@ -60,19 +54,19 @@ git hosts). orgs = "mbtests" ) get_files_content( - gitstats_obj = my_gitstats, + gitstats = my_gitstats, file_path = c("LICENSE", "DESCRIPTION") ) # example with files structure files_structure <- get_files_structure( - gitstats_obj = my_gitstats, + gitstats = my_gitstats, pattern = "\\\\.Rmd", depth = 2L ) # get_files_content() will make use of pulled earlier files structure files_content <- get_files_content( - gitstats_obj = my_gitstats + gitstats = my_gitstats ) } } diff --git a/man/get_files_structure.Rd b/man/get_files_structure.Rd index f02be215..db33b9a4 100644 --- a/man/get_files_structure.Rd +++ b/man/get_files_structure.Rd @@ -51,7 +51,7 @@ Pulls file structure for a given repository. orgs = "mbtests" ) get_files_structure( - gitstats_obj = my_gitstats, + gitstats = my_gitstats, pattern = "\\\\.md" ) } diff --git a/tests/testthat/_snaps/01-get_repos-GitHub.md b/tests/testthat/_snaps/01-get_repos-GitHub.md index e51ab31c..043a672d 100644 --- a/tests/testthat/_snaps/01-get_repos-GitHub.md +++ b/tests/testthat/_snaps/01-get_repos-GitHub.md @@ -5,10 +5,74 @@ Output [1] "\n query GetReposByOrg($login: String!) {\n repositoryOwner(login: $login) {\n ... on Organization {\n \n repositories(first: 100) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n \n }\n }\n }" -# `get_all_repos()` prints proper message +# `get_repos_with_code_from_orgs()` pulls raw response Code - gh_repos_table <- github_testhost_priv$get_all_repos(verbose = TRUE) + repos_with_code_from_orgs_raw <- github_testhost_priv$ + get_repos_with_code_from_orgs(code = "shiny", in_files = c("DESCRIPTION", + "NAMESPACE"), output = "raw", verbose = TRUE) Message - i [Host:GitHub][Engine:GraphQl][Scope:test-org] Pulling repositories... + i [Host:GitHub][Engine:REST][Scope:test_org] Pulling repositories... + +# `get_repos_with_code_from_host()` pulls and parses output into table + + Code + repos_with_code_from_host_table <- github_testhost_priv$ + get_repos_with_code_from_host(code = "DESCRIPTION", in_path = TRUE, output = "table_full", + verbose = TRUE) + Message + i [Host:GitHub][Engine:REST] Pulling repositories... + +# `get_repos_with_code_from_repos()` works + + Code + repos_with_code_from_repos_full <- github_testhost_priv$ + get_repos_with_code_from_repos(code = "tests", output = "table_full", + verbose = TRUE) + Message + i [Host:GitHub][Engine:REST][Scope:] Pulling repositories... + i Preparing repositories table... + +# `get_repos_with_code_from_repos()` pulls minimum version of table + + Code + repos_with_code_from_repos_min <- github_testhost_priv$ + get_repos_with_code_from_repos(code = "tests", in_files = "DESCRIPTION", + output = "table_min", verbose = TRUE) + Message + i [Host:GitHub][Engine:REST][Scope:] Pulling repositories... + i Preparing repositories table... + +# `get_repos_with_code_from_host()` pulls raw response + + Code + repos_with_code_from_host_raw <- github_testhost_priv$ + get_repos_with_code_from_host(code = "shiny", in_files = c("DESCRIPTION", + "NAMESPACE"), output = "raw", verbose = TRUE) + Message + i [Host:GitHub][Engine:REST] Pulling repositories... + +# get_repos_from_repos works + + Code + gh_repos_individual <- github_testhost_priv$get_repos_from_repos(verbose = TRUE, + progress = FALSE) + Message + i [Host:GitHub][Engine:GraphQl][Scope:test_org] Pulling repositories... + +# `get_all_repos()` is set to scan whole git host + + Code + gh_repos <- github_testhost_all_priv$get_all_repos(verbose = TRUE, progress = FALSE) + Message + i [Host:GitHub][Engine:GraphQl] Pulling all organizations... + +# `get_repos_contributors()` works on GitHost level + + Code + gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( + repos_table = test_mocker$use("gh_repos_table_with_platform"), verbose = TRUE, + progress = FALSE) + Message + i [Host:GitHub][Engine:REST] Pulling contributors... diff --git a/tests/testthat/_snaps/01-get_repos-GitLab.md b/tests/testthat/_snaps/01-get_repos-GitLab.md index 632c8019..f13ca202 100644 --- a/tests/testthat/_snaps/01-get_repos-GitLab.md +++ b/tests/testthat/_snaps/01-get_repos-GitLab.md @@ -3,5 +3,29 @@ Code gl_repos_by_org_query Output - [1] "\n query GetReposByOrg($org: ID! $repo_cursor: String!) {\n group(fullPath: $org) {\n projects(first: 100 after: $repo_cursor) {\n \n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n namespace {\n path\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" + [1] "\n query GetReposByOrg($org: ID! $repo_cursor: String!) {\n group(fullPath: $org) {\n projects(first: 100 after: $repo_cursor) {\n \n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n namespace {\n path: fullPath\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" + +# `search_for_code()` works + + Code + gl_search_repos_by_code <- test_rest_gitlab_priv$search_for_code(code = "test", + filename = "TESTFILE", verbose = TRUE, page_max = 2) + Message + i Searching for code [test]... + +# `search_repos_for_code()` works + + Code + gl_search_repos_by_code <- test_rest_gitlab_priv$search_repos_for_code(code = "test", + repos = "TestRepo", filename = "TESTFILE", verbose = TRUE, page_max = 2) + Message + i Searching for code [test]... + +# get_repos_from_org prints proper message + + Code + gl_repos_from_orgs <- gitlab_testhost_priv$get_repos_from_orgs(verbose = TRUE, + progress = FALSE) + Message + i [Host:GitLab][Engine:GraphQl][Scope:mbtests] Pulling repositories... diff --git a/tests/testthat/_snaps/01-get_commits-GitHub.md b/tests/testthat/_snaps/02-get_commits-GitHub.md similarity index 84% rename from tests/testthat/_snaps/01-get_commits-GitHub.md rename to tests/testthat/_snaps/02-get_commits-GitHub.md index 5fe49647..8c59addf 100644 --- a/tests/testthat/_snaps/01-get_commits-GitHub.md +++ b/tests/testthat/_snaps/02-get_commits-GitHub.md @@ -9,3 +9,11 @@ i GraphQL response error +# `get_commits()` is set to scan whole git host + + Code + gh_commits_table <- github_testhost_all$get_commits(since = "2023-01-01", + until = "2023-02-28", verbose = TRUE, progress = FALSE) + Message + i [GitHub][Engine:GraphQL] Pulling all organizations... + diff --git a/tests/testthat/_snaps/02-get_commits-GitLab.md b/tests/testthat/_snaps/02-get_commits-GitLab.md new file mode 100644 index 00000000..13418dda --- /dev/null +++ b/tests/testthat/_snaps/02-get_commits-GitLab.md @@ -0,0 +1,16 @@ +# get_commits_from_orgs works + + Code + gl_commits_table <- gitlab_testhost_priv$get_commits_from_orgs(since = "2023-03-01", + until = "2023-04-01", verbose = TRUE, progress = FALSE) + Message + i [Host:GitLab][Engine:REST][Scope:mbtests] Pulling commits... + +# get_commits_from_repos works + + Code + gl_commits_table <- gitlab_testhost_priv$get_commits_from_repos(since = "2023-03-01", + until = "2023-04-01", verbose = TRUE, progress = FALSE) + Message + i [Host:GitLab][Engine:REST][Scope:test_org/TestRepo] Pulling commits... + diff --git a/tests/testthat/_snaps/01-get_commits-GitStats.md b/tests/testthat/_snaps/02-get_commits-GitStats.md similarity index 100% rename from tests/testthat/_snaps/01-get_commits-GitStats.md rename to tests/testthat/_snaps/02-get_commits-GitStats.md diff --git a/tests/testthat/_snaps/get_files_structure-GitHub.md b/tests/testthat/_snaps/get_files_structure-GitHub.md index 9a747432..e00cecfe 100644 --- a/tests/testthat/_snaps/get_files_structure-GitHub.md +++ b/tests/testthat/_snaps/get_files_structure-GitHub.md @@ -8,19 +8,22 @@ # when files_structure is empty, appropriate message is returned Code - github_testhost_priv$get_files_structure_from_orgs(pattern = "\\.png", depth = 1L, + github_testhost_priv$get_files_structure_from_repos(pattern = "\\.png", depth = 1L, verbose = TRUE) Message - i [Host:GitHub][Engine:GraphQl][Scope:r-world-devs] Pulling files structure...[files matching pattern: '\.png']... - i [Host:GitHub][Engine:GraphQl][Scope:openpharma] Pulling files structure...[files matching pattern: '\.png']... ! For GitHub no files structure found. Output named list() +# get_files_structure aborts when scope to scan whole host + + x This feature is not applicable to scan whole Git Host (time consuming). + i Set `orgs` or `repos` arguments in `set_*_host()` if you wish to run this function. + # get_files_content makes use of files_structure Code - files_content <- github_testhost_priv$get_files_content_from_orgs(file_path = NULL, + files_content <- github_testhost_priv$get_files_content_from_files_structure( host_files_structure = test_mocker$use("gh_files_structure_from_orgs")) Message i I will make use of files structure stored in GitStats. diff --git a/tests/testthat/_snaps/get_files_structure-GitLab.md b/tests/testthat/_snaps/get_files_structure-GitLab.md index 342d0ead..6f4c26ad 100644 --- a/tests/testthat/_snaps/get_files_structure-GitLab.md +++ b/tests/testthat/_snaps/get_files_structure-GitLab.md @@ -8,7 +8,7 @@ # get_files_content makes use of files_structure Code - files_content <- gitlab_testhost_priv$get_files_content_from_orgs(file_path = NULL, + files_content <- gitlab_testhost_priv$get_files_content_from_files_structure( host_files_structure = test_mocker$use("gl_files_structure_from_orgs")) Message i I will make use of files structure stored in GitStats. diff --git a/tests/testthat/_snaps/get_release-GitHub.md b/tests/testthat/_snaps/get_release-GitHub.md index a46f4267..94bc9e8e 100644 --- a/tests/testthat/_snaps/get_release-GitHub.md +++ b/tests/testthat/_snaps/get_release-GitHub.md @@ -5,11 +5,11 @@ Output [1] "query GetReleasesFromRepo ($org: String!, $repo: String!) {\n repository(owner:$org, name:$repo){\n name\n url\n releases (last: 100) {\n nodes {\n name\n tagName\n publishedAt\n url\n description\n }\n }\n }\n }" -# `get_release_logs()` prints proper message when running +# `get_release_logs()` is set to scan whole git host Code - releases_table <- github_testhost$get_release_logs(since = "2023-05-01", until = "2023-09-30", - verbose = TRUE, progress = FALSE) + gh_releases_table <- github_testhost_all$get_release_logs(since = "2023-01-01", + until = "2023-02-28", verbose = TRUE, progress = FALSE) Message - i [Host:GitHub][Engine:GraphQl][Scope:test-org] Pulling release logs... + i [GitHub][Engine:GraphQL] Pulling all organizations... diff --git a/tests/testthat/_snaps/get_release-GitLab.md b/tests/testthat/_snaps/get_release-GitLab.md index 7c4fb7a8..b456d7ca 100644 --- a/tests/testthat/_snaps/get_release-GitLab.md +++ b/tests/testthat/_snaps/get_release-GitLab.md @@ -5,3 +5,11 @@ Output [1] "query GetReleasesFromRepo($project_path: ID!) {\n project(fullPath: $project_path) {\n name\n webUrl\n \t\t\t\t\t\treleases {\n nodes{\n name\n tagName\n releasedAt\n links {\n selfUrl\n }\n description\n }\n }\n }\n }" +# `get_release_logs_from_repos()` works + + Code + releases_from_repos <- gitlab_testhost_priv$get_release_logs_from_repos(since = "2023-05-01", + until = "2023-09-30", verbose = TRUE, progress = FALSE) + Message + i [Host:GitLab][Engine:GraphQl][Scope:test_org/TestRepo] Pulling release logs... + diff --git a/tests/testthat/_snaps/get_urls_repos-GitHub.md b/tests/testthat/_snaps/get_urls_repos-GitHub.md new file mode 100644 index 00000000..34365484 --- /dev/null +++ b/tests/testthat/_snaps/get_urls_repos-GitHub.md @@ -0,0 +1,21 @@ +# get_repos_urls_from_orgs prepares web repo_urls vector + + Code + gh_repos_urls_from_orgs <- github_testhost_priv$get_repos_urls_from_orgs(type = "web", + verbose = TRUE, progress = FALSE) + Message + i [Host:GitHub][Engine:REST][Scope:test_org] Pulling repositories (URLs)... + +# get_repos_urls_from_repos prepares web repo_urls vector + + Code + gh_repos_urls <- github_testhost_priv$get_repos_urls_from_repos(type = "web", + verbose = TRUE, progress = FALSE) + Message + i [Host:GitHub][Engine:REST][Scope:test_org] Pulling repositories (URLs)... + +# get_all_repos_urls prepares web repo_urls vector + + Code + gh_repos_urls <- github_testhost_priv$get_all_repos_urls(type = "web", verbose = TRUE) + diff --git a/tests/testthat/_snaps/helpers.md b/tests/testthat/_snaps/helpers.md index aec3df67..896199b3 100644 --- a/tests/testthat/_snaps/helpers.md +++ b/tests/testthat/_snaps/helpers.md @@ -3,16 +3,20 @@ Code gitlab_testhost_priv$set_searching_scope(orgs = "mbtests", repos = NULL, verbose = TRUE) - Message - i Searching scope set to [org]. --- Code gitlab_testhost_priv$set_searching_scope(orgs = NULL, repos = "mbtests/GitStatsTesting", verbose = TRUE) + +# `set_searching_scope` sets scope to whole host + + Code + gitlab_testhost_priv$set_searching_scope(orgs = NULL, repos = NULL, verbose = TRUE) Message - i Searching scope set to [repo]. + i No `orgs` nor `repos` specified. + i Searching scope set to [all]. # When token is empty throw error @@ -50,9 +54,3 @@ Message i Using PAT from GITLAB_PAT envar. -# `set_searching_scope` throws error when both `orgs` and `repos` are defined - - Do not specify `orgs` while specifing `repos`. - x Host will not be added. - i Specify `orgs` or `repos`. - diff --git a/tests/testthat/_snaps/set_host.md b/tests/testthat/_snaps/set_host.md index 19211cfd..0cbab877 100644 --- a/tests/testthat/_snaps/set_host.md +++ b/tests/testthat/_snaps/set_host.md @@ -4,7 +4,6 @@ set_github_host(gitstats = test_gitstats, token = Sys.getenv("GITHUB_PAT"), orgs = c("openpharma", "r-world-devs")) Message - i Searching scope set to [org]. i Checking organizations... v Set connection to GitHub. @@ -14,7 +13,6 @@ test_gitstats %>% set_gitlab_host(token = Sys.getenv("GITLAB_PAT_PUBLIC"), orgs = c("mbtests")) Message - i Searching scope set to [org]. i Checking organizations... v Set connection to GitLab. @@ -25,7 +23,6 @@ "r-world-devs")) Message i Using PAT from GITHUB_PAT envar. - i Searching scope set to [org]. i Checking organizations... v Set connection to GitHub. @@ -37,7 +34,6 @@ }) Message i Using PAT from GITLAB_PAT envar. - i Searching scope set to [org]. i Checking organizations... v Set connection to GitLab. @@ -48,7 +44,6 @@ "r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder", "openpharma/GithubMetrics", "openpharma/DataFakeR")) Message - i Searching scope set to [repo]. i Checking repositories... v Set connection to GitHub. @@ -58,21 +53,14 @@ test_gitstats %>% set_gitlab_host(token = Sys.getenv("GITLAB_PAT_PUBLIC"), repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2")) Message - i Searching scope set to [repo]. i Checking repositories... v Set connection to GitLab. -# Set host prints error when repos and orgs are defined and host is not passed to GitStats - - Do not specify `orgs` while specifing `repos`. - x Host will not be added. - i Specify `orgs` or `repos`. - # Error shows if organizations are not specified and host is not passed - You need to specify `orgs` for public Git Host. + You need to specify `orgs` or/and `repos` for public Git Host. x Host will not be added. - i Add organizations to your `orgs` parameter. + i Add organizations to your `orgs` and/or repositories to `repos` parameter. # Error shows, when wrong input is passed when setting connection and host is not passed @@ -86,10 +74,8 @@ test_gitstats %>% set_github_host(token = Sys.getenv("GITHUB_PAT"), orgs = "pharmaverse") %>% set_github_host(token = Sys.getenv("GITHUB_PAT"), orgs = "openpharma") Message - i Searching scope set to [org]. i Checking organizations... v Set connection to GitHub. - i Searching scope set to [org]. i Checking organizations... v Set connection to GitHub. Condition @@ -102,7 +88,6 @@ test_gitstats <- create_gitstats() %>% set_github_host(token = Sys.getenv( "GITHUB_PAT"), orgs = c("openparma")) Message - i Searching scope set to [org]. i Checking organizations... Condition Error in `purrr::map()`: @@ -118,7 +103,6 @@ test_gitstats <- create_gitstats() %>% set_gitlab_host(token = Sys.getenv( "GITLAB_PAT_PUBLIC"), orgs = c("openparma", "mbtests")) Message - i Searching scope set to [org]. i Checking organizations... Condition Error in `purrr::map()`: @@ -134,7 +118,6 @@ test_gitstats <- create_gitstats() %>% set_github_host(token = Sys.getenv( "GITHUB_PAT"), orgs = c("openpharma", "r_world_devs")) Message - i Searching scope set to [org]. i Checking organizations... Condition Error in `purrr::map()`: diff --git a/tests/testthat/helper-expect-responses.R b/tests/testthat/helper-expect-responses.R index 32dba117..acffaad3 100644 --- a/tests/testthat/helper-expect-responses.R +++ b/tests/testthat/helper-expect-responses.R @@ -96,7 +96,7 @@ expect_gl_commit_rest_response <- function(object) { "list" ) expect_list_contains( - object[[1]], + object, c( "id", "short_id", "created_at", "parent_ids", "title", "message", "author_name", "author_email", "authored_date", "committer_name", diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 873d30e2..a464927c 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -34,10 +34,10 @@ test_fixtures$github_repository_rest_response <- list( "id" = 627452680, "node_id" = "R_kgDOJWYrCA", "name" = "testRepo", - "full_name" = "test-org/TestRepo", + "full_name" = "test_org/TestRepo", "private" = FALSE, "owner" = list( - "login" = "test-org", + "login" = "test_org", "id" = 103638913, "node_id" = "O_kgDOBi1ngQ", "avatar_url" = "https://avatars.githubusercontent.com/u/103638913?v=4" @@ -96,40 +96,42 @@ test_fixtures$gitlab_repositories_rest_response <- list( ) ) -github_repository_node <- list( - "repo_id" = "xyz", - "repo_name" = "TestRepo", - "default_branch" = list( - "name" = "main" - ), - "stars" = 10, - "forks" = 2, - "created_at" = "2022-04-20T00:00:00Z", - "last_activity_at" = "2023-04-20T00:00:00Z", - "languages" = list( - "nodes" = list( - list( - "name" = "R" - ), - list( - "name" = "CSS" - ), - list( - "name" = "JavaScript" +github_repository_node <- function(repo_name) { + list( + "repo_id" = "xyz", + "repo_name" = repo_name, + "default_branch" = list( + "name" = "main" + ), + "stars" = 10, + "forks" = 2, + "created_at" = "2022-04-20T00:00:00Z", + "last_activity_at" = "2023-04-20T00:00:00Z", + "languages" = list( + "nodes" = list( + list( + "name" = "R" + ), + list( + "name" = "CSS" + ), + list( + "name" = "JavaScript" + ) ) - ) - ), - "issues_open" = list( - "totalCount" = 10 - ), - "issues_closed" = list( - "totalCount" = 5 - ), - "organization" = list( - "login" = "test_org" - ), - "repo_url" = "https://test_url" -) + ), + "issues_open" = list( + "totalCount" = 10 + ), + "issues_closed" = list( + "totalCount" = 5 + ), + "organization" = list( + "login" = "test_org" + ), + "repo_url" = "https://test_url" + ) +} test_fixtures$github_repos_by_org_response <- list( "data" = list( @@ -141,11 +143,11 @@ test_fixtures$github_repos_by_org_response <- list( "hasNextPage" = FALSE ), "nodes" = list( - github_repository_node, - github_repository_node, - github_repository_node, - github_repository_node, - github_repository_node + github_repository_node("TestRepo"), + github_repository_node("TestRepo1"), + github_repository_node("TestRepo2"), + github_repository_node("TestRepo3"), + github_repository_node("TestRepo4") ) ) ) @@ -162,11 +164,11 @@ test_fixtures$github_repos_by_user_response <- list( "hasNextPage" = FALSE ), "nodes" = list( - github_repository_node, - github_repository_node, - github_repository_node, - github_repository_node, - github_repository_node + github_repository_node("TestRepo"), + github_repository_node("TestRepo1"), + github_repository_node("TestRepo2"), + github_repository_node("TestRepo3"), + github_repository_node("TestRepo4") ) ) ) @@ -309,9 +311,7 @@ gitlab_commit <- list( ) ) -test_fixtures$gitlab_commits_response <- list( - rep(gitlab_commit, 5) -) +test_fixtures$gitlab_commits_response <- rep(list(gitlab_commit), 5) test_fixtures$github_file_response <- list( "data" = list( @@ -417,20 +417,6 @@ test_fixtures$gitlab_file_repo_response <- list( ) ) -test_fixtures$github_png_file_response <- list( - "data" = list( - "repository" = list( - "repo_id" = "01010101", - "repo_name" = "TestProject", - "repo_url" = "https://github.com/r-world-devs/GitStats", - "file" = list( - "text" = NULL, - "byteSize" = 50L - ) - ) - ) -) - test_fixtures$gitlab_search_response <- list( list( "basename" = "test", @@ -527,7 +513,7 @@ test_fixtures$github_files_tree_response <- list( "repository" = list( "id" = "R_kgD0Ivtxsg", "name" = "TestRepo", - "url" = "https://github.com/test-org/TestRepo", + "url" = "https://github.com/test_org/TestRepo", "object" = list( "entries" = list( list( diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 9704d42a..69811c67 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -30,24 +30,10 @@ test_graphql_gitlab <- EngineGraphQLGitLab$new( ) test_graphql_gitlab_priv <- environment(test_graphql_gitlab$initialize)$private -github_testhost <- create_github_testhost(orgs = "test-org") +github_testhost <- create_github_testhost(orgs = "test_org") -github_testhost_priv <- create_github_testhost(orgs = "test-org", mode = "private") - -github_testhost_repos <- create_github_testhost( - repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder") -) - -github_testhost_repos_priv <- create_github_testhost( - repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder"), - mode = "private" -) +github_testhost_priv <- create_github_testhost(orgs = "test_org", mode = "private") gitlab_testhost <- create_gitlab_testhost(orgs = "mbtests") gitlab_testhost_priv <- create_gitlab_testhost(orgs = "mbtests", mode = "private") - -gitlab_testhost_repos <- create_gitlab_testhost( - repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2") -) - diff --git a/tests/testthat/test-01-get_repos-GitHub.R b/tests/testthat/test-01-get_repos-GitHub.R index 6fca8d8c..ee7798f6 100644 --- a/tests/testthat/test-01-get_repos-GitHub.R +++ b/tests/testthat/test-01-get_repos-GitHub.R @@ -128,48 +128,73 @@ test_that("Mapping search result to repositories works", { test_mocker$cache(gh_mapped_repos) }) -test_that("`get_repos_by_code()` returns repos output for code search in files", { +test_that("`search_for_code()` returns repos output for code search in files", { mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_for_code, "self$response", list("total_count" = 3L) ) mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_for_code, "private$search_response", test_fixtures$github_search_response ) mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_for_code, "private$map_search_into_repos", test_mocker$use("gh_mapped_repos") ) - gh_repos_by_code <- test_rest_github$get_repos_by_code( + gh_search_for_code <- test_rest_github_priv$search_for_code( code = "test_code", filename = "test_file", + in_path = FALSE, org = "test_org", - verbose = FALSE + verbose = FALSE, + progress = FALSE ) - expect_gh_repos_rest_response(gh_repos_by_code) - test_mocker$cache(gh_repos_by_code) + expect_gh_search_response(gh_search_for_code$items) + test_mocker$cache(gh_search_for_code) }) -test_that("`get_repos_by_code()` for GitHub prepares a raw search response", { +test_that("`search_repos_for_code()` returns repos output for code search in files", { mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_repos_for_code, "self$response", list("total_count" = 3L) ) mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_repos_for_code, "private$search_response", test_fixtures$github_search_response ) mockery::stub( - test_rest_github$get_repos_by_code, + test_rest_github_priv$search_repos_for_code, "private$map_search_into_repos", test_mocker$use("gh_mapped_repos") ) + gh_search_repos_for_code <- test_rest_github_priv$search_repos_for_code( + code = "test_code", + filename = "test_file", + in_path = FALSE, + repos = c("TestRepo", "TestRepo1"), + verbose = FALSE, + progress = FALSE + ) + expect_gh_search_response(gh_search_repos_for_code$items) + test_mocker$cache(gh_search_repos_for_code) +}) + +test_that("`get_repos_by_code()` for GitHub prepares a raw search response", { + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_for_code", + test_mocker$use("gh_search_for_code") + ) + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_repos_for_code", + test_mocker$use("gh_search_repos_for_code") + ) gh_repos_by_code_raw <- test_rest_github$get_repos_by_code( code = "test_code", org = "test_org", @@ -180,6 +205,32 @@ test_that("`get_repos_by_code()` for GitHub prepares a raw search response", { test_mocker$cache(gh_repos_by_code_raw) }) +test_that("`get_repos_by_code()` for GitHub prepares a repository output", { + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_for_code", + test_mocker$use("gh_search_for_code") + ) + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_repos_for_code", + test_mocker$use("gh_search_repos_for_code") + ) + mockery::stub( + test_rest_github$get_repos_by_code, + "private$map_search_into_repos", + test_mocker$use("gh_mapped_repos") + ) + gh_repos_by_code <- test_rest_github$get_repos_by_code( + code = "test_code", + org = "test_org", + output = "table_min", + verbose = FALSE + ) + expect_gh_repos_rest_response(gh_repos_by_code) + test_mocker$cache(gh_repos_by_code) +}) + test_that("GitHub tailors precisely `repos_list`", { gh_repos_by_code <- test_mocker$use("gh_repos_by_code") gh_repos_by_code_tailored <- @@ -320,30 +371,116 @@ test_that("`get_repos_with_code_from_orgs()` pulls raw response", { "rest_engine$get_repos_by_code", test_mocker$use("gh_repos_by_code_raw") ) - repos_with_code_from_orgs_raw <- github_testhost_priv$get_repos_with_code_from_orgs( - code = "shiny", - in_files = c("DESCRIPTION", "NAMESPACE"), - output = "raw", - verbose = FALSE + expect_snapshot( + repos_with_code_from_orgs_raw <- github_testhost_priv$get_repos_with_code_from_orgs( + code = "shiny", + in_files = c("DESCRIPTION", "NAMESPACE"), + output = "raw", + verbose = TRUE + ) ) expect_type(repos_with_code_from_orgs_raw, "list") expect_gt(length(repos_with_code_from_orgs_raw), 0) }) +test_that("`get_repos_with_code_from_host()` pulls and parses output into table", { + mockery::stub( + github_testhost_priv$get_repos_with_code_from_host, + "rest_engine$get_repos_by_code", + test_mocker$use("gh_repos_by_code_raw") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_host, + "rest_engine$prepare_repos_table", + test_mocker$use("gh_repos_by_code_table") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_host, + "rest_engine$get_repos_issues", + test_mocker$use("gh_repos_by_code_table") + ) + expect_snapshot( + repos_with_code_from_host_table <- github_testhost_priv$get_repos_with_code_from_host( + code = "DESCRIPTION", + in_path = TRUE, + output = "table_full", + verbose = TRUE + ) + ) + expect_repos_table(repos_with_code_from_host_table) +}) + +test_that("`get_repos_with_code_from_repos()` works", { + github_testhost_priv <- create_github_testhost( + repos = c("TestRepo1", "TestRepo2"), + mode = "private" + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_repos, + "rest_engine$get_repos_by_code", + test_mocker$use("gh_repos_by_code") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_repos, + "rest_engine$get_repos_issues", + test_mocker$use("gh_repos_by_code_table") + ) + github_testhost_priv$searching_scope <- c("repo") + expect_snapshot( + repos_with_code_from_repos_full <- github_testhost_priv$get_repos_with_code_from_repos( + code = "tests", + output = "table_full", + verbose = TRUE + ) + ) + expect_repos_table(repos_with_code_from_repos_full) +}) + +test_that("`get_repos_with_code_from_repos()` pulls minimum version of table", { + github_testhost_priv <- create_github_testhost( + repos = c("TestRepo1", "TestRepo2"), + mode = "private" + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_repos, + "rest_engine$get_repos_by_code", + test_mocker$use("gh_repos_by_code") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_repos, + "rest_engine$get_repos_issues", + test_mocker$use("gh_repos_by_code_table") + ) + github_testhost_priv$searching_scope <- c("org", "repo") + expect_snapshot( + repos_with_code_from_repos_min <- github_testhost_priv$get_repos_with_code_from_repos( + code = "tests", + in_files = "DESCRIPTION", + output = "table_min", + verbose = TRUE + ) + ) + expect_repos_table(repos_with_code_from_repos_min, + repo_cols = repo_min_colnames) +}) + test_that("`get_repos_with_code_from_host()` pulls raw response", { mockery::stub( github_testhost_priv$get_repos_with_code_from_host, "rest_engine$get_repos_by_code", test_mocker$use("gh_repos_by_code_raw") ) - repos_with_code_from_host_raw <- github_testhost_priv$get_repos_with_code_from_host( - code = "shiny", - in_files = c("DESCRIPTION", "NAMESPACE"), - output = "raw", - verbose = FALSE + expect_snapshot( + repos_with_code_from_host_raw <- github_testhost_priv$get_repos_with_code_from_host( + code = "shiny", + in_files = c("DESCRIPTION", "NAMESPACE"), + output = "raw", + verbose = TRUE + ) ) expect_type(repos_with_code_from_host_raw, "list") expect_gt(length(repos_with_code_from_host_raw), 0) + test_mocker$cache(repos_with_code_from_host_raw) }) test_that("get_repos_with_code() works", { @@ -381,6 +518,25 @@ test_that("get_repos_with_code() works", { test_mocker$cache(github_repos_with_code_min) }) + +test_that("get_repos_with_code() scans whole host", { + mockery::stub( + github_testhost_priv$get_repos_with_code, + "private$get_repos_with_code_from_host", + test_mocker$use("repos_with_code_from_host_raw") + ) + github_testhost_priv$scan_all <- TRUE + github_repos_with_code_raw <- github_testhost_priv$get_repos_with_code( + code = "test-code", + output = "raw", + verbose = FALSE, + progress = FALSE + ) + expect_type(github_repos_with_code_raw, "list") + expect_gt(length(github_repos_with_code_raw), 0) + github_testhost_priv$scan_all <- FALSE +}) + test_that("GitHub prepares repos table from repositories response", { gh_repos_table <- test_graphql_github$prepare_repos_table( repos_list = test_mocker$use("gh_repos_from_org") @@ -391,14 +547,63 @@ test_that("GitHub prepares repos table from repositories response", { test_mocker$cache(gh_repos_table) }) -test_that("`get_all_repos()` works as expected", { +test_that("get_repos_from_org works", { mockery::stub( - github_testhost_priv$get_all_repos, + github_testhost_priv$get_repos_from_orgs, + "graphql_engine$prepare_repos_table", + test_mocker$use("gh_repos_table") + ) + gh_repos_from_orgs <- github_testhost_priv$get_repos_from_orgs( + verbose = FALSE, + progress = FALSE + ) + expect_repos_table( + gh_repos_from_orgs + ) + test_mocker$cache(gh_repos_from_orgs) +}) + +test_that("get_repos_from_repos works", { + test_org <- "test_org" + attr(test_org, "type") <- "organization" + mockery::stub( + github_testhost_priv$get_repos_from_repos, + "private$set_owner_type", + test_org + ) + mockery::stub( + github_testhost_priv$get_repos_from_repos, "graphql_engine$prepare_repos_table", test_mocker$use("gh_repos_table") ) + github_testhost_priv$searching_scope <- c("org", "repo") + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + expect_snapshot( + gh_repos_individual <- github_testhost_priv$get_repos_from_repos( + verbose = TRUE, + progress = FALSE + ) + ) + expect_repos_table( + gh_repos_individual + ) + test_mocker$cache(gh_repos_individual) +}) + +test_that("`get_all_repos()` works as expected", { + mockery::stub( + github_testhost_priv$get_all_repos, + "private$get_repos_from_orgs", + test_mocker$use("gh_repos_from_orgs") + ) + mockery::stub( + github_testhost_priv$get_all_repos, + "private$get_repos_from_repos", + test_mocker$use("gh_repos_individual") + ) gh_repos_table <- github_testhost_priv$get_all_repos( - verbose = FALSE + verbose = FALSE, + progress = FALSE ) expect_repos_table( gh_repos_table @@ -406,15 +611,30 @@ test_that("`get_all_repos()` works as expected", { test_mocker$cache(gh_repos_table) }) -test_that("`get_all_repos()` prints proper message", { +test_that("`get_all_repos()` is set to scan whole git host", { + github_testhost_all_priv <- create_github_testhost_all( + orgs = "test_org", + mode = "private" + ) mockery::stub( - github_testhost_priv$get_all_repos, - "graphql_engine$prepare_repos_table", - test_mocker$use("gh_repos_table") + github_testhost_all_priv$get_all_repos, + "graphql_engine$get_orgs", + "test_org" + ) + mockery::stub( + github_testhost_all_priv$get_all_repos, + "private$get_repos_from_orgs", + test_mocker$use("gh_repos_from_orgs") + ) + mockery::stub( + github_testhost_all_priv$get_all_repos, + "private$get_repos_from_repos", + test_mocker$use("gh_repos_individual") ) expect_snapshot( - gh_repos_table <- github_testhost_priv$get_all_repos( - verbose = TRUE + gh_repos <- github_testhost_all_priv$get_all_repos( + verbose = TRUE, + progress = FALSE ) ) }) @@ -461,7 +681,7 @@ test_that("`get_repos_contributors()` adds contributors to repos table", { ) gh_repos_with_contributors <- test_rest_github$get_repos_contributors( repos_table = test_mocker$use("gh_repos_table_with_platform"), - progress = FALSE + progress = FALSE ) expect_repos_table( gh_repos_with_contributors, @@ -480,10 +700,12 @@ test_that("`get_repos_contributors()` works on GitHost level", { "rest_engine$get_repos_contributors", test_mocker$use("gh_repos_with_contributors") ) - gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( - repos_table = test_mocker$use("gh_repos_table_with_platform"), - verbose = FALSE, - progress = FALSE + expect_snapshot( + gh_repos_with_contributors <- github_testhost_priv$get_repos_contributors( + repos_table = test_mocker$use("gh_repos_table_with_platform"), + verbose = TRUE, + progress = FALSE + ) ) expect_repos_table( gh_repos_with_contributors, @@ -552,5 +774,16 @@ test_that("`get_repos()` pulls table in minimalist version", { repo_cols = repo_min_colnames, with_cols = c("api_url", "platform") ) + gh_repos_table_min <- github_testhost$get_repos( + add_contributors = FALSE, + with_file = "test_file", + output = "table_min", + verbose = FALSE + ) + expect_repos_table( + gh_repos_table_min, + repo_cols = repo_min_colnames, + with_cols = c("api_url", "platform") + ) test_mocker$cache(gh_repos_table_min) }) diff --git a/tests/testthat/test-01-get_repos-GitLab.R b/tests/testthat/test-01-get_repos-GitLab.R index fffd5153..695eb38d 100644 --- a/tests/testthat/test-01-get_repos-GitLab.R +++ b/tests/testthat/test-01-get_repos-GitLab.R @@ -119,6 +119,39 @@ test_that("`get_repos_from_org()` does not fail when GraphQL response is not com ) }) +test_that("`search_for_code()` works", { + mockery::stub( + test_rest_gitlab_priv$search_for_code, + "self$response", + list() + ) + expect_snapshot( + gl_search_repos_by_code <- test_rest_gitlab_priv$search_for_code( + code = "test", + filename = "TESTFILE", + verbose = TRUE, + page_max = 2 + ) + ) +}) + +test_that("`search_repos_for_code()` works", { + mockery::stub( + test_rest_gitlab_priv$search_repos_for_code, + "self$response", + list() + ) + expect_snapshot( + gl_search_repos_by_code <- test_rest_gitlab_priv$search_repos_for_code( + code = "test", + repos = "TestRepo", + filename = "TESTFILE", + verbose = TRUE, + page_max = 2 + ) + ) +}) + test_that("`map_search_into_repos()` works", { gl_search_response <- test_fixtures$gitlab_search_response test_mocker$cache(gl_search_response) @@ -132,7 +165,7 @@ test_that("`map_search_into_repos()` works", { test_mocker$cache(gl_search_repos_by_code) }) -test_that("`get_repos_languages` works", { +test_that("`get_repos_languages()` works", { repos_list <- test_mocker$use("gl_search_repos_by_code") repos_list[[1]]$id <- "45300912" mockery::stub( @@ -140,12 +173,63 @@ test_that("`get_repos_languages` works", { "self$response", test_fixtures$gitlab_languages_response ) - repos_list_with_languages <- test_rest_gitlab_priv$get_repos_languages( + gl_repos_list_with_languages <- test_rest_gitlab_priv$get_repos_languages( repos_list = repos_list, progress = FALSE ) - purrr::walk(repos_list_with_languages, ~ expect_list_contains(., "languages")) - expect_equal(repos_list_with_languages[[1]]$languages, c("Python", "R")) + purrr::walk(gl_repos_list_with_languages, ~ expect_list_contains(., "languages")) + expect_equal(gl_repos_list_with_languages[[1]]$languages, c("Python", "R")) + test_mocker$cache(gl_repos_list_with_languages) +}) + +test_that("`get_repos_by_code()` works", { + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$search_for_code", + test_fixtures$gitlab_search_response + ) + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$map_search_into_repos", + test_mocker$use("gl_search_repos_by_code") + ) + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$get_repos_languages", + test_mocker$use("gl_repos_list_with_languages") + ) + gl_repos_by_code <- test_rest_gitlab$get_repos_by_code( + code = "test", + org = "test_org" + ) + expect_gl_repos_rest_response( + gl_repos_by_code + ) +}) + +test_that("`get_repos_by_code()` works", { + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$search_repos_for_code", + test_fixtures$gitlab_search_response + ) + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$map_search_into_repos", + test_mocker$use("gl_search_repos_by_code") + ) + mockery::stub( + test_rest_gitlab$get_repos_by_code, + "private$get_repos_languages", + test_mocker$use("gl_repos_list_with_languages") + ) + gl_repos_by_code <- test_rest_gitlab$get_repos_by_code( + code = "test", + repos = c("TestRepo1", "TestRepo2") + ) + expect_gl_repos_rest_response( + gl_repos_by_code + ) }) test_that("`prepare_repos_table()` prepares repos table", { @@ -158,6 +242,24 @@ test_that("`prepare_repos_table()` prepares repos table", { test_mocker$cache(gl_repos_table) }) +test_that("get_repos_from_org prints proper message", { + mockery::stub( + gitlab_testhost_priv$get_repos_from_orgs, + "graphql_engine$prepare_repos_table", + test_mocker$use("gl_repos_table") + ) + expect_snapshot( + gl_repos_from_orgs <- gitlab_testhost_priv$get_repos_from_orgs( + verbose = TRUE, + progress = FALSE + ) + ) + expect_repos_table( + gl_repos_from_orgs + ) + test_mocker$cache(gl_repos_from_orgs) +}) + test_that("GitHost adds `repo_api_url` column to GitLab repos table", { repos_table <- test_mocker$use("gl_repos_table") gl_repos_table_with_api_url <- gitlab_testhost_priv$add_repo_api_url(repos_table) @@ -168,7 +270,10 @@ test_that("GitHost adds `repo_api_url` column to GitLab repos table", { test_that("`tailor_repos_response()` tailors precisely `repos_list`", { gl_repos_by_code <- test_mocker$use("gl_search_repos_by_code") gl_repos_by_code_tailored <- - test_rest_gitlab$tailor_repos_response(gl_repos_by_code) + test_rest_gitlab$tailor_repos_response( + repos_response = gl_repos_by_code, + output = "table_full" + ) gl_repos_by_code_tailored %>% expect_type("list") %>% expect_length(length(gl_repos_by_code)) @@ -188,6 +293,27 @@ test_that("`tailor_repos_response()` tailors precisely `repos_list`", { test_mocker$cache(gl_repos_by_code_tailored) }) +test_that("`tailor_repos_response()` tailors precisely to minimal `repos_list`", { + gl_repos_by_code <- test_mocker$use("gl_search_repos_by_code") + gl_repos_by_code_tailored <- + test_rest_gitlab$tailor_repos_response( + repos_response = gl_repos_by_code, + output = "table_min" + ) + gl_repos_by_code_tailored %>% + expect_type("list") %>% + expect_length(length(gl_repos_by_code)) + + expect_list_contains_only( + gl_repos_by_code_tailored[[1]], + c("repo_id", "repo_name", "created_at", "default_branch", "organization") + ) + expect_lt( + length(gl_repos_by_code_tailored[[1]]), + length(gl_repos_by_code[[1]]) + ) +}) + test_that("REST client prepares table from GitLab repositories response", { gl_repos_by_code_table <- test_rest_gitlab$prepare_repos_table( repos_list = test_mocker$use("gl_repos_by_code_tailored"), diff --git a/tests/testthat/test-01-get_commits-GitHub.R b/tests/testthat/test-02-get_commits-GitHub.R similarity index 57% rename from tests/testthat/test-01-get_commits-GitHub.R rename to tests/testthat/test-02-get_commits-GitHub.R index 91140a8f..38f0c5e1 100644 --- a/tests/testthat/test-01-get_commits-GitHub.R +++ b/tests/testthat/test-02-get_commits-GitHub.R @@ -98,60 +98,120 @@ test_that("fill_empty_authors() works as expected", { ) }) +test_that("`get_repos_names` works", { + mockery::stub( + github_testhost_priv$get_repos_names, + "graphql_engine$get_repos_from_org", + test_mocker$use("gh_repos_from_org") + ) + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + github_testhost_priv$searching_scope <- "org" + gh_repos_names <- github_testhost_priv$get_repos_names( + org = "test_org" + ) + expect_type(gh_repos_names, "character") + expect_gt(length(gh_repos_names), 0) + test_mocker$cache(gh_repos_names) +}) + test_that("get_commits_from_orgs for GitHub works", { mockery::stub( - github_testhost_repos_priv$get_commits_from_orgs, + github_testhost_priv$get_commits_from_orgs, "graphql_engine$prepare_commits_table", test_mocker$use("gh_commits_table") ) - suppressMessages( - gh_commits_table <- github_testhost_repos_priv$get_commits_from_orgs( - since = "2023-03-01", - until = "2023-04-01", - verbose = FALSE, - progress = FALSE - ) + mockery::stub( + github_testhost_priv$get_commits_from_orgs, + "private$get_repos_names", + test_mocker$use("gh_repos_names") + ) + github_testhost_priv$searching_scope <- "org" + gh_commits_from_orgs <- github_testhost_priv$get_commits_from_orgs( + since = "2023-03-01", + until = "2023-04-01", + verbose = FALSE, + progress = FALSE ) expect_commits_table( - gh_commits_table + gh_commits_from_orgs ) - test_mocker$cache(gh_commits_table) + test_mocker$cache(gh_commits_from_orgs) +}) + + +test_that("get_commits_from_repos for GitHub works", { + mockery::stub( + github_testhost_priv$get_commits_from_repos, + "graphql_engine$prepare_commits_table", + test_mocker$use("gh_commits_table") + ) + github_testhost_priv$searching_scope <- "repo" + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + test_org <- "test_org" + attr(test_org, "type") <- "organization" + mockery::stub( + github_testhost_priv$get_commits_from_repos, + "private$set_owner_type", + test_org + ) + gh_commits_from_repos <- github_testhost_priv$get_commits_from_repos( + since = "2023-03-01", + until = "2023-04-01", + verbose = FALSE, + progress = FALSE + ) + expect_commits_table( + gh_commits_from_repos + ) + test_mocker$cache(gh_commits_from_repos) }) test_that("`get_commits()` retrieves commits in the table format", { mockery::stub( github_testhost$get_commits, "private$get_commits_from_orgs", - test_mocker$use("gh_commits_table") + test_mocker$use("gh_commits_from_orgs") ) - suppressMessages( - commits_table <- github_testhost$get_commits( - since = "2023-01-01", - until = "2023-02-28", - verbose = FALSE, - progress = FALSE - ) + mockery::stub( + github_testhost$get_commits, + "private$get_commits_from_repos", + test_mocker$use("gh_commits_from_repos") + ) + gh_commits_table <- github_testhost$get_commits( + since = "2023-01-01", + until = "2023-02-28", + verbose = FALSE, + progress = FALSE ) expect_commits_table( - commits_table + gh_commits_table ) + test_mocker$cache(gh_commits_table) }) -test_that("get_commits for GitHub repositories works", { +test_that("`get_commits()` is set to scan whole git host", { + github_testhost_all <- create_github_testhost_all(orgs = "test_org") mockery::stub( - github_testhost_repos$get_commits, + github_testhost_all$get_commits, + "graphql_engine$get_orgs", + "test_org" + ) + mockery::stub( + github_testhost_all$get_commits, "private$get_commits_from_orgs", - test_mocker$use("gh_commits_table") + test_mocker$use("gh_commits_from_orgs") ) - suppressMessages( - gh_commits_table <- github_testhost_repos$get_commits( - since = "2023-03-01", - until = "2023-04-01", - verbose = FALSE, + mockery::stub( + github_testhost_all$get_commits, + "private$get_commits_from_repos", + test_mocker$use("gh_commits_from_repos") + ) + expect_snapshot( + gh_commits_table <- github_testhost_all$get_commits( + since = "2023-01-01", + until = "2023-02-28", + verbose = TRUE, progress = FALSE ) ) - expect_commits_table( - gh_commits_table - ) }) diff --git a/tests/testthat/test-01-get_commits-GitLab.R b/tests/testthat/test-02-get_commits-GitLab.R similarity index 72% rename from tests/testthat/test-01-get_commits-GitLab.R rename to tests/testthat/test-02-get_commits-GitLab.R index 5f904133..8ee44a3f 100644 --- a/tests/testthat/test-01-get_commits-GitLab.R +++ b/tests/testthat/test-02-get_commits-GitLab.R @@ -1,17 +1,34 @@ -test_that("`get_commits_from_repos()` pulls commits from repo", { +test_that("`get_commits_from_one_repo()` pulls commits from repository", { + mockery::stub( + test_rest_gitlab_priv$get_commits_from_one_repo, + "private$paginate_results", + test_fixtures$gitlab_commits_response + ) + gl_commits_repo <- test_rest_gitlab_priv$get_commits_from_one_repo( + repo_path = "TestRepo", + since = "2023-01-01", + until = "2023-04-20" + ) + expect_gt(length(gl_commits_repo), 1) + purrr::walk(gl_commits_repo, ~ expect_gl_commit_rest_response(.)) + test_mocker$cache(gl_commits_repo) +}) + +test_that("`get_commits_from_repos()` pulls commits from repositories", { mockery::stub( test_rest_gitlab$get_commits_from_repos, "private$get_commits_from_one_repo", - test_fixtures$gitlab_commits_response + test_mocker$use("gl_commits_repo") ) - repos_names <- c("mbtests%2Fgitstatstesting", "mbtests%2Fgitstats-testing-2") + repos_names <- c("test_org/TestRepo1", "test_org/TestRepo2") gl_commits_org <- test_rest_gitlab$get_commits_from_repos( repos_names = repos_names, since = "2023-01-01", until = "2023-04-20", progress = FALSE ) - purrr::walk(gl_commits_org, ~ expect_gl_commit_rest_response(.)) + expect_equal(names(gl_commits_org), c("test_org/TestRepo1", "test_org/TestRepo2")) + purrr::walk(gl_commits_org[[1]], ~ expect_gl_commit_rest_response(.)) test_mocker$cache(gl_commits_org) }) @@ -126,11 +143,11 @@ test_that("get_commits_from_orgs works", { "rest_engine$get_commits_authors_handles_and_names", test_mocker$use("gl_commits_table") ) - suppressMessages( + expect_snapshot( gl_commits_table <- gitlab_testhost_priv$get_commits_from_orgs( since = "2023-03-01", until = "2023-04-01", - verbose = FALSE, + verbose = TRUE, progress = FALSE ) ) @@ -140,16 +157,32 @@ test_that("get_commits_from_orgs works", { test_mocker$cache(gl_commits_table) }) -test_that("get_commits for GitLab works with repos implied", { +test_that("get_commits_from_repos works", { + gitlab_testhost_priv <- create_gitlab_testhost( + repos = "TestRepo", + mode = "private" + ) + test_org <- "test_org" + attr(test_org, "type") <- "organization" mockery::stub( - gitlab_testhost_repos$get_commits, - "private$get_commits_from_orgs", + gitlab_testhost_priv$get_repos_from_repos, + "private$set_owner_type", + test_org + ) + mockery::stub( + gitlab_testhost_priv$get_commits_from_repos, + "rest_engine$get_commits_authors_handles_and_names", test_mocker$use("gl_commits_table") ) - gl_commits_table <- gitlab_testhost_repos$get_commits( - since = "2023-01-01", - until = "2023-06-01", - verbose = FALSE + gitlab_testhost_priv$searching_scope <- "repo" + gitlab_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + expect_snapshot( + gl_commits_table <- gitlab_testhost_priv$get_commits_from_repos( + since = "2023-03-01", + until = "2023-04-01", + verbose = TRUE, + progress = FALSE + ) ) expect_commits_table( gl_commits_table diff --git a/tests/testthat/test-01-get_commits-GitStats.R b/tests/testthat/test-02-get_commits-GitStats.R similarity index 100% rename from tests/testthat/test-01-get_commits-GitStats.R rename to tests/testthat/test-02-get_commits-GitStats.R diff --git a/tests/testthat/test-get_files_content-GitHub.R b/tests/testthat/test-get_files_content-GitHub.R index ec9183c8..54c896ce 100644 --- a/tests/testthat/test-get_files_content-GitHub.R +++ b/tests/testthat/test-get_files_content-GitHub.R @@ -47,23 +47,6 @@ test_that("GitHub GraphQL Engine pulls file response", { test_mocker$cache(github_file_response) }) -test_that("GitHub GraphQL Engine pulls png file response", { - mockery::stub( - test_graphql_github_priv$get_file_response, - "self$gql_response", - test_fixtures$github_png_file_response - ) - github_png_file_response <- test_graphql_github_priv$get_file_response( - org = "r-world-devs", - repo = "GitStats", - def_branch = "master", - file_path = "man/figures/logo.png", - files_query = test_mocker$use("gh_file_blobs_from_repo_query") - ) - expect_github_files_response(github_png_file_response) - test_mocker$cache(github_png_file_response) -}) - test_that("get_repositories_with_files works", { mockery::stub( test_graphql_github_priv$get_repositories_with_files, @@ -73,11 +56,10 @@ test_that("get_repositories_with_files works", { gh_repositories_with_files <- test_graphql_github_priv$get_repositories_with_files( repositories = c("test_repo_1", "test_repo_2", "test_repo_3", "test_repo_4", "test_repo_5"), def_branches = c("test_branch_1", "test_branch_2", "test_branch_3", "test_branch_4", "test_branch_5"), - org = "test-org", - file_paths = "test_files", - only_text_files = TRUE, + org = "test_org", + file_paths = "test_files.txt", host_files_structure = NULL, - progress = FALSE + progress = FALSE ) expect_type(gh_repositories_with_files, "list") test_mocker$cache(gh_repositories_with_files) @@ -95,13 +77,12 @@ test_that("GitHub GraphQL Engine pulls files from organization", { test_mocker$use("gh_repositories_with_files") ) github_files_response <- test_graphql_github$get_files_from_org( - org = "test_org", - repos = NULL, - file_paths = "test_files", - only_text_files = TRUE, + org = "test_org", + repos = NULL, + file_paths = "test_files.txt", host_files_structure = NULL, - verbose = FALSE, - progress = FALSE + verbose = FALSE, + progress = FALSE ) expect_github_files_response(github_files_response) test_mocker$cache(github_files_response) @@ -110,51 +91,65 @@ test_that("GitHub GraphQL Engine pulls files from organization", { test_that("GitHubHost prepares table from files response", { gh_files_table <- test_graphql_github$prepare_files_table( files_response = test_mocker$use("github_files_response"), - org = "r-world-devs", - file_path = "LICENSE" + org = "r-world-devs" ) expect_files_table(gh_files_table) test_mocker$cache(gh_files_table) }) -test_that("GitHubHost prepares table from files with no content", { - empty_files_response <- test_mocker$use("github_files_response") %>% - purrr::map(function(test_repo) { - test_repo$test_files$file$text <- NULL - return(test_repo) - }) - gh_empty_files_table <- test_graphql_github$prepare_files_table( - files_response = empty_files_response, - org = "test_org", - file_path = "test_files" +test_that("get_files_content_from_orgs for GitHub works", { + mockery::stub( + github_testhost_priv$get_files_content_from_orgs, + "graphql_engine$prepare_files_table", + test_mocker$use("gh_files_table") + ) + github_testhost_priv$searching_scope <- "org" + gh_files_from_orgs <- github_testhost_priv$get_files_content_from_orgs( + file_path = "DESCRIPTION", + verbose = FALSE ) - expect_files_table(gh_empty_files_table) - expect_true(all(is.na(gh_empty_files_table$file_content))) - test_mocker$cache(gh_empty_files_table) + expect_files_table( + gh_files_from_orgs, + with_cols = "api_url" + ) + test_mocker$cache(gh_files_from_orgs) }) -test_that("get_files_content_from_orgs for GitHub works", { +test_that("get_files_content_from_repos for GitHub works", { + test_org <- "test_org" + attr(test_org, "type") <- "organization" mockery::stub( - github_testhost_priv$get_files_content_from_orgs, + github_testhost_priv$get_files_content_from_repos, + "private$set_owner_type", + test_org + ) + mockery::stub( + github_testhost_priv$get_files_content_from_repos, "graphql_engine$prepare_files_table", test_mocker$use("gh_files_table") ) - gh_files_table <- github_testhost_priv$get_files_content_from_orgs( + github_testhost_priv$searching_scope <- "repo" + gh_files_from_repos <- github_testhost_priv$get_files_content_from_repos( file_path = "DESCRIPTION", verbose = FALSE ) expect_files_table( - gh_files_table, + gh_files_from_repos, with_cols = "api_url" ) - test_mocker$cache(gh_files_table) + test_mocker$cache(gh_files_from_repos) }) test_that("`get_files_content()` pulls files in the table format", { mockery::stub( github_testhost$get_files_content, "private$get_files_content_from_orgs", - test_mocker$use("gh_files_table") + test_mocker$use("gh_files_from_orgs") + ) + mockery::stub( + github_testhost$get_files_content, + "private$get_files_content_from_repos", + test_mocker$use("gh_files_from_repos") ) gh_files_table <- github_testhost$get_files_content( file_path = "DESCRIPTION" diff --git a/tests/testthat/test-get_files_content-GitLab.R b/tests/testthat/test-get_files_content-GitLab.R index 1c84606a..acacb91e 100644 --- a/tests/testthat/test-get_files_content-GitLab.R +++ b/tests/testthat/test-get_files_content-GitLab.R @@ -60,7 +60,6 @@ test_that("GitLab GraphQL Engine pulls files from a group", { type = "organization", repos = NULL, file_paths = "meta_data.yaml", - only_text_files = TRUE, host_files_structure = NULL ) expect_gitlab_files_from_org_response(gitlab_files_response) @@ -78,15 +77,16 @@ test_that("GitLab GraphQL Engine pulls files from org by iterating over repos", "private$get_file_blobs_response", test_mocker$use("gl_file_blobs_response") ) - gl_files_from_org <- test_graphql_gitlab$get_files_from_org_per_repo( - org = "mbtests", - repos = "graphql_tests", + gl_files_from_org_per_repo <- test_graphql_gitlab$get_files_from_org_per_repo( + org = "test_org", + repos = "TestProject", file_paths = c("project_metadata.yaml", "README.md") ) expect_gitlab_files_from_org_by_repos_response( - response = gl_files_from_org, + response = gl_files_from_org_per_repo, expected_files = c("project_metadata.yaml", "README.md") ) + test_mocker$cache(gl_files_from_org_per_repo) }) test_that("is query error is FALSE when response is empty (non query error)", { @@ -111,19 +111,23 @@ test_that("Gitlab GraphQL switches to pulling files per repositories when query "private$is_complexity_error", TRUE ) + mockery::stub( + test_graphql_gitlab$get_files_from_org, + "self$get_files_from_org_per_repo", + test_mocker$use("gl_files_from_org_per_repo") + ) gitlab_files_response_by_repos <- test_graphql_gitlab$get_files_from_org( org = "mbtests", type = "organization", repos = NULL, - file_paths = c("DESCRIPTION", "project_metadata.yaml", "README.md"), + file_paths = c("project_metadata.yaml", "README.md"), host_files_structure = NULL, - only_text_files = TRUE, verbose = FALSE, progress = FALSE ) expect_gitlab_files_from_org_by_repos_response( response = gitlab_files_response_by_repos, - expected_files = c("DESCRIPTION", "project_metadata.yaml", "README.md") + expected_files = c("project_metadata.yaml", "README.md") ) test_mocker$cache(gitlab_files_response_by_repos) }) @@ -144,8 +148,7 @@ test_that("checker properly identifies gitlab files responses", { test_that("GitLab prepares table from files response", { gl_files_table <- test_graphql_gitlab$prepare_files_table( files_response = test_mocker$use("gitlab_files_response"), - org = "mbtests", - file_path = "meta_data.yaml" + org = "mbtests" ) expect_files_table(gl_files_table) test_mocker$cache(gl_files_table) @@ -154,8 +157,7 @@ test_that("GitLab prepares table from files response", { test_that("GitLab prepares table from files response prepared in alternative way", { gl_files_table <- test_graphql_gitlab$prepare_files_table( files_response = test_mocker$use("gitlab_files_response_by_repos"), - org = "mbtests", - file_path = "meta_data.yaml" + org = "mbtests" ) expect_files_table(gl_files_table) }) @@ -177,16 +179,3 @@ test_that("get_files_content_from_orgs for GitLab works", { ) test_mocker$cache(gl_files_table) }) - -test_that("`get_files_content()` pulls files in the table format", { - mockery::stub( - gitlab_testhost$get_files_content, - "super$get_files_content", - test_mocker$use("gl_files_table") - ) - gl_files_table <- gitlab_testhost$get_files_content( - file_path = "README.md" - ) - expect_files_table(gl_files_table, with_cols = "api_url") - test_mocker$cache(gl_files_table) -}) diff --git a/tests/testthat/test-get_files_structure-GitHub.R b/tests/testthat/test-get_files_structure-GitHub.R index 0c63da20..cc6fc82c 100644 --- a/tests/testthat/test-get_files_structure-GitHub.R +++ b/tests/testthat/test-get_files_structure-GitHub.R @@ -14,7 +14,7 @@ test_that("get_file_response works", { test_fixtures$github_files_tree_response ) gh_files_tree_response <- test_graphql_github_priv$get_file_response( - org = "test-org", + org = "test_org", repo = "TestRepo", def_branch = "master", file_path = "", @@ -55,7 +55,7 @@ test_that("get_files_structure_from_repo returns list with files and dirs vector files_and_dirs ) files_structure <- test_graphql_github_priv$get_files_structure_from_repo( - org = "test-org", + org = "test_org", repo = "TestRepo", def_branch = "master" ) @@ -77,13 +77,13 @@ test_that("get_files_structure_from_repo returns list of files up to 2 tier of d files_and_dirs <- test_mocker$use("files_and_dirs_list") ) files_structure_very_shallow <- test_graphql_github_priv$get_files_structure_from_repo( - org = "test-org", + org = "test_org", repo = "TestRepo", def_branch = "master", depth = 1L ) files_structure_shallow <- test_graphql_github_priv$get_files_structure_from_repo( - org = "test-org", + org = "test_org", repo = "TestRepo", def_branch = "master", depth = 2L @@ -127,12 +127,12 @@ test_that("GitHub GraphQL Engine pulls files structure from repositories", { ) gh_files_structure <- test_graphql_github$get_files_structure_from_org( org = "test_org", - repos = rep("TestRepo", 5) + repos = c("TestRepo", "TestRepo1", "TestRepo2", "TestRepo3", "TestRepo4") ) purrr::walk(gh_files_structure, ~ expect_true(length(.) > 0)) expect_equal( names(gh_files_structure), - rep("TestRepo", 5) + c("TestRepo", "TestRepo1", "TestRepo2", "TestRepo3", "TestRepo4") ) test_mocker$cache(gh_files_structure) }) @@ -149,7 +149,7 @@ test_that("GitHub GraphQL Engine pulls files structure with pattern from reposit test_mocker$use("md_files_structure") ) gh_md_files_structure <- test_graphql_github$get_files_structure_from_org( - org = "test-org", + org = "test_org", repos = "TestRepo", pattern = "\\.md|\\.qmd|\\.Rmd" ) @@ -163,6 +163,7 @@ test_that("get_files_structure_from_orgs", { "graphql_engine$get_files_structure_from_org", test_mocker$use("gh_md_files_structure") ) + github_testhost_priv$searching_scope <- "org" gh_files_structure_from_orgs <- github_testhost_priv$get_files_structure_from_orgs( pattern = "\\.md|\\.qmd|\\.Rmd", depth = Inf, @@ -174,7 +175,7 @@ test_that("get_files_structure_from_orgs", { ) expect_equal( names(gh_files_structure_from_orgs), - "test-org" + "test_org" ) test_mocker$cache(gh_files_structure_from_orgs) }) @@ -196,13 +197,14 @@ test_that("when files_structure is empty, appropriate message is returned", { mode = "private" ) mockery::stub( - github_testhost_priv$get_files_structure_from_orgs, + github_testhost_priv$get_files_structure_from_repos, "graphql_engine$get_files_structure_from_org", list() |> purrr::set_names() ) + github_testhost_priv$searching_scope <- "repo" expect_snapshot( - github_testhost_priv$get_files_structure_from_orgs( + github_testhost_priv$get_files_structure_from_repos( pattern = "\\.png", depth = 1L, verbose = TRUE @@ -218,8 +220,7 @@ test_that("get_path_from_files_structure gets file path from files structure", { test_graphql_github <- environment(test_graphql_github$initialize)$private file_path <- test_graphql_github$get_path_from_files_structure( host_files_structure = test_mocker$use("gh_files_structure_from_orgs"), - only_text_files = FALSE, - org = "test-org", + org = "test_org", repo = "TestRepo" ) expect_equal(typeof(file_path), "character") @@ -239,7 +240,7 @@ test_that("get_files_structure pulls files structure for repositories in orgs", ) expect_equal( names(gh_files_structure_from_orgs), - "test-org" + "test_org" ) purrr::walk(gh_files_structure_from_orgs[[1]], function(repo_files) { expect_true(any(grepl("\\.md|\\.Rmd", repo_files))) @@ -247,15 +248,25 @@ test_that("get_files_structure pulls files structure for repositories in orgs", test_mocker$cache(gh_files_structure_from_orgs) }) +test_that("get_files_structure aborts when scope to scan whole host", { + github_testhost$.__enclos_env__$private$scan_all <- TRUE + expect_snapshot_error( + github_testhost$get_files_structure( + pattern = "\\.md|\\.qmd", + depth = 1L, + verbose = FALSE + ) + ) +}) + test_that("get_files_content makes use of files_structure", { mockery::stub( - github_testhost_priv$get_files_content_from_orgs, + github_testhost_priv$get_files_content_from_files_structure, "private$add_repo_api_url", test_mocker$use("gh_files_table") ) expect_snapshot( - files_content <- github_testhost_priv$get_files_content_from_orgs( - file_path = NULL, + files_content <- github_testhost_priv$get_files_content_from_files_structure( host_files_structure = test_mocker$use("gh_files_structure_from_orgs") ) ) diff --git a/tests/testthat/test-get_files_structure-GitLab.R b/tests/testthat/test-get_files_structure-GitLab.R index e2bf6450..64f08825 100644 --- a/tests/testthat/test-get_files_structure-GitLab.R +++ b/tests/testthat/test-get_files_structure-GitLab.R @@ -198,7 +198,6 @@ test_that("get_path_from_files_structure gets file path from files structure", { test_graphql_gitlab <- environment(test_graphql_gitlab$initialize)$private file_path <- test_graphql_gitlab$get_path_from_files_structure( host_files_structure = test_mocker$use("gl_files_structure_from_orgs"), - only_text_files = TRUE, org = "mbtests" # this will need fixing and repo parameter must come back ) expect_equal(typeof(file_path), "character") @@ -229,13 +228,12 @@ test_that("get_files_structure pulls files structure for repositories in orgs", test_that("get_files_content makes use of files_structure", { mockery::stub( - gitlab_testhost_priv$get_files_content_from_orgs, + gitlab_testhost_priv$get_files_content_from_files_structure, "private$add_repo_api_url", test_mocker$use("gl_files_table") ) expect_snapshot( - files_content <- gitlab_testhost_priv$get_files_content_from_orgs( - file_path = NULL, + files_content <- gitlab_testhost_priv$get_files_content_from_files_structure( host_files_structure = test_mocker$use("gl_files_structure_from_orgs") ) ) diff --git a/tests/testthat/test-get_files_structure-GitStats.R b/tests/testthat/test-get_files_structure-GitStats.R index 280f5873..50d6c46c 100644 --- a/tests/testthat/test-get_files_structure-GitStats.R +++ b/tests/testthat/test-get_files_structure-GitStats.R @@ -11,7 +11,7 @@ test_that("get_files_structure_from_hosts works as expected", { ) expect_equal(names(files_structure_from_hosts), c("github.com", "gitlab.com")) - expect_equal(names(files_structure_from_hosts[[1]]), c("test-org")) + expect_equal(names(files_structure_from_hosts[[1]]), c("test_org")) files_structure_from_hosts[[2]] <- test_mocker$use("gl_files_structure_from_orgs") test_mocker$cache(files_structure_from_hosts) }) diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R index 60a0e56b..19b8f49a 100644 --- a/tests/testthat/test-get_release-GitHub.R +++ b/tests/testthat/test-get_release-GitHub.R @@ -23,9 +23,9 @@ test_that("`get_releases_from_org()` pulls releases from the repositories", { test_that("`prepare_releases_table()` prepares releases table", { releases_table <- test_graphql_github$prepare_releases_table( releases_response = test_mocker$use("releases_from_repos"), - org = "r-world-devs", - date_from = "2023-05-01", - date_until = "2023-09-30" + org = "r-world-devs", + since = "2023-05-01", + until = "2023-09-30" ) expect_releases_table(releases_table) expect_gt(min(releases_table$published_at), as.POSIXct("2023-05-01")) @@ -33,28 +33,63 @@ test_that("`prepare_releases_table()` prepares releases table", { test_mocker$cache(releases_table) }) -test_that("`set_repositories` works", { +test_that("`get_release_logs_from_orgs()` works", { mockery::stub( - github_testhost_priv$set_repositories, - "private$get_all_repos", - test_mocker$use("gh_repos_table") - ) - repos_names <- github_testhost_priv$set_repositories() - expect_type(repos_names, "character") - expect_gt(length(repos_names), 0) - test_mocker$cache(repos_names) + github_testhost_priv$get_release_logs_from_orgs, + "graphql_engine$prepare_releases_table", + test_mocker$use("releases_table") + ) + mockery::stub( + github_testhost_priv$get_release_logs_from_orgs, + "private$get_repos_names", + test_mocker$use("gh_repos_names") + ) + github_testhost_priv$searching_scope <- "org" + releases_from_orgs <- github_testhost_priv$get_release_logs_from_orgs( + since = "2023-05-01", + until = "2023-09-30", + verbose = FALSE, + progress = FALSE + ) + expect_releases_table(releases_from_orgs) + test_mocker$cache(releases_from_orgs) }) -test_that("`get_release_logs()` pulls release logs in the table format", { +test_that("`get_release_logs_from_repos()` works", { mockery::stub( - github_testhost$get_release_logs, + github_testhost_priv$get_release_logs_from_repos, "graphql_engine$prepare_releases_table", test_mocker$use("releases_table") ) + test_org <- "test_org" + attr(test_org, "type") <- "organization" + mockery::stub( + github_testhost_priv$get_release_logs_from_repos, + "private$set_owner_type", + test_org + ) + github_testhost_priv$searching_scope <- "repo" + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + releases_from_repos <- github_testhost_priv$get_release_logs_from_repos( + since = "2023-05-01", + until = "2023-09-30", + verbose = FALSE, + progress = FALSE + ) + expect_releases_table(releases_from_repos) + test_mocker$cache(releases_from_repos) +}) + +test_that("`get_release_logs()` pulls release logs in the table format", { + mockery::stub( + github_testhost$get_release_logs, + "private$get_release_logs_from_repos", + test_mocker$use("releases_from_repos") + ) mockery::stub( github_testhost$get_release_logs, - "private$set_repositories", - test_mocker$use("repos_names") + "private$get_release_logs_from_orgs", + test_mocker$use("releases_from_orgs") ) releases_table <- github_testhost$get_release_logs( since = "2023-05-01", @@ -68,21 +103,27 @@ test_that("`get_release_logs()` pulls release logs in the table format", { test_mocker$cache(releases_table) }) -test_that("`get_release_logs()` prints proper message when running", { +test_that("`get_release_logs()` is set to scan whole git host", { + github_testhost_all <- create_github_testhost_all(orgs = "test_org") mockery::stub( - github_testhost$get_release_logs, - "graphql_engine$prepare_releases_table", - test_mocker$use("releases_table") + github_testhost_all$get_release_logs, + "graphql_engine$get_orgs", + "test_org" ) mockery::stub( - github_testhost$get_release_logs, - "private$set_repositories", - test_mocker$use("repos_names") + github_testhost_all$get_release_logs, + "private$get_release_logs_from_repos", + test_mocker$use("releases_from_repos") + ) + mockery::stub( + github_testhost_all$get_release_logs, + "private$get_release_logs_from_orgs", + test_mocker$use("releases_from_orgs") ) expect_snapshot( - releases_table <- github_testhost$get_release_logs( - since = "2023-05-01", - until = "2023-09-30", + gh_releases_table <- github_testhost_all$get_release_logs( + since = "2023-01-01", + until = "2023-02-28", verbose = TRUE, progress = FALSE ) diff --git a/tests/testthat/test-get_release-GitLab.R b/tests/testthat/test-get_release-GitLab.R index fe79a434..05d8eb4f 100644 --- a/tests/testthat/test-get_release-GitLab.R +++ b/tests/testthat/test-get_release-GitLab.R @@ -24,9 +24,9 @@ test_that("`get_releases_from_org()` pulls releases from the repositories", { test_that("`prepare_releases_table()` prepares releases table", { releases_table <- test_graphql_gitlab$prepare_releases_table( releases_response = test_mocker$use("releases_from_repos"), - org = "test_org", - date_from = "2023-08-01", - date_until = "2024-06-30" + org = "test_org", + since = "2023-08-01", + until = "2024-06-30" ) expect_releases_table(releases_table) expect_gt(min(releases_table$published_at), as.POSIXct("2023-08-01")) @@ -34,12 +34,75 @@ test_that("`prepare_releases_table()` prepares releases table", { test_mocker$cache(releases_table) }) -test_that("`get_release_logs()` pulls release logs in the table format", { +test_that("`get_repos_names` works", { mockery::stub( - gitlab_testhost$get_release_logs, + gitlab_testhost_priv$get_repos_names, + "graphql_engine$get_repos_from_org", + test_mocker$use("gl_repos_from_org") + ) + gitlab_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + gitlab_testhost_priv$searching_scope <- "org" + repos_names <- gitlab_testhost_priv$get_repos_names( + org = "test_org" + ) + expect_type(repos_names, "character") + expect_gt(length(repos_names), 0) + test_mocker$cache(repos_names) +}) + +test_that("`get_release_logs_from_orgs()` works", { + mockery::stub( + gitlab_testhost_priv$get_release_logs_from_orgs, "graphql_engine$prepare_releases_table", test_mocker$use("releases_table") ) + mockery::stub( + gitlab_testhost_priv$get_release_logs_from_orgs, + "private$get_repos_names", + test_mocker$use("repos_names") + ) + gitlab_testhost_priv$searching_scope <- "org" + releases_from_orgs <- gitlab_testhost_priv$get_release_logs_from_orgs( + since = "2023-05-01", + until = "2023-09-30", + verbose = FALSE, + progress = FALSE + ) + expect_releases_table(releases_from_orgs) + test_mocker$cache(releases_from_orgs) +}) + +test_that("`get_release_logs_from_repos()` works", { + mockery::stub( + gitlab_testhost_priv$get_release_logs_from_repos, + "graphql_engine$prepare_releases_table", + test_mocker$use("releases_table") + ) + gitlab_testhost_priv$searching_scope <- "repo" + gitlab_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + expect_snapshot( + releases_from_repos <- gitlab_testhost_priv$get_release_logs_from_repos( + since = "2023-05-01", + until = "2023-09-30", + verbose = TRUE, + progress = FALSE + ) + ) + expect_releases_table(releases_from_repos) + test_mocker$cache(releases_from_repos) +}) + +test_that("`get_release_logs()` pulls release logs in the table format", { + mockery::stub( + gitlab_testhost$get_release_logs, + "private$get_release_logs_from_repos", + test_mocker$use("releases_from_repos") + ) + mockery::stub( + gitlab_testhost$get_release_logs, + "private$get_release_logs_from_orgs", + test_mocker$use("releases_from_orgs") + ) releases_table <- gitlab_testhost$get_release_logs( since = "2023-08-01", until = "2024-06-30", diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R index fd0c2e75..e474b445 100644 --- a/tests/testthat/test-get_urls_repos-GitHub.R +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -1,12 +1,12 @@ test_that("get_repos_urls() works for whole orgs", { mockery::stub( test_rest_github$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$github_repositories_rest_response ) gh_repos_urls <- test_rest_github$get_repos_urls( type = "web", - org = "test-org", + org = "test_org", repos = NULL ) expect_length( @@ -19,12 +19,12 @@ test_that("get_repos_urls() works for whole orgs", { test_that("get_repos_urls() works for individual repos", { mockery::stub( test_rest_github$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$github_repositories_rest_response ) gh_repos_urls <- test_rest_github$get_repos_urls( type = "web", - org = "test-org", + org = "test_org", repos = c("testRepo", "testRepo2") ) expect_length( @@ -34,15 +34,14 @@ test_that("get_repos_urls() works for individual repos", { test_mocker$cache(gh_repos_urls) }) -test_that("get_all_repos_urls prepares api repo_urls vector", { - github_testhost_priv <- create_github_testhost(orgs = "test-org", - mode = "private") +test_that("get_repos_urls prepares api repo_urls vector", { mockery::stub( test_rest_github$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$github_repositories_rest_response ) gh_api_repos_urls <- test_rest_github$get_repos_urls( + org = "test_org", repos = NULL, type = "api" ) @@ -52,21 +51,106 @@ test_that("get_all_repos_urls prepares api repo_urls vector", { test_mocker$cache(gh_api_repos_urls) }) -test_that("get_all_repos_urls prepares web repo_urls vector", { +test_that("get_repos_urls_from_orgs prepares web repo_urls vector", { mockery::stub( - github_testhost_priv$get_all_repos_urls, + github_testhost_priv$get_repos_urls_from_orgs, + "rest_engine$get_repos_urls", + test_mocker$use("gh_repos_urls") + ) + github_testhost_priv$searching_scope <- "org" + github_testhost_priv$orgs <- "test_org" + expect_snapshot( + gh_repos_urls_from_orgs <- github_testhost_priv$get_repos_urls_from_orgs( + type = "web", + verbose = TRUE, + progress = FALSE + ) + ) + expect_gt(length(gh_repos_urls_from_orgs), 0) + expect_true(any(grepl("test-org", gh_repos_urls_from_orgs))) + expect_true(all(grepl("https://testhost.com/", gh_repos_urls_from_orgs))) + test_mocker$cache(gh_repos_urls_from_orgs) +}) + +test_that("get_repos_urls_from_repos prepares web repo_urls vector", { + test_org <- "test_org" + attr(test_org, "type") <- "organization" + mockery::stub( + github_testhost_priv$get_repos_urls_from_repos, + "private$set_owner_type", + test_org + ) + mockery::stub( + github_testhost_priv$get_repos_urls_from_repos, "rest_engine$get_repos_urls", test_mocker$use("gh_repos_urls"), depth = 2L ) - gh_repos_urls <- github_testhost_priv$get_all_repos_urls( + github_testhost_priv$searching_scope <- c("repo") + github_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + expect_snapshot( + gh_repos_urls <- github_testhost_priv$get_repos_urls_from_repos( + type = "web", + verbose = TRUE, + progress = FALSE + ) + ) + expect_gt(length(gh_repos_urls), 0) + expect_true(any(grepl("test-org", gh_repos_urls))) + expect_true(all(grepl("https://testhost.com/", gh_repos_urls))) + test_mocker$cache(gh_repos_urls) +}) + +test_that("get_all_repos_urls prepares web repo_urls vector", { + mockery::stub( + github_testhost_priv$get_all_repos_urls, + "private$get_repos_urls_from_orgs", + test_mocker$use("gh_repos_urls_from_orgs") + ) + mockery::stub( + github_testhost_priv$get_all_repos_urls, + "private$get_repos_urls_from_repos", + test_mocker$use("gh_repos_urls") + ) + expect_snapshot( + gh_repos_urls <- github_testhost_priv$get_all_repos_urls( + type = "web", + verbose = TRUE + ) + ) + expect_gt(length(gh_repos_urls), 0) + expect_true(any(grepl("test-org", gh_repos_urls))) + expect_true(all(grepl("https://testhost.com/", gh_repos_urls))) + test_mocker$cache(gh_repos_urls) +}) + +test_that("get_all_repos_urls is set to scan whole host", { + github_testhost_all_priv <- create_github_testhost_all( + orgs = "test_org", + mode = "private" + ) + mockery::stub( + github_testhost_all_priv$get_all_repos_urls, + "graphql_engine$get_orgs", + "test_org" + ) + mockery::stub( + github_testhost_all_priv$get_all_repos_urls, + "private$get_repos_urls_from_orgs", + test_mocker$use("gh_repos_urls_from_orgs") + ) + mockery::stub( + github_testhost_all_priv$get_all_repos_urls, + "private$get_repos_urls_from_repos", + test_mocker$use("gh_repos_urls") + ) + gh_repos_urls <- github_testhost_all_priv$get_all_repos_urls( type = "web", verbose = FALSE ) expect_gt(length(gh_repos_urls), 0) expect_true(any(grepl("test-org", gh_repos_urls))) expect_true(all(grepl("https://testhost.com/", gh_repos_urls))) - test_mocker$cache(gh_repos_urls) }) test_that("get_repo_url_from_response retrieves repositories URLS", { @@ -86,19 +170,116 @@ test_that("get_repo_url_from_response retrieves repositories URLS", { test_mocker$cache(gh_repo_web_urls) }) -test_that("get_repos_urls returns repositories URLS", { +test_that("get_repos_urls_with_code_from_orgs returns repositories URLS", { mockery::stub( - github_testhost$get_repos_urls, + github_testhost_priv$get_repos_urls_with_code_from_orgs, "private$get_repo_url_from_response", test_mocker$use("gh_repo_web_urls") ) + gh_repos_urls_with_code_from_orgs <- github_testhost_priv$get_repos_urls_with_code_from_orgs( + type = "web", + code = "shiny", + in_files = "DESCRIPTION", + in_path = FALSE, + verbose = FALSE, + progress= FALSE + ) + expect_type(gh_repos_urls_with_code_from_orgs, "character") + expect_gt(length(gh_repos_urls_with_code_from_orgs), 0) + test_mocker$cache(gh_repos_urls_with_code_from_orgs) +}) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + github_testhost_priv$get_repos_urls_with_code_from_repos, + "private$get_repo_url_from_response", + test_mocker$use("gh_repo_web_urls") + ) + gh_repos_urls_with_code_from_repos <- github_testhost_priv$get_repos_urls_with_code_from_repos( + type = "web", + code = "shiny", + in_files = "DESCRIPTION", + in_path = FALSE, + verbose = FALSE, + progress = FALSE + ) + expect_type(gh_repos_urls_with_code_from_repos, "character") + expect_gt(length(gh_repos_urls_with_code_from_repos), 0) + test_mocker$cache(gh_repos_urls_with_code_from_repos) +}) + +test_that("get_repos_urls_with_code returns repositories URLS", { + mockery::stub( + github_testhost_priv$get_repos_urls_with_code, + "private$get_repos_urls_with_code_from_orgs", + test_mocker$use("gh_repos_urls_with_code_from_orgs") + ) + mockery::stub( + github_testhost_priv$get_repos_urls_with_code, + "private$get_repos_urls_with_code_from_repos", + test_mocker$use("gh_repos_urls_with_code_from_repos") + ) + gh_repos_urls_with_code_in_files <- github_testhost_priv$get_repos_urls_with_code( + type = "web", + code = "shiny", + in_files = "DESCRIPTION", + in_path = FALSE, + verbose = FALSE, + progress = FALSE + ) + expect_type(gh_repos_urls_with_code_in_files, "character") + expect_gt(length(gh_repos_urls_with_code_in_files), 0) + test_mocker$cache(gh_repos_urls_with_code_in_files) +}) + +test_that("get_repos_urls returns repositories URLS", { + mockery::stub( + github_testhost$get_repos_urls, + "private$get_repos_urls_with_code", + test_mocker$use("gh_repos_urls_with_code_in_files") + ) gh_repos_urls_with_code_in_files <- github_testhost$get_repos_urls( type = "web", with_code = "shiny", in_files = "DESCRIPTION", - verbose = FALSE + with_file = NULL, + verbose = FALSE, + progress = FALSE ) expect_type(gh_repos_urls_with_code_in_files, "character") expect_gt(length(gh_repos_urls_with_code_in_files), 0) test_mocker$cache(gh_repos_urls_with_code_in_files) }) + +test_that("get_repos_urls returns repositories URLS", { + mockery::stub( + github_testhost$get_repos_urls, + "private$get_repos_urls_with_code", + test_mocker$use("gh_repos_urls_with_code_in_files") + ) + gh_repos_urls_with_code_in_files <- github_testhost$get_repos_urls( + type = "web", + with_file = "DESCRIPTION", + verbose = FALSE, + progress = FALSE + ) + expect_type(gh_repos_urls_with_code_in_files, "character") + expect_gt(length(gh_repos_urls_with_code_in_files), 0) + test_mocker$cache(gh_repos_urls_with_code_in_files) +}) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + github_testhost$get_repos_urls, + "private$get_all_repos_urls", + test_mocker$use("gh_repos_urls") + ) + gh_repos_urls <- github_testhost$get_repos_urls( + type = "web", + verbose = FALSE, + progress = FALSE + ) + expect_type(gh_repos_urls, "character") + expect_gt(length(gh_repos_urls), 0) + test_mocker$cache(gh_repos_urls) +}) diff --git a/tests/testthat/test-get_urls_repos-GitLab.R b/tests/testthat/test-get_urls_repos-GitLab.R index 75f28141..91fb72dc 100644 --- a/tests/testthat/test-get_urls_repos-GitLab.R +++ b/tests/testthat/test-get_urls_repos-GitLab.R @@ -1,12 +1,12 @@ test_that("get_repos_urls() works for org", { mockery::stub( test_rest_gitlab$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$gitlab_repositories_rest_response ) gl_api_repos_urls <- test_rest_gitlab$get_repos_urls( type = "api", - org = "mbtests", + org = "test_org", repos = NULL ) expect_length( @@ -16,7 +16,7 @@ test_that("get_repos_urls() works for org", { test_mocker$cache(gl_api_repos_urls) gl_web_repos_urls <- test_rest_gitlab$get_repos_urls( type = "web", - org = "mbtests", + org = "test_org", repos = NULL ) expect_length( @@ -29,12 +29,12 @@ test_that("get_repos_urls() works for org", { test_that("get_repos_urls() works for individual repos", { mockery::stub( test_rest_gitlab$get_repos_urls, - "self$response", + "private$paginate_results", test_fixtures$gitlab_repositories_rest_response ) gl_api_repos_urls <- test_rest_gitlab$get_repos_urls( type = "api", - org = "mbtests", + org = "test_org", repos = c("testRepo1", "testRepo2") ) expect_length( @@ -54,44 +54,60 @@ test_that("get_repos_urls() works for individual repos", { test_mocker$cache(gl_web_repos_urls) }) -test_that("get_all_repos_urls prepares api repo_urls vector", { +test_that("get_repos_urls_from_orgs prepares api repo_urls vector", { mockery::stub( - gitlab_testhost_priv$get_all_repos_urls, + gitlab_testhost_priv$get_repos_urls_from_orgs, "rest_engine$get_repos_urls", test_mocker$use("gl_api_repos_urls") ) - gl_api_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( + gitlab_testhost_priv$searching_scope <- "org" + gitlab_testhost_priv$orgs <- "test_org" + gl_api_repos_urls <- gitlab_testhost_priv$get_repos_urls_from_orgs( type = "api", - verbose = FALSE + verbose = FALSE, + progress = FALSE ) expect_gt(length(gl_api_repos_urls), 0) expect_true(all(grepl("api", gl_api_repos_urls))) test_mocker$cache(gl_api_repos_urls) +}) + +test_that("get_repos_urls_from_repos prepares api repo_urls vector", { mockery::stub( - gitlab_testhost_priv$get_all_repos_urls, + gitlab_testhost_priv$get_repos_urls_from_repos, "rest_engine$get_repos_urls", test_mocker$use("gl_web_repos_urls") ) - gl_web_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( + gitlab_testhost_priv$searching_scope <- c("repo") + gitlab_testhost_priv$orgs_repos <- list("test_org" = "TestRepo") + gl_web_repos_urls <- gitlab_testhost_priv$get_repos_urls_from_repos( type = "web", - verbose = FALSE + verbose = FALSE, + progress = FALSE ) expect_gt(length(gl_web_repos_urls), 0) test_mocker$cache(gl_web_repos_urls) }) -test_that("get_all_repos_urls prepares web repo_urls vector", { + +test_that("get_all_repos_urls prepares api repo_urls vector", { mockery::stub( gitlab_testhost_priv$get_all_repos_urls, - "rest_engine$get_repos_urls", - test_mocker$use("gl_web_repos_urls") + "private$get_repos_urls_from_orgs", + test_mocker$use("gl_api_repos_urls") ) - gl_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( - type = "web", + mockery::stub( + gitlab_testhost_priv$get_all_repos_urls, + "private$get_repos_urls_from_repos", + NULL + ) + gl_api_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( + type = "api", verbose = FALSE ) - expect_gt(length(gl_repos_urls), 0) - expect_true(all(!grepl("api", gl_repos_urls))) + expect_true(all(grepl("api", gl_api_repos_urls))) + expect_gt(length(gl_api_repos_urls), 0) + test_mocker$cache(gl_api_repos_urls) }) test_that("`get_repo_url_from_response()` works", { @@ -105,17 +121,62 @@ test_that("`get_repo_url_from_response()` works", { test_mocker$cache(gl_repo_web_urls) }) -test_that("get_repos_urls returns repositories URLS", { +test_that("get_repos_urls_with_code_from_orgs returns repositories URLS", { mockery::stub( - gitlab_testhost$get_repos_urls, + gitlab_testhost_priv$get_repos_urls_with_code_from_orgs, "private$get_repo_url_from_response", test_mocker$use("gl_repo_web_urls") ) - gl_repos_urls_with_code_in_files <- gitlab_testhost$get_repos_urls( + gl_repos_urls_with_code_from_orgs <- gitlab_testhost_priv$get_repos_urls_with_code_from_orgs( type = "web", - with_code = "shiny", + code = "shiny", in_files = "DESCRIPTION", - verbose = FALSE + in_path = FALSE, + verbose = FALSE, + progress = FALSE + ) + expect_type(gl_repos_urls_with_code_from_orgs, "character") + expect_gt(length(gl_repos_urls_with_code_from_orgs), 0) + test_mocker$cache(gl_repos_urls_with_code_from_orgs) +}) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + gitlab_testhost_priv$get_repos_urls_with_code_from_repos, + "private$get_repo_url_from_response", + test_mocker$use("gl_repo_web_urls") + ) + gl_repos_urls_with_code_from_repos <- gitlab_testhost_priv$get_repos_urls_with_code_from_repos( + type = "web", + code = "shiny", + in_files = "DESCRIPTION", + in_path = FALSE, + verbose = FALSE, + progress = FALSE + ) + expect_type(gl_repos_urls_with_code_from_repos, "character") + expect_gt(length(gl_repos_urls_with_code_from_repos), 0) + test_mocker$cache(gl_repos_urls_with_code_from_repos) +}) + +test_that("get_repos_urls_with_code_from_repos returns repositories URLS", { + mockery::stub( + gitlab_testhost_priv$get_repos_urls_with_code, + "private$get_repos_urls_with_code_from_orgs", + test_mocker$use("gl_repos_urls_with_code_from_orgs") + ) + mockery::stub( + gitlab_testhost_priv$get_repos_urls_with_code, + "private$get_repos_urls_with_code_from_repos", + test_mocker$use("gl_repos_urls_with_code_from_repos") + ) + gl_repos_urls_with_code_in_files <- gitlab_testhost_priv$get_repos_urls_with_code( + type = "web", + code = "shiny", + in_files = "DESCRIPTION", + in_path = FALSE, + verbose = FALSE, + progress = FALSE ) expect_type(gl_repos_urls_with_code_in_files, "character") expect_gt(length(gl_repos_urls_with_code_in_files), 0) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R index 09bb8488..827bcd41 100644 --- a/tests/testthat/test-helpers.R +++ b/tests/testthat/test-helpers.R @@ -7,6 +7,16 @@ test_that("`set_searching_scope` does not throw error when `orgs` or `repos` are ) }) +test_that("`set_searching_scope` sets scope to whole host", { + gitlab_testhost_priv$is_public <- FALSE + expect_snapshot( + gitlab_testhost_priv$set_searching_scope(orgs = NULL, repos = NULL, verbose = TRUE) + ) + expect_true( + gitlab_testhost_priv$scan_all + ) +}) + test_that("`extract_repos_and_orgs` extracts fullnames vector into a list of GitLab organizations with assigned repositories", { repos_fullnames <- c( "mbtests/gitstatstesting", "mbtests/gitstats-testing-2", "mbtests/subgroup/test-project-in-subgroup" @@ -192,13 +202,3 @@ test_that("`set_default_token` sets default token for GitLab", { 200 ) }) - -test_that("`set_searching_scope` throws error when both `orgs` and `repos` are defined", { - expect_snapshot_error( - gitlab_testhost_priv$set_searching_scope( - orgs = "mbtests", - repos = "mbtests/GitStatsTesting", - verbose = TRUE - ) - ) -}) diff --git a/tests/testthat/test-set_host.R b/tests/testthat/test-set_host.R index 956d28c5..c26f8308 100644 --- a/tests/testthat/test-set_host.R +++ b/tests/testthat/test-set_host.R @@ -71,23 +71,6 @@ test_that("Set GitLab host with particular repos vector instead of orgs", { ) }) -test_that("Set host prints error when repos and orgs are defined and host is not passed to GitStats", { - skip_on_cran() - test_gitstats <- create_gitstats() - expect_snapshot_error( - test_gitstats %>% - set_github_host( - token = Sys.getenv("GITHUB_PAT"), - orgs = c('r-world-devs', "openpharma"), - repos = c("r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder", "openpharma/GithubMetrics", "openpharma/DataFakeR") - ) - ) - expect_length( - test_gitstats$.__enclos_env__$private$hosts, - 0 - ) -}) - test_that("Error shows if organizations are not specified and host is not passed", { skip_on_cran() test_gitstats <- create_gitstats() diff --git a/vignettes/set_hosts.Rmd b/vignettes/set_hosts.Rmd index 2e4d35c9..ec7386e8 100644 --- a/vignettes/set_hosts.Rmd +++ b/vignettes/set_hosts.Rmd @@ -21,40 +21,40 @@ To make `GitStats` work you need to set hosts after creating `gitstats`. You can set GitLab host with `set_gitlab_host()` and GitHub host with `set_github_host()` or both. -When setting hosts you need to take into account: +When setting hosts you need to consider: * Do you wish to connect to `private` or `public` hosts? -* What `scanning scope` do you wish to set? Do you want to scan specific `organizations` or `repositories` or maybe whole git platforms? +* What `scanning scope` do you wish to set? Do you want to scan specific `organizations` and/or `repositories` or maybe whole git platforms? * Do you have `tokens` set up and stored in your environment variables that grant you access to APIs? ## Public and private hosts -If you connect to public hosts you simply call `set_github_host()` or `set_gitlab_host()` function without specifying `host` parameter. +If you connect to **public** hosts you simply call `set_github_host()` or `set_gitlab_host()` function without specifying `host` parameter. ```{r, eval = FALSE} library(GitStats) -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( orgs = c("r-world-devs", "openpharma"), token = Sys.getenv("GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( orgs = c("mbtests"), token = Sys.getenv("GITLAB_PAT_PUBLIC") ) ``` -If you wish to connect to internal GitHub or GitLab, you need to pass names of the hosts to `host` parameter. Remember also to have tokens set up properly for these hosts (on tokens read below). +If you wish to connect to **internal** GitHub or GitLab, you need to pass names of the hosts to `host` parameter. Remember also to have tokens set up properly for these hosts (on tokens read below). ```r -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( host = "github.internal.com", orgs = c("org_1", "org_2", "org_3"), token = Sys.getenv("YOUR_GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( host = "internal.host.com", orgs = c("internal_org"), @@ -64,20 +64,20 @@ git_stats <- create_gitstats() %>% ### API versions -GitStats is configured to connect to GitHub API (version 3) and GitLab API (version 4). +`GitStats` is configured to connect to GitHub API (version 3) and GitLab API (version 4). ## Scanning scope -When setting hosts you choose what scanning scope of your GitStats will be: +When setting hosts you choose what scanning scope of your `GitStats` will be: * `organizations/groups` - in this case you need to pass character arguments (names of organizations (in case of GitHub) or groups (in case of GitLab)) to `orgs` parameter. ```{r, eval = FALSE} -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( orgs = c("r-world-devs", "openpharma"), token = Sys.getenv("GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( orgs = c("mbtests"), token = Sys.getenv("GITLAB_PAT_PUBLIC") @@ -87,25 +87,36 @@ git_stats <- create_gitstats() %>% * `repositories` - in this case you need to pass full names of repositories (`{org_name}/{repo_name}`) to the `repos` parameter. ```{r, eval = FALSE} -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( repos = c("r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder", "openpharma/DataFakeR"), token = Sys.getenv("GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( repos = "mbtests/gitstatstesting", token = Sys.getenv("GITLAB_PAT_PUBLIC") ) ``` -* `whole hosts` - this is possible for the time being only in case of private hosts, as public ones are deemed to be too large. To set whole Git platform to be scanned just set hosts without specifying `orgs` or `repos`. On the other hand, remember that to connect with internal host, you need to pass argument to `host` parameter. +* `organizations/groups` and `repositories` - you can define both at the same time: + +```{r, eval = FALSE} +git_stats <- create_gitstats() |> + set_github_host( + orgs = "openpharma", + repos = c("r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder"), + token = Sys.getenv("GITHUB_PAT") + ) +``` + +* `whole hosts` - this is possible for the time being only in case of private hosts, as public ones are deemed to be too large. To set whole Git platform to be scanned just set hosts **without specifying** `orgs` or `repos`. On the other hand, remember that to connect with internal host, you need to pass argument to `host` parameter. ```r -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( host = "github.internal.com", token = Sys.getenv("YOUR_GITHUB_PAT") - ) %>% + ) |> set_gitlab_host( host = "internal.host.com", token = Sys.getenv("YOUR_GITLAB_PAT") @@ -128,10 +139,10 @@ When creating tokens you will be asked to set access scopes of the tokens. For ` If you have your access tokens stored in environment variables with such names as `GITHUB_PAT` or `GITHUB_PAT_*` and `GITLAB_PAT` or `GITLAB_PAT_*` you do not need to specify them in `set_*_host()` functions, `GitStats` will automatically find them. ```{r, eval = FALSE} -git_stats <- create_gitstats() %>% +git_stats <- create_gitstats() |> set_github_host( orgs = c("r-world-devs", "openpharma") - ) %>% + ) |> set_gitlab_host( orgs = c("mbtests") )