diff --git a/.Rbuildignore b/.Rbuildignore index 549fb04c..d31234c8 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -12,3 +12,4 @@ ^docs$ ^pkgdown$ ^project_metadata.yaml +^.lintr diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..e252d22b --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,28 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: [push, pull_request] + +name: lint + +jobs: + lint: + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4.1.4 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::lintr, local::. + needs: lint + + - name: Lint + run: lintr::lint_package() + shell: Rscript {0} + env: + LINTR_ERROR_ON_LINT: true diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml new file mode 100644 index 00000000..7d1ee754 --- /dev/null +++ b/.github/workflows/pkgdown.yaml @@ -0,0 +1,51 @@ +# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples +# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] + release: + types: [published] + workflow_dispatch: + +name: pkgdown.yaml + +permissions: read-all + +jobs: + pkgdown: + runs-on: ubuntu-latest + # Only restrict concurrency for non-PR jobs + concurrency: + group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }} + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + GITLAB_PAT_PUBLIC: ${{ secrets.GITLAB_PAT}} + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + + - uses: r-lib/actions/setup-pandoc@v2 + + - uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + + - uses: r-lib/actions/setup-r-dependencies@v2 + with: + extra-packages: any::pkgdown, local::. + needs: website + + - name: Build site + run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) + shell: Rscript {0} + + - name: Deploy to GitHub pages 🚀 + if: github.event_name != 'pull_request' + uses: JamesIves/github-pages-deploy-action@v4.5.0 + with: + clean: false + branch: gh-pages + folder: docs diff --git a/.lintr b/.lintr new file mode 100644 index 00000000..2a131d9a --- /dev/null +++ b/.lintr @@ -0,0 +1,11 @@ +linters: linters_with_defaults( + quotes_linter = NULL, + line_length_linter = NULL, + object_usage_linter = NULL, + object_length_linter = NULL, + object_name_linter = NULL, + cyclocomp_linter = NULL + ) +exclusions: list( + "inst", + "tests") diff --git a/DESCRIPTION b/DESCRIPTION index 4b833d85..bca8688e 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: GitStats Title: Get Statistics from GitHub and GitLab -Version: 2.0.2 +Version: 2.1.0 Authors@R: c( person(given = "Maciej", family = "Banas", email = "banasmaciek@gmail.com", role = c("aut", "cre")), person(given = "Kamil", family = "Koziej", email = "koziej.k@gmail.com", role = "aut"), diff --git a/NAMESPACE b/NAMESPACE index ec0ae1a8..e3906012 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -5,7 +5,8 @@ export(create_gitstats) export(get_R_package_usage) export(get_commits) export(get_commits_stats) -export(get_files) +export(get_files_content) +export(get_files_structure) export(get_release_logs) export(get_repos) export(get_repos_urls) diff --git a/NEWS.md b/NEWS.md index 26928922..be7a48fa 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,17 @@ +# GitStats 2.1.0 + +## New features: + +- Added new `get_files_structure()` function to pull files structure for a given repository with possibility to control level of directories (`depth` parameter) and to limit output to files matching regex argument passed to `pattern` parameter ([#338](https://github.com/r-world-devs/GitStats/issues/338)). Together with that, `get_files()` function was renamed to `get_files_content()` to better reflect its purpose. +- Adjusted `get_files_content()` so it can make use of `files_structure` pulled to `GitStats` storage with `get_files_structure()` function - if `file_path` is set to `NULL` and `use_files_structure()` parameter to `TRUE` (both are by default)([#467](https://github.com/r-world-devs/GitStats/issues/467)). +- Added `progress` parameter to user functions to control showing of `cli` progress bar separately from messages (which are controlled with `verbose`) ([#465](https://github.com/r-world-devs/GitStats/issues/465)). + +## Other: + +- Changed message when searching scope is set to scan whole git host (no `orgs` nor `repos` specified) from warning to info ([#456](https://github.com/r-world-devs/GitStats/issues/456)). +- Added new CI/CD jobs: deploy to `gh-pages`, lint and check for bumping version. +- Mocked extensively API responses to improve tests and checks progress ([#481](https://github.com/r-world-devs/GitStats/issues/481)]. + # GitStats 2.0.2 This is a patch release with substantial improvements to some functions (`get_repos()`, `get_files()` and `get_R_package_usage()`), adding `with_files` and `in_files` parameters, fixing `cache` feature and introducing new `get_repos_urls()` function, a minimalist version of `get_repos()`: @@ -19,7 +33,7 @@ This is a patch release with some hot issues that needed to be addressed, notabl ## Features: -- Getting files feature has been speeded up when `GitStats` is set to scan whole hosts, with switching to `Search API` instead of pulling files via `GraphQL` (with iteration over organizations and repositories) ([#411](https://github.com/r-world-devs/GitStats/issues/411)). +- Getting files feature has been sped up when `GitStats` is set to scan whole hosts, with switching to `Search API` instead of pulling files via `GraphQL` (with iteration over organizations and repositories) ([#411](https://github.com/r-world-devs/GitStats/issues/411)). - When setting hosts to be scanned in whole (without specifying `orgs` or `repos`) GitStats does not pull no more all organizations. Pulling all organizations from host is triggered only when user decides to pull repositories from organizations. If he decides, e.g. to pull repositories by code, there is no need to pull all organizations (which may be a time consuming process), as GitStats uses then `Search API` ([#393](https://github.com/r-world-devs/GitStats/issues/393)). - It is now possible to mute messages also from `set_*_host()` functions with `verbose_off()` or `verbose` parameter ([#413](https://github.com/r-world-devs/GitStats/issues/413)). - Setting `verbose` to `FALSE` does not lead to hiding output of the `get_*()` functions - i.e. a glimpse of table will always appear after pulling data, even if the `verbose` is switched off. `verbose` parameter serves now only the purpose to show and hide messages to user ([#423](https://github.com/r-world-devs/GitStats/issues/423)). diff --git a/R/EngineGraphQL.R b/R/EngineGraphQL.R index ab1c1c91..aff83114 100644 --- a/R/EngineGraphQL.R +++ b/R/EngineGraphQL.R @@ -1,62 +1,91 @@ #' @noRd #' @description A class for methods wrapping GitHub's GraphQL API responses. -EngineGraphQL <- R6::R6Class("EngineGraphQL", - inherit = Engine, +EngineGraphQL <- R6::R6Class( + "EngineGraphQL", + inherit = Engine, + public = list( - public = list( + #' @field gql_api_url A character, url of GraphQL API. + gql_api_url = NULL, - #' @field gql_api_url A character, url of GraphQL API. - gql_api_url = NULL, + #' @field gql_query An environment for GraphQL queries. + gql_query = NULL, - #' @field gql_query An environment for GraphQL queries. - gql_query = NULL, + #' Create `EngineGraphQL` object. + initialize = function(gql_api_url = NA, + token = NA, + scan_all = FALSE) { + private$engine <- "graphql" + self$gql_api_url <- gql_api_url + private$token <- token + private$scan_all <- scan_all + }, - #' Create `EngineGraphQL` object. - initialize = function(gql_api_url = NA, - token = NA, - scan_all = FALSE) { - private$engine <- "graphql" - self$gql_api_url <- gql_api_url - private$token <- token - private$scan_all <- scan_all - }, + #' Wrapper of GraphQL API request and response. + gql_response = function(gql_query, vars = "null") { + response <- private$perform_request( + gql_query = gql_query, + vars = vars + ) + response_list <- httr2::resp_body_json(response) + return(response_list) + }, - #' Wrapper of GraphQL API request and response. - gql_response = function(gql_query, vars = "null") { - response <- private$perform_request( - gql_query = gql_query, - vars = vars - ) - response_list <- httr2::resp_body_json(response) - return(response_list) - }, + # A method to pull information on user. + get_user = function(username) { + response <- tryCatch( + { + self$gql_response( + gql_query = self$gql_query$user(), + vars = list("user" = username) + ) + }, + error = function(e) { + NULL + } + ) + return(response) + } + ), + private = list( - # A method to pull information on user. - pull_user = function(username) { - response <- tryCatch({ - self$gql_response( - gql_query = self$gql_query$user(), - vars = list("user" = username) - ) - }, error = function(e) { - NULL - }) - return(response) - } - ), - private = list( - - # GraphQL method for pulling response from API - perform_request = function(gql_query, vars) { - response <- httr2::request(paste0(self$gql_api_url, "?")) %>% - httr2::req_headers("Authorization" = paste0("Bearer ", private$token)) %>% - httr2::req_body_json(list(query = gql_query, variables = vars)) %>% - httr2::req_retry( - is_transient = ~ httr2::resp_status(.x) %in% c(400, 500, 502), - max_seconds = 60 - ) %>% - httr2::req_perform() - return(response) - } - ) + # GraphQL method for pulling response from API + perform_request = function(gql_query, vars) { + response <- httr2::request(paste0(self$gql_api_url, "?")) %>% + httr2::req_headers("Authorization" = paste0("Bearer ", private$token)) %>% + httr2::req_body_json(list(query = gql_query, variables = vars)) %>% + httr2::req_retry( + is_transient = ~ httr2::resp_status(.x) %in% c(400, 502), + max_seconds = 30 + ) %>% + httr2::req_perform() + return(response) + }, + is_query_error = function(response) { + check <- FALSE + if (length(response) > 0) { + check <- names(response) == "errors" + } + return(check) + }, + filter_files_by_pattern = function(files_structure, pattern) { + files_structure[grepl(pattern, files_structure)] + }, + get_path_from_files_structure = function(host_files_structure, + only_text_files, + org, + repo = NULL) { + if (is.null(repo)) { + file_path <- host_files_structure[[org]] %>% + unlist(use.names = FALSE) %>% + unique() + } else { + file_path <- host_files_structure[[org]][[repo]] + } + if (only_text_files) { + file_path <- file_path[!grepl(non_text_files_pattern, file_path)] + } + return(file_path) + } + ) ) diff --git a/R/EngineGraphQLGitHub.R b/R/EngineGraphQLGitHub.R index 95c1b974..e837945a 100644 --- a/R/EngineGraphQLGitHub.R +++ b/R/EngineGraphQLGitHub.R @@ -1,8 +1,9 @@ #' @noRd #' @description A class for methods wrapping GitHub's GraphQL API responses. -EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", - inherit = EngineGraphQL, - public = list( +EngineGraphQLGitHub <- R6::R6Class( + classname = "EngineGraphQLGitHub", + inherit = EngineGraphQL, + public = list( #' Create `EngineGraphQLGitHub` object. initialize = function(gql_api_url, @@ -15,7 +16,7 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", }, #' Get all orgs from GitHub. - pull_orgs = function() { + get_orgs = function() { end_cursor <- NULL has_next_page <- TRUE full_orgs_list <- list() @@ -40,12 +41,12 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", }, # Pull all repositories from organization - pull_repos_from_org = function(org = NULL) { + get_repos_from_org = function(org = NULL) { full_repos_list <- list() next_page <- TRUE repo_cursor <- "" while (next_page) { - repos_response <- private$pull_repos_page( + repos_response <- private$get_repos_page( org = org, repo_cursor = repo_cursor ) @@ -65,19 +66,19 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", }, # Iterator over pulling commits from all repositories. - pull_commits_from_repos = function(org, - repos_names, - since, - until, - verbose) { + get_commits_from_repos = function(org, + repos_names, + since, + until, + progress) { repos_list_with_commits <- purrr::map(repos_names, function(repo) { - private$pull_commits_from_one_repo( - org = org, - repo = repo, + private$get_commits_from_one_repo( + org = org, + repo = repo, since = since, until = until ) - }, .progress = !private$scan_all && verbose) + }, .progress = !private$scan_all && progress) names(repos_list_with_commits) <- repos_names repos_list_with_commits <- repos_list_with_commits %>% purrr::discard(~ length(.) == 0) @@ -85,38 +86,79 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", }, # Pull all given files from all repositories of an organization. - pull_files_from_org = function(org, repos, file_path) { - repos_list <- self$pull_repos_from_org( - org = org + get_files_from_org = function(org, + repos, + file_paths, + host_files_structure, + only_text_files, + verbose = TRUE, + progress = TRUE) { + repo_data <- private$get_repos_data( + org = org, + repos = repos ) - if (!is.null(repos)) { - repos_list <- purrr::keep(repos_list, ~ .$repo_name %in% repos) - } - repositories <- purrr::map(repos_list, ~ .$repo_name) - def_branches <- purrr::map(repos_list, ~ .$default_branch$name) - files_list <- purrr::map(file_path, function(file_path) { - files_list <- purrr::map2(repositories, def_branches, function(repository, def_branch) { - files_query <- self$gql_query$files_by_repo() - files_response <- self$gql_response( - gql_query = files_query, - vars = list( - "org" = org, - "repo" = repository, - "file_path" = paste0(def_branch, ":", file_path) - ) + repositories <- repo_data[["repositories"]] + def_branches <- repo_data[["def_branches"]] + org_files_list <- purrr::map2(repositories, def_branches, function(repo, def_branch) { + if (!is.null(host_files_structure)) { + file_paths <- private$get_path_from_files_structure( + host_files_structure = host_files_structure, + only_text_files = only_text_files, + org = org, + repo = repo + ) + } else if (is.null(host_files_structure) && only_text_files) { + file_paths <- file_paths[!grepl(non_text_files_pattern, file_paths)] + } + repo_files_list <- purrr::map(file_paths, function(file_path) { + private$get_file_response( + org = org, + repo = repo, + def_branch = def_branch, + file_path = file_path, + files_query = self$gql_query$file_blob_from_repo() ) }) %>% purrr::map(~ .$data$repository) - names(files_list) <- repositories - files_list <- purrr::discard(files_list, ~ length(.$object) == 0) - return(files_list) - }) - names(files_list) <- file_path - return(files_list) + names(repo_files_list) <- file_paths + return(repo_files_list) + }, .progress = progress) + names(org_files_list) <- repositories + for (file_path in file_paths) { + org_files_list <- purrr::discard(org_files_list, ~ length(.[[file_path]]$file) == 0) + } + return(org_files_list) + }, + + # Pull all files from all repositories of an organization. + get_files_structure_from_org = function(org, + repos, + pattern = NULL, + depth = Inf, + verbose = FALSE, + progress = TRUE) { + repo_data <- private$get_repos_data( + org = org, + repos = repos + ) + repositories <- repo_data[["repositories"]] + def_branches <- repo_data[["def_branches"]] + files_structure <- purrr::map2(repositories, def_branches, function(repo, def_branch) { + private$get_files_structure_from_repo( + org = org, + repo = repo, + def_branch = def_branch, + pattern = pattern, + depth = depth + ) + }, .progress = progress) + names(files_structure) <- repositories + files_structure <- purrr::discard(files_structure, ~ length(.) == 0) + return(files_structure) }, # Pull release logs from organization - pull_release_logs_from_org = function(repos_names, org) { + get_release_logs_from_org = function(repos_names, org) { release_responses <- purrr::map(repos_names, function(repository) { releases_from_repo_query <- self$gql_query$releases_from_repo() response <- self$gql_response( @@ -132,70 +174,169 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub", return(release_responses) } ), - private = list( + private = list( - # Wrapper over building GraphQL query and response. - pull_repos_page = function(org = NULL, - repo_cursor = "") { - repos_query <- self$gql_query$repos_by_org( - repo_cursor = repo_cursor + # Wrapper over building GraphQL query and response. + get_repos_page = function(org = NULL, + repo_cursor = "") { + repos_query <- self$gql_query$repos_by_org( + repo_cursor = repo_cursor + ) + response <- self$gql_response( + gql_query = repos_query, + vars = list("org" = org) + ) + return(response) + }, + + # An iterator over pulling commit pages from one repository. + get_commits_from_one_repo = function(org, + repo, + since, + until) { + next_page <- TRUE + full_commits_list <- list() + commits_cursor <- "" + while (next_page) { + commits_response <- private$get_commits_page_from_repo( + org = org, + repo = repo, + since = since, + until = until, + commits_cursor = commits_cursor ) - response <- self$gql_response( - gql_query = repos_query, - vars = list("org" = org) + commits_list <- commits_response$data$repository$defaultBranchRef$target$history$edges + next_page <- commits_response$data$repository$defaultBranchRef$target$history$pageInfo$hasNextPage + if (is.null(next_page)) next_page <- FALSE + if (is.null(commits_list)) commits_list <- list() + if (next_page) { + commits_cursor <- commits_response$data$repository$defaultBranchRef$target$history$pageInfo$endCursor + } else { + commits_cursor <- "" + } + full_commits_list <- append(full_commits_list, commits_list) + } + return(full_commits_list) + }, + + # Wrapper over building GraphQL query and response. + get_commits_page_from_repo = function(org, + repo, + since, + until, + commits_cursor = "", + author_id = "") { + commits_by_org_query <- self$gql_query$commits_by_repo( + org = org, + repo = repo, + since = date_to_gts(since), + until = date_to_gts(until), + commits_cursor = commits_cursor, + author_id = author_id + ) + response <- self$gql_response( + gql_query = commits_by_org_query + ) + return(response) + }, + + get_repos_data = function(org, repos = NULL) { + repos_list <- self$get_repos_from_org( + org = org + ) + if (!is.null(repos)) { + repos_list <- purrr::keep(repos_list, ~ .$repo_name %in% repos) + } + result <- list( + "repositories" = purrr::map(repos_list, ~ .$repo_name), + "def_branches" = purrr::map(repos_list, ~ .$default_branch$name) + ) + return(result) + }, + + get_file_response = function(org, repo, def_branch, file_path, files_query) { + expression <- paste0(def_branch, ":", file_path) + files_response <- self$gql_response( + gql_query = files_query, + vars = list( + "org" = org, + "repo" = repo, + "expression" = expression ) - return(response) - }, + ) + return(files_response) + }, - # An iterator over pulling commit pages from one repository. - pull_commits_from_one_repo = function(org, - repo, - since, - until) { - next_page <- TRUE - full_commits_list <- list() - commits_cursor <- "" - while (next_page) { - commits_response <- private$pull_commits_page_from_repo( + get_files_structure_from_repo = function(org, repo, def_branch, pattern = NULL, depth = Inf) { + files_tree_response <- private$get_file_response( + org = org, + repo = repo, + def_branch = def_branch, + file_path = "", + files_query = self$gql_query$files_tree_from_repo() + ) + files_and_dirs_list <- private$get_files_and_dirs( + files_tree_response = files_tree_response + ) + if (length(files_and_dirs_list$dirs) > 0) { + folders_exist <- TRUE + } else { + folders_exist <- FALSE + } + all_files_and_dirs_list <- files_and_dirs_list + dirs <- files_and_dirs_list$dirs + tier <- 1 + while (folders_exist && tier < depth) { + new_dirs_list <- c() + for (dir in dirs) { + files_tree_response <- private$get_file_response( org = org, repo = repo, - since = since, - until = until, - commits_cursor = commits_cursor + def_branch = def_branch, + file_path = dir, + files_query = self$gql_query$files_tree_from_repo() + ) + files_and_dirs_list <- private$get_files_and_dirs( + files_tree_response = files_tree_response ) - commits_list <- commits_response$data$repository$defaultBranchRef$target$history$edges - next_page <- commits_response$data$repository$defaultBranchRef$target$history$pageInfo$hasNextPage - if (is.null(next_page)) next_page <- FALSE - if (is.null(commits_list)) commits_list <- list() - if (next_page) { - commits_cursor <- commits_response$data$repository$defaultBranchRef$target$history$pageInfo$endCursor - } else { - commits_cursor <- "" + all_files_and_dirs_list$files <- append( + all_files_and_dirs_list$files, + paste0(dir, "/", files_and_dirs_list$files) + ) + if (length(files_and_dirs_list$dirs) > 0) { + new_dirs_list <- c(new_dirs_list, paste0(dir, "/", files_and_dirs_list$dirs)) } - full_commits_list <- append(full_commits_list, commits_list) } - return(full_commits_list) - }, - - # Wrapper over building GraphQL query and response. - pull_commits_page_from_repo = function(org, - repo, - since, - until, - commits_cursor = "", - author_id = "") { - commits_by_org_query <- self$gql_query$commits_by_repo( - org = org, - repo = repo, - since = date_to_gts(since), - until = date_to_gts(until), - commits_cursor = commits_cursor, - author_id = author_id - ) - response <- self$gql_response( - gql_query = commits_by_org_query + if (length(new_dirs_list) > 0) { + dirs <- new_dirs_list + folders_exist <- TRUE + tier <- tier + 1 + } else { + folders_exist <- FALSE + } + } + if (!is.null(pattern)) { + files_structure <- private$filter_files_by_pattern( + files_structure = all_files_and_dirs_list$files, + pattern = pattern ) - return(response) + } else { + files_structure <- all_files_and_dirs_list$files } - ) + return(files_structure) + }, + + get_files_and_dirs = function(files_tree_response) { + entries <- files_tree_response$data$repository$object$entries + dirs <- purrr::keep(entries, ~ .$type == "tree") %>% + purrr::map_vec(~ .$name) + files <- purrr::discard(entries, ~ .$type == "tree") %>% + purrr::map_vec(~ .$name) + result <- list( + "dirs" = dirs, + "files" = files + ) + return(result) + } + ) ) diff --git a/R/EngineGraphQLGitLab.R b/R/EngineGraphQLGitLab.R index 742fbb52..7dc0b38a 100644 --- a/R/EngineGraphQLGitLab.R +++ b/R/EngineGraphQLGitLab.R @@ -1,162 +1,388 @@ #' @noRd #' @description A class for methods wrapping GitLab's GraphQL API responses. -EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab", - inherit = EngineGraphQL, - public = list( +EngineGraphQLGitLab <- R6::R6Class( + classname = "EngineGraphQLGitLab", + inherit = EngineGraphQL, + public = list( - #' Create `EngineGraphQLGitLab` object. - initialize = function(gql_api_url, - token, - scan_all = FALSE) { - super$initialize(gql_api_url = gql_api_url, - token = token, - scan_all = scan_all) - self$gql_query <- GQLQueryGitLab$new() - }, + #' Create `EngineGraphQLGitLab` object. + initialize = function(gql_api_url, + token, + scan_all = FALSE) { + super$initialize( + gql_api_url = gql_api_url, + token = token, + scan_all = scan_all + ) + self$gql_query <- GQLQueryGitLab$new() + }, - #' Get all groups from GitLab. - pull_orgs = function() { - group_cursor <- "" - has_next_page <- TRUE - full_orgs_list <- list() - while(has_next_page) { - response <- self$gql_response( - gql_query = self$gql_query$groups(), - vars = list("groupCursor" = group_cursor) - ) - if (length(response$data$groups$edges) == 0) { - cli::cli_abort( - c( - "x" = "Empty response.", - "!" = "Your token probably does not cover scope to pull organizations.", - "i" = "Set `read_api` scope when creating GitLab token." - ) - ) - } - orgs_list <- purrr::map(response$data$groups$edges, ~.$node$fullPath) - full_orgs_list <- append(full_orgs_list, orgs_list) - has_next_page <- response$data$groups$pageInfo$hasNextPage - group_cursor <- response$data$groups$pageInfo$endCursor - } - all_orgs <- unlist(full_orgs_list) - return(all_orgs) - }, + #' Get all groups from GitLab. + get_orgs = function() { + group_cursor <- "" + has_next_page <- TRUE + full_orgs_list <- list() + while (has_next_page) { + response <- self$gql_response( + gql_query = self$gql_query$groups(), + vars = list("groupCursor" = group_cursor) + ) + if (length(response$data$groups$edges) == 0) { + cli::cli_abort( + c( + "x" = "Empty response.", + "!" = "Your token probably does not cover scope to pull organizations.", + "i" = "Set `read_api` scope when creating GitLab token." + ) + ) + } + orgs_list <- purrr::map(response$data$groups$edges, ~ .$node$fullPath) + full_orgs_list <- append(full_orgs_list, orgs_list) + has_next_page <- response$data$groups$pageInfo$hasNextPage + group_cursor <- response$data$groups$pageInfo$endCursor + } + all_orgs <- unlist(full_orgs_list) + return(all_orgs) + }, - # Iterator over pulling pages of repositories. - pull_repos_from_org = function(org = NULL) { - full_repos_list <- list() - next_page <- TRUE - repo_cursor <- "" - while (next_page) { - repos_response <- private$pull_repos_page( - org = org, - repo_cursor = repo_cursor - ) - if (length(repos_response$data$group) == 0) { - cli::cli_abort("Empty") - } - core_response <- repos_response$data$group$projects - repos_list <- core_response$edges - next_page <- core_response$pageInfo$hasNextPage - if (is.null(next_page)) next_page <- FALSE - if (is.null(repos_list)) repos_list <- list() - if (length(repos_list) == 0) next_page <- FALSE - if (next_page) { - repo_cursor <- core_response$pageInfo$endCursor - } else { - repo_cursor <- "" - } - full_repos_list <- append(full_repos_list, repos_list) - } - return(full_repos_list) - }, + # Iterator over pulling pages of repositories. + get_repos_from_org = function(org = NULL) { + full_repos_list <- list() + next_page <- TRUE + repo_cursor <- "" + while (next_page) { + repos_response <- private$get_repos_page( + org = org, + repo_cursor = repo_cursor + ) + if (length(repos_response$data$group) == 0) { + cli::cli_abort("Empty") + } + core_response <- repos_response$data$group$projects + repos_list <- core_response$edges + next_page <- core_response$pageInfo$hasNextPage + if (is.null(next_page)) next_page <- FALSE + if (is.null(repos_list)) repos_list <- list() + if (length(repos_list) == 0) next_page <- FALSE + if (next_page) { + repo_cursor <- core_response$pageInfo$endCursor + } else { + repo_cursor <- "" + } + full_repos_list <- append(full_repos_list, repos_list) + } + return(full_repos_list) + }, - # Pull all given files from all repositories of a group. - pull_files_from_org = function(org, repos, file_path) { - org <- URLdecode(org) - full_files_list <- list() - next_page <- TRUE - end_cursor <- "" - while (next_page) { - files_query <- self$gql_query$files_by_org( - end_cursor = end_cursor - ) - files_response <- self$gql_response( - gql_query = files_query, - vars = list( - "org" = org, - "file_paths" = file_path - ) - ) - if (length(files_response$data$group) == 0) { - cli::cli_alert_danger("Empty") - } - projects <- files_response$data$group$projects - files_list <- purrr::map(projects$edges, function(edge) { - edge$node - }) %>% - purrr::discard(~ length(.$repository$blobs$nodes) == 0) - if (is.null(files_list)) files_list <- list() - if (length(files_list) > 0) { - next_page <- files_response$pageInfo$hasNextPage - } else { - next_page <- FALSE - } - if (is.null(next_page)) next_page <- FALSE - if (next_page) { - end_cursor <- files_response$pageInfo$endCursor - } else { - end_cursor <- "" - } - full_files_list <- append(full_files_list, files_list) - } - if (!is.null(repos)) { - full_files_list <- purrr::keep(full_files_list, function(project) { - repo_name <- private$get_repo_name_from_url(project$webUrl) - repo_name %in% repos - }) - } - return(full_files_list) - }, + # Pull all given files from all repositories of a group. + # This is a one query way to get all the necessary info. + # However it may fail if query is too complex (too many files in file_paths). + # This may be especially the case when trying to pull the data from earlier + # pulled files_structure. In such a case GitStats will switch from this function + # to iterator over repositories (multiple queries), as it is done for GitHub. + get_files_from_org = function(org, + repos, + file_paths, + host_files_structure, + only_text_files, + verbose = FALSE, + progress = FALSE) { + org <- URLdecode(org) + full_files_list <- list() + next_page <- TRUE + end_cursor <- "" + if (!is.null(host_files_structure)) { + file_paths <- private$get_path_from_files_structure( + host_files_structure = host_files_structure, + only_text_files = only_text_files, + org = org + ) + } else if (is.null(host_files_structure) && only_text_files) { + file_paths <- file_paths[!grepl(non_text_files_pattern, file_paths)] + } + while (next_page) { + files_query <- self$gql_query$files_by_org( + end_cursor = end_cursor + ) + files_response <- tryCatch( + { + self$gql_response( + gql_query = files_query, + vars = list( + "org" = org, + "file_paths" = file_paths + ) + ) + }, + error = function(e) { + list() + } + ) + if (private$is_query_error(files_response)) { + if (verbose) { + purrr::walk(files_response$errors, ~ cli::cli_alert_warning(.)) + } + if (private$is_complexity_error(files_response)) { + if (verbose) { + cli::cli_alert_info( + cli::col_br_cyan("I will switch to pulling files per repository.") + ) + } + full_files_list <- self$get_files_from_org_per_repo( + org = org, + repos = repos, + file_paths = file_paths, + host_files_structure = host_files_structure, + only_text_files = only_text_files, + verbose = verbose, + progress = progress + ) + return(full_files_list) + } + } + if (length(files_response$data$group) == 0 && verbose) { + cli::cli_alert_danger("Empty response.") + } + projects <- files_response$data$group$projects + files_list <- purrr::map(projects$edges, function(edge) { + edge$node + }) %>% + purrr::discard(~ length(.$repository$blobs$nodes) == 0) + if (is.null(files_list)) files_list <- list() + if (length(files_list) > 0) { + next_page <- files_response$pageInfo$hasNextPage + } else { + next_page <- FALSE + } + if (is.null(next_page)) next_page <- FALSE + if (next_page) { + end_cursor <- files_response$pageInfo$endCursor + } else { + end_cursor <- "" + } + full_files_list <- append(full_files_list, files_list) + } + if (!is.null(repos)) { + full_files_list <- purrr::keep(full_files_list, function(project) { + repo_name <- private$get_repo_name_from_url(project$webUrl) + repo_name %in% repos + }) + } + return(full_files_list) + }, - # Pull all releases from all repositories of an organization. - pull_release_logs_from_org = function(repos_names, org) { - release_responses <- purrr::map(repos_names, function(repository) { - releases_from_repo_query <- self$gql_query$releases_from_repo() - response <- self$gql_response( - gql_query = releases_from_repo_query, - vars = list( - "project_path" = utils::URLdecode(repository) - ) - ) - return(response) - }) %>% - purrr::discard(~ length(.$data$project$releases$nodes) == 0) - return(release_responses) - } + # This method is a kind of support to the method above. It is only run when + # one query way applied with get_files_from_org() fails due to its complexity. + # For more info see docs above. + get_files_from_org_per_repo = function(org, + repos, + file_paths = NULL, + host_files_structure = NULL, + only_text_files = TRUE, + verbose = FALSE, + progress = FALSE) { + if (is.null(repos)) { + repo_data <- private$get_repos_data( + org = org, + repos = repos + ) + repos <- repo_data[["repositories"]] + } + org_files_list <- purrr::map(repos, function(repo) { + if (!is.null(host_files_structure)) { + file_paths <- private$get_path_from_files_structure( + host_files_structure = host_files_structure, + only_text_files = only_text_files, + org = org, + repo = repo + ) + } + files_response <- tryCatch( + { + private$get_file_blobs_response( + org = org, + repo = repo, + file_paths = file_paths + ) + }, + error = function(e) { + list() + } + ) + }, .progress = progress) + return(org_files_list) + }, + get_files_structure_from_org = function(org, + repos, + pattern = NULL, + depth = Inf, + verbose = TRUE, + progress = TRUE) { + repo_data <- private$get_repos_data( + org = org, + repos = repos + ) + repositories <- repo_data[["repositories"]] + files_structure <- purrr::map(repositories, function(repo) { + private$get_files_structure_from_repo( + org = org, + repo = repo, + pattern = pattern, + depth = depth + ) + }, .progress = progress) + names(files_structure) <- repositories + files_structure <- purrr::discard(files_structure, ~ length(.) == 0) + return(files_structure) + }, - ), + # Pull all releases from all repositories of an organization. + get_release_logs_from_org = function(repos_names, org) { + release_responses <- purrr::map(repos_names, function(repository) { + releases_from_repo_query <- self$gql_query$releases_from_repo() + response <- self$gql_response( + gql_query = releases_from_repo_query, + vars = list( + "project_path" = utils::URLdecode(repository) + ) + ) + return(response) + }) %>% + purrr::discard(~ length(.$data$project$releases$nodes) == 0) + return(release_responses) + } + ), + private = list( + is_complexity_error = function(response) { + any(purrr::map_lgl(response$errors, ~ grepl("Query has complexity", .$message))) + }, - private = list( + # Wrapper over building GraphQL query and response. + get_repos_page = function(org = NULL, + repo_cursor = "") { + repos_query <- self$gql_query$repos_by_org( + repo_cursor = repo_cursor + ) + response <- self$gql_response( + gql_query = repos_query, + vars = list("org" = org) + ) + return(response) + }, - # Wrapper over building GraphQL query and response. - pull_repos_page = function(org = NULL, - repo_cursor = "") { - repos_query <- self$gql_query$repos_by_org( - repo_cursor = repo_cursor - ) - response <- self$gql_response( - gql_query = repos_query, - vars = list("org" = org) - ) - return(response) - }, + # Helper + get_repo_name_from_url = function(web_url) { + url_split <- stringr::str_split(web_url, ":|/")[[1]] + repo_name <- url_split[length(url_split)] + return(repo_name) + }, - # Helper - get_repo_name_from_url = function(web_url) { - url_split <- stringr::str_split(web_url, ":|/")[[1]] - repo_name <- url_split[length(url_split)] - return(repo_name) - } - ) + get_repos_data = function(org, repos = NULL) { + repos_list <- self$get_repos_from_org( + org = org + ) + if (!is.null(repos)) { + repos_list <- purrr::keep(repos_list, ~ .$node$repo_path %in% repos) + } + result <- list( + "repositories" = purrr::map_vec(repos_list, ~ .$node$repo_path) + ) + return(result) + }, + + get_file_blobs_response = function(org, repo, file_paths) { + file_blobs_response <- self$gql_response( + gql_query = self$gql_query$file_blob_from_repo(), + vars = list( + "fullPath" = paste0(org, "/", repo), + "file_paths" = file_paths + ) + ) + return(file_blobs_response) + }, + + get_files_tree_response = function(org, repo, file_path) { + files_tree_response <- self$gql_response( + gql_query = self$gql_query$files_tree_from_repo(), + vars = list( + "fullPath" = paste0(org, "/", repo), + "file_path" = file_path + ) + ) + return(files_tree_response) + }, + + get_files_structure_from_repo = function(org, repo, pattern = NULL, depth = Inf) { + files_tree_response <- private$get_files_tree_response( + org = org, + repo = repo, + file_path = "" + ) + files_and_dirs_list <- private$get_files_and_dirs( + files_tree_response = files_tree_response + ) + if (length(files_and_dirs_list$dirs) > 0) { + folders_exist <- TRUE + } else { + folders_exist <- FALSE + } + all_files_and_dirs_list <- files_and_dirs_list + dirs <- files_and_dirs_list$dirs + tier <- 1 + while (folders_exist && tier < depth) { + new_dirs_list <- c() + for (dir in dirs) { + files_tree_response <- private$get_files_tree_response( + org = org, + repo = repo, + file_path = dir + ) + files_and_dirs_list <- private$get_files_and_dirs( + files_tree_response = files_tree_response + ) + if (length(files_and_dirs_list$files) > 0) { + all_files_and_dirs_list$files <- append( + all_files_and_dirs_list$files, + paste0(dir, "/", files_and_dirs_list$files) + ) + } + if (length(files_and_dirs_list$dirs) > 0) { + new_dirs_list <- c(new_dirs_list, paste0(dir, "/", files_and_dirs_list$dirs)) + } + } + if (length(new_dirs_list) > 0) { + dirs <- new_dirs_list + folders_exist <- TRUE + tier <- tier + 1 + } else { + folders_exist <- FALSE + } + } + if (!is.null(pattern)) { + files_structure <- private$filter_files_by_pattern( + files_structure = all_files_and_dirs_list$files, + pattern = pattern + ) + } else { + files_structure <- all_files_and_dirs_list$files + } + return(files_structure) + }, + + get_files_and_dirs = function(files_tree_response) { + tree_nodes <- files_tree_response$data$project$repository$tree$trees$nodes + blob_nodes <- files_tree_response$data$project$repository$tree$blobs$nodes + dirs <- purrr::map_vec(tree_nodes, ~ .$name) %>% + unlist() %>% + unname() + files <- purrr::map_vec(blob_nodes, ~ .$name) %>% + unlist() %>% + unname() + result <- list( + "dirs" = dirs, + "files" = files + ) + return(result) + } + ) ) diff --git a/R/EngineRestGitHub.R b/R/EngineRestGitHub.R index 37164ec5..b85b42a7 100644 --- a/R/EngineRestGitHub.R +++ b/R/EngineRestGitHub.R @@ -1,11 +1,12 @@ #' @noRd #' @description A class for methods wrapping GitHub's REST API responses. -EngineRestGitHub <- R6::R6Class("EngineRestGitHub", +EngineRestGitHub <- R6::R6Class( + classname = "EngineRestGitHub", inherit = EngineRest, public = list( # Pull repositories with files - pull_files = function(files) { + get_files = function(files) { files_list <- list() for (filename in files) { search_file_endpoint <- paste0(private$endpoints[["search"]], "filename:", filename) @@ -24,13 +25,13 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", }, # Pulling repositories where code appears - pull_repos_by_code = function(code, - org = NULL, - filename = NULL, - in_path = FALSE, - raw_output = FALSE, - verbose) { - private$set_verbose(verbose) + get_repos_by_code = function(code, + org = NULL, + filename = NULL, + in_path = FALSE, + raw_output = FALSE, + verbose = TRUE, + progress = TRUE) { user_query <- if (!is.null(org)) { paste0('+user:', org) } else { @@ -54,7 +55,8 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", ) if (!raw_output) { search_output <- private$map_search_into_repos( - search_response = search_result + search_response = search_result, + progress = progress ) } else { search_output <- search_result @@ -66,7 +68,7 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", }, #' Pull all repositories URLS from organization - pull_repos_urls = function(type, org) { + get_repos_urls = function(type, org) { repos_urls <- self$response( endpoint = paste0(private$endpoints[["organizations"]], org, "/repos") ) %>% @@ -81,7 +83,7 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", }, #' A method to add information on open and closed issues of a repository. - pull_repos_issues = function(repos_table) { + get_repos_issues = function(repos_table, progress) { if (nrow(repos_table) > 0) { repos_iterator <- paste0(repos_table$organization, "/", repos_table$repo_name) issues <- purrr::map_dfr(repos_iterator, function(repo_path) { @@ -93,7 +95,7 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", "open" = length(purrr::keep(issues, ~ .$state == "open")), "closed" = length(purrr::keep(issues, ~ .$state == "closed")) ) - }, .progress = if (private$verbose) { + }, .progress = if (progress) { "Pulling repositories issues..." } else { FALSE @@ -105,24 +107,23 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", }, #' Add information on repository contributors. - pull_repos_contributors = function(repos_table, settings) { + get_repos_contributors = function(repos_table, progress) { if (nrow(repos_table) > 0) { repo_iterator <- paste0(repos_table$organization, "/", repos_table$repo_name) user_name <- rlang::expr(.$login) repos_table$contributors <- purrr::map_chr(repo_iterator, function(repos_id) { tryCatch({ - contributors_endpoint <- paste0(private$endpoints[["repositories"]], repos_id, "/contributors") - contributors_vec <- private$pull_contributors_from_repo( - contributors_endpoint = contributors_endpoint, - user_name = user_name - ) - return(contributors_vec) - }, - error = function(e) { - NA - } - ) - }, .progress = if (private$scan_all && private$verbose) { + contributors_endpoint <- paste0(private$endpoints[["repositories"]], repos_id, "/contributors") + contributors_vec <- private$pull_contributors_from_repo( + contributors_endpoint = contributors_endpoint, + user_name = user_name + ) + return(contributors_vec) + }, + error = function(e) { + NA + }) + }, .progress = if (progress) { "[GitHost:GitHub] Pulling contributors..." } else { FALSE @@ -198,10 +199,15 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", NULL } else if ((n_count - 1) %/% 100 > 0) { for (page in (1:(n_count %/% 100) + 1)) { - resp_list <- self$response(paste0(search_endpoint, size_formula, "&page=", page, "&per_page=100"))[["items"]] %>% append(resp_list, .) + resp_list <- self$response(paste0(search_endpoint, + size_formula, + "&page=", + page, + "&per_page=100"))[["items"]] |> + append(resp_list, .) } } else if ((n_count - 1) %/% 100 == 0) { - resp_list <- self$response(paste0(search_endpoint, size_formula, "&page=1&per_page=100"))[["items"]] %>% + resp_list <- self$response(paste0(search_endpoint, size_formula, "&page=1&per_page=100"))[["items"]] |> append(resp_list, .) } index[1] <- index[2] @@ -224,18 +230,18 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub", }, # Parse search response into repositories output - map_search_into_repos = function(search_response) { + map_search_into_repos = function(search_response, progress) { repos_ids <- purrr::map_chr(search_response, ~ as.character(.$repository$id)) %>% unique() repos_list <- purrr::map(repos_ids, function(repo_id) { content <- self$response( endpoint = paste0(self$rest_api_url, "/repositories/", repo_id) ) - }, .progress = if (private$verbose) { + }, .progress = if (progress) { "Parsing search response into respositories output..." - } else { - FALSE - }) + } else { + FALSE + }) repos_list }, diff --git a/R/EngineRestGitLab.R b/R/EngineRestGitLab.R index 9f7dea55..2018b670 100644 --- a/R/EngineRestGitLab.R +++ b/R/EngineRestGitLab.R @@ -1,26 +1,36 @@ #' @noRd #' @description A class for methods wrapping GitLab's REST API responses. -EngineRestGitLab <- R6::R6Class("EngineRestGitLab", +EngineRestGitLab <- R6::R6Class( + classname = "EngineRestGitLab", inherit = EngineRest, public = list( # Pull repositories with files - pull_files = function(files, verbose = TRUE) { + get_files = function(file_paths = NULL, + org = NULL, + clean_files_content = TRUE, + verbose = TRUE, + progress = TRUE) { files_list <- list() - for (filename in files) { + file_paths <- utils::URLencode(file_paths, reserved = TRUE) + files_list <- purrr::map(file_paths, function(filename) { files_search_result <- private$search_for_code( - code = filename, + code = filename, in_path = TRUE, - settings = list(), + org = org, verbose = verbose ) %>% purrr::keep(~ .$path == filename) - files_content <- private$add_file_content( + files_content <- private$add_file_info( files_search_result = files_search_result, - filename = filename + clean_file_content = clean_files_content, + filename = filename, + verbose = verbose, + progress = progress ) - files_list <- append(files_list, files_content) - } + return(files_content) + }, .progress = progress) %>% + purrr::list_flatten() return(files_list) }, @@ -28,13 +38,13 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", # @details For the time being there is no possibility to search GitLab with # filtering by language. For more information look here: # https://gitlab.com/gitlab-org/gitlab/-/issues/340333 - pull_repos_by_code = function(code, - org = NULL, - filename = NULL, - in_path = FALSE, - raw_output = FALSE, - verbose) { - private$set_verbose(verbose) + get_repos_by_code = function(code, + org = NULL, + filename = NULL, + in_path = FALSE, + raw_output = FALSE, + verbose = TRUE, + progress = TRUE) { search_response <- private$search_for_code( code = code, filename = filename, @@ -46,14 +56,18 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", search_output <- search_response } else { search_output <- search_response %>% - private$map_search_into_repos() %>% - private$pull_repos_languages() + private$map_search_into_repos( + progress = progress + ) %>% + private$pull_repos_languages( + progress = progress + ) } return(search_output) }, # Pull all repositories URLs from organization - pull_repos_urls = function(type, org) { + get_repos_urls = function(type, org) { repos_urls <- self$response( endpoint = paste0(private$endpoints[["organizations"]], utils::URLencode(org, reserved = TRUE), "/projects") ) %>% @@ -68,7 +82,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", }, # Add information on open and closed issues of a repository. - pull_repos_issues = function(repos_table) { + get_repos_issues = function(repos_table, progress) { if (nrow(repos_table) > 0) { issues <- purrr::map(repos_table$repo_id, function(repos_id) { id <- gsub("gid://gitlab/Project/", "", repos_id) @@ -77,7 +91,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", self$response( endpoint = issues_endpoint )[["statistics"]][["counts"]] - }, .progress = if (private$verbose) { + }, .progress = if (progress) { "Pulling repositories issues..." } else { FALSE @@ -89,7 +103,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", }, #' Add information on repository contributors. - pull_repos_contributors = function(repos_table, settings) { + get_repos_contributors = function(repos_table, progress) { if (nrow(repos_table) > 0) { repo_urls <- repos_table$api_url user_name <- rlang::expr(.$name) @@ -103,12 +117,12 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", contributors_endpoint = contributors_endpoint, user_name = user_name ) - }, - error = function(e) { - NA + }, + error = function(e) { + NA }) return(contributors_vec) - }, .progress = if (private$scan_all && private$verbose) { + }, .progress = if (progress) { "[GitHost:GitLab] Pulling contributors..." } else { FALSE @@ -118,18 +132,18 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", }, # Pull all commits from give repositories. - pull_commits_from_repos = function(repos_names, - since, - until, - verbose) { + get_commits_from_repos = function(repos_names, + since, + until, + progress) { repos_list_with_commits <- purrr::map(repos_names, function(repo_path) { - commits_from_repo <- private$pull_commits_from_one_repo( + commits_from_repo <- private$get_commits_from_one_repo( repo_path = repo_path, since = since, until = until ) return(commits_from_repo) - }, .progress = !private$scan_all && verbose) + }, .progress = !private$scan_all && progress) names(repos_list_with_commits) <- repos_names repos_list_with_commits <- repos_list_with_commits %>% purrr::discard(~ length(.) == 0) @@ -137,7 +151,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", }, # A method to get separately GL logins and display names - get_commits_authors_handles_and_names = function(commits_table, verbose) { + get_commits_authors_handles_and_names = function(commits_table, + verbose = TRUE, + progress = verbose) { if (nrow(commits_table) > 0) { if (verbose) { cli::cli_alert_info("Looking up for authors' names and logins...") @@ -179,7 +195,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", ) } return(user_tbl) - }, .progress = TRUE) %>% + }, .progress = progress) %>% purrr::list_rbind() commits_table <- commits_table %>% @@ -251,14 +267,14 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", }, # Iterator over pulling pages of repositories. - pull_repos_from_org = function(org, settings) { + get_repos_from_org = function(org, progress) { repo_endpoint <- paste0(self$rest_api_url, "/groups/", org, "/projects") repos_response <- private$paginate_results( endpoint = repo_endpoint ) full_repos_list <- repos_response %>% private$pull_repos_languages( - verbose = settings$verbose + progress = progress ) return(full_repos_list) }, @@ -304,7 +320,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", }, # Parse search response into repositories output - map_search_into_repos = function(search_response) { + map_search_into_repos = function(search_response, progress) { repos_ids <- purrr::map_chr(search_response, ~ as.character(.$project_id)) %>% unique() @@ -312,7 +328,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", content <- self$response( endpoint = paste0(private$endpoints[["projects"]], repo_id) ) - }, .progress = if (private$verbose) { + }, .progress = if (progress) { "Parsing search response into repositories output..." } else { FALSE @@ -321,13 +337,13 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", }, # Pull languages of repositories. - pull_repos_languages = function(repos_list) { + pull_repos_languages = function(repos_list, progress) { repos_list_with_languages <- purrr::map(repos_list, function(repo) { id <- repo$id repo$languages <- names(self$response(paste0(private$endpoints[["projects"]], id, "/languages"))) repo - }, .progress = if (private$verbose) { - "Pulling reposiotories languages..." + }, .progress = if (progress) { + "Pulling repositories languages..." } else { FALSE }) @@ -335,9 +351,9 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", }, # Iterator over pages of commits response. - pull_commits_from_one_repo = function(repo_path, - since, - until) { + get_commits_from_one_repo = function(repo_path, + since, + until) { commits_endpoint <- paste0( private$endpoints$projects, repo_path, @@ -352,7 +368,7 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", endpoint = commits_endpoint, joining_sign = "&" ) - }, error = function (e) { + }, error = function(e) { list() }) return(all_commits_in_repo) @@ -364,7 +380,11 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", }, # Add file content to files search result - add_file_content = function(files_search_result, filename) { + add_file_info = function(files_search_result, + filename, + clean_file_content = FALSE, + verbose = FALSE, + progress = FALSE) { purrr::map(files_search_result, function(file_data) { repo_data <- self$response( paste0( @@ -392,9 +412,16 @@ EngineRestGitLab <- R6::R6Class("EngineRestGitLab", file_data$repo_fullname <- repo_data$path_with_namespace file_data$repo_id <- repo_data$id file_data$repo_url <- repo_data$web_url + if (clean_file_content) { + file_data$content <- NA + } } return(file_data) - }, .progress = glue::glue("Adding file [{filename}] info...")) %>% + }, .progress = if (progress) { + glue::glue("Adding file [{filename}] info...") + } else { + FALSE + }) |> purrr::discard(is.null) } ) diff --git a/R/GQLQueryGitHub.R b/R/GQLQueryGitHub.R index 5ede1141..38548b86 100644 --- a/R/GQLQueryGitHub.R +++ b/R/GQLQueryGitHub.R @@ -15,22 +15,23 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub", pagination_phrase <- paste0('after: "', end_cursor, '"') } paste0( - 'query { - search(first: 100, type: USER, query: "type:org" ', pagination_phrase, ') { - pageInfo { - hasNextPage - endCursor - } - edges { - node{ - ... on Organization { - name - url + 'query { + search(first: 100, type: USER, query: "type:org" ', pagination_phrase, ') { + pageInfo { + hasNextPage + endCursor + } + edges { + node{ + ... on Organization { + name + url + } } } } - } - }') + }' + ) }, #' @description Prepare query to get repositories from GitHub. @@ -48,7 +49,7 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub", repositoryOwner(login: $org) { ... on Organization { repositories(first: 100 ', after_cursor, ') { - ', private$repository_field(),' + ', private$repository_field(), ' } } } @@ -143,14 +144,13 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub", #' @description Prepare query to get files in a standard filepath from #' GitHub repositories. #' @return A query. - files_by_repo = function(){ - paste0( - 'query GetFilesByRepo($org: String!, $repo: String!, $file_path: String!) { + file_blob_from_repo = function() { + 'query GetFileBlobFromRepo($org: String!, $repo: String!, $expression: String!) { repository(owner: $org, name: $repo) { - id - name - url - object(expression: $file_path) { + repo_id: id + repo_name: name + repo_url: url + file: object(expression: $expression) { ... on Blob { text byteSize @@ -158,7 +158,24 @@ GQLQueryGitHub <- R6::R6Class("GQLQueryGitHub", } } }' - ) + }, + + files_tree_from_repo = function() { + 'query GetFilesFromRepo($org: String!, $repo: String!, $expression: String!) { + repository(owner: $org, name: $repo) { + id + name + url + object(expression: $expression) { + ... on Tree { + entries { + name + type + } + } + } + } + }' }, #' @description Prepare query to get releases from GitHub repositories. diff --git a/R/GQLQueryGitLab.R b/R/GQLQueryGitLab.R index 60e7f72f..e6777dc6 100644 --- a/R/GQLQueryGitLab.R +++ b/R/GQLQueryGitLab.R @@ -44,6 +44,7 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab", node { repo_id: id repo_name: name + repo_path: path ... on Project { repository { rootRef @@ -103,7 +104,7 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab", #' GitLab repositories. #' @param end_cursor An endCursor. #' @return A query. - files_by_org = function(end_cursor = ""){ + files_by_org = function(end_cursor = "") { if (nchar(end_cursor) == 0) { after_cursor <- end_cursor } else { @@ -113,35 +114,87 @@ GQLQueryGitLab <- R6::R6Class("GQLQueryGitLab", 'query GetFilesByOrg($org: ID!, $file_paths: [String!]!) { group(fullPath: $org) { projects(first: 100', - after_cursor, - ') { - count - pageInfo { - hasNextPage - endCursor - } - edges { - node { + after_cursor, + ') { + count + pageInfo { + hasNextPage + endCursor + } + edges { + node { + name + id + webUrl + repository { + blobs(paths: $file_paths) { + nodes { name - id - webUrl - repository { - blobs(paths: $file_paths) { - nodes { - name - rawBlob - size - } - } - } + rawBlob + size } } } } - }' + } + } + } + }' ) }, + file_blob_from_repo = function() { + ' + query GetFilesByRepo($fullPath: ID!, $file_paths: [String!]!) { + project(fullPath: $fullPath) { + name + id + webUrl + repository { + blobs(paths: $file_paths) { + nodes { + name + rawBlob + size + } + } + } + } + } + ' + }, + + files_tree_from_repo = function() { + ' + query GetFilesTree ($fullPath: ID!, $file_path: String!) { + project(fullPath: $fullPath) { + repository { + tree(path: $file_path) { + trees (first: 100) { + pageInfo{ + endCursor + hasNextPage + } + nodes { + name + } + } + blobs (first: 100) { + pageInfo{ + endCursor + hasNextPage + } + nodes { + name + } + } + } + } + } + } + ' + }, + #' @description Prepare query to get releases from GitHub repositories. #' @return A query. releases_from_repo = function() { diff --git a/R/GitHost.R b/R/GitHost.R index 1ab85405..a7b3169d 100644 --- a/R/GitHost.R +++ b/R/GitHost.R @@ -1,6 +1,7 @@ #' @noRd #' @description A class to manage which engine to use for pulling data -GitHost <- R6::R6Class("GitHost", +GitHost <- R6::R6Class( + classname = "GitHost", public = list( #' @description Create a new `GitHost` object. @@ -12,84 +13,106 @@ GitHost <- R6::R6Class("GitHost", #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing #' output is switched off. #' @return A new `GitHost` object. - initialize = function(orgs = NA, - repos = NA, - token = NA, - host = NA, + initialize = function(orgs = NA, + repos = NA, + token = NA, + host = NA, verbose = NA) { - private$set_verbose(verbose) private$set_api_url(host) + private$set_web_url(host) private$set_endpoints() private$check_if_public(host) - private$set_token(token) + private$set_token( + token = token, + verbose = verbose + ) private$set_graphql_url() - private$set_searching_scope(orgs, repos) + private$set_searching_scope( + orgs = orgs, + repos = repos, + verbose = verbose + ) private$setup_engines() - private$set_orgs_and_repos(orgs, repos) + private$set_orgs_and_repos( + orgs = orgs, + repos = repos, + verbose = verbose + ) }, # Pull repositories method get_repos = function(add_contributors = TRUE, - with_code = NULL, - in_files = NULL, - with_file = NULL, - verbose = TRUE, - settings) { - private$set_verbose(verbose) + with_code = NULL, + in_files = NULL, + with_file = NULL, + verbose = TRUE, + progress = TRUE) { if (is.null(with_code) && is.null(with_file)) { repos_table <- private$get_all_repos( - settings = settings + verbose = verbose, + progress = progress ) } if (!is.null(with_code)) { repos_table <- private$get_repos_with_code( - code = with_code, - in_files = in_files + code = with_code, + in_files = in_files, + verbose = verbose, + progress = progress ) } else if (!is.null(with_file)) { repos_table <- private$get_repos_with_code( - code = with_file, - in_path = TRUE + code = with_file, + in_path = TRUE, + verbose = verbose, + progress = progress ) } repos_table <- private$add_repo_api_url(repos_table) if (add_contributors) { - repos_table <- private$pull_repos_contributors( + repos_table <- private$get_repos_contributors( repos_table = repos_table, - settings = settings + verbose = verbose, + progress = progress ) } return(repos_table) }, # Get repositories URLS from the Git host - get_repos_urls = function(type = "web", + get_repos_urls = function(type = "web", with_code = NULL, - in_files = NULL, + in_files = NULL, with_file = NULL, - verbose) { - private$set_verbose(verbose) + verbose = TRUE, + progress = TRUE) { if (!is.null(with_code)) { repo_urls <- private$get_repos_with_code( - code = with_code, - in_files = in_files, - raw_output = TRUE + code = with_code, + in_files = in_files, + raw_output = TRUE, + verbose = verbose ) %>% private$get_repo_url_from_response( - type = type + type = type, + progress = progress ) } else if (!is.null(with_file)) { repo_urls <- private$get_repos_with_code( - code = with_file, - in_path = TRUE, - raw_output = TRUE + code = with_file, + in_path = TRUE, + raw_output = TRUE, + verbose = verbose ) %>% private$get_repo_url_from_response( - type = type + type = type, + progress = progress ) } else { repo_urls <- private$get_all_repos_urls( - type = type + type = type, + verbose = verbose, + progress = progress ) } return(repo_urls) @@ -97,21 +120,21 @@ GitHost <- R6::R6Class("GitHost", #' Pull commits method get_commits = function(since, - until = Sys.Date(), - verbose = TRUE, - settings) { - private$set_verbose(verbose) - if (private$scan_all && is.null(private$orgs)) { + until = Sys.Date(), + verbose = TRUE, + progress = TRUE) { + if (private$scan_all && is.null(private$orgs) && verbose) { cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") - private$orgs <- private$engines$graphql$pull_orgs() + private$orgs <- private$engines$graphql$get_orgs() } if (is.null(until)) { until <- Sys.time() } - commits_table <- private$get_commits_from_host( - since = since, - until = until, - settings = settings + commits_table <- private$get_commits_from_orgs( + since = since, + until = until, + verbose = verbose, + progress = progress ) return(commits_table) }, @@ -120,7 +143,7 @@ GitHost <- R6::R6Class("GitHost", get_users = function(users) { graphql_engine <- private$engines$graphql users_table <- purrr::map(users, function(user) { - graphql_engine$pull_user(user) %>% + graphql_engine$get_user(user) %>% private$prepare_user_table() }) %>% purrr::list_rbind() @@ -129,26 +152,54 @@ GitHost <- R6::R6Class("GitHost", #' Retrieve content of given text files from all repositories for a host in #' a table format. - get_files = function(file_path, verbose = TRUE) { + get_files_content = function(file_path, + host_files_structure = NULL, + only_text_files = TRUE, + verbose = TRUE, + progress = TRUE) { files_table <- if (!private$scan_all) { - private$get_files_from_orgs( - file_path = file_path, - verbose = verbose + private$get_files_content_from_orgs( + file_path = file_path, + host_files_structure = host_files_structure, + only_text_files = only_text_files, + verbose = verbose, + progress = progress ) } else { - private$get_files_from_host( + private$get_files_content_from_host( file_path = file_path, - verbose = verbose + verbose = verbose, + progress = progress ) } return(files_table) }, + #' Get files structure + get_files_structure = function(pattern, + depth, + verbose = TRUE, + progress = TRUE) { + if (private$scan_all) { + cli::cli_abort(c( + "x" = "This feature is not applicable to scan whole Git Host (time consuming).", + "i" = "Set `orgs` or `repos` arguments in `set_*_host()` if you wish to run this function." + ), call = NULL) + } + files_structure <- private$get_files_structure_from_orgs( + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress + ) + return(files_structure) + }, + #' Iterator over pulling release logs from engines - get_release_logs = function(since, until, verbose, settings) { - if (private$scan_all && is.null(private$orgs)) { + get_release_logs = function(since, until, verbose, progress) { + if (private$scan_all && is.null(private$orgs) && verbose) { cli::cli_alert_info("[{private$host_name}][Engine:{cli::col_yellow('GraphQL')}] Pulling all organizations...") - private$orgs <- private$engines$graphql$pull_orgs() + private$orgs <- private$engines$graphql$get_orgs() } until <- until %||% Sys.time() release_logs_table <- purrr::map(private$orgs, function(org) { @@ -163,12 +214,11 @@ GitHost <- R6::R6Class("GitHost", ) } repos_names <- private$set_repositories( - org = org, - settings = settings + org = org ) gql_engine <- private$engines$graphql if (length(repos_names) > 0) { - release_logs_table_org <- gql_engine$pull_release_logs_from_org( + release_logs_table_org <- gql_engine$get_release_logs_from_org( org = org, repos_names = repos_names ) %>% @@ -177,7 +227,7 @@ GitHost <- R6::R6Class("GitHost", releases_logs_table_org <- NULL } return(release_logs_table_org) - }, .progress = if (private$scan_all && verbose) { + }, .progress = if (progress) { glue::glue("[GitHost:{private$host_name}] Pulling release logs...") } else { FALSE @@ -191,6 +241,9 @@ GitHost <- R6::R6Class("GitHost", # A REST API URL. api_url = NULL, + # Web URL. + web_url = NULL, + # A GraphQL API url. graphql_api_url = NULL, @@ -231,11 +284,6 @@ GitHost <- R6::R6Class("GitHost", # Show messages or not. verbose = TRUE, - # Set verbose mode - set_verbose = function(verbose) { - private$verbose <- verbose - }, - # engines A placeholder for REST and GraphQL Engine classes. engines = list(), @@ -252,6 +300,17 @@ GitHost <- R6::R6Class("GitHost", } }, + # Set web url + set_custom_web_url = function(host) { + private$web_url <- if (!grepl("https://", host)) { + glue::glue( + "https://{host}" + ) + } else { + host + } + }, + # Set endpoints set_endpoints = function() { private$set_test_endpoint() @@ -261,9 +320,11 @@ GitHost <- R6::R6Class("GitHost", }, # Set authorizing token - set_token = function(token) { - if (is.null(token)){ - token <- private$set_default_token() + set_token = function(token, verbose) { + if (is.null(token)) { + token <- private$set_default_token( + verbose = verbose + ) } else { token <- private$check_token(token) } @@ -289,7 +350,7 @@ GitHost <- R6::R6Class("GitHost", }, # Check if both repos and orgs are defined or not. - set_searching_scope = function(orgs, repos) { + set_searching_scope = function(orgs, repos, verbose) { if (is.null(repos) && is.null(orgs)) { if (private$is_public) { cli::cli_abort(c( @@ -299,9 +360,9 @@ GitHost <- R6::R6Class("GitHost", ), call = NULL) } else { - if (private$verbose) { - cli::cli_alert_warning(cli::col_yellow( - "No `orgs` specified." + if (verbose) { + cli::cli_alert_info(cli::col_grey( + "No `orgs` nor `repos` specified." )) cli::cli_alert_info(cli::col_grey( "Searching scope set to [all]." @@ -312,13 +373,13 @@ GitHost <- R6::R6Class("GitHost", } } if (!is.null(repos) && is.null(orgs)) { - if (private$verbose) { + if (verbose) { cli::cli_alert_info(cli::col_grey("Searching scope set to [repo].")) } private$searching_scope <- "repo" } if (is.null(repos) && !is.null(orgs)) { - if (private$verbose) { + if (verbose) { cli::cli_alert_info(cli::col_grey("Searching scope set to [org].")) } private$searching_scope <- "org" @@ -334,13 +395,19 @@ GitHost <- R6::R6Class("GitHost", }, # Set organization or repositories - set_orgs_and_repos = function(orgs, repos) { + set_orgs_and_repos = function(orgs, repos, verbose) { if (!private$scan_all) { if (!is.null(orgs)) { - private$orgs <- private$check_organizations(orgs) + private$orgs <- private$check_organizations( + orgs = orgs, + verbose = verbose + ) } if (!is.null(repos)) { - repos <- private$check_repositories(repos) + repos <- private$check_repositories( + repos = repos, + verbose = verbose + ) private$repos_fullnames <- repos orgs_repos <- private$extract_repos_and_orgs(repos) private$orgs <- names(orgs_repos) @@ -351,12 +418,12 @@ GitHost <- R6::R6Class("GitHost", }, # Check if repositories exist - check_repositories = function(repos) { - if (private$verbose) { - cli::cli_alert_info(cli::col_grey("Checking host data...")) + check_repositories = function(repos, verbose) { + if (verbose) { + cli::cli_alert_info(cli::col_grey("Checking repositories...")) } repos <- purrr::map(repos, function(repo) { - repo_endpoint = glue::glue("{private$endpoints$repositories}/{repo}") + repo_endpoint <- glue::glue("{private$endpoints$repositories}/{repo}") check <- private$check_endpoint( endpoint = repo_endpoint, type = "Repository" @@ -375,12 +442,12 @@ GitHost <- R6::R6Class("GitHost", }, # Check if organizations exist - check_organizations = function(orgs) { - if (private$verbose) { - cli::cli_alert_info(cli::col_grey("Checking host data...")) + check_organizations = function(orgs, verbose) { + if (verbose) { + cli::cli_alert_info(cli::col_grey("Checking organizations...")) } orgs <- purrr::map(orgs, function(org) { - org_endpoint = glue::glue("{private$endpoints$orgs}/{org}") + org_endpoint <- glue::glue("{private$endpoints$orgs}/{org}") check <- private$check_endpoint( endpoint = org_endpoint, type = "Organization" @@ -415,9 +482,12 @@ GitHost <- R6::R6Class("GitHost", if (grepl("404", e)) { cli::cli_abort( c( - "x" = "{type} you provided does not exist or its name was passed in a wrong way: {cli::col_red({endpoint})}", - "!" = "Please type your {tolower(type)} name as you see it in web URL.", - "i" = "E.g. do not use spaces. {type} names as you see on the page may differ from their web 'address'." + "x" = "{type} you provided does not exist or its name was passed + in a wrong way: {cli::col_red({endpoint})}", + "!" = "Please type your {tolower(type)} name as you see it in + web URL.", + "i" = "E.g. do not use spaces. {type} names as you see on the + page may differ from their web 'address'." ), call = NULL ) @@ -434,10 +504,10 @@ GitHost <- R6::R6Class("GitHost", }, # Set default token if none exists. - set_default_token = function() { + set_default_token = function(verbose) { primary_token_name <- private$token_name token <- Sys.getenv(primary_token_name) - if (private$test_token(token) && private$verbose) { + if (private$test_token(token) && verbose) { cli::cli_alert_info("Using PAT from {primary_token_name} envar.") } else { pat_names <- names(Sys.getenv()[grepl(primary_token_name, names(Sys.getenv()))]) @@ -445,7 +515,7 @@ GitHost <- R6::R6Class("GitHost", for (token_name in possible_tokens) { if (private$test_token(Sys.getenv(token_name))) { token <- Sys.getenv(token_name) - if (private$verbose) { + if (verbose) { cli::cli_alert_info("Using PAT from {token_name} envar.") } break @@ -459,9 +529,9 @@ GitHost <- R6::R6Class("GitHost", test_token = function(token) { response <- NULL test_endpoint <- private$test_endpoint - try(response <- httr2::request(test_endpoint) %>% - httr2::req_headers("Authorization" = paste0("Bearer ", token)) %>% - httr2::req_perform(), silent = TRUE) + try(response <- httr2::request(test_endpoint) |> + httr2::req_headers("Authorization" = paste0("Bearer ", token)) |> + httr2::req_perform(), silent = TRUE) if (!is.null(response)) { private$check_token_scopes(response, token) TRUE @@ -473,7 +543,7 @@ GitHost <- R6::R6Class("GitHost", # Helper to extract organizations and repositories from vector of full names # of repositories extract_repos_and_orgs = function(repos_fullnames = NULL) { - repos_fullnames <-URLdecode(repos_fullnames) + repos_fullnames <- URLdecode(repos_fullnames) repos_vec <- stringr::str_split(repos_fullnames, "/") %>% purrr::map(~ paste0(.[length(.)], collapse = "/")) %>% unlist() @@ -490,7 +560,7 @@ GitHost <- R6::R6Class("GitHost", }, # Set repositories - set_repos = function(settings, org) { + set_repos = function(org) { if (private$searching_scope == "repo") { repos <- private$orgs_repos[[org]] } else { @@ -508,30 +578,30 @@ GitHost <- R6::R6Class("GitHost", }, #' Retrieve all repositories for an organization in a table format. - get_all_repos = function(settings, verbose = private$verbose) { + get_all_repos = function(verbose = TRUE, progress = TRUE) { if (private$scan_all && is.null(private$orgs)) { if (verbose) { show_message( - host = private$host_name, - engine = "graphql", + host = private$host_name, + engine = "graphql", information = "Pulling all organizations" ) } - private$orgs <- private$engines$graphql$pull_orgs() + private$orgs <- private$engines$graphql$get_orgs() } graphql_engine <- private$engines$graphql repos_table <- purrr::map(private$orgs, function(org) { org <- utils::URLdecode(org) if (!private$scan_all && verbose) { show_message( - host = private$host_name, - engine = "graphql", - scope = org, + host = private$host_name, + engine = "graphql", + scope = org, information = "Pulling repositories" ) } - repos <- private$set_repos(settings, org) - repos_table <- graphql_engine$pull_repos_from_org( + repos <- private$set_repos(org) + repos_table <- graphql_engine$get_repos_from_org( org = org ) %>% private$prepare_repos_table_from_graphql() @@ -540,93 +610,107 @@ GitHost <- R6::R6Class("GitHost", dplyr::filter(repo_name %in% repos) } return(repos_table) - }, .progress = private$scan_all) %>% + }, .progress = progress) %>% purrr::list_rbind() return(repos_table) }, # Pull repositories with specific code - get_repos_with_code = function(code, in_files = NULL, in_path = FALSE, raw_output = FALSE, verbose = private$verbose) { + get_repos_with_code = function(code, + in_files = NULL, + in_path = FALSE, + raw_output = FALSE, + verbose = TRUE, + progress = TRUE) { if (private$scan_all) { repos_table <- private$get_repos_with_code_from_host( - code = code, - in_files = in_files, - in_path = in_path, + code = code, + in_files = in_files, + in_path = in_path, raw_output = raw_output, - verbose = verbose + verbose = verbose, + progress = progress ) } if (!private$scan_all) { repos_table <- private$get_repos_with_code_from_orgs( - code = code, - in_files = in_files, - in_path = in_path, + code = code, + in_files = in_files, + in_path = in_path, raw_output = raw_output, - verbose = verbose + verbose = verbose, + progress = progress ) } return(repos_table) }, # Pull all repositories URLs from organizations - get_all_repos_urls = function(type, verbose = private$verbose) { + get_all_repos_urls = function(type, verbose = TRUE, progress = TRUE) { if (private$scan_all && is.null(private$orgs)) { if (verbose) { show_message( - host = private$host_name, - engine = "graphql", + host = private$host_name, + engine = "graphql", information = "Pulling all organizations" ) } - private$orgs <- private$engines$graphql$pull_orgs() + private$orgs <- private$engines$graphql$get_orgs() } rest_engine <- private$engines$rest repos_vector <- purrr::map(private$orgs, function(org) { org <- utils::URLdecode(org) if (!private$scan_all && verbose) { show_message( - host = private$host_name, - engine = "rest", - scope = org, + host = private$host_name, + engine = "rest", + scope = org, information = "Pulling repositories (URLS)" ) } - repos <- private$set_repos(settings, org) - repos_urls <- rest_engine$pull_repos_urls( + repos <- private$set_repos(org) + repos_urls <- rest_engine$get_repos_urls( type = type, - org = org + org = org ) return(repos_urls) - }, .progress = private$scan_all) %>% + }, .progress = progress) %>% unlist() return(repos_vector) }, # Pull repositories with code from whole Git Host get_repos_with_code_from_host = function(code, - in_files = NULL, - in_path = FALSE, + in_files = NULL, + in_path = FALSE, raw_output = FALSE, - verbose = private$verbose) { + verbose = TRUE, + progress = TRUE) { if (verbose) { show_message( - host = private$host_name, - engine = "rest", + host = private$host_name, + engine = "rest", information = "Pulling repositories" ) } repos_response <- private$get_repos_response_with_code( code = code, - in_files = in_files, - in_path = in_path, - raw_output = raw_output + in_files = in_files, + in_path = in_path, + raw_output = raw_output, + verbose = verbose, + progress = progress ) if (!raw_output) { rest_engine <- private$engines$rest repos_table <- repos_response %>% private$tailor_repos_response() %>% - private$prepare_repos_table_from_rest() %>% - rest_engine$pull_repos_issues() + private$prepare_repos_table_from_rest( + verbose = verbose + ) %>% + rest_engine$get_repos_issues( + progress = progress + ) return(repos_table) } else { return(repos_response) @@ -635,10 +719,11 @@ GitHost <- R6::R6Class("GitHost", # Pull repositories with code from given organizations get_repos_with_code_from_orgs = function(code, - in_files = NULL, - in_path = FALSE, + in_files = NULL, + in_path = FALSE, raw_output = FALSE, - verbose = private$verbose) { + verbose = TRUE, + progress = TRUE) { repos_list <- purrr::map(private$orgs, function(org) { if (verbose) { show_message( @@ -650,23 +735,29 @@ GitHost <- R6::R6Class("GitHost", ) } repos_response <- private$get_repos_response_with_code( - org = org, - code = code, - in_files = in_files, - in_path = in_path, - raw_output = raw_output + org = org, + code = code, + in_files = in_files, + in_path = in_path, + raw_output = raw_output, + verbose = verbose, + progress = progress ) if (!raw_output) { rest_engine <- private$engines$rest repos_table <- repos_response %>% private$tailor_repos_response() %>% - private$prepare_repos_table_from_rest() %>% - rest_engine$pull_repos_issues() + private$prepare_repos_table_from_rest( + verbose = verbose + ) %>% + rest_engine$get_repos_issues( + progress = progress + ) return(repos_table) } else { return(repos_response) } - }, .progress = private$scan_all) + }, .progress = progress) if (!raw_output) { repos_output <- purrr::list_rbind(repos_list) } else { @@ -676,26 +767,33 @@ GitHost <- R6::R6Class("GitHost", }, # Wrapper in case in_files is fed. - get_repos_response_with_code = function(org = NULL, code, in_files, in_path, raw_output) { + get_repos_response_with_code = function(org = NULL, + code, + in_files, + in_path, + raw_output, + verbose, + progress) { rest_engine <- private$engines$rest if (is.null(in_files)) { - repos_response <- rest_engine$pull_repos_by_code( - org = org, - code = code, - in_path = in_path, + repos_response <- rest_engine$get_repos_by_code( + org = org, + code = code, + in_path = in_path, raw_output = raw_output, - verbose = private$verbose + verbose = verbose, + progress = progress ) } else { repos_response <- purrr::map(in_files, function(filename) { - cli::cli_alert_info("In file: {filename}") - rest_engine$pull_repos_by_code( - org = org, - code = code, - filename = filename, - in_path = in_path, + rest_engine$get_repos_by_code( + org = org, + code = code, + filename = filename, + in_path = in_path, raw_output = raw_output, - verbose = private$verbose + verbose = verbose, + progress = progress ) }) %>% purrr::list_flatten() @@ -704,9 +802,9 @@ GitHost <- R6::R6Class("GitHost", }, #' Add information on repository contributors. - pull_repos_contributors = function(repos_table, settings) { + get_repos_contributors = function(repos_table, verbose, progress) { if (!is.null(repos_table) && nrow(repos_table) > 0) { - if (!private$scan_all && private$verbose) { + if (!private$scan_all && verbose) { show_message( host = private$host_name, engine = "rest", @@ -715,16 +813,16 @@ GitHost <- R6::R6Class("GitHost", } repos_table <- private$filter_repos_by_host(repos_table) rest_engine <- private$engines$rest - repos_table <- rest_engine$pull_repos_contributors( + repos_table <- rest_engine$get_repos_contributors( repos_table = repos_table, - settings = settings + progress = progress ) return(repos_table) } }, # Prepare table for repositories content - prepare_repos_table_from_rest = function(repos_list) { + prepare_repos_table_from_rest = function(repos_list, verbose = TRUE) { repos_dt <- purrr::map(repos_list, function(repo) { repo <- purrr::map(repo, function(attr) { attr <- attr %||% "" @@ -732,41 +830,68 @@ GitHost <- R6::R6Class("GitHost", data.frame(repo) }) %>% purrr::list_rbind() - if (private$verbose) { + if (verbose) { cli::cli_alert_info("Preparing repositories table...") } if (length(repos_dt) > 0) { - repos_dt <- dplyr::mutate(repos_dt, - repo_id = as.character(repo_id), - created_at = as.POSIXct(created_at), - last_activity_at = as.POSIXct(last_activity_at), - forks = as.integer(forks), - issues_open = as.integer(issues_open), - issues_closed = as.integer(issues_closed) + repos_dt <- dplyr::mutate( + repos_dt, + repo_id = as.character(repo_id), + created_at = as.POSIXct(created_at), + last_activity_at = as.POSIXct(last_activity_at), + forks = as.integer(forks), + issues_open = as.integer(issues_open), + issues_closed = as.integer(issues_closed) ) } return(repos_dt) }, # Pull files content from organizations - get_files_from_orgs = function(file_path, verbose) { + get_files_content_from_orgs = function(file_path, + host_files_structure = NULL, + only_text_files = TRUE, + verbose = TRUE, + progress = TRUE) { graphql_engine <- private$engines$graphql - files_table <- purrr::map(private$orgs, function(org) { + if (!is.null(host_files_structure)) { if (verbose) { + cli::cli_alert_info(cli::col_green("I will make use of files structure stored in GitStats.")) + } + result <- private$get_orgs_and_repos_from_files_structure( + host_files_structure = host_files_structure + ) + orgs <- result$orgs + repos <- result$repos + } else { + orgs <- private$orgs + repos <- private$repos + } + files_table <- purrr::map(orgs, function(org) { + if (verbose) { + user_msg <- if (!is.null(host_files_structure)) { + "Pulling files from files structure" + } else { + glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") + } show_message( - host = private$host_name, - engine = "graphql", - scope = org, - information = glue::glue("Pulling files: [{paste0(file_path, collapse = ', ')}]") + host = private$host_name, + engine = "graphql", + scope = org, + information = user_msg ) } - graphql_engine$pull_files_from_org( - org = org, - repos = private$repos, - file_path = file_path + graphql_engine$get_files_from_org( + org = org, + repos = repos, + file_paths = file_path, + host_files_structure = host_files_structure, + only_text_files = only_text_files, + verbose = verbose, + progress = progress ) %>% private$prepare_files_table( - org = org, + org = org, file_path = file_path ) }) %>% @@ -775,8 +900,59 @@ GitHost <- R6::R6Class("GitHost", return(files_table) }, + get_orgs_and_repos_from_files_structure = function(host_files_structure) { + result <- list( + "orgs" = names(host_files_structure), + "repos" = purrr::map(host_files_structure, ~names(.)) %>% unlist() %>% unname() + ) + return(result) + }, + + get_files_structure_from_orgs = function(pattern, + depth, + verbose = TRUE, + progress = TRUE) { + graphql_engine <- private$engines$graphql + files_structure_list <- purrr::map(private$orgs, function(org) { + if (verbose) { + user_info <- if (!is.null(pattern)) { + glue::glue("Pulling files structure...[files matching pattern: '{pattern}']") + } else { + glue::glue("Pulling files structure...") + } + show_message( + host = private$host_name, + engine = "graphql", + scope = org, + information = user_info + ) + } + graphql_engine$get_files_structure_from_org( + org = org, + repos = private$repos, + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress + ) + }) + names(files_structure_list) <- private$orgs + files_structure_list <- files_structure_list %>% + purrr::discard(~ length(.) == 0) + if (length(files_structure_list) == 0 && verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "For {private$host_name} no files structure found." + ) + ) + } + return(files_structure_list) + }, + # Pull files from host - get_files_from_host = function(file_path, verbose) { + get_files_content_from_host = function(file_path, + verbose = TRUE, + progress = TRUE) { rest_engine <- private$engines$rest if (verbose) { show_message( @@ -785,9 +961,10 @@ GitHost <- R6::R6Class("GitHost", information = glue::glue("Pulling files: [{paste0(file_path, collapse = ', ')}]") ) } - files_table <- rest_engine$pull_files( - files = file_path, - verbose = verbose + files_table <- rest_engine$get_files( + file_paths = file_path, + verbose = verbose, + progress = progress ) %>% private$prepare_files_table_from_rest() %>% private$add_repo_api_url() diff --git a/R/GitHostGitHub.R b/R/GitHostGitHub.R index a24e1346..b2df0367 100644 --- a/R/GitHostGitHub.R +++ b/R/GitHostGitHub.R @@ -1,5 +1,6 @@ #' @noRd -GitHostGitHub <- R6::R6Class("GitHostGitHub", +GitHostGitHub <- R6::R6Class( + classname = "GitHostGitHub", inherit = GitHost, public = list( initialize = function(orgs = NA, @@ -12,7 +13,7 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", token = token, host = host, verbose = verbose) - if (private$verbose) { + if (verbose) { cli::cli_alert_success("Set connection to GitHub.") } } @@ -44,7 +45,7 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", ) ), - # Set API url + # Set API URL set_api_url = function(host) { if (is.null(host)) { private$api_url <- "https://api.github.com" @@ -53,6 +54,15 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", } }, + # Set web URL + set_web_url = function(host) { + if (is.null(host)) { + private$web_url <- "https://github.com" + } else { + private$set_custom_web_url(host) + } + }, + # Check whether Git platform is public or internal. check_if_public = function(host) { private$is_public <- is.null(host) || grepl("github.com", host) @@ -60,22 +70,22 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", # Set endpoint for basic checks set_test_endpoint = function() { - private$test_endpoint = private$api_url + private$test_endpoint <- private$api_url }, # Set tokens endpoint set_tokens_endpoint = function() { - private$endpoints$tokens = NULL + private$endpoints$tokens <- NULL }, # Set groups endpoint set_orgs_endpoint = function() { - private$endpoints$orgs = glue::glue("{private$api_url}/orgs") + private$endpoints$orgs <- glue::glue("{private$api_url}/orgs") }, # Set projects endpoint set_repositories_endpoint = function() { - private$endpoints$repositories = glue::glue("{private$api_url}/repos") + private$endpoints$repositories <- glue::glue("{private$api_url}/repos") }, # Setup REST and GraphQL engines @@ -96,7 +106,8 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", # token parameter only for need of super method check_token_scopes = function(response, token = NULL) { token_scopes <- response$headers$`x-oauth-scopes` %>% - stringr::str_split(", ") %>% unlist() + stringr::str_split(", ") %>% + unlist() all(private$access_scopes %in% token_scopes) }, @@ -110,7 +121,11 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", "stars" = repo$stargazers_count, "forks" = repo$forks_count, "created_at" = gts_to_posixt(repo$created_at), - "last_activity_at" = if (!is.null(repo$pushed_at)) gts_to_posixt(repo$pushed_at) else gts_to_posixt(repo$created_at), + "last_activity_at" = if (!is.null(repo$pushed_at)) { + gts_to_posixt(repo$pushed_at) + } else { + gts_to_posixt(repo$created_at) + }, "languages" = repo$language, "issues_open" = repo$issues_open, "issues_closed" = repo$issues_closed, @@ -125,7 +140,7 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", prepare_repos_table_from_graphql = function(repos_list) { if (length(repos_list) > 0) { repos_table <- purrr::map_dfr(repos_list, function(repo) { - repo$default_branch <- if(!is.null(repo$default_branch)) { + repo$default_branch <- if (!is.null(repo$default_branch)) { repo$default_branch$name } else { "" @@ -154,18 +169,18 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", }, # Add `api_url` column to table. - add_repo_api_url = function(repos_table){ + add_repo_api_url = function(repos_table) { if (!is.null(repos_table) && nrow(repos_table) > 0) { repos_table <- dplyr::mutate( - repos_table, - api_url = paste0(private$endpoints$repositories, "/", organization, "/", repo_name), - ) + repos_table, + api_url = paste0(private$endpoints$repositories, "/", organization, "/", repo_name), + ) } return(repos_table) }, # Get projects URL from search response - get_repo_url_from_response = function(search_response, type) { + get_repo_url_from_response = function(search_response, type, progress = TRUE) { purrr::map_vec(search_response, function(project) { if (type == "api") { project$repository$url @@ -176,32 +191,31 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", }, # Pull commits from GitHub - get_commits_from_host = function(since, until, settings) { + get_commits_from_orgs = function(since, until, verbose, progress) { graphql_engine <- private$engines$graphql commits_table <- purrr::map(private$orgs, function(org) { commits_table_org <- NULL - if (!private$scan_all && private$verbose) { + if (!private$scan_all && verbose) { show_message( - host = private$host_name, - engine = "graphql", - scope = org, + host = private$host_name, + engine = "graphql", + scope = org, information = "Pulling commits" ) } repos_names <- private$set_repositories( - org = org, - settings = settings + org = org ) - commits_table_org <- graphql_engine$pull_commits_from_repos( - org = org, + commits_table_org <- graphql_engine$get_commits_from_repos( + org = org, repos_names = repos_names, - since = since, - until = until, - verbose = private$verbose + since = since, + until = until, + progress = progress ) %>% private$prepare_commits_table(org) return(commits_table_org) - }, .progress = if (private$scan_all && private$verbose) { + }, .progress = if (private$scan_all && progress) { "[GitHost:GitHub] Pulling commits..." } else { FALSE @@ -211,13 +225,12 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", }, # Use repositories either from parameter or, if not set, pull them from API - set_repositories = function(org, settings) { + set_repositories = function(org) { if (private$searching_scope == "repo") { repos_names <- private$orgs_repos[[org]] } else { repos_table <- private$get_all_repos( - verbose = FALSE, - settings = settings + verbose = FALSE ) repos_names <- repos_table$repo_name } @@ -289,16 +302,16 @@ GitHostGitHub <- R6::R6Class("GitHostGitHub", # Prepare files table. prepare_files_table = function(files_response, org, file_path) { if (!is.null(files_response)) { - files_table <- purrr::map(file_path, function(file) { - purrr::imap(files_response[[file]], function(repository, name) { + files_table <- purrr::map(files_response, function(repository) { + purrr::imap(repository, function(file_data, file_name) { data.frame( - "repo_name" = repository$name, - "repo_id" = repository$id, + "repo_name" = file_data$repo_name, + "repo_id" = file_data$repo_id, "organization" = org, - "file_path" = file, - "file_content" = repository$object$text, - "file_size" = repository$object$byteSize, - "repo_url" = repository$url + "file_path" = file_name, + "file_content" = file_data$file$text %||% NA, + "file_size" = file_data$file$byteSize, + "repo_url" = file_data$repo_url ) }) %>% purrr::list_rbind() diff --git a/R/GitHostGitLab.R b/R/GitHostGitLab.R index 273a60a3..5e444564 100644 --- a/R/GitHostGitLab.R +++ b/R/GitHostGitLab.R @@ -18,9 +18,59 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", token = token, host = host, verbose = verbose) - if (private$verbose) { + if (verbose) { cli::cli_alert_success("Set connection to GitLab.") } + }, + + # Retrieve content of given text files from all repositories for a host in + # a table format. + get_files_content = function(file_path, + host_files_structure = NULL, + only_text_files = TRUE, + verbose = TRUE, + progress = TRUE) { + if (!private$scan_all && private$are_non_text_files(file_path, host_files_structure)) { + if (only_text_files) { + files_table <- private$get_files_content_from_orgs( + file_path = file_path, + host_files_structure = host_files_structure, + only_text_files = only_text_files, + verbose = verbose, + progress = progress + ) + } else { + text_files_table <- private$get_files_content_from_orgs( + file_path = file_path, + host_files_structure = host_files_structure, + only_text_files = TRUE, + verbose = verbose, + progress = progress + ) + non_text_files_table <- private$get_files_content_from_orgs_via_rest( + file_path = file_path, + host_files_structure = host_files_structure, + clean_files_content = FALSE, + only_non_text_files = TRUE, + verbose = verbose, + progress = progress + ) + files_table <- purrr::list_rbind( + list( + text_files_table, + non_text_files_table + ) + ) + } + } else { + files_table <- super$get_files_content( + file_path = file_path, + host_files_structure = host_files_structure, + verbose = verbose, + progress = progress + ) + } + return(files_table) } ), private = list( @@ -50,7 +100,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", ) ), - # Set API url + # Set API URL set_api_url = function(host) { if (is.null(host)) { private$api_url <- glue::glue( @@ -61,6 +111,17 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", } }, + # Set web URL + set_web_url = function(host) { + if (is.null(host)) { + private$web_url <- glue::glue( + "https://gitlab.com" + ) + } else { + private$set_custom_web_url(host) + } + }, + # Check whether Git platform is public or internal. check_if_public = function(host) { private$is_public <- is.null(host) || grepl("gitlab.com", host) @@ -68,35 +129,35 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", # Set endpoint for basic checks set_test_endpoint = function() { - private$test_endpoint = glue::glue("{private$api_url}/projects") + private$test_endpoint <- glue::glue("{private$api_url}/projects") }, # Set tokens endpoint set_tokens_endpoint = function() { - private$endpoints$tokens = glue::glue("{private$api_url}/personal_access_tokens") + private$endpoints$tokens <- glue::glue("{private$api_url}/personal_access_tokens") }, # Set groups endpoint set_orgs_endpoint = function() { - private$endpoints$orgs = glue::glue("{private$api_url}/groups") + private$endpoints$orgs <- glue::glue("{private$api_url}/groups") }, # Set projects endpoint set_repositories_endpoint = function() { - private$endpoints$repositories = glue::glue("{private$api_url}/projects") + private$endpoints$repositories <- glue::glue("{private$api_url}/projects") }, # Setup REST and GraphQL engines setup_engines = function() { private$engines$rest <- EngineRestGitLab$new( - rest_api_url = private$api_url, - token = private$token, - scan_all = private$scan_all + rest_api_url = private$api_url, + token = private$token, + scan_all = private$scan_all ) private$engines$graphql <- EngineGraphQLGitLab$new( - gql_api_url = private$graphql_api_url, - token = private$token, - scan_all = private$scan_all + gql_api_url = private$graphql_api_url, + token = private$token, + scan_all = private$scan_all ) }, @@ -161,6 +222,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", repo$last_activity_at <- as.POSIXct(repo$last_activity_at) repo$organization <- repo$group$path repo$group <- NULL + repo$repo_path <- NULL # temporary to close issue 338 data.frame(repo) }) %>% dplyr::relocate( @@ -178,20 +240,20 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", }, # Add `api_url` column to table. - add_repo_api_url = function(repos_table){ + add_repo_api_url = function(repos_table) { if (!is.null(repos_table) && nrow(repos_table) > 0) { repos_table <- dplyr::mutate( - repos_table, - api_url = paste0(private$endpoints$repositories, - "/", - stringr::str_match(repo_id, "[0-9].*")) - ) + repos_table, + api_url = paste0(private$endpoints$repositories, + "/", + stringr::str_match(repo_id, "[0-9].*")) + ) } return(repos_table) }, # Get projects API URL from search response - get_repo_url_from_response = function(search_response, type) { + get_repo_url_from_response = function(search_response, type, progress = TRUE) { purrr::map_vec(search_response, function(response) { api_url <- paste0(private$api_url, "/projects/", response$project_id) if (type == "api") { @@ -204,7 +266,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", web_url <- project_response$web_url return(web_url) } - }, .progress = if (type != "api") { + }, .progress = if (progress && type != "api") { "Mapping api URL to web URL..." } else { FALSE @@ -212,35 +274,38 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", }, # Pull commits from GitHub - get_commits_from_host = function(since, until, settings) { + get_commits_from_orgs = function(since, + until, + verbose = TRUE, + progress = verbose) { rest_engine <- private$engines$rest commits_table <- purrr::map(private$orgs, function(org) { commits_table_org <- NULL - if (!private$scan_all && private$verbose) { + if (!private$scan_all && verbose) { show_message( - host = private$host_name, - engine = "rest", - scope = utils::URLdecode(org), + host = private$host_name, + engine = "rest", + scope = utils::URLdecode(org), information = "Pulling commits" ) } repos_names <- private$set_repositories( - org = org, - settings = settings + org = org ) - commits_table_org <- rest_engine$pull_commits_from_repos( + commits_table_org <- rest_engine$get_commits_from_repos( repos_names = repos_names, - since = since, - until = until, - verbose = private$verbose + since = since, + until = until, + progress = progress ) %>% private$tailor_commits_info(org = org) %>% private$prepare_commits_table() %>% rest_engine$get_commits_authors_handles_and_names( - verbose = private$verbose + verbose = verbose, + progress = progress ) return(commits_table_org) - }, .progress = if (private$scan_all && private$verbose) { + }, .progress = if (private$scan_all && progress) { "[GitHost:GitLab] Pulling commits..." } else { FALSE @@ -256,8 +321,7 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", repos_names <- paste0(org, "%2f", repos) } else { repos_table <- private$get_all_repos( - verbose = FALSE, - settings = settings + verbose = FALSE ) gitlab_web_url <- stringr::str_extract(private$api_url, "^.*?(?=api)") repos <- stringr::str_remove(repos_table$repo_url, gitlab_web_url) @@ -294,7 +358,8 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", commits_dt <- purrr::map(commits_list, function(x) { purrr::map(x, ~ data.frame(.)) %>% purrr::list_rbind() - }) %>% purrr::list_rbind() + }) %>% + purrr::list_rbind() if (length(commits_dt) > 0) { commits_dt <- dplyr::mutate( @@ -305,6 +370,70 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", return(commits_dt) }, + are_non_text_files = function(file_path, host_files_structure) { + if (!is.null(file_path)) { + any(grepl(non_text_files_pattern, file_path)) + } else if (!is.null(host_files_structure)) { + any(grepl(non_text_files_pattern, unlist(host_files_structure, use.names = FALSE))) + } else { + FALSE + } + }, + + # Pull files from orgs via rest + get_files_content_from_orgs_via_rest = function(file_path, + host_files_structure, + only_non_text_files, + clean_files_content, + verbose, + progress) { + rest_engine <- private$engines$rest + if (!is.null(host_files_structure)) { + if (verbose) { + cli::cli_alert_info(cli::col_green("I will make use of files structure stored in GitStats.")) + } + result <- private$get_orgs_and_repos_from_files_structure( + host_files_structure = host_files_structure + ) + orgs <- result$orgs + repos <- result$repos + } else { + orgs <- private$orgs + repos <- private$repos + } + if (verbose) { + user_msg <- if (!is.null(host_files_structure)) { + "Pulling files from files structure" + } else { + glue::glue("Pulling files content: [{paste0(file_path, collapse = ', ')}]") + } + show_message( + host = private$host_name, + engine = "rest", + information = user_msg + ) + } + files_table <- purrr::map(orgs, function(org) { + if (!is.null(host_files_structure)) { + file_path <- host_files_structure[[org]] %>% unlist(use.names = FALSE) %>% unique() + } + if (only_non_text_files) { + file_path <- file_path[grepl(non_text_files_pattern, file_path)] + } + files_table <- rest_engine$get_files( + file_paths = file_path, + clean_files_content = clean_files_content, + org = org, + verbose = FALSE, + progress = progress + ) %>% + private$prepare_files_table_from_rest() + }, .progress = progress) %>% + purrr::list_rbind() %>% + private$add_repo_api_url() + return(files_table) + }, + # Prepare user table. prepare_user_table = function(user_response) { if (!is.null(user_response$data$user)) { @@ -330,27 +459,49 @@ GitHostGitLab <- R6::R6Class("GitHostGitLab", # Prepare files table. prepare_files_table = function(files_response, org, file_path) { if (!is.null(files_response)) { - files_table <- purrr::map(files_response, function(project) { - purrr::map(project$repository$blobs$nodes, function(file) { - data.frame( - "repo_name" = project$name, - "repo_id" = project$id, - "organization" = org, - "file_path" = file$name, - "file_content" = file$rawBlob, - "file_size" = as.integer(file$size), - "repo_url" = project$webUrl - ) + if (private$response_prepared_by_iteration(files_response)) { + files_table <- purrr::map(files_response, function(response_data) { + purrr::map(response_data$data$project$repository$blobs$nodes, function(file) { + data.frame( + "repo_name" = response_data$data$project$name, + "repo_id" = response_data$data$project$id, + "organization" = org, + "file_path" = file$name, + "file_content" = file$rawBlob, + "file_size" = as.integer(file$size), + "repo_url" = response_data$data$project$webUrl + ) + }) %>% + purrr::list_rbind() }) %>% purrr::list_rbind() - }) %>% - purrr::list_rbind() + } else { + files_table <- purrr::map(files_response, function(project) { + purrr::map(project$repository$blobs$nodes, function(file) { + data.frame( + "repo_name" = project$name, + "repo_id" = project$id, + "organization" = org, + "file_path" = file$name, + "file_content" = file$rawBlob, + "file_size" = as.integer(file$size), + "repo_url" = project$webUrl + ) + }) %>% + purrr::list_rbind() + }) %>% + purrr::list_rbind() + } } else { files_table <- NULL } return(files_table) }, + response_prepared_by_iteration = function(files_response) { + !all(purrr::map_lgl(files_response, ~ all(c("name", "id", "webUrl", "repository") %in% names(.)))) + }, + # Prepare files table from REST API. prepare_files_table_from_rest = function(files_list) { files_table <- NULL diff --git a/R/GitStats.R b/R/GitStats.R index 70d886d4..39058e4a 100644 --- a/R/GitStats.R +++ b/R/GitStats.R @@ -1,6 +1,7 @@ #' @noRd #' @description A highest-level class to manage data pulled from different hosts. -GitStats <- R6::R6Class("GitStats", +GitStats <- R6::R6Class( + classname = "GitStats", public = list( #' @description Method to set connections to Git platforms. @@ -76,12 +77,15 @@ GitStats <- R6::R6Class("GitStats", #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and #' printing output is switched off. + #' @param progress A logical, by default set to `verbose` value. If `FALSE` no + #' `cli` progress bar will be displayed. get_repos = function(add_contributors = FALSE, - with_code = NULL, - in_files = NULL, - with_files = NULL, - cache = TRUE, - verbose = TRUE) { + with_code = NULL, + in_files = NULL, + with_files = NULL, + cache = TRUE, + verbose = TRUE, + progress = TRUE) { private$check_for_host() private$check_params_conflict( with_code = with_code, @@ -92,22 +96,22 @@ GitStats <- R6::R6Class("GitStats", "in_files" = in_files, "with_files" = with_files) trigger <- private$trigger_pulling( - cache = cache, - storage = "repositories", + cache = cache, + storage = "repositories", args_list = args_list, - verbose = verbose + verbose = verbose ) if (trigger) { - repositories <- private$get_repos_table( + repositories <- private$get_repos_from_hosts( add_contributors = add_contributors, - with_code = with_code, - in_files = in_files, - with_files = with_files, - verbose = verbose, - settings = private$settings + with_code = with_code, + in_files = in_files, + with_files = with_files, + verbose = verbose, + progress = progress ) %>% private$set_object_class( - class = "repos_table", + class = "repos_table", attr_list = args_list ) private$save_to_storage( @@ -115,7 +119,7 @@ GitStats <- R6::R6Class("GitStats", ) } else { repositories <- private$get_from_storage( - table = "repositories", + table = "repositories", verbose = verbose ) } @@ -128,8 +132,8 @@ GitStats <- R6::R6Class("GitStats", #' returned. #' @param with_code A character vector, if defined, GitStats will pull #' repositories with specified code phrases in code blobs. - #' @param in_files A character vector of file names. Works when `with_code` is - #' set - then it searches code blobs only in files passed to `in_files` + #' @param in_files A character vector of file names. Works when `with_code` + #' is set - then it searches code blobs only in files passed to `in_files` #' parameter. #' @param with_files A character vector, if defined, GitStats will pull #' repositories with specified files. @@ -137,36 +141,40 @@ GitStats <- R6::R6Class("GitStats", #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and #' printing output is switched off. + #' @param progress A logical, by default set to `verbose` value. If `FALSE` + #' no `cli` progress bar will be displayed. #' @return A character vector. - get_repos_urls = function(type = "web", - with_code = NULL, - in_files = NULL, + get_repos_urls = function(type = "web", + with_code = NULL, + in_files = NULL, with_files = NULL, - cache = TRUE, - verbose = TRUE) { + cache = TRUE, + verbose = TRUE, + progress = TRUE) { private$check_for_host() private$check_params_conflict( with_code = with_code, in_files = in_files, with_files = with_files ) - args_list <- list("type" = type, - "with_code" = with_code, - "in_files" = in_files, + args_list <- list("type" = type, + "with_code" = with_code, + "in_files" = in_files, "with_files" = with_files) trigger <- private$trigger_pulling( - cache = cache, - storage = "repos_urls", + cache = cache, + storage = "repos_urls", args_list = args_list, - verbose = verbose + verbose = verbose ) if (trigger) { repos_urls <- private$get_repos_urls_from_hosts( - type = type, - with_code = with_code, - in_files = in_files, + type = type, + with_code = with_code, + in_files = in_files, with_files = with_files, - verbose = verbose + verbose = verbose, + progress = progress ) %>% private$set_object_class( class = "repos_urls", @@ -191,27 +199,31 @@ GitStats <- R6::R6Class("GitStats", #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and #' printing output is switched off. + #' @param progress A logical, by default set to `verbose` value. If `FALSE` + #' no `cli` progress bar will be displayed. get_commits = function(since, until, - cache = TRUE, - verbose = TRUE) { + cache = TRUE, + verbose = TRUE, + progress = TRUE) { private$check_for_host() args_list <- list("since" = since, "until" = until) trigger <- private$trigger_pulling( - cache = cache, - storage = "commits", + cache = cache, + storage = "commits", args_list = args_list, - verbose = verbose + verbose = verbose ) if (trigger) { - commits <- private$get_commits_table( - since = since, - until = until, - verbose = verbose + commits <- private$get_commits_from_hosts( + since = since, + until = until, + verbose = verbose, + progress = progress ) %>% private$set_object_class( - class = "commits_data", + class = "commits_data", attr_list = args_list ) private$save_to_storage(commits) @@ -230,7 +242,7 @@ GitStats <- R6::R6Class("GitStats", #' @param time_interval A character, specifying time interval to show #' statistics. #' @return A table of `commits_stats` class. - get_commits_stats = function(time_interval = c("month", "day", "week")){ + get_commits_stats = function(time_interval = c("month", "day", "week")) { commits <- private$storage[["commits"]] if (is.null(commits)) { cli::cli_abort(c( @@ -264,7 +276,7 @@ GitStats <- R6::R6Class("GitStats", verbose = verbose ) if (trigger) { - users <- private$get_users_table(logins) %>% + users <- private$get_users_from_hosts(logins) %>% private$set_object_class( class = "users_data", attr_list = args_list @@ -280,14 +292,35 @@ GitStats <- R6::R6Class("GitStats", }, #' @description Pull text content of a file from all repositories. - #' @param file_path A file path, may be a character vector. + #' @param file_path Optional. A standardized path to file(s) in + #' repositories. May be a character vector if multiple files are to be + #' pulled. If set to `NULL` and `use_files_structure` parameter is set to + #' `TRUE`, `GitStats` will try to pull data from `files_structure` (see + #' below). + #' @param use_files_structure Logical. If `TRUE` and `file_path` is set to + #' `NULL`, will iterate over `files_structure` pulled by + #' `get_files_structure()` function and kept in storage. If there is no + #' `files_structure` in storage, an error will be returned. If `file_path` + #' is defined, it will override `use_files_structure` parameter. + #' @param only_text_files A logical, `TRUE` by default. If set to `FALSE`, + #' apart from files with text content shows in table output also non-text + #' files with `NA` value for text content. #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and #' printing output is switched off. - get_files = function(file_path, cache = TRUE, verbose = TRUE) { + #' @param progress A logical, by default set to `verbose` value. If `FALSE` + #' no `cli` progress bar will be displayed. + get_files_content = function(file_path = NULL, + use_files_structure = TRUE, + only_text_files = TRUE, + cache = TRUE, + verbose = TRUE, + progress = verbose) { private$check_for_host() - args_list <- list("file_path" = file_path) + args_list <- list("file_path" = file_path, + "use_files_structure" = use_files_structure, + "only_text_files" = only_text_files) trigger <- private$trigger_pulling( cache = cache, storage = "files", @@ -295,9 +328,12 @@ GitStats <- R6::R6Class("GitStats", verbose = verbose ) if (trigger) { - files <- private$get_files_table( + files <- private$get_files_content_from_hosts( file_path = file_path, - verbose = verbose + use_files_structure = use_files_structure, + only_text_files = only_text_files, + verbose = verbose, + progress = progress ) %>% private$set_object_class( class = "files_data", @@ -313,6 +349,59 @@ GitStats <- R6::R6Class("GitStats", return(files) }, + #' @name get_files_structure + #' @description Pulls file structure for a given repository. + #' @param gitstats_object A GitStats object. + #' @param pattern An optional regular expression. If defined, it pulls file + #' structure for a repository matching this pattern. + #' @param depth An optional integer. Defines level of directories to retrieve + #' files from. E.g. if set to `0`, it will pull files only from root, if `1`, + #' will take data from `root` directory and directories visible in `root` + #' directory. If left with no argument, will pull files from all directories. + #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last + #' result from its storage. + #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing + #' output is switched off. + #' @param progress A logical, by default set to `verbose` value. If `FALSE` + #' no `cli` progress bar will be displayed. + get_files_structure = function(pattern, + depth, + cache = TRUE, + verbose = TRUE, + progress = verbose) { + private$check_for_host() + args_list <- list("pattern" = pattern, + "depth" = depth) + trigger <- private$trigger_pulling( + cache = cache, + storage = "files_structure", + args_list = args_list, + verbose = verbose + ) + if (trigger) { + files_structure <- private$get_files_structure_from_hosts( + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress + ) + if (!is.null(files_structure)) { + files_structure <- private$set_object_class( + object = files_structure, + class = "files_structure", + attr_list = args_list + ) + private$save_to_storage(files_structure) + } + } else { + files_structure <- private$get_from_storage( + table = "files_structure", + verbose = verbose + ) + } + return(files_structure) + }, + #' @description Get release logs of repositories. #' @param since A starting date for release logs. #' @param until An end date for release logs. @@ -320,29 +409,33 @@ GitStats <- R6::R6Class("GitStats", #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and #' printing output is switched off. + #' @param progress A logical, by default set to `verbose` value. If `FALSE` + #' no `cli` progress bar will be displayed. get_release_logs = function(since, until, - cache = TRUE, - verbose = TRUE) { + cache = TRUE, + verbose = TRUE, + progress = TRUE) { private$check_for_host() args_list <- list( "since" = since, "until" = until ) trigger <- private$trigger_pulling( - storage = "release_logs", - cache = cache, + storage = "release_logs", + cache = cache, args_list = args_list, - verbose = verbose + verbose = verbose ) if (trigger) { - release_logs <- private$get_release_logs_table( - since = since, - until = until, - verbose = verbose + release_logs <- private$get_release_logs_from_hosts( + since = since, + until = until, + verbose = verbose, + progress = progress ) %>% private$set_object_class( - class = "release_logs", + class = "release_logs", attr_list = args_list ) private$save_to_storage(release_logs) @@ -365,8 +458,8 @@ GitStats <- R6::R6Class("GitStats", #' printing output is switched off. get_R_package_usage = function(package_name, only_loading = FALSE, - cache = TRUE, - verbose = TRUE) { + cache = TRUE, + verbose = TRUE) { private$check_for_host() if (is.null(package_name)) { cli::cli_abort("You need to define `package_name`.", call = NULL) @@ -376,25 +469,25 @@ GitStats <- R6::R6Class("GitStats", "only_loading" = only_loading ) trigger <- private$trigger_pulling( - storage = "R_package_usage", - cache = cache, + storage = "R_package_usage", + cache = cache, args_list = args_list, - verbose = verbose + verbose = verbose ) if (trigger) { - R_package_usage <- private$get_R_package_usage_table( + R_package_usage <- private$get_R_package_usage_from_hosts( package_name = package_name, only_loading = only_loading, - verbose = verbose + verbose = verbose ) %>% private$set_object_class( - class = "R_package_usage", + class = "R_package_usage", attr_list = args_list ) private$save_to_storage(R_package_usage) } else { R_package_usage <- private$get_from_storage( - table = "R_package_usage", + table = "R_package_usage", verbose = verbose ) } @@ -406,7 +499,8 @@ GitStats <- R6::R6Class("GitStats", purrr::map(private$hosts, function(host) { orgs <- host$.__enclos_env__$private$orgs purrr::map_vec(orgs, ~ URLdecode(.)) - }) %>% unlist() + }) %>% + unlist() }, #' @description switch on verbose mode @@ -425,7 +519,7 @@ GitStats <- R6::R6Class("GitStats", #' @description A print method for a GitStats object. print = function() { - cat(paste0("A ", cli::col_blue('GitStats'), " object for ", length(private$hosts)," hosts: \n")) + cat(paste0("A ", cli::col_blue('GitStats'), " object for ", length(private$hosts), " hosts: \n")) private$print_hosts() cat(cli::col_blue("Scanning scope: \n")) private$print_orgs_and_repos() @@ -440,17 +534,18 @@ GitStats <- R6::R6Class("GitStats", # @field settings List of search preferences. settings = list( verbose = TRUE, - cache = TRUE + cache = TRUE ), # @field storage for results storage = list( - repositories = NULL, - commits = NULL, - users = NULL, - files = NULL, + repositories = NULL, + commits = NULL, + users = NULL, + files = NULL, + files_structure = NULL, R_package_usage = NULL, - release_logs = NULL + release_logs = NULL ), # Add new host @@ -475,14 +570,17 @@ GitStats <- R6::R6Class("GitStats", cli::cli_abort(c( "x" = "Both `with_code` and `with_files` parameters are defined.", "!" = "Use either `with_code` of `with_files` parameter.", - "i" = "If you want to search for [{with_code}] code in given files - use `in_files` parameter together with `with_code` instead." - ), call = NULL) + "i" = "If you want to search for [{with_code}] code in given files + - use `in_files` parameter together with `with_code` instead." + ), call = NULL) } if (!is.null(in_files) && is.null(with_code)) { cli::cli_abort(c( - "!" = "Passing files to `in_files` parameter works only when you search code with `with_code` parameter.", - "i" = "If you want to search for repositories with [{in_files}] files you should instead use `with_files` parameter." - ), call = NULL) + "!" = "Passing files to `in_files` parameter works only when you + search code with `with_code` parameter.", + "i" = "If you want to search for repositories with [{in_files}] files + you should instead use `with_files` parameter." + ), call = NULL) } }, @@ -557,44 +655,45 @@ GitStats <- R6::R6Class("GitStats", !all(purrr::map2_lgl(new_params, stored_params, ~ identical(.x, .y))) }, - # Save dates parameters as attributes of the object + # Set object class with attributes set_object_class = function(object, class, attr_list) { class(object) <- append(class, class(object)) purrr::iwalk(attr_list, function(attrib, attrib_name) { attr(object, attrib_name) <<- attrib }) - # standardize_dates() return(object) }, # Pull repositories tables from hosts and bind them into one - get_repos_table = function(add_contributors = FALSE, - with_code, - in_files = NULL, - with_files, - verbose, - settings) { + get_repos_from_hosts = function(add_contributors = FALSE, + with_code, + in_files = NULL, + with_files, + verbose = TRUE, + progress = TRUE) { repos_table <- purrr::map(private$hosts, function(host) { if (!is.null(with_code)) { private$get_repos_from_host_with_code( - host = host, + host = host, add_contributors = add_contributors, - with_code = with_code, - in_files = in_files, - verbose = verbose + with_code = with_code, + in_files = in_files, + verbose = verbose, + progress = progress ) } else if (!is.null(with_files)) { privater$get_repos_from_host_with_files( - host = host, + host = host, add_contributors = add_contributors, - with_files = with_files, - verbose = verbose + with_files = with_files, + verbose = verbose, + progress = progress ) } else { host$get_repos( add_contributors = add_contributors, - verbose = verbose, - settings = settings + verbose = verbose, + progress = progress ) } }) %>% @@ -609,14 +708,14 @@ GitStats <- R6::R6Class("GitStats", with_code, in_files, verbose, - settings) { + progress) { purrr::map(with_code, function(with_code) { host$get_repos( add_contributors = add_contributors, - with_code = with_code, - in_files = in_files, - verbose = verbose, - settings = settings + with_code = with_code, + in_files = in_files, + verbose = verbose, + progress = progress ) }) %>% purrr::list_rbind() @@ -627,40 +726,48 @@ GitStats <- R6::R6Class("GitStats", add_contributors, with_files, verbose, - settings) { + progress) { purrr::map(with_files, function(with_file) { host$get_repos( add_contributors = add_contributors, - with_file = with_file, - verbose = verbose, - settings = settings + with_file = with_file, + verbose = verbose, + progress = progress ) }) %>% purrr::list_rbind() }, # Get repositories character vectors from hosts and bind them into one - get_repos_urls_from_hosts = function(type, with_code, in_files, with_files, verbose) { + get_repos_urls_from_hosts = function(type, + with_code, + in_files, + with_files, + verbose, + progress) { purrr::map(private$hosts, function(host) { if (!is.null(with_code)) { private$get_repos_urls_from_host_with_code( - host = host, - type = type, + host = host, + type = type, with_code = with_code, - in_files = in_files, - verbose = verbose + in_files = in_files, + verbose = verbose, + progress = progress ) } else if (!is.null(with_files)) { private$get_repos_urls_from_host_with_files( - host = host, - type = type, + host = host, + type = type, with_files = with_files, - verbose = verbose + verbose = verbose, + progress = progress ) } else { host$get_repos_urls( - type = type, - verbose = verbose + type = type, + verbose = verbose, + progress = progress ) } }) %>% @@ -673,13 +780,15 @@ GitStats <- R6::R6Class("GitStats", type, with_code, in_files, - verbose) { + verbose, + progress) { purrr::map(with_code, function(code) { host$get_repos_urls( - type = type, + type = type, with_code = code, - in_files = in_files, - verbose = verbose + in_files = in_files, + verbose = verbose, + progress = progress ) }) %>% unlist() @@ -689,25 +798,27 @@ GitStats <- R6::R6Class("GitStats", get_repos_urls_from_host_with_files = function(host, type, with_files, - verbose) { + verbose, + progress) { purrr::map(with_files, function(file) { host$get_repos_urls( - type = type, + type = type, with_file = file, - verbose = verbose + verbose = verbose, + progress = progress ) }) %>% unlist() }, # Get commits tables from hosts and bind them into one - get_commits_table = function(since, until, verbose) { + get_commits_from_hosts = function(since, until, verbose, progress) { commits_table <- purrr::map(private$hosts, function(host) { host$get_commits( - since = since, - until = until, - verbose = verbose, - settings = private$settings + since = since, + until = until, + verbose = verbose, + progress = progress ) }) %>% purrr::list_rbind() @@ -715,7 +826,7 @@ GitStats <- R6::R6Class("GitStats", }, # Pull information on unique users in a table form - get_users_table = function(logins) { + get_users_from_hosts = function(logins) { purrr::map(private$hosts, function(host) { host$get_users(logins) }) %>% @@ -724,42 +835,120 @@ GitStats <- R6::R6Class("GitStats", }, # Pull content of a text file in a table form - get_files_table = function(file_path, verbose) { + get_files_content_from_hosts = function(file_path, + use_files_structure, + only_text_files, + verbose, + progress) { purrr::map(private$hosts, function(host) { - host$get_files( - file_path = file_path, - verbose = verbose - ) + if (is.null(file_path) && use_files_structure) { + host_files_structure <- private$get_host_files_structure( + host = host, + verbose = FALSE + ) + } else { + host_files_structure <- NULL + } + if (is.null(file_path) && is.null(host_files_structure)) { + host_name <- host$.__enclos_env__$private$host_name + if (verbose) { + cli::cli_alert_warning( + cli::col_yellow("I will skip pulling data for {host_name}: files structure is empty.") + ) + } + NULL + } else { + host$get_files_content( + file_path = file_path, + host_files_structure = host_files_structure, + only_text_files = only_text_files, + verbose = verbose, + progress = progress + ) + } }) %>% purrr::list_rbind() }, + get_host_files_structure = function(host, verbose) { + files_structure <- private$get_from_storage( + table = "files_structure", + verbose = verbose + ) + if (is.null(files_structure)) { + cli::cli_abort(c( + "x" = "No files_structure object found in GitStats.", + "i" = "Run `get_files_structure()` function first, then `get_files_content()`." + ), + call = NULL + ) + } + host_name <- host$.__enclos_env__$private$web_url + return(files_structure[[gsub("https://", "", host_name)]]) + }, + + get_files_structure_from_hosts = function(pattern, depth, verbose, progress) { + files_structure_from_hosts <- purrr::map(private$hosts, function(host) { + host$get_files_structure( + pattern = pattern, + depth = depth, + verbose = verbose, + progress = progress + ) + }) + names(files_structure_from_hosts) <- private$get_host_urls() + files_structure_from_hosts <- files_structure_from_hosts %>% + purrr::discard(~ length(.) == 0) + if (length(files_structure_from_hosts) == 0) { + files_structure_from_hosts <- NULL + if (verbose) { + cli::cli_alert_warning( + cli::col_yellow( + "No files structure found for matching pattern {pattern} in {depth} level of dirs." + ) + ) + cli::cli_alert_warning( + cli::col_yellow( + "Files structure will not be saved in GitStats." + ) + ) + } + } + return(files_structure_from_hosts) + }, + + get_host_urls = function() { + purrr::map_vec(private$hosts, ~ gsub("https://", "", .$.__enclos_env__$private$web_url)) + }, + # Pull release logs tables from hosts and bind them into one - get_release_logs_table = function(since, until, verbose) { + get_release_logs_from_hosts = function(since, until, verbose, progress) { purrr::map(private$hosts, function(host) { host$get_release_logs( - since = since, - until = until, - verbose = verbose, - settings = private$settings + since = since, + until = until, + verbose = verbose, + progress = progress ) }) %>% purrr::list_rbind() }, # Pull information on package usage in a table form - get_R_package_usage_table = function(package_name, only_loading, verbose) { + get_R_package_usage_from_hosts = function(package_name, + only_loading, + verbose) { if (!only_loading) { repos_with_package_as_dependency <- private$get_R_package_as_dependency( package_name = package_name, - verbose = verbose + verbose = verbose ) } else { repos_with_package_as_dependency <- NULL } repos_using_package <- private$get_R_package_loading( package_name = package_name, - verbose = verbose + verbose = verbose ) package_usage_table <- purrr::list_rbind( list( @@ -768,12 +957,12 @@ GitStats <- R6::R6Class("GitStats", ) ) duplicated_repos <- package_usage_table$api_url[duplicated(package_usage_table$api_url)] - package_usage_table <- package_usage_table[!duplicated(package_usage_table$api_url),] + package_usage_table <- package_usage_table[!duplicated(package_usage_table$api_url), ] package_usage_table <- package_usage_table %>% dplyr::mutate( package_usage = ifelse(api_url %in% duplicated_repos, "import, library", package_usage) ) - rownames(package_usage_table) <- c(1:nrow(package_usage_table)) + rownames(package_usage_table) <- seq_len(nrow(package_usage_table)) return(package_usage_table) }, @@ -787,9 +976,10 @@ GitStats <- R6::R6Class("GitStats", paste0("require(", package_name, ")") ) repos_using_package <- purrr::map(package_usage_phrases, ~ { - repos_using_package <- private$get_repos_table( + repos_using_package <- private$get_repos_from_hosts( with_code = ., - verbose = FALSE + verbose = FALSE, + progress = FALSE ) if (!is.null(repos_using_package)) { repos_using_package$package_usage <- "library" @@ -809,14 +999,14 @@ GitStats <- R6::R6Class("GitStats", if (verbose) { cli::cli_alert_info("Checking where [{package_name}] is used as a dependency...") } - repos_with_package <- private$get_repos_table( + repos_with_package <- private$get_repos_from_hosts( with_code = package_name, in_files = c("DESCRIPTION", "NAMESPACE"), verbose = FALSE, - settings = private$settings + progress = FALSE ) if (nrow(repos_with_package) > 0) { - repos_with_package <- repos_with_package[!duplicated(repos_with_package$api_url),] + repos_with_package <- repos_with_package[!duplicated(repos_with_package$api_url), ] repos_with_package$package_usage <- "import" } repos_with_package <- repos_with_package %>% @@ -965,10 +1155,15 @@ GitStats <- R6::R6Class("GitStats", # print storage print_storage = function() { - gitstats_storage <- purrr::imap(private$storage, function(storage_table, storage_name) { - if (!is.null(storage_table)) { + gitstats_storage <- purrr::imap(private$storage, function(storage_object, storage_name) { + if (!is.null(storage_object)) { + storage_size <- if (inherits(storage_object, "data.frame")) { + nrow(storage_object) + } else { + length(storage_object) + } glue::glue( - "{stringr::str_to_title(storage_name)}: {nrow(storage_table)} {private$print_storage_attribute(storage_table, storage_name)}" + "{stringr::str_to_title(storage_name)}: {storage_size} {private$print_storage_attribute(storage_object, storage_name)}" ) } }) %>% @@ -976,7 +1171,7 @@ GitStats <- R6::R6Class("GitStats", if (length(gitstats_storage) == 0) { private$print_item( "Storage", - cli::col_grey("") + cli::col_grey("") ) } else { cat(paste0( @@ -988,9 +1183,10 @@ GitStats <- R6::R6Class("GitStats", # print storage attribute print_storage_attribute = function(storage_table, storage_name) { - if (storage_name != "repositories") { + if (storage_name != "repositories") { storage_attr <- switch(storage_name, "files" = "file_path", + "files_structure" = "pattern", "commits" = "dates_range", "release_logs" = "dates_range", "users" = "logins", @@ -998,6 +1194,7 @@ GitStats <- R6::R6Class("GitStats", attr_data <- attr(storage_table, storage_attr) attr_name <- switch(storage_attr, "file_path" = "files", + "pattern" = "files matching pattern", "dates_range" = "date range", "package_name" = "package", "logins" = "logins") diff --git a/R/gitstats_functions.R b/R/gitstats_functions.R index f3f2efed..d98afac4 100644 --- a/R/gitstats_functions.R +++ b/R/gitstats_functions.R @@ -114,6 +114,8 @@ set_gitlab_host <- function(gitstats_object, #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing #' output is switched off. +#' @param progress A logical, by default set to `verbose` value. If `FALSE` no +#' `cli` progress bar will be displayed. #' @return A data.frame. #' @examples #' \dontrun{ @@ -134,18 +136,20 @@ set_gitlab_host <- function(gitstats_object, #' @export get_repos <- function(gitstats_object, add_contributors = TRUE, - with_code = NULL, - in_files = NULL, - with_files = NULL, - cache = TRUE, - verbose = is_verbose(gitstats_object)) { + with_code = NULL, + in_files = NULL, + with_files = NULL, + cache = TRUE, + verbose = is_verbose(gitstats_object), + progress = verbose) { gitstats_object$get_repos( add_contributors = add_contributors, - with_code = with_code, - in_files = in_files, - with_files = with_files, - cache = cache, - verbose = verbose + with_code = with_code, + in_files = in_files, + with_files = with_files, + cache = cache, + verbose = verbose, + progress = progress ) } @@ -168,6 +172,8 @@ get_repos <- function(gitstats_object, #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing #' output is switched off. +#' @param progress A logical, by default set to `verbose` value. If `FALSE` no +#' `cli` progress bar will be displayed. #' @return A character vector. #' @examples #' \dontrun{ @@ -185,19 +191,21 @@ get_repos <- function(gitstats_object, #' } #' @export get_repos_urls <- function(gitstats_object, - type = "web", - with_code = NULL, - in_files = NULL, + type = "web", + with_code = NULL, + in_files = NULL, with_files = NULL, - cache = TRUE, - verbose = TRUE) { + cache = TRUE, + verbose = is_verbose(gitstats_object), + progress = verbose) { gitstats_object$get_repos_urls( - type = type, - with_code = with_code, - in_files = in_files, + type = type, + with_code = with_code, + in_files = in_files, with_files = with_files, - cache = cache, - verbose = verbose + cache = cache, + verbose = verbose, + progress = progress ) } @@ -210,8 +218,10 @@ get_repos_urls <- function(gitstats_object, #' @param until An end date. #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last #' result from its storage. -#' @param verbose A logical, `TRUE` by default. If `FALSE` messages and -#' printing output is switched off. +#' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing +#' output is switched off. +#' @param progress A logical, by default set to `verbose` value. If `FALSE` no +#' `cli` progress bar will be displayed. #' @return A data.frame. #' @examples #' \dontrun{ @@ -228,18 +238,20 @@ get_repos_urls <- function(gitstats_object, #' } #' @export get_commits <- function(gitstats_object, - since = NULL, - until = NULL, - cache = TRUE, - verbose = is_verbose(gitstats_object)) { + since = NULL, + until = NULL, + cache = TRUE, + verbose = is_verbose(gitstats_object), + progress = verbose) { if (is.null(since)) { cli::cli_abort(cli::col_red("You need to pass date to `since` parameter."), call = NULL) } gitstats_object$get_commits( - since = since, - until = until, - cache = cache, - verbose = verbose + since = since, + until = until, + cache = cache, + verbose = verbose, + progress = progress ) } @@ -263,8 +275,8 @@ get_commits <- function(gitstats_object, #' get_commits_stats(my_gitstats, time_interval = "week") #' } #' @export -get_commits_stats = function(gitstats_object, - time_interval = c("month", "day", "week")) { +get_commits_stats <- function(gitstats_object, + time_interval = c("month", "day", "week")) { gitstats_object$get_commits_stats( time_interval = time_interval ) @@ -295,26 +307,38 @@ get_commits_stats = function(gitstats_object, #' @export get_users <- function(gitstats_object, logins, - cache = TRUE, - verbose = is_verbose(gitstats_object)){ + cache = TRUE, + verbose = is_verbose(gitstats_object)) { gitstats_object$get_users( - logins = logins, - cache = cache, + logins = logins, + cache = cache, verbose = verbose ) } -#' @title Get data on files -#' @name get_files +#' @title Get content of files +#' @name get_files_content #' @description Pull text files content for a given scope (orgs, repos or whole #' git hosts). #' @param gitstats_object A GitStats object. -#' @param file_path A standardized path to file(s) in repositories. May be a -#' character vector if multiple files are to be pulled. +#' @param file_path Optional. A standardized path to file(s) in repositories. +#' May be a character vector if multiple files are to be pulled. If set to +#' `NULL` and `use_files_structure` parameter is set to `TRUE`, `GitStats` +#' will try to pull data from `files_structure` (see below). +#' @param use_files_structure Logical. If `TRUE` and `file_path` is set to +#' `NULL`, will iterate over `files_structure` pulled by +#' `get_files_structure()` function and kept in storage. If there is no +#' `files_structure` in storage, an error will be returned. If `file_path` is +#' defined, it will override `use_files_structure` parameter. +#' @param only_text_files A logical, `TRUE` by default. If set to `FALSE`, apart +#' from files with text content shows in table output also non-text files with +#' `NA` value for text content. #' @param cache A logical, if set to `TRUE` GitStats will retrieve the last #' result from its storage. #' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing #' output is switched off. +#' @param progress A logical, by default set to `verbose` value. If `FALSE` no +#' `cli` progress bar will be displayed. #' @examples #' \dontrun{ #' my_gitstats <- create_gitstats() %>% @@ -326,18 +350,87 @@ get_users <- function(gitstats_object, #' token = Sys.getenv("GITLAB_PAT_PUBLIC"), #' orgs = "mbtests" #' ) -#' get_files(my_gitstats, c("LICENSE", "DESCRIPTION")) +#' get_files_content( +#' gitstats_obj = my_gitstats, +#' file_path = c("LICENSE", "DESCRIPTION") +#' ) +#' +#' # example with files structure +#' files_structure <- get_files_structure( +#' gitstats_obj = my_gitstats, +#' pattern = "\\.Rmd", +#' depth = 2L +#' ) +#' # get_files_content() will make use of pulled earlier files structure +#' files_content <- get_files_content( +#' gitstats_obj = my_gitstats +#' ) #' } #' @return A data.frame. #' @export -get_files <- function(gitstats_object, - file_path, - cache = TRUE, - verbose = is_verbose(gitstats_object)){ - gitstats_object$get_files( - file_path = file_path, - cache = cache, - verbose = verbose +get_files_content <- function(gitstats_object, + file_path = NULL, + use_files_structure = TRUE, + only_text_files = TRUE, + cache = TRUE, + verbose = is_verbose(gitstats_object), + progress = verbose) { + gitstats_object$get_files_content( + file_path = file_path, + use_files_structure = use_files_structure, + only_text_files = only_text_files, + cache = cache, + verbose = verbose, + progress = progress + ) +} + +#' @title Get structure of files +#' @name get_files_structure +#' @description Pulls file structure for a given repository. +#' @param gitstats_object A GitStats object. +#' @param pattern An optional regular expression. If defined, it pulls file +#' structure for a repository matching this pattern. +#' @param depth An optional integer. Defines level of directories to retrieve +#' files from. E.g. if set to `0`, it will pull files only from root, if `1`, +#' will take data from `root` directory and directories visible in `root` +#' directory. If left with no argument, will pull files from all directories. +#' @param cache A logical, if set to `TRUE` GitStats will retrieve the last +#' result from its storage. +#' @param verbose A logical, `TRUE` by default. If `FALSE` messages and printing +#' output is switched off. +#' @param progress A logical, by default set to `verbose` value. If `FALSE` no +#' `cli` progress bar will be displayed. +#' @examples +#' \dontrun{ +#' my_gitstats <- create_gitstats() %>% +#' set_github_host( +#' token = Sys.getenv("GITHUB_PAT"), +#' orgs = c("r-world-devs") +#' ) %>% +#' set_gitlab_host( +#' token = Sys.getenv("GITLAB_PAT_PUBLIC"), +#' orgs = "mbtests" +#' ) +#' get_files_structure( +#' gitstats_obj = my_gitstats, +#' pattern = "\\.md" +#' ) +#' } +#' @return A list of vectors. +#' @export +get_files_structure <- function(gitstats_object, + pattern = NULL, + depth = Inf, + cache = TRUE, + verbose = is_verbose(gitstats_object), + progress = verbose) { + gitstats_object$get_files_structure( + pattern = pattern, + depth = depth, + cache = cache, + verbose = verbose, + progress = progress ) } @@ -366,18 +459,16 @@ get_files <- function(gitstats_object, #' get_R_package_usage(my_gitstats, "Shiny") #' } #' @export -get_R_package_usage <- function( - gitstats_object, - package_name, - only_loading = FALSE, - cache = TRUE, - verbose = is_verbose(gitstats_object) - ) { +get_R_package_usage <- function(gitstats_object, + package_name, + only_loading = FALSE, + cache = TRUE, + verbose = is_verbose(gitstats_object)) { gitstats_object$get_R_package_usage( package_name = package_name, only_loading = only_loading, - cache = cache, - verbose = verbose + cache = cache, + verbose = verbose ) } @@ -396,21 +487,21 @@ get_R_package_usage <- function( #' get_release_logs(my_gistats, since = "2024-01-01") #' } #' @export -get_release_logs <- function( - gitstats_object, - since = NULL, - until = NULL, - cache = TRUE, - verbose = is_verbose(gitstats_object) -) { +get_release_logs <- function(gitstats_object, + since = NULL, + until = NULL, + cache = TRUE, + verbose = is_verbose(gitstats_object), + progress = verbose) { if (is.null(since)) { cli::cli_abort(cli::col_red("You need to pass date to `since` parameter."), call = NULL) } gitstats_object$get_release_logs( - since = since, - until = until, - cache = cache, - verbose = verbose + since = since, + until = until, + cache = cache, + verbose = verbose, + progress = progress ) } @@ -422,7 +513,7 @@ get_release_logs <- function( #' @param gitstats_object A GitStats object. #' @return A vector of organizations. #' @export -show_orgs <- function(gitstats_object){ +show_orgs <- function(gitstats_object) { gitstats_object$show_orgs() } diff --git a/R/global.R b/R/global.R index 6e7e9df7..23d1fe37 100644 --- a/R/global.R +++ b/R/global.R @@ -6,3 +6,5 @@ globalVariables(c( "repository", "stars", "forks", "languages", "issues_open", "issues_closed", "contributors_n" )) + +non_text_files_pattern <- "\\.(png||.jpg||.jpeg||.bmp||.gif||.tiff)$" diff --git a/R/message_handler.R b/R/message_handler.R index da0bd7d9..7d578126 100644 --- a/R/message_handler.R +++ b/R/message_handler.R @@ -11,7 +11,7 @@ show_message <- function(host, } else if (engine == "rest") { msg_rest } else if (engine == "both") { - paste0(msg_rest, "&",msg_graphql) + paste0(msg_rest, "&", msg_graphql) } message <- if (is.null(scope)) { glue::glue("[Host:{host}][Engine:{engine_msg}] {information}...") diff --git a/R/test_helpers.R b/R/test_helpers.R index be66396d..d022630f 100644 --- a/R/test_helpers.R +++ b/R/test_helpers.R @@ -22,7 +22,8 @@ Mocker <- R6::R6Class("Mocker", #' @noRd #' @description A helper class for use in tests - it does not throw superfluous #' messages and does exactly what is needed for in tests. -GitHostGitHubTest <- R6::R6Class("GitHostGitHubTest", +GitHostGitHubTest <- R6::R6Class( + classname = "GitHostGitHubTest", inherit = GitHostGitHub, public = list( initialize = function(orgs = NA, @@ -32,23 +33,23 @@ GitHostGitHubTest <- R6::R6Class("GitHostGitHubTest", private$set_api_url(host) private$set_endpoints() private$check_if_public(host) - private$set_token(token) + private$set_token(token, verbose = FALSE) private$set_graphql_url() - private$set_orgs_and_repos(orgs, repos) + private$set_orgs_and_repos(orgs, repos, verbose = FALSE) private$setup_test_engines() - private$set_searching_scope(orgs, repos) + private$set_searching_scope(orgs, repos, verbose = FALSE) } ), private = list( setup_test_engines = function() { - private$engines$rest <- TestEngineRestGitHub$new( - token = private$token, - rest_api_url = private$api_url - ) - private$engines$graphql <- EngineGraphQLGitHub$new( - token = private$token, - gql_api_url = private$set_graphql_url() - ) + private$engines$rest <- TestEngineRestGitHub$new( + token = private$token, + rest_api_url = private$api_url + ) + private$engines$graphql <- EngineGraphQLGitHub$new( + token = private$token, + gql_api_url = private$set_graphql_url() + ) } ) ) @@ -56,35 +57,36 @@ GitHostGitHubTest <- R6::R6Class("GitHostGitHubTest", #' @noRd #' @description A helper class for use in tests - it does not throw superfluous #' messages and does exactly what is needed for in tests. -GitHostGitLabTest <- R6::R6Class("GitHostGitLabTest", - inherit = GitHostGitLab, - public = list( - initialize = function(orgs = NA, - repos = NA, - token = NA, - host = NA) { - private$set_api_url(host) - private$set_endpoints() - private$check_if_public(host) - private$set_token(token) - private$set_graphql_url() - private$set_orgs_and_repos(orgs, repos) - private$setup_test_engines() - private$set_searching_scope(orgs, repos) - } - ), - private = list( - setup_test_engines = function() { - private$engines$rest <- TestEngineRestGitLab$new( - token = private$token, - rest_api_url = private$api_url - ) - private$engines$graphql <- EngineGraphQLGitLab$new( - token = private$token, - gql_api_url = private$set_graphql_url() - ) - } - ) +GitHostGitLabTest <- R6::R6Class( + classname = "GitHostGitLabTest", + inherit = GitHostGitLab, + public = list( + initialize = function(orgs = NA, + repos = NA, + token = NA, + host = NA) { + private$set_api_url(host) + private$set_endpoints() + private$check_if_public(host) + private$set_token(token, verbose = FALSE) + private$set_graphql_url() + private$set_orgs_and_repos(orgs, repos, verbose = FALSE) + private$setup_test_engines() + private$set_searching_scope(orgs, repos, verbose = FALSE) + } + ), + private = list( + setup_test_engines = function() { + private$engines$rest <- TestEngineRestGitLab$new( + token = private$token, + rest_api_url = private$api_url + ) + private$engines$graphql <- EngineGraphQLGitLab$new( + token = private$token, + gql_api_url = private$set_graphql_url() + ) + } + ) ) #' @noRd diff --git a/R/utils.R b/R/utils.R index c7fce52e..3d8311d5 100644 --- a/R/utils.R +++ b/R/utils.R @@ -45,7 +45,7 @@ retrieve_platform <- function(api_url) { commits_stats <- function(object, time_interval) { stopifnot(inherits(object, "grouped_df")) object <- dplyr::ungroup(object) - class(object) = append(class(object), "commits_stats") + class(object) <- append(class(object), "commits_stats") attr(object, "time_interval") <- time_interval object } diff --git a/README.Rmd b/README.Rmd index cb12ff36..d8319f04 100644 --- a/README.Rmd +++ b/README.Rmd @@ -27,6 +27,7 @@ With GitStats you can pull git data in a uniform way (table format) from GitHub * commits, * users, * release logs, +* repository files structure, * text files content, * R package usage. diff --git a/README.md b/README.md index 93f21768..f0b46433 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ GitHub and GitLab. For the time-being you can get data on: - commits, - users, - release logs, +- repository files structure, - text files content, - R package usage. diff --git a/_pkgdown.yml b/_pkgdown.yml index bfcc25f2..538aa70f 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -19,4 +19,5 @@ articles: - set_hosts - get_data - get_repos_with_code + - get_files diff --git a/inst/WORDLIST b/inst/WORDLIST index 6d88cef3..d87512cf 100644 --- a/inst/WORDLIST +++ b/inst/WORDLIST @@ -35,3 +35,4 @@ POSIXct plotly repo repos +regex diff --git a/inst/get_files_workflow.R b/inst/get_files_workflow.R new file mode 100644 index 00000000..960e7427 --- /dev/null +++ b/inst/get_files_workflow.R @@ -0,0 +1,29 @@ +devtools::load_all(".") + +test_gitstats <- create_gitstats() |> + set_github_host( + orgs = c("r-world-devs", "openpharma") + ) |> + set_gitlab_host( + orgs = c("mbtests", "mbtestapps") + ) + +get_files_content( + gitstats_obj = test_gitstats, + file_path = c("LICENSE", "DESCRIPTION") +) + +md_files_structure <- get_files_structure( + gitstats_obj = test_gitstats, + pattern = "\\.md|.R", + depth = 2L +) + +get_files_content(test_gitstats) + +md_files_structure <- get_files_structure( + gitstats_obj = test_gitstats, + pattern = "\\.md|\\.qmd|\\.Rmd", + depth = 2L, + verbose = FALSE +) diff --git a/man/get_commits.Rd b/man/get_commits.Rd index 8c40d59a..8748ea58 100644 --- a/man/get_commits.Rd +++ b/man/get_commits.Rd @@ -9,7 +9,8 @@ get_commits( since = NULL, until = NULL, cache = TRUE, - verbose = is_verbose(gitstats_object) + verbose = is_verbose(gitstats_object), + progress = verbose ) } \arguments{ @@ -22,8 +23,11 @@ get_commits( \item{cache}{A logical, if set to \code{TRUE} GitStats will retrieve the last result from its storage.} -\item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and -printing output is switched off.} +\item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and printing +output is switched off.} + +\item{progress}{A logical, by default set to \code{verbose} value. If \code{FALSE} no +\code{cli} progress bar will be displayed.} } \value{ A data.frame. diff --git a/man/get_files.Rd b/man/get_files.Rd deleted file mode 100644 index 04450b55..00000000 --- a/man/get_files.Rd +++ /dev/null @@ -1,46 +0,0 @@ -% Generated by roxygen2: do not edit by hand -% Please edit documentation in R/gitstats_functions.R -\name{get_files} -\alias{get_files} -\title{Get data on files} -\usage{ -get_files( - gitstats_object, - file_path, - cache = TRUE, - verbose = is_verbose(gitstats_object) -) -} -\arguments{ -\item{gitstats_object}{A GitStats object.} - -\item{file_path}{A standardized path to file(s) in repositories. May be a -character vector if multiple files are to be pulled.} - -\item{cache}{A logical, if set to \code{TRUE} GitStats will retrieve the last -result from its storage.} - -\item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and printing -output is switched off.} -} -\value{ -A data.frame. -} -\description{ -Pull text files content for a given scope (orgs, repos or whole -git hosts). -} -\examples{ -\dontrun{ - my_gitstats <- create_gitstats() \%>\% - set_github_host( - token = Sys.getenv("GITHUB_PAT"), - orgs = c("r-world-devs") - ) \%>\% - set_gitlab_host( - token = Sys.getenv("GITLAB_PAT_PUBLIC"), - orgs = "mbtests" - ) - get_files(my_gitstats, c("LICENSE", "DESCRIPTION")) -} -} diff --git a/man/get_files_content.Rd b/man/get_files_content.Rd new file mode 100644 index 00000000..8a2b0292 --- /dev/null +++ b/man/get_files_content.Rd @@ -0,0 +1,78 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gitstats_functions.R +\name{get_files_content} +\alias{get_files_content} +\title{Get content of files} +\usage{ +get_files_content( + gitstats_object, + file_path = NULL, + use_files_structure = TRUE, + only_text_files = TRUE, + cache = TRUE, + verbose = is_verbose(gitstats_object), + progress = verbose +) +} +\arguments{ +\item{gitstats_object}{A GitStats object.} + +\item{file_path}{Optional. A standardized path to file(s) in repositories. +May be a character vector if multiple files are to be pulled. If set to +\code{NULL} and \code{use_files_structure} parameter is set to \code{TRUE}, \code{GitStats} +will try to pull data from \code{files_structure} (see below).} + +\item{use_files_structure}{Logical. If \code{TRUE} and \code{file_path} is set to +\code{NULL}, will iterate over \code{files_structure} pulled by +\code{get_files_structure()} function and kept in storage. If there is no +\code{files_structure} in storage, an error will be returned. If \code{file_path} is +defined, it will override \code{use_files_structure} parameter.} + +\item{only_text_files}{A logical, \code{TRUE} by default. If set to \code{FALSE}, apart +from files with text content shows in table output also non-text files with +\code{NA} value for text content.} + +\item{cache}{A logical, if set to \code{TRUE} GitStats will retrieve the last +result from its storage.} + +\item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and printing +output is switched off.} + +\item{progress}{A logical, by default set to \code{verbose} value. If \code{FALSE} no +\code{cli} progress bar will be displayed.} +} +\value{ +A data.frame. +} +\description{ +Pull text files content for a given scope (orgs, repos or whole +git hosts). +} +\examples{ +\dontrun{ + my_gitstats <- create_gitstats() \%>\% + set_github_host( + token = Sys.getenv("GITHUB_PAT"), + orgs = c("r-world-devs") + ) \%>\% + set_gitlab_host( + token = Sys.getenv("GITLAB_PAT_PUBLIC"), + orgs = "mbtests" + ) + get_files_content( + gitstats_obj = my_gitstats, + file_path = c("LICENSE", "DESCRIPTION") + ) + + # example with files structure + files_structure <- get_files_structure( + gitstats_obj = my_gitstats, + pattern = "\\\\.Rmd", + depth = 2L + ) + # get_files_content() will make use of pulled earlier files structure + files_content <- get_files_content( + gitstats_obj = my_gitstats + ) +} +} diff --git a/man/get_files_structure.Rd b/man/get_files_structure.Rd new file mode 100644 index 00000000..ec3fd1c7 --- /dev/null +++ b/man/get_files_structure.Rd @@ -0,0 +1,58 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/gitstats_functions.R +\name{get_files_structure} +\alias{get_files_structure} +\title{Get structure of files} +\usage{ +get_files_structure( + gitstats_object, + pattern = NULL, + depth = Inf, + cache = TRUE, + verbose = is_verbose(gitstats_object), + progress = verbose +) +} +\arguments{ +\item{gitstats_object}{A GitStats object.} + +\item{pattern}{An optional regular expression. If defined, it pulls file +structure for a repository matching this pattern.} + +\item{depth}{An optional integer. Defines level of directories to retrieve +files from. E.g. if set to \code{0}, it will pull files only from root, if \code{1}, +will take data from \code{root} directory and directories visible in \code{root} +directory. If left with no argument, will pull files from all directories.} + +\item{cache}{A logical, if set to \code{TRUE} GitStats will retrieve the last +result from its storage.} + +\item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and printing +output is switched off.} + +\item{progress}{A logical, by default set to \code{verbose} value. If \code{FALSE} no +\code{cli} progress bar will be displayed.} +} +\value{ +A list of vectors. +} +\description{ +Pulls file structure for a given repository. +} +\examples{ +\dontrun{ + my_gitstats <- create_gitstats() \%>\% + set_github_host( + token = Sys.getenv("GITHUB_PAT"), + orgs = c("r-world-devs") + ) \%>\% + set_gitlab_host( + token = Sys.getenv("GITLAB_PAT_PUBLIC"), + orgs = "mbtests" + ) + get_files_structure( + gitstats_obj = my_gitstats, + pattern = "\\\\.md" + ) +} +} diff --git a/man/get_release_logs.Rd b/man/get_release_logs.Rd index ef945bc4..bd664194 100644 --- a/man/get_release_logs.Rd +++ b/man/get_release_logs.Rd @@ -9,7 +9,8 @@ get_release_logs( since = NULL, until = NULL, cache = TRUE, - verbose = is_verbose(gitstats_object) + verbose = is_verbose(gitstats_object), + progress = verbose ) } \arguments{ @@ -22,8 +23,11 @@ get_release_logs( \item{cache}{A logical, if set to \code{TRUE} GitStats will retrieve the last result from its storage.} -\item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and -printing output is switched off.} +\item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and printing +output is switched off.} + +\item{progress}{A logical, by default set to \code{verbose} value. If \code{FALSE} no +\code{cli} progress bar will be displayed.} } \value{ A data.frame. diff --git a/man/get_repos.Rd b/man/get_repos.Rd index c4a5ecf6..ac33c371 100644 --- a/man/get_repos.Rd +++ b/man/get_repos.Rd @@ -11,7 +11,8 @@ get_repos( in_files = NULL, with_files = NULL, cache = TRUE, - verbose = is_verbose(gitstats_object) + verbose = is_verbose(gitstats_object), + progress = verbose ) } \arguments{ @@ -43,6 +44,9 @@ result from its storage.} \item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and printing output is switched off.} + +\item{progress}{A logical, by default set to \code{verbose} value. If \code{FALSE} no +\code{cli} progress bar will be displayed.} } \value{ A data.frame. diff --git a/man/get_repos_urls.Rd b/man/get_repos_urls.Rd index 18ee3a2a..5dfeedbf 100644 --- a/man/get_repos_urls.Rd +++ b/man/get_repos_urls.Rd @@ -11,7 +11,8 @@ get_repos_urls( in_files = NULL, with_files = NULL, cache = TRUE, - verbose = TRUE + verbose = is_verbose(gitstats_object), + progress = verbose ) } \arguments{ @@ -35,6 +36,9 @@ result from its storage.} \item{verbose}{A logical, \code{TRUE} by default. If \code{FALSE} messages and printing output is switched off.} + +\item{progress}{A logical, by default set to \code{verbose} value. If \code{FALSE} no +\code{cli} progress bar will be displayed.} } \value{ A character vector. diff --git a/tests/spelling.R b/tests/spelling.R index 6713838f..14bc8976 100644 --- a/tests/spelling.R +++ b/tests/spelling.R @@ -1,3 +1,3 @@ -if(requireNamespace('spelling', quietly = TRUE)) +if (requireNamespace('spelling', quietly = TRUE)) spelling::spell_check_test(vignettes = TRUE, error = FALSE, skip_on_cran = TRUE) diff --git a/tests/testthat/_snaps/01-GQLQueryGitHub.md b/tests/testthat/_snaps/01-GQLQueryGitHub.md deleted file mode 100644 index c91686d8..00000000 --- a/tests/testthat/_snaps/01-GQLQueryGitHub.md +++ /dev/null @@ -1,35 +0,0 @@ -# commits_by_repo query is built properly - - Code - gh_commits_by_repo_query - Output - [1] "{\n repository(name: \"GitStats\", owner: \"r-world-devs\") {\n defaultBranchRef {\n target {\n ... on Commit {\n history(since: \"2023-01-01T00:00:00Z\"\n until: \"2023-02-28T00:00:00Z\"\n \n ) {\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n ... on Commit {\n id\n committed_date: committedDate\n author {\n name\n user {\n name\n login\n }\n }\n additions\n deletions\n }\n }\n }\n }\n }\n }\n }\n }\n }" - -# repos_by_org query is built properly - - Code - gh_repos_by_org_query - Output - [1] "\n query GetReposByOrg($org: String!) {\n repositoryOwner(login: $org) {\n ... on Organization {\n repositories(first: 100 ) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n }\n }\n }" - -# user query is built properly - - Code - gh_user_query - Output - [1] "\n query GetUser($user: String!) {\n user(login: $user) {\n id\n name\n login\n email\n location\n starred_repos: starredRepositories {\n totalCount\n }\n contributions: contributionsCollection {\n totalIssueContributions\n totalCommitContributions\n totalPullRequestContributions\n totalPullRequestReviewContributions\n }\n avatar_url: avatarUrl\n web_url: websiteUrl\n }\n }" - -# file query is built properly - - Code - gh_files_query - Output - [1] "query GetFilesByRepo($org: String!, $repo: String!, $file_path: String!) {\n repository(owner: $org, name: $repo) {\n id\n name\n url\n object(expression: $file_path) {\n ... on Blob {\n text\n byteSize\n }\n }\n }\n }" - -# releases query is built properly - - Code - gh_releases_query - Output - [1] "query GetReleasesFromRepo ($org: String!, $repo: String!) {\n repository(owner:$org, name:$repo){\n name\n url\n releases (last: 100) {\n nodes {\n name\n tagName\n publishedAt\n url\n description\n }\n }\n }\n }" - diff --git a/tests/testthat/_snaps/01-GQLQueryGitLab.md b/tests/testthat/_snaps/01-GQLQueryGitLab.md deleted file mode 100644 index 97b595bf..00000000 --- a/tests/testthat/_snaps/01-GQLQueryGitLab.md +++ /dev/null @@ -1,28 +0,0 @@ -# repos queries are built properly - - Code - gl_repos_by_org_query - Output - [1] "query GetReposByOrg($org: ID!) {\n group(fullPath: $org) {\n projects(first: 100 ) {\n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n group {\n path\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" - -# user query is built properly - - Code - gl_user_query - Output - [1] "\n query GetUser($user: String!) {\n user(username: $user) {\n id\n name\n login: username\n email: publicEmail\n location\n starred_repos: starredProjects {\n count\n }\n pull_requests: authoredMergeRequests {\n count\n }\n reviews: reviewRequestedMergeRequests {\n count\n }\n avatar_url: avatarUrl\n web_url: webUrl\n }\n }\n " - -# file queries are built properly - - Code - gl_files_query - Output - [1] "query GetFilesByOrg($org: ID!, $file_paths: [String!]!) {\n group(fullPath: $org) {\n projects(first: 100) {\n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n name\n id\n webUrl\n repository {\n blobs(paths: $file_paths) {\n nodes {\n name\n rawBlob\n size\n }\n }\n }\n }\n }\n }\n }\n }" - -# releases query is built properly - - Code - gl_releases_query - Output - [1] "query GetReleasesFromRepo($project_path: ID!) {\n project(fullPath: $project_path) {\n name\n webUrl\n \t\t\t\t\t\treleases {\n nodes{\n name\n tagName\n releasedAt\n links {\n selfUrl\n }\n description\n }\n }\n }\n }" - diff --git a/tests/testthat/_snaps/04-GitHost.md b/tests/testthat/_snaps/04-GitHost.md deleted file mode 100644 index 3deae094..00000000 --- a/tests/testthat/_snaps/04-GitHost.md +++ /dev/null @@ -1,56 +0,0 @@ -# `set_searching_scope` does not throw error when `orgs` or `repos` are defined - - Code - test_host$set_searching_scope(orgs = "mbtests", repos = NULL) - Message - i Searching scope set to [org]. - ---- - - Code - test_host$set_searching_scope(orgs = NULL, repos = "mbtests/GitStatsTesting") - Message - i Searching scope set to [repo]. - -# `prepare_repos_table()` prepares repos table - - Code - gh_repos_by_code_table <- test_host$prepare_repos_table_from_rest(repos_list = test_mocker$ - use("gh_repos_by_code_tailored")) - Message - i Preparing repositories table... - -# `get_files()` pulls files in the table format - - Code - gh_files_table <- test_host$get_files(file_path = "LICENSE") - ---- - - Code - gl_files_table <- test_host_gitlab$get_files(file_path = "README.md") - -# `get_files()` pulls files only for the repositories specified - - Code - gh_files_table <- test_host$get_files(file_path = "renv.lock") - Message - i [Host:GitHub][Engine:GraphQl][Scope:r-world-devs] Pulling files: [renv.lock]... - i [Host:GitHub][Engine:GraphQl][Scope:openpharma] Pulling files: [renv.lock]... - -# GitHost prepares table from GitLab repositories response - - Code - gl_repos_by_code_table <- test_host_gitlab$prepare_repos_table_from_rest( - repos_list = test_mocker$use("gl_repos_by_code_tailored")) - Message - i Preparing repositories table... - -# `get_files()` pulls two files in the table format - - Code - gl_files_table <- test_host_gitlab$get_files(file_path = c("meta_data.yaml", - "README.md")) - Message - i [Host:GitLab][Engine:GraphQl][Scope:mbtests] Pulling files: [meta_data.yaml, README.md]... - diff --git a/tests/testthat/_snaps/05-GitHostGitHub.md b/tests/testthat/_snaps/05-GitHostGitHub.md deleted file mode 100644 index 8dfa6e3f..00000000 --- a/tests/testthat/_snaps/05-GitHostGitHub.md +++ /dev/null @@ -1,34 +0,0 @@ -# When token is empty throw error - - Code - test_host$check_token("") - Condition - Error in `test_host$check_token()`: - i No token provided. - -# `check_token()` prints error when token exists but does not grant access - - x Token exists but does not grant access. - i Check if you use correct token. Check scopes your token is using. - -# check_endpoint returns error if they are not correct - - x Repository you provided does not exist or its name was passed in a wrong way: https://api.github.com/repos/r-worlddevs/GitStats - ! Please type your repository name as you see it in web URL. - i E.g. do not use spaces. Repository names as you see on the page may differ from their web 'address'. - -# `set_default_token` sets default token for public GitHub - - Code - default_token <- test_host$set_default_token() - Message - i Using PAT from GITHUB_PAT envar. - -# `get_all_repos()` works as expected - - Code - gh_repos_table <- test_host$get_all_repos() - Message - i [Host:GitHub][Engine:GraphQl][Scope:openpharma] Pulling repositories... - i [Host:GitHub][Engine:GraphQl][Scope:r-world-devs] Pulling repositories... - diff --git a/tests/testthat/_snaps/05-GitHostGitLab.md b/tests/testthat/_snaps/05-GitHostGitLab.md deleted file mode 100644 index f4ce317a..00000000 --- a/tests/testthat/_snaps/05-GitHostGitLab.md +++ /dev/null @@ -1,21 +0,0 @@ -# `set_default_token` sets default token for GitLab - - Code - withr::with_envvar(new = c(GITLAB_PAT = Sys.getenv("GITLAB_PAT_PUBLIC")), { - default_token <- test_host$set_default_token() - }) - Message - i Using PAT from GITLAB_PAT envar. - -# `set_searching_scope` throws error when both `orgs` and `repos` are defined - - Do not specify `orgs` while specifing `repos`. - x Host will not be added. - i Specify `orgs` or `repos`. - -# get_commits for GitLab works with repos implied - - Code - gl_commits_table <- test_host$get_commits(since = "2023-01-01", until = "2023-06-01", - settings = test_settings_repo) - diff --git a/tests/testthat/_snaps/06-EngineRestGitHub-contributors.md b/tests/testthat/_snaps/06-EngineRestGitHub-contributors.md deleted file mode 100644 index c6ae5e70..00000000 --- a/tests/testthat/_snaps/06-EngineRestGitHub-contributors.md +++ /dev/null @@ -1,6 +0,0 @@ -# `pull_repos_contributors()` adds contributors to repos table - - Code - gh_repos_by_code_table <- test_rest$pull_repos_contributors(repos_table = test_mocker$ - use("gh_repos_by_code_table"), settings = test_settings) - diff --git a/tests/testthat/_snaps/06-EngineRestGitLab-contributors.md b/tests/testthat/_snaps/06-EngineRestGitLab-contributors.md deleted file mode 100644 index 3a3cfd04..00000000 --- a/tests/testthat/_snaps/06-EngineRestGitLab-contributors.md +++ /dev/null @@ -1,14 +0,0 @@ -# `pull_repos_contributors()` adds contributors to repos table - - Code - gl_repos_table_with_contributors <- test_rest$pull_repos_contributors( - test_mocker$use("gl_repos_table_with_api_url"), settings = test_settings) - -# `get_commits_authors_handles_and_names()` adds author logis and names to commits table - - Code - gl_commits_table <- test_rest$get_commits_authors_handles_and_names( - commits_table = test_mocker$use("gl_commits_table"), verbose = TRUE) - Message - i Looking up for authors' names and logins... - diff --git a/tests/testthat/_snaps/get_commits-GitHub.md b/tests/testthat/_snaps/get_commits-GitHub.md new file mode 100644 index 00000000..637aace0 --- /dev/null +++ b/tests/testthat/_snaps/get_commits-GitHub.md @@ -0,0 +1,7 @@ +# commits_by_repo GitHub query is built properly + + Code + gh_commits_by_repo_query + Output + [1] "{\n repository(name: \"GitStats\", owner: \"r-world-devs\") {\n defaultBranchRef {\n target {\n ... on Commit {\n history(since: \"2023-01-01T00:00:00Z\"\n until: \"2023-02-28T00:00:00Z\"\n \n ) {\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n ... on Commit {\n id\n committed_date: committedDate\n author {\n name\n user {\n name\n login\n }\n }\n additions\n deletions\n }\n }\n }\n }\n }\n }\n }\n }\n }" + diff --git a/tests/testthat/_snaps/get_commits-GitLab.md b/tests/testthat/_snaps/get_commits-GitLab.md new file mode 100644 index 00000000..439f03b4 --- /dev/null +++ b/tests/testthat/_snaps/get_commits-GitLab.md @@ -0,0 +1,8 @@ +# `get_commits_authors_handles_and_names()` adds author logis and names to commits table + + Code + gl_commits_table <- test_rest_gitlab$get_commits_authors_handles_and_names( + commits_table = test_mocker$use("gl_commits_table"), verbose = TRUE) + Message + i Looking up for authors' names and logins... + diff --git a/tests/testthat/_snaps/get_files_content-GitHub.md b/tests/testthat/_snaps/get_files_content-GitHub.md new file mode 100644 index 00000000..6646a97c --- /dev/null +++ b/tests/testthat/_snaps/get_files_content-GitHub.md @@ -0,0 +1,7 @@ +# file queries for GitHub are built properly + + Code + gh_file_blobs_from_repo_query + Output + [1] "query GetFileBlobFromRepo($org: String!, $repo: String!, $expression: String!) {\n repository(owner: $org, name: $repo) {\n repo_id: id\n repo_name: name\n repo_url: url\n file: object(expression: $expression) {\n ... on Blob {\n text\n byteSize\n }\n }\n }\n }" + diff --git a/tests/testthat/_snaps/get_files_content-GitLab.md b/tests/testthat/_snaps/get_files_content-GitLab.md new file mode 100644 index 00000000..3dcd9476 --- /dev/null +++ b/tests/testthat/_snaps/get_files_content-GitLab.md @@ -0,0 +1,24 @@ +# file queries for GitLab are built properly + + Code + gl_files_query + Output + [1] "query GetFilesByOrg($org: ID!, $file_paths: [String!]!) {\n group(fullPath: $org) {\n projects(first: 100) {\n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n name\n id\n webUrl\n repository {\n blobs(paths: $file_paths) {\n nodes {\n name\n rawBlob\n size\n }\n }\n }\n }\n }\n }\n }\n }" + +--- + + Code + gl_file_blobs_from_repo_query + Output + [1] "\n query GetFilesByRepo($fullPath: ID!, $file_paths: [String!]!) {\n project(fullPath: $fullPath) {\n name\n id\n webUrl\n repository {\n blobs(paths: $file_paths) {\n nodes {\n name\n rawBlob\n size\n }\n }\n }\n }\n }\n " + +# Gitlab GraphQL switches to pulling files per repositories when query is too complex + + Code + gitlab_files_response_by_repos <- test_graphql_gitlab$get_files_from_org(org = "mbtests", + repos = NULL, file_paths = c("DESCRIPTION", "project_metadata.yaml", + "README.md"), host_files_structure = NULL, only_text_files = TRUE, verbose = TRUE, + progress = FALSE) + Message + i I will switch to pulling files per repository. + diff --git a/tests/testthat/_snaps/get_files_structure-GitHub.md b/tests/testthat/_snaps/get_files_structure-GitHub.md new file mode 100644 index 00000000..37945d6e --- /dev/null +++ b/tests/testthat/_snaps/get_files_structure-GitHub.md @@ -0,0 +1,36 @@ +# files tree query for GitHub are built properly + + Code + gh_files_tree_query + Output + [1] "query GetFilesFromRepo($org: String!, $repo: String!, $expression: String!) {\n repository(owner: $org, name: $repo) {\n id\n name\n url\n object(expression: $expression) {\n ... on Tree {\n entries {\n name\n type\n }\n }\n }\n }\n }" + +# get_files_structure_from_orgs pulls files structure for repositories in orgs + + Code + gh_files_structure_from_orgs <- github_testhost_priv$ + get_files_structure_from_orgs(pattern = NULL, depth = 2L, verbose = TRUE) + Message + i [Host:GitHub][Engine:GraphQl][Scope:r-world-devs] Pulling files structure...... + i [Host:GitHub][Engine:GraphQl][Scope:openpharma] Pulling files structure...... + +# when files_structure is empty, appropriate message is returned + + Code + github_testhost_priv$get_files_structure_from_orgs(pattern = "\\.png", depth = 1L, + verbose = TRUE) + Message + i [Host:GitHub][Engine:GraphQl][Scope:r-world-devs] Pulling files structure...[files matching pattern: '\.png']... + i [Host:GitHub][Engine:GraphQl][Scope:openpharma] Pulling files structure...[files matching pattern: '\.png']... + ! For GitHub no files structure found. + Output + named list() + +# get_files_content makes use of files_structure + + Code + files_content <- github_testhost_priv$get_files_content_from_orgs(file_path = NULL, + host_files_structure = test_mocker$use("gh_files_structure_from_orgs")) + Message + i I will make use of files structure stored in GitStats. + diff --git a/tests/testthat/_snaps/get_files_structure-GitLab.md b/tests/testthat/_snaps/get_files_structure-GitLab.md new file mode 100644 index 00000000..a4c86492 --- /dev/null +++ b/tests/testthat/_snaps/get_files_structure-GitLab.md @@ -0,0 +1,24 @@ +# files tree query for GitLab are built properly + + Code + gl_files_tree_query + Output + [1] "\n query GetFilesTree ($fullPath: ID!, $file_path: String!) {\n project(fullPath: $fullPath) {\n repository {\n tree(path: $file_path) {\n trees (first: 100) {\n pageInfo{\n endCursor\n hasNextPage\n }\n nodes {\n name\n }\n }\n blobs (first: 100) {\n pageInfo{\n endCursor\n hasNextPage\n }\n nodes {\n name\n }\n }\n }\n }\n }\n }\n " + +# get_files_structure_from_orgs pulls files structure for repositories in orgs + + Code + gl_files_structure_from_orgs <- gitlab_testhost_priv$ + get_files_structure_from_orgs(pattern = "\\.md|\\.R", depth = 1L, verbose = TRUE, + progress = FALSE) + Message + i [Host:GitLab][Engine:GraphQl][Scope:mbtests] Pulling files structure...[files matching pattern: '\.md|\.R']... + +# get_files_content makes use of files_structure + + Code + files_content <- gitlab_testhost_priv$get_files_content_from_orgs(file_path = NULL, + host_files_structure = test_mocker$use("gl_files_structure_from_orgs")) + Message + i I will make use of files structure stored in GitStats. + diff --git a/tests/testthat/_snaps/get_files_structure-GitStats.md b/tests/testthat/_snaps/get_files_structure-GitStats.md new file mode 100644 index 00000000..1f0e3190 --- /dev/null +++ b/tests/testthat/_snaps/get_files_structure-GitStats.md @@ -0,0 +1,15 @@ +# if returned files_structure is empty, do not store it and give proper message + + Code + files_structure <- test_gitstats_priv$get_files_structure_from_hosts(pattern = "\\.png", + depth = 1L, verbose = TRUE) + Message + ! No files structure found for matching pattern \.png in 1 level of dirs. + ! Files structure will not be saved in GitStats. + +# get_files_structure works as expected + + Code + files_structure <- test_gitstats$get_files_structure(pattern = "\\.md", depth = 2L, + verbose = TRUE) + diff --git a/tests/testthat/_snaps/get_release-GitHub.md b/tests/testthat/_snaps/get_release-GitHub.md new file mode 100644 index 00000000..116312b0 --- /dev/null +++ b/tests/testthat/_snaps/get_release-GitHub.md @@ -0,0 +1,7 @@ +# releases query is built properly + + Code + gh_releases_query + Output + [1] "query GetReleasesFromRepo ($org: String!, $repo: String!) {\n repository(owner:$org, name:$repo){\n name\n url\n releases (last: 100) {\n nodes {\n name\n tagName\n publishedAt\n url\n description\n }\n }\n }\n }" + diff --git a/tests/testthat/_snaps/get_release-GitLab.md b/tests/testthat/_snaps/get_release-GitLab.md new file mode 100644 index 00000000..7c4fb7a8 --- /dev/null +++ b/tests/testthat/_snaps/get_release-GitLab.md @@ -0,0 +1,7 @@ +# releases query is built properly + + Code + gl_releases_query + Output + [1] "query GetReleasesFromRepo($project_path: ID!) {\n project(fullPath: $project_path) {\n name\n webUrl\n \t\t\t\t\t\treleases {\n nodes{\n name\n tagName\n releasedAt\n links {\n selfUrl\n }\n description\n }\n }\n }\n }" + diff --git a/tests/testthat/_snaps/get_repos-GitHub.md b/tests/testthat/_snaps/get_repos-GitHub.md new file mode 100644 index 00000000..904780b0 --- /dev/null +++ b/tests/testthat/_snaps/get_repos-GitHub.md @@ -0,0 +1,28 @@ +# repos_by_org query is built properly + + Code + gh_repos_by_org_query + Output + [1] "\n query GetReposByOrg($org: String!) {\n repositoryOwner(login: $org) {\n ... on Organization {\n repositories(first: 100 ) {\n totalCount\n pageInfo {\n endCursor\n hasNextPage\n }\n nodes {\n repo_id: id\n repo_name: name\n default_branch: defaultBranchRef {\n name\n }\n stars: stargazerCount\n forks: forkCount\n created_at: createdAt\n last_activity_at: pushedAt\n languages (first: 5) { nodes {name} }\n issues_open: issues (first: 100 states: [OPEN]) {\n totalCount\n }\n issues_closed: issues (first: 100 states: [CLOSED]) {\n totalCount\n }\n organization: owner {\n login\n }\n repo_url: url\n }\n }\n }\n }\n }" + +# `prepare_repos_table()` prepares repos table + + Code + gh_repos_by_code_table <- github_testhost_priv$prepare_repos_table_from_rest( + repos_list = test_mocker$use("gh_repos_by_code_tailored")) + Message + i Preparing repositories table... + +# `get_all_repos()` works as expected + + Code + gh_repos_table <- github_testhost_priv$get_all_repos() + Message + i [Host:GitHub][Engine:GraphQl][Scope:r-world-devs] Pulling repositories... + +# `get_repos_contributors()` adds contributors to repos table + + Code + gh_repos_by_code_table <- test_rest_github$get_repos_contributors(repos_table = test_mocker$ + use("gh_repos_by_code_table"), progress = FALSE) + diff --git a/tests/testthat/_snaps/get_repos-GitLab.md b/tests/testthat/_snaps/get_repos-GitLab.md new file mode 100644 index 00000000..5335a073 --- /dev/null +++ b/tests/testthat/_snaps/get_repos-GitLab.md @@ -0,0 +1,15 @@ +# repos queries are built properly + + Code + gl_repos_by_org_query + Output + [1] "query GetReposByOrg($org: ID!) {\n group(fullPath: $org) {\n projects(first: 100 ) {\n count\n pageInfo {\n hasNextPage\n endCursor\n }\n edges {\n node {\n repo_id: id\n repo_name: name\n repo_path: path\n ... on Project {\n repository {\n rootRef\n }\n }\n stars: starCount\n forks: forksCount\n created_at: createdAt\n last_activity_at: lastActivityAt\n languages {\n name\n }\n issues: issueStatusCounts {\n all\n closed\n opened\n }\n group {\n path\n }\n repo_url: webUrl\n }\n }\n }\n }\n }" + +# GitHost prepares table from GitLab repositories response + + Code + gl_repos_by_code_table <- gitlab_testhost_priv$prepare_repos_table_from_rest( + repos_list = test_mocker$use("gl_repos_by_code_tailored")) + Message + i Preparing repositories table... + diff --git a/tests/testthat/_snaps/get_repos-GitStats.md b/tests/testthat/_snaps/get_repos-GitStats.md new file mode 100644 index 00000000..a9bca13f --- /dev/null +++ b/tests/testthat/_snaps/get_repos-GitStats.md @@ -0,0 +1,8 @@ +# get_repos works properly and for the second time uses cache + + Code + repos_table <- test_gitstats$get_repos() + Message + ! Retrieving repositories from the GitStats storage. + i If you wish to pull the data from API once more, set `cache` parameter to `FALSE`. + diff --git a/tests/testthat/_snaps/get_user-GitHub.md b/tests/testthat/_snaps/get_user-GitHub.md new file mode 100644 index 00000000..890cad8b --- /dev/null +++ b/tests/testthat/_snaps/get_user-GitHub.md @@ -0,0 +1,7 @@ +# user query is built properly + + Code + gh_user_query + Output + [1] "\n query GetUser($user: String!) {\n user(login: $user) {\n id\n name\n login\n email\n location\n starred_repos: starredRepositories {\n totalCount\n }\n contributions: contributionsCollection {\n totalIssueContributions\n totalCommitContributions\n totalPullRequestContributions\n totalPullRequestReviewContributions\n }\n avatar_url: avatarUrl\n web_url: websiteUrl\n }\n }" + diff --git a/tests/testthat/_snaps/get_user-GitLab.md b/tests/testthat/_snaps/get_user-GitLab.md new file mode 100644 index 00000000..099ea46a --- /dev/null +++ b/tests/testthat/_snaps/get_user-GitLab.md @@ -0,0 +1,7 @@ +# user query is built properly + + Code + gl_user_query + Output + [1] "\n query GetUser($user: String!) {\n user(username: $user) {\n id\n name\n login: username\n email: publicEmail\n location\n starred_repos: starredProjects {\n count\n }\n pull_requests: authoredMergeRequests {\n count\n }\n reviews: reviewRequestedMergeRequests {\n count\n }\n avatar_url: avatarUrl\n web_url: webUrl\n }\n }\n " + diff --git a/tests/testthat/_snaps/helpers.md b/tests/testthat/_snaps/helpers.md new file mode 100644 index 00000000..84602e49 --- /dev/null +++ b/tests/testthat/_snaps/helpers.md @@ -0,0 +1,57 @@ +# `set_searching_scope` does not throw error when `orgs` or `repos` are defined + + Code + gitlab_testhost_priv$set_searching_scope(orgs = "mbtests", repos = NULL, + verbose = TRUE) + Message + i Searching scope set to [org]. + +--- + + Code + gitlab_testhost_priv$set_searching_scope(orgs = NULL, repos = "mbtests/GitStatsTesting", + verbose = TRUE) + Message + i Searching scope set to [repo]. + +# When token is empty throw error + + Code + github_testhost_priv$check_token("") + Condition + Error in `github_testhost_priv$check_token()`: + i No token provided. + +# `check_token()` prints error when token exists but does not grant access + + x Token exists but does not grant access. + i Check if you use correct token. Check scopes your token is using. + +# check_endpoint returns error if they are not correct + + x Repository you provided does not exist or its name was passed in a wrong way: https://api.github.com/repos/r-worlddevs/GitStats + ! Please type your repository name as you see it in web URL. + i E.g. do not use spaces. Repository names as you see on the page may differ from their web 'address'. + +# `set_default_token` sets default token for public GitHub + + Code + default_token <- github_testhost_priv$set_default_token(verbose = TRUE) + Message + i Using PAT from GITHUB_PAT envar. + +# `set_default_token` sets default token for GitLab + + Code + withr::with_envvar(new = c(GITLAB_PAT = Sys.getenv("GITLAB_PAT_PUBLIC")), { + default_token <- gitlab_testhost_priv$set_default_token(verbose = TRUE) + }) + Message + i Using PAT from GITLAB_PAT envar. + +# `set_searching_scope` throws error when both `orgs` and `repos` are defined + + Do not specify `orgs` while specifing `repos`. + x Host will not be added. + i Specify `orgs` or `repos`. + diff --git a/tests/testthat/_snaps/set_host.md b/tests/testthat/_snaps/set_host.md index 4808f72d..c459840b 100644 --- a/tests/testthat/_snaps/set_host.md +++ b/tests/testthat/_snaps/set_host.md @@ -5,7 +5,7 @@ orgs = c("openpharma", "r-world-devs")) Message i Searching scope set to [org]. - i Checking host data... + i Checking organizations... v Set connection to GitHub. --- @@ -15,7 +15,7 @@ orgs = c("mbtests")) Message i Searching scope set to [org]. - i Checking host data... + i Checking organizations... v Set connection to GitLab. # When empty token for GitHub, GitStats pulls default token @@ -26,7 +26,7 @@ Message i Using PAT from GITHUB_PAT envar. i Searching scope set to [org]. - i Checking host data... + i Checking organizations... v Set connection to GitHub. # When empty token for GitLab, GitStats pulls default token @@ -38,7 +38,7 @@ Message i Using PAT from GITLAB_PAT envar. i Searching scope set to [org]. - i Checking host data... + i Checking organizations... v Set connection to GitLab. # Set GitHub host with particular repos vector instead of orgs @@ -49,7 +49,7 @@ "openpharma/GithubMetrics", "openpharma/DataFakeR")) Message i Searching scope set to [repo]. - i Checking host data... + i Checking repositories... v Set connection to GitHub. # Set GitLab host with particular repos vector instead of orgs @@ -59,7 +59,7 @@ repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2")) Message i Searching scope set to [repo]. - i Checking host data... + i Checking repositories... v Set connection to GitLab. # Set host prints error when repos and orgs are defined and host is not passed to GitStats @@ -86,7 +86,7 @@ "r_world_devs")) Message i Searching scope set to [org]. - i Checking host data... + i Checking organizations... Condition Error in `purrr::map()`: i In index: 1. @@ -100,10 +100,10 @@ set_github_host(token = Sys.getenv("GITHUB_PAT"), orgs = "openpharma") Message i Searching scope set to [org]. - i Checking host data... + i Checking organizations... v Set connection to GitHub. i Searching scope set to [org]. - i Checking host data... + i Checking organizations... v Set connection to GitHub. Condition Error: @@ -116,7 +116,7 @@ "GITHUB_PAT"), orgs = c("openparma")) Message i Searching scope set to [org]. - i Checking host data... + i Checking organizations... Condition Error in `purrr::map()`: i In index: 1. @@ -132,7 +132,7 @@ "GITLAB_PAT_PUBLIC"), orgs = c("openparma", "mbtests")) Message i Searching scope set to [org]. - i Checking host data... + i Checking organizations... Condition Error in `purrr::map()`: i In index: 1. @@ -148,7 +148,7 @@ "GITHUB_PAT"), orgs = c("openpharma", "r_world_devs")) Message i Searching scope set to [org]. - i Checking host data... + i Checking organizations... Condition Error in `purrr::map()`: i In index: 2. diff --git a/tests/testthat/_snaps/07-GitStats.md b/tests/testthat/_snaps/z-GitStats.md similarity index 80% rename from tests/testthat/_snaps/07-GitStats.md rename to tests/testthat/_snaps/z-GitStats.md index 4371c115..a460e5d0 100644 --- a/tests/testthat/_snaps/07-GitStats.md +++ b/tests/testthat/_snaps/z-GitStats.md @@ -8,7 +8,7 @@ Scanning scope: Organizations: [0] Repositories: [0] - Storage: + Storage: # GitStats prints the proper info when connections are added. @@ -20,7 +20,7 @@ Scanning scope: Organizations: [2] r-world-devs, mbtests Repositories: [0] - Storage: + Storage: # GitStats prints the proper info when repos are passed instead of orgs. @@ -32,7 +32,7 @@ Scanning scope: Organizations: [0] Repositories: [4] r-world-devs/GitStats, openpharma/GithubMetrics, mbtests/gitstatstesting, mbtests/gitstats-testing-2 - Storage: + Storage: # check_for_host returns error when no hosts are passed @@ -49,14 +49,6 @@ ! Use either `with_code` of `with_files` parameter. i If you want to search for [shiny] code in given files - use `in_files` parameter together with `with_code` instead. -# get_repos works properly and for the second time uses cache - - Code - repos_table <- test_gitstats$get_repos() - Message - ! Retrieving repositories from the GitStats storage. - i If you wish to pull the data from API once more, set `cache` parameter to `FALSE`. - # subgroups are cleanly printed in GitStats Code @@ -67,5 +59,5 @@ Scanning scope: Organizations: [1] mbtests/subgroup Repositories: [0] - Storage: + Storage: diff --git a/tests/testthat/helper-expect-responses.R b/tests/testthat/helper-expect-responses.R index d5c9b567..883e4269 100644 --- a/tests/testthat/helper-expect-responses.R +++ b/tests/testthat/helper-expect-responses.R @@ -123,6 +123,33 @@ expect_user_gql_response <- function(object) { ) } +expect_github_files_raw_response <- function(object) { + purrr::walk(object$data$repository$object$entries, function(entry) { + expect_equal( + names(entry), + c("name", "type") + ) + }) +} + +expect_gitlab_files_blob_response <- function(object) { + purrr::walk(object$data$project$repository$blobs$nodes, function(node) { + expect_equal( + names(node), + c("name", "rawBlob", "size") + ) + }) +} + +expect_gitlab_files_tree_response <- function(object) { + purrr::walk(object$data$project$repository$tree$blobs$nodes, function(node) { + expect_equal( + names(node), + c("name") + ) + }) +} + expect_github_files_response <- function(object) { expect_type( object, @@ -132,21 +159,21 @@ expect_github_files_response <- function(object) { length(object[[1]]), 0 ) - purrr::walk(object, function(file_path) { - purrr::walk(file_path, function(repository) { + purrr::walk(object, function(repository) { + purrr::walk(repository, function(file_path) { expect_list_contains( - repository, - c("name", "id", "object") + file_path, + c("repo_name", "repo_id", "repo_url", "object") ) expect_list_contains( - repository$object, + file_path$file, c("text", "byteSize") ) }) }) } -expect_gitlab_files_response <- function(object) { +expect_gitlab_files_from_org_response <- function(object) { expect_type( object, "list" @@ -171,6 +198,31 @@ expect_gitlab_files_response <- function(object) { }) } +expect_gitlab_files_from_org_by_repos_response <- function(response, expected_files) { + expect_type( + response, + "list" + ) + expect_gt( + length(response), + 0 + ) + purrr::walk(response, function(repo) { + purrr::walk(repo$data$project$repository$blobs$nodes, function(file) { + expect_equal( + names(file), + c("name", "rawBlob", "size") + ) + }) + }) + files_vec <- purrr::map(response, ~ purrr::map_vec(.$data$project$repository$blobs$nodes, ~ .$name)) %>% + unlist() %>% + unique() + expect_true( + all(expected_files %in% files_vec) + ) +} + expect_github_releases_response <- function(object) { expect_type( object, diff --git a/tests/testthat/helper-expect-tables.R b/tests/testthat/helper-expect-tables.R index b06dfa2b..65fa3c79 100644 --- a/tests/testthat/helper-expect-tables.R +++ b/tests/testthat/helper-expect-tables.R @@ -5,8 +5,8 @@ repo_gitstats_colnames <- c( ) repo_host_colnames <- c('repo_id', 'repo_name', 'default_branch', 'stars', 'forks', - 'created_at', 'last_activity_at', 'languages', 'issues_open', - 'issues_closed', 'organization', 'repo_url') + 'created_at', 'last_activity_at', 'languages', 'issues_open', + 'issues_closed', 'organization', 'repo_url') expect_package_usage_table <- function(object, with_cols = NULL) { expect_s3_class(object, "data.frame") @@ -31,7 +31,7 @@ expect_repos_table <- function(repos_object, repo_cols = repo_host_colnames, wit expect_gt(nrow(repos_object), 0) } -expect_commits_table <- function(pull_commits_object, with_stats = TRUE, exp_author = TRUE) { +expect_commits_table <- function(get_commits_object, with_stats = TRUE, exp_author = TRUE) { commit_cols <- if (exp_author) { c( "id", "committed_date", "author", "author_login", "author_name", "additions", "deletions", @@ -43,13 +43,13 @@ expect_commits_table <- function(pull_commits_object, with_stats = TRUE, exp_aut "repository", "organization", "api_url" ) } - expect_s3_class(pull_commits_object, "data.frame") - expect_named(pull_commits_object, commit_cols) - expect_gt(nrow(pull_commits_object), 0) - expect_s3_class(pull_commits_object$committed_date, "POSIXt") + expect_s3_class(get_commits_object, "data.frame") + expect_named(get_commits_object, commit_cols) + expect_gt(nrow(get_commits_object), 0) + expect_s3_class(get_commits_object$committed_date, "POSIXt") if (with_stats) { - expect_type(pull_commits_object$additions, "integer") - expect_type(pull_commits_object$deletions, "integer") + expect_type(get_commits_object$additions, "integer") + expect_type(get_commits_object$deletions, "integer") } } diff --git a/tests/testthat/helper-fixtures.R b/tests/testthat/helper-fixtures.R index 865c3275..13298240 100644 --- a/tests/testthat/helper-fixtures.R +++ b/tests/testthat/helper-fixtures.R @@ -30,6 +30,268 @@ test_fixtures$half_empty_gql_response <- list( ) ) -test_fixtures$gl_search_response <- list( +test_fixtures$github_file_response <- list( + "data" = list( + "repository" = list( + "repo_id" = "01010101", + "repo_name" = "TestProject", + "repo_url" = "https://github.com/r-world-devs/GitStats", + "file" = list( + "text" = "Some interesting text.", + "byteSize" = 50L + ) + ) + ) +) +test_fixtures$gitlab_file_org_response <- list( + "data" = list( + "group" = list( + "projects" = list( + "count" = 3, + "pageInfo" = list( + "hasNextPage" = FALSE, + "endCursor" = "xxxxxx" + ), + "edges" = list( + list( + "node" = list( + "name" = "graphql_tests", + "id" = "gid://gitlab/Project/61399846", + "webUrl" = "https://gitlab.com/mbtests/graphql_tests", + "repository" = list( + "blobs" = list( + "nodes" = list() # empty response for query, no files found in org + ) + ) + ) + ), + list( + "node" = list( + "name" = "RM Tests 3", + "id" = "gid://gitlab/Project/44346961", + "webUrl" = "https://gitlab.com/mbtests/rm-tests-3", + "repository" = list( + "blobs" = list( + "nodes" = list( + list( + "name" = "meta_data.yaml", + "rawBlob" = "Some interesting text", + "size" = 4 + ) + ) + ) + ) + ) + ), + list( + "node" = list( + "name" = "RM Tests 2", + "id" = "gid://gitlab/Project/44293594", + "webUrl" = "https://gitlab.com/mbtests/rm-tests-2", + "repository" = list( + "blobs" = list( + "nodes" = list( + list( + "name" = "meta_data.yaml", + "rawBlob" = "Some interesting text", + "size" = 5 + ) + ) + ) + ) + ) + ) + ) + ) + ) + ) +) + +test_fixtures$gitlab_file_repo_response <- list( + "data" = list( + "project" = list( + "name" = "TestProject", + "id" = "1010101", + "webUrl" = "https://gitlab.com/mbtests/graphql_tests", + "repository" = list( + "blobs" = list( + "nodes" = list( + list( + "name" = "README.md", + "rawBlob" = "# graphql_tests\n\nThis project is for testing GraphQL capabilities.\n", + "size" = "67" + ), + list( + "name" = "project_metadata.yaml", + "rawBlob" = "Name: GraphQL Tests", + "size" = "19" + ) + ) + ) + ) + ) + ) +) + +test_fixtures$github_png_file_response <- list( + "data" = list( + "repository" = list( + "repo_id" = "01010101", + "repo_name" = "TestProject", + "repo_url" = "https://github.com/r-world-devs/GitStats", + "file" = list( + "text" = NULL, + "byteSize" = 50L + ) + ) + ) +) + +test_fixtures$gitlab_search_response <- list( + list( + "basename" = "test", + "data" = "some text with searched phrase", + "path" = "test.R", + "filename" = "test.R", + "id" = NULL, + "ref" = "main", + "startline" = 10, + "project_id" = 43398933 + ), + list( + "basename" = "test", + "data" = "some text with searched phrase", + "path" = "test.R", + "filename" = "test.R", + "id" = NULL, + "ref" = "main", + "startline" = 15, + "project_id" = 43400864 + ) +) + +test_fixtures$github_search_response <- list( + "total_count" = 3, + "incomplete_results" = FALSE, + "items" = list( + list( + "name" = "test1.R", + "path" = "examples/test1.R", + "sha" = "0d42b9d23ddfc0bca1", + "url" = "test_url", + "git_url" = "test_git_url", + "html_url" = "test_html_url", + "repository" = list( + "id" = 627452680, + "url" = "https://api.github.com/repos/r-world-devs/GitStats", + "html" = "https://github.com/r-world-devs/GitStats" + ), + "score" = 1 + ), + list( + "name" = "test2.R", + "path" = "tests/test2.R", + "sha" = "01238xb", + "url" = "test_url", + "git_url" = "test_git_url", + "html_url" = "test_html_url", + "repository" = list( + "id" = 604718884, + "url" = "https://api.github.com/repos/r-world-devs/GitStats", + "html" = "https://github.com/r-world-devs/GitStats" + ), + "score" = 1 + ), + list( + "name" = "test3.R", + "path" = "R/test3.R", + "sha" = "20e19af2dda26d04f6", + "url" = "test_url", + "git_url" = "test_git_url", + "html_url" = "test_html_url", + "repository" = list( + "id" = 495151911, + "url" = "https://api.github.com/repos/r-world-devs/GitStats", + "html" = "https://github.com/r-world-devs/GitStats" + ), + "score" = 1 + ) + ) +) + +test_fixtures$gitlab_files_tree_response <- list( + "data" = list( + "project" = list( + "repository" = list( + "tree" = list( + "trees" = list( + "nodes" = list( + list( + "name" = "R" + ), + list( + "name" = "tests" + ) + ) + ), + "blobs" = list( + "nodes" = list( + list( + "name" = "DESCRIPTION" + ), + list( + "name" = "README.md" + ), + list( + "name" = "project_metadata.yaml" + ) + ) + ) + ) + ) + ) + ) +) + +test_fixtures$github_files_tree_response <- list( + "data" = list( + "repository" = list( + "id" = "R_kgD0Ivtxsg", + "name" = "GitStats", + "url" = "https://github.com/r-world-devs/GitStats", + "object" = list( + "entries" = list( + list( + "name" = ".Rbuildignore", + "type" = "blob" + ), + list( + "name" = ".Renviron", + "type" = "blob" + ), + list( + "name" = ".Rprofile", + "type" = "blob" + ), + list( + "name" = ".covrignore", + "type" = "blob" + ), + list( + "name" = "renv", + "type" = "tree" + ), + list( + "name" = "tests", + "type" = "tree" + ), + list( + "name" = "vignettes", + "type" = "tree" + ) + ) + ) + ) + ) ) diff --git a/tests/testthat/helper.R b/tests/testthat/helper.R index bfa57bca..d8f5d027 100644 --- a/tests/testthat/helper.R +++ b/tests/testthat/helper.R @@ -1,13 +1,12 @@ #' Create GitStats object for tests -create_test_gitstats <- function( - hosts = 0, - priv_mode = FALSE, - inject_repos = NULL, - inject_commits = NULL, - inject_files = NULL, - inject_users = NULL, - inject_package_usage = NULL -) { +create_test_gitstats <- function(hosts = 0, + priv_mode = FALSE, + inject_repos = NULL, + inject_commits = NULL, + inject_files = NULL, + inject_files_structure = NULL, + inject_users = NULL, + inject_package_usage = NULL) { test_gitstats <- create_gitstats() if (hosts == 1) { @@ -41,6 +40,9 @@ create_test_gitstats <- function( if (!is.null(inject_files)) { test_gitstats$.__enclos_env__$private$storage$files <- test_mocker$use(inject_files) } + if (!is.null(inject_files_structure)) { + test_gitstats$.__enclos_env__$private$storage$files_structure <- test_mocker$use(inject_files_structure) + } if (!is.null(inject_users)) { test_gitstats$.__enclos_env__$private$storage$users <- test_mocker$use(inject_users) } diff --git a/tests/testthat/setup.R b/tests/testthat/setup.R index 67dd4a1b..235256a8 100644 --- a/tests/testthat/setup.R +++ b/tests/testthat/setup.R @@ -1,15 +1,54 @@ test_mocker <- Mocker$new() -test_team <- list( - "Member1" = list( - logins = "galachad" - ), - "Member2" = list( - logins = "Cotau" - ), - "Member3" = list( - logins = c("maciekbanas", "banasm") - ) +test_gitstats <- create_test_gitstats(hosts = 2) +test_gitstats_priv <- create_test_gitstats(hosts = 2, priv_mode = TRUE) + +test_gqlquery_gh <- GQLQueryGitHub$new() +test_gqlquery_gl <- GQLQueryGitLab$new() + +test_rest_github <- EngineRestGitHub$new( + rest_api_url = "https://api.github.com", + token = Sys.getenv("GITHUB_PAT") +) +test_rest_github_priv <- environment(test_rest_github$initialize)$private + +test_graphql_github <- EngineGraphQLGitHub$new( + gql_api_url = "https://api.github.com/graphql", + token = Sys.getenv("GITHUB_PAT") +) +test_graphql_github_priv <- environment(test_graphql_github$initialize)$private + +test_rest_gitlab <- EngineRestGitLab$new( + rest_api_url = "https://gitlab.com/api/v4", + token = Sys.getenv("GITLAB_PAT_PUBLIC") +) +test_rest_gitlab_priv <- environment(test_rest_gitlab$initialize)$private + +test_graphql_gitlab <- EngineGraphQLGitLab$new( + gql_api_url = "https://gitlab.com/api/graphql", + token = Sys.getenv("GITLAB_PAT_PUBLIC") +) +test_graphql_gitlab_priv <- environment(test_graphql_gitlab$initialize)$private + +github_testhost <- create_github_testhost(orgs = "r-world-devs") + +github_testhost_priv <- create_github_testhost(orgs = "r-world-devs", mode = "private") + +github_testhost_repos <- create_github_testhost( + repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder") +) + +github_testhost_repos_priv <- create_github_testhost( + repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder"), + mode = "private" +) + +gitlab_testhost <- create_gitlab_testhost(orgs = "mbtests") + +gitlab_testhost_priv <- create_gitlab_testhost(orgs = "mbtests", mode = "private") + +gitlab_testhost_repos <- create_gitlab_testhost( + repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2") ) test_settings <- list( @@ -22,29 +61,26 @@ test_settings_silence <- list( storage = TRUE ) -test_settings_repo <- list( - searching_scope = "repo", - verbose = TRUE, - storage = TRUE -) - if (nchar(Sys.getenv("GITHUB_PAT")) == 0) { cli::cli_abort(c( "x" = "You did not set up your GITHUB_PAT environment variable.", - "i" = "If you wish to run tests for GitHub - set up your GITHUB_PAT enviroment variable (as a GitHub access token on github.com)." + "i" = "If you wish to run tests for GitHub - set up your GITHUB_PAT + enviroment variable (as a GitHub access token on github.com)." )) } if (nchar(Sys.getenv("GITHUB_PAT")) > 0) { tryCatch({ - httr2::request("https://api.github.com") %>% - httr2::req_headers("Authorization" = paste0("Bearer ", Sys.getenv("GITHUB_PAT"))) %>% - httr2::req_perform() - }, + httr2::request("https://api.github.com") %>% + httr2::req_headers("Authorization" = paste0("Bearer ", Sys.getenv("GITHUB_PAT"))) %>% + httr2::req_perform() + }, error = function(e) { if (grepl("401", e$message)) { cli::cli_abort(c( - "x" = "Your GITHUB_PAT enviroment variable does not grant access. Please setup your GITHUB_PAT before running tests.", - "i" = "If you wish to run tests for GitHub - set up your GITHUB_PAT enviroment variable (as a GitHub access token on github.com)." + "x" = "Your GITHUB_PAT enviroment variable does not grant access. Please + setup your GITHUB_PAT before running tests.", + "i" = "If you wish to run tests for GitHub - set up your GITHUB_PAT + enviroment variable (as a GitHub access token on github.com)." )) } }) @@ -52,20 +88,24 @@ if (nchar(Sys.getenv("GITHUB_PAT")) > 0) { if (nchar(Sys.getenv("GITLAB_PAT_PUBLIC")) == 0) { cli::cli_abort(c( "x" = "You did not set up your GITLAB_PAT_PUBLIC environment variable.", - "i" = "If you wish to run tests for GitLab - set up your GITLAB_PAT_PUBLIC enviroment variable (as a GitLab access token on gitlab.com)." + "i" = "If you wish to run tests for GitLab - set up your GITLAB_PAT_PUBLIC + enviroment variable (as a GitLab access token on gitlab.com)." )) } if (nchar(Sys.getenv("GITLAB_PAT_PUBLIC")) > 0) { tryCatch({ httr2::request("https://gitlab.com/api/v4/projects") %>% - httr2::req_headers("Authorization" = paste0("Bearer ", Sys.getenv("GITLAB_PAT_PUBLIC"))) %>% + httr2::req_headers("Authorization" = paste0("Bearer ", + Sys.getenv("GITLAB_PAT_PUBLIC"))) %>% httr2::req_perform() }, error = function(e) { if (grepl("401", e$message)) { cli::cli_abort(c( - "x" = "Your GITLAB_PAT_PUBLIC enviroment variable does not grant access. Please setup your GITLAB_PAT_PUBLIC before running tests.", - "i" = "If you wish to run tests for GitLab - set up your GITLAB_PAT_PUBLIC enviroment variable (as a GitLab access token on gitlab.com)." + "x" = "Your GITLAB_PAT_PUBLIC enviroment variable does not grant access. + Please setup your GITLAB_PAT_PUBLIC before running tests.", + "i" = "If you wish to run tests for GitLab - set up your GITLAB_PAT_PUBLIC + enviroment variable (as a GitLab access token on gitlab.com)." )) } }) diff --git a/tests/testthat/test-01-GQLQueryGitHub.R b/tests/testthat/test-01-GQLQueryGitHub.R deleted file mode 100644 index 986863c1..00000000 --- a/tests/testthat/test-01-GQLQueryGitHub.R +++ /dev/null @@ -1,50 +0,0 @@ -test_gqlquery_gh <- GQLQueryGitHub$new() - -test_that("commits_by_repo query is built properly", { - gh_commits_by_repo_query <- - test_gqlquery_gh$commits_by_repo( - org = "r-world-devs", - repo = "GitStats", - since = "2023-01-01T00:00:00Z", - until = "2023-02-28T00:00:00Z" - ) - expect_snapshot( - gh_commits_by_repo_query - ) - test_mocker$cache(gh_commits_by_repo_query) -}) - -test_that("repos_by_org query is built properly", { - gh_repos_by_org_query <- - test_gqlquery_gh$repos_by_org() - expect_snapshot( - gh_repos_by_org_query - ) - test_mocker$cache(gh_repos_by_org_query) -}) - -test_that("user query is built properly", { - gh_user_query <- - test_gqlquery_gh$user() - expect_snapshot( - gh_user_query - ) - test_mocker$cache(gh_user_query) -}) - - -test_that("file query is built properly", { - gh_files_query <- - test_gqlquery_gh$files_by_repo() - expect_snapshot( - gh_files_query - ) -}) - -test_that("releases query is built properly", { - gh_releases_query <- - test_gqlquery_gh$releases_from_repo() - expect_snapshot( - gh_releases_query - ) -}) diff --git a/tests/testthat/test-01-GQLQueryGitLab.R b/tests/testthat/test-01-GQLQueryGitLab.R deleted file mode 100644 index a827db63..00000000 --- a/tests/testthat/test-01-GQLQueryGitLab.R +++ /dev/null @@ -1,35 +0,0 @@ -test_gqlquery_gl <- GQLQueryGitLab$new() - -test_that("repos queries are built properly", { - gl_repos_by_org_query <- - test_gqlquery_gl$repos_by_org() - expect_snapshot( - gl_repos_by_org_query - ) - test_mocker$cache(gl_repos_by_org_query) -}) - -test_that("user query is built properly", { - gl_user_query <- - test_gqlquery_gl$user() - expect_snapshot( - gl_user_query - ) - test_mocker$cache(gl_user_query) -}) - -test_that("file queries are built properly", { - gl_files_query <- - test_gqlquery_gl$files_by_org() - expect_snapshot( - gl_files_query - ) -}) - -test_that("releases query is built properly", { - gl_releases_query <- - test_gqlquery_gl$releases_from_repo() - expect_snapshot( - gl_releases_query - ) -}) diff --git a/tests/testthat/test-02-EngineGraphQL.R b/tests/testthat/test-02-EngineGraphQL.R deleted file mode 100644 index 8d2c6f15..00000000 --- a/tests/testthat/test-02-EngineGraphQL.R +++ /dev/null @@ -1,92 +0,0 @@ -### GitHub - -test_gql <- EngineGraphQL$new( - gql_api_url = "https://api.github.com/graphql", - token = Sys.getenv("GITHUB_PAT") -) - -# public methods - -test_that("`gql_response()` work as expected for GitHub", { - gh_commits_by_repo_gql_response <- test_gql$gql_response( - test_mocker$use("gh_commits_by_repo_query") - ) - expect_gh_commit_gql_response( - gh_commits_by_repo_gql_response$data$repository$defaultBranchRef$target$history$edges[[1]] - ) - test_mocker$cache(gh_commits_by_repo_gql_response) - - gh_repos_by_org_gql_response <- test_gql$gql_response( - test_mocker$use("gh_repos_by_org_query"), - vars = list(org = "r-world-devs") - ) - expect_gh_repos_gql_response( - gh_repos_by_org_gql_response - ) - test_mocker$cache(gh_repos_by_org_gql_response) - - gh_user_gql_response <- test_gql$gql_response( - test_mocker$use("gh_user_query"), - vars = list(user = "maciekbanas") - ) - expect_user_gql_response( - gh_user_gql_response - ) - test_mocker$cache(gh_user_gql_response) -}) - -test_that("pull_user pulls GitHub user response", { - mockery::stub( - test_gql$pull_user, - "self$gql_response", - test_mocker$use("gh_user_gql_response") - ) - gh_user_response <- test_gql$pull_user(username = "maciekbanas") - expect_user_gql_response( - gh_user_response - ) - test_mocker$cache(gh_user_response) -}) - -### GitLab - -test_gql <- EngineGraphQL$new( - gql_api_url = "https://gitlab.com/api/graphql", - token = Sys.getenv("GITLAB_PAT_PUBLIC") -) - -# public methods - -test_that("`gql_response()` work as expected for GitLab", { - gl_repos_by_org_gql_response <- test_gql$gql_response( - gql_query = test_mocker$use("gl_repos_by_org_query"), - vars = list(org = "mbtests") - ) - expect_gl_repos_gql_response( - gl_repos_by_org_gql_response - ) - test_mocker$cache(gl_repos_by_org_gql_response) - - gl_user_gql_response <- test_gql$gql_response( - test_mocker$use("gl_user_query"), - vars = list(user = "maciekbanas") - ) - expect_user_gql_response( - gl_user_gql_response - ) - test_mocker$cache(gl_user_gql_response) -}) - - -test_that("pull_user pulls GitLab user response", { - mockery::stub( - test_gql$pull_user, - "self$gql_response", - test_mocker$use("gl_user_gql_response") - ) - gl_user_response <- test_gql$pull_user(username = "maciekbanas") - expect_user_gql_response( - gl_user_response - ) - test_mocker$cache(gl_user_response) -}) diff --git a/tests/testthat/test-02-EngineRest.R b/tests/testthat/test-02-EngineRest.R deleted file mode 100644 index 82c6cacb..00000000 --- a/tests/testthat/test-02-EngineRest.R +++ /dev/null @@ -1,92 +0,0 @@ -test_rest <- TestEngineRest$new( - rest_api_url = "https://api.github.com", - token = Sys.getenv("GITHUB_PAT") -) - -# private methods - -test_rest_priv <- environment(test_rest$response)$private - -test_that("`perform_request()` returns proper status when token is empty or invalid", { - wrong_tokens <- c("", "bad_token") - purrr::walk( - wrong_tokens, - ~ expect_message( - test_rest_priv$perform_request( - endpoint = "https://api.github.com/org/openpharma", - token = . - ), - "HTTP 401 Unauthorized." - ) - ) -}) - -test_that("`perform_request()` throws error on bad host", { - bad_host <- "https://github.bad_host.com" - expect_error( - suppressMessages( - test_rest_priv$perform_request( - endpoint = paste0(bad_host), - token = Sys.getenv("GITHUB_PAT") - ) - ), - "Could not resolve host" - ) -}) - -test_that("`perform_request()` returns proper status", { - bad_endpoint <- "https://api.github.com/orgs/everybody_loves_somebody" - expect_error( - test_rest_priv$perform_request( - endpoint = bad_endpoint, - token = Sys.getenv("GITHUB_PAT") - ), - "HTTP 404 Not Found" - ) -}) - -# public methods - -test_that("`response()` returns search response from GitHub's REST API", { - search_endpoint <- "https://api.github.com/search/code?q=shiny+user:openpharma" - test_mocker$cache(search_endpoint) - gh_search_response_raw <- test_rest$response(search_endpoint) - expect_gh_search_response(gh_search_response_raw[["items"]]) - test_mocker$cache(gh_search_response_raw) -}) - -test_that("`response()` returns search response from GitHub's REST API", { - search_endpoint <- "https://api.github.com/search/code?q=shiny+user:openpharma+in:file+filename:DESCRIPTION" - gh_search_response_in_file <- test_rest$response(search_endpoint)[["items"]] - expect_gh_search_response(gh_search_response_in_file) - test_mocker$cache(gh_search_response_in_file) -}) - -test_rest <- create_testrest( - rest_api_url = "https://gitlab.com/api/v4", - token = Sys.getenv("GITLAB_PAT_PUBLIC") -) - -test_that("`response()` returns responses from GitLab's REST API", { - gl_search_response <- test_rest$response( - "https://gitlab.com/api/v4/groups/9970/search?scope=blobs&search=git" - ) - expect_gl_search_response(gl_search_response) - test_mocker$cache(gl_search_response) - - gl_commits_rest_response_repo_1 <- test_rest$response( - "https://gitlab.com/api/v4/projects/44293594/repository/commits?since='2023-01-01T00:00:00'&until='2023-04-20T00:00:00'&with_stats=true" - ) - expect_gl_commit_rest_response( - gl_commits_rest_response_repo_1 - ) - test_mocker$cache(gl_commits_rest_response_repo_1) - - gl_commits_rest_response_repo_2 <- test_rest$response( - "https://gitlab.com/api/v4/projects/44346961/repository/commits?since='2023-01-01T00:00:00'&until='2023-04-20T00:00:00'&with_stats=true" - ) - expect_gl_commit_rest_response( - gl_commits_rest_response_repo_2 - ) - test_mocker$cache(gl_commits_rest_response_repo_2) -}) diff --git a/tests/testthat/test-03-EngineGraphQLGitHub.R b/tests/testthat/test-03-EngineGraphQLGitHub.R deleted file mode 100644 index c7d1c3d7..00000000 --- a/tests/testthat/test-03-EngineGraphQLGitHub.R +++ /dev/null @@ -1,149 +0,0 @@ -test_graphql_github <- EngineGraphQLGitHub$new( - gql_api_url = "https://api.github.com/graphql", - token = Sys.getenv("GITHUB_PAT") -) - -# private methods - -test_graphql_github <- environment(test_graphql_github$initialize)$private - -test_that("`pull_commits_page_from_repo()` pulls commits page from repository", { - mockery::stub( - test_graphql_github$pull_commits_page_from_repo, - "self$gql_response", - test_mocker$use("gh_commits_by_repo_gql_response") - ) - commits_page <- test_graphql_github$pull_commits_page_from_repo( - org = "r-world-devs", - repo = "GitStats", - since = "2023-01-01", - until = "2023-02-28" - ) - expect_gh_commit_gql_response( - commits_page$data$repository$defaultBranchRef$target$history$edges[[1]] - ) - test_mocker$cache(commits_page) -}) - -test_that("`pull_repos_page_from_org()` pulls repos page from GitHub organization", { - mockery::stub( - test_graphql_github$pull_repos_page, - "self$gql_response", - test_mocker$use("gh_repos_by_org_gql_response") - ) - gh_repos_page <- test_graphql_github$pull_repos_page( - org = "r-world-devs" - ) - expect_gh_repos_gql_response( - gh_repos_page - ) - test_mocker$cache(gh_repos_page) -}) - -test_that("`pull_commits_from_one_repo()` prepares formatted list", { - # overcome of infinite loop in pull_commits_from_repo - commits_page <- test_mocker$use("commits_page") - commits_page$data$repository$defaultBranchRef$target$history$pageInfo$hasNextPage <- FALSE - - mockery::stub( - test_graphql_github$pull_commits_from_one_repo, - "private$pull_commits_page_from_repo", - commits_page - ) - commits_from_repo <- test_graphql_github$pull_commits_from_one_repo( - org = "r-world-devs", - repo = "GitStats", - since = "2023-01-01", - until = "2023-02-28" - ) - expect_gh_commit_gql_response( - commits_from_repo[[1]] - ) - test_mocker$cache(commits_from_repo) -}) - -# public methods - -test_graphql_github <- EngineGraphQLGitHub$new( - gql_api_url = "https://api.github.com/graphql", - token = Sys.getenv("GITHUB_PAT") -) - -test_that("`pull_repos_from_org()` prepares formatted list", { - mockery::stub( - test_graphql_github$pull_repos_from_org, - "private$pull_repos_page", - test_mocker$use("gh_repos_page") - ) - gh_repos_from_org <- test_graphql_github$pull_repos_from_org( - org = "r-world-devs" - ) - expect_list_contains( - gh_repos_from_org[[1]], - c( - "id", "name", "stars", "forks", "created_at", - "last_activity_at", "languages", "issues_open", "issues_closed", - "contributors", "repo_url" - ) - ) - test_mocker$cache(gh_repos_from_org) -}) - -test_that("`pull_commits_from_repos()` pulls commits from repos", { - commits_from_repos <- test_graphql_github$pull_commits_from_repos( - org = "r-world-devs", - repo = "GitStats", - since = "2023-01-01", - until = "2023-02-28", - verbose = FALSE - ) - expect_gh_commit_gql_response( - commits_from_repos[[1]][[1]] - ) - test_mocker$cache(commits_from_repos) -}) - -test_that("`pull_releases_from_org()` pulls releases from the repositories", { - releases_from_repos <- test_graphql_github$pull_release_logs_from_org( - repos_names = c("GitStats", "shinyCohortBuilder"), - org = "r-world-devs" - ) - expect_github_releases_response(releases_from_repos) - test_mocker$cache(releases_from_repos) -}) - -test_that("GitHub GraphQL Engine pulls files from organization", { - github_files_response <- test_graphql_github$pull_files_from_org( - org = "r-world-devs", - repos = NULL, - file_path = "LICENSE" - ) - expect_github_files_response(github_files_response) - test_mocker$cache(github_files_response) -}) - -test_that("GitHub GraphQL Engine pulls files from defined repositories", { - github_files_response <- test_graphql_github$pull_files_from_org( - org = "openpharma", - repos = c("DataFakeR", "visR"), - file_path = "README.md" - ) - expect_github_files_response(github_files_response) - expect_equal(length(github_files_response[["README.md"]]), 2) -}) - -test_that("GitHub GraphQL Engine pulls two files from a group", { - github_files_response <- test_graphql_github$pull_files_from_org( - org = "r-world-devs", - repos = NULL, - file_path = c("DESCRIPTION", "NAMESPACE") - ) - expect_github_files_response(github_files_response) - expect_true( - all( - c("DESCRIPTION", "NAMESPACE") %in% - names(github_files_response) - ) - ) -}) - diff --git a/tests/testthat/test-03-EngineGraphQLGitLab.R b/tests/testthat/test-03-EngineGraphQLGitLab.R deleted file mode 100644 index 5265f6d9..00000000 --- a/tests/testthat/test-03-EngineGraphQLGitLab.R +++ /dev/null @@ -1,120 +0,0 @@ -test_gql_gl <- EngineGraphQLGitLab$new( - gql_api_url = "https://gitlab.com/api/graphql", - token = Sys.getenv("GITLAB_PAT_PUBLIC") -) - -# private methods - -test_gql_gl <- environment(test_gql_gl$initialize)$private - -test_that("`pull_repos_page()` pulls repos page from GitLab group", { - mockery::stub( - test_gql_gl$pull_repos_page, - "self$gql_response", - test_mocker$use("gl_repos_by_org_gql_response") - ) - gl_repos_page <- test_gql_gl$pull_repos_page( - org = "mbtests" - ) - expect_gl_repos_gql_response( - gl_repos_page - ) - test_mocker$cache(gl_repos_page) -}) - -# public methods - -test_gql_gl <- EngineGraphQLGitLab$new( - gql_api_url = "https://gitlab.com/api/graphql", - token = Sys.getenv("GITLAB_PAT_PUBLIC") -) - -test_that("`pull_repos_from_org()` prepares formatted list", { - mockery::stub( - test_gql_gl$pull_repos_from_org, - "private$pull_repos_page", - test_mocker$use("gl_repos_page") - ) - gl_repos_from_org <- test_gql_gl$pull_repos_from_org( - org = "mbtests" - ) - expect_list_contains( - gl_repos_from_org[[1]]$node, - c( - "id", "name", "stars", "forks", "created_at", - "last_activity_at", "languages", "issues", - "repo_url" - ) - ) - test_mocker$cache(gl_repos_from_org) -}) - -test_that("`pull_repos_from_org()` does not fail when GraphQL response is not complete", { - mockery::stub( - test_gql_gl$pull_repos_from_org, - "private$pull_repos_page", - test_fixtures$empty_gql_response - ) - gl_repos_from_org <- test_gql_gl$pull_repos_from_org( - org = "mbtests" - ) - expect_type( - gl_repos_from_org, - "list" - ) - expect_length( - gl_repos_from_org, - 0 - ) - mockery::stub( - test_gql_gl$pull_repos_from_org, - "private$pull_repos_page", - test_fixtures$half_empty_gql_response - ) - gl_repos_from_org <- test_gql_gl$pull_repos_from_org( - org = "mbtests" - ) - expect_type( - gl_repos_from_org, - "list" - ) - expect_length( - gl_repos_from_org, - 0 - ) -}) - -test_that("GitLab GraphQL Engine pulls files from a group", { - gitlab_files_response <- test_gql_gl$pull_files_from_org( - org = "mbtests", - repos = NULL, - file_path = "meta_data.yaml" - ) - expect_gitlab_files_response(gitlab_files_response) - test_mocker$cache(gitlab_files_response) -}) - -test_that("GitLab GraphQL Engine pulls files only from defined projects", { - gitlab_files_response <- test_gql_gl$pull_files_from_org( - org = "mbtests", - repos = c("gitstatstesting", "gitstats-testing-2", "gitstatstesting3"), - file_path = "README.md" - ) - expect_gitlab_files_response(gitlab_files_response) - expect_equal(length(gitlab_files_response), 3) -}) - -test_that("GitLab GraphQL Engine pulls two files from a group", { - gitlab_files_response <- test_gql_gl$pull_files_from_org( - org = "mbtests", - repos = NULL, - file_path = c("meta_data.yaml", "README.md") - ) - expect_gitlab_files_response(gitlab_files_response) - expect_true( - all( - c("meta_data.yaml", "README.md") %in% - purrr::map_vec(gitlab_files_response, ~.$repository$blobs$nodes[[1]]$name) - ) - ) -}) diff --git a/tests/testthat/test-03-EngineRestGitHub.R b/tests/testthat/test-03-EngineRestGitHub.R deleted file mode 100644 index 7c0edba2..00000000 --- a/tests/testthat/test-03-EngineRestGitHub.R +++ /dev/null @@ -1,79 +0,0 @@ -test_rest <- EngineRestGitHub$new( - rest_api_url = "https://api.github.com", - token = Sys.getenv("GITHUB_PAT") -) - -# private methods - -test_rest_priv <- environment(test_rest$initialize)$private - -test_that("`search_response()` performs search with limit under 100", { - total_n <- test_mocker$use("gh_search_response_raw")[["total_count"]] - mockery::stub( - test_rest_priv$search_response, - "private$rest_response", - test_mocker$use("gh_search_response_raw") - ) - gh_search_repos_response <- test_rest_priv$search_response( - search_endpoint = test_mocker$use("search_endpoint"), - total_n = total_n, - byte_max = 384000 - ) - expect_gh_search_response(gh_search_repos_response) - test_mocker$cache(gh_search_repos_response) -}) - -test_that("Mapping search result to repositories works", { - suppressMessages( - result <- test_rest_priv$map_search_into_repos( - search_response = test_mocker$use("gh_search_repos_response") - ) - ) - expect_gh_repos_rest_response(result) -}) - -# public methods - -test_that("`pull_repos_by_code()` returns repos output for code search in files", { - mockery::stub( - test_rest$pull_repos_by_code, - "private$search_response", - test_mocker$use("gh_search_response_in_file") - ) - gh_repos_by_code <- test_rest$pull_repos_by_code( - code = "shiny", - filename = "DESCRIPTION", - org = "openpharma", - verbose = FALSE - ) - expect_gh_repos_rest_response(gh_repos_by_code) - test_mocker$cache(gh_repos_by_code) -}) - -test_that("`pull_repos_by_code()` for GitHub prepares a raw (raw_output = TRUE) search response", { - mockery::stub( - test_rest$pull_repos_by_code, - "private$search_response", - test_mocker$use("gh_search_repos_response") - ) - gh_repos_by_code_raw <- test_rest$pull_repos_by_code( - code = "shiny", - org = "openpharma", - raw_output = TRUE, - verbose = FALSE - ) - expect_gh_search_response(gh_repos_by_code_raw) - test_mocker$cache(gh_repos_by_code_raw) -}) - -test_that("pull_repos_urls() works", { - gh_repos_urls <- test_rest$pull_repos_urls( - type = "web", - org = "r-world-devs" - ) - expect_gt( - length(gh_repos_urls), - 0 - ) - test_mocker$cache(gh_repos_urls) -}) diff --git a/tests/testthat/test-03-EngineRestGitLab.R b/tests/testthat/test-03-EngineRestGitLab.R deleted file mode 100644 index a2a2a720..00000000 --- a/tests/testthat/test-03-EngineRestGitLab.R +++ /dev/null @@ -1,75 +0,0 @@ -test_rest <- EngineRestGitLab$new( - rest_api_url = "https://gitlab.com/api/v4", - token = Sys.getenv("GITLAB_PAT_PUBLIC") -) - -# private methods - -test_rest_priv <- environment(test_rest$initialize)$private - -test_that("`get_group_id()` gets group's id", { - gl_group_id <- test_rest_priv$get_group_id("mbtests") - expect_equal(gl_group_id, 63684059) -}) - -test_that("`map_search_into_repos()` works", { - gl_search_response <- test_mocker$use("gl_search_response") - suppressMessages( - gl_search_repos_by_code <- test_rest_priv$map_search_into_repos( - gl_search_response - ) - ) - expect_gl_repos_rest_response( - gl_search_repos_by_code - ) - test_mocker$cache(gl_search_repos_by_code) -}) - -test_that("`pull_repos_languages` works", { - repos_list <- test_mocker$use("gl_search_repos_by_code") - repos_list[[1]]$id <- "45300912" - suppressMessages( - repos_list_with_languages <- test_rest_priv$pull_repos_languages( - repos_list = repos_list - ) - ) - purrr::walk(repos_list_with_languages, ~ expect_list_contains(., "languages")) -}) - -# public methods - -test_rest <- EngineRestGitLab$new( - rest_api_url = "https://gitlab.com/api/v4", - token = Sys.getenv("GITLAB_PAT_PUBLIC") -) - -test_that("`pull_commits_from_repos()` pulls commits from repo", { - gl_commits_repo_1 <- test_mocker$use("gl_commits_rest_response_repo_1") - - mockery::stub( - test_rest$pull_commits_from_repos, - "private$pull_commits_from_one_repo", - gl_commits_repo_1 - ) - repos_names <- c("mbtests%2Fgitstatstesting", "mbtests%2Fgitstats-testing-2") - gl_commits_org <- test_rest$pull_commits_from_repos( - repos_names = repos_names, - since = "2023-01-01", - until = "2023-04-20", - verbose = FALSE - ) - purrr::walk(gl_commits_org, ~ expect_gl_commit_rest_response(.)) - test_mocker$cache(gl_commits_org) -}) - -test_that("pull_repos_urls() works", { - gl_repos_urls <- test_rest$pull_repos_urls( - type = "api", - org = "mbtests" - ) - expect_gt( - length(gl_repos_urls), - 0 - ) - test_mocker$cache(gl_repos_urls) -}) diff --git a/tests/testthat/test-04-GitHost.R b/tests/testthat/test-04-GitHost.R deleted file mode 100644 index 50cb5ee9..00000000 --- a/tests/testthat/test-04-GitHost.R +++ /dev/null @@ -1,368 +0,0 @@ -test_host <- create_github_testhost(orgs = "r-world-devs", mode = "private") - -test_that("`set_searching_scope` does not throw error when `orgs` or `repos` are defined", { - expect_snapshot( - test_host$set_searching_scope(orgs = "mbtests", repos = NULL) - ) - expect_snapshot( - test_host$set_searching_scope(orgs = NULL, repos = "mbtests/GitStatsTesting") - ) -}) - -test_that("`extract_repos_and_orgs` extracts fullnames vector into a list of GitLab organizations with assigned repositories", { - repos_fullnames <- c( - "mbtests/gitstatstesting", "mbtests/gitstats-testing-2", "mbtests/subgroup/test-project-in-subgroup" - ) - expect_equal( - test_host$extract_repos_and_orgs(repos_fullnames), - list( - "mbtests" = c("gitstatstesting", "gitstats-testing-2"), - "mbtests/subgroup" = c("test-project-in-subgroup") - ) - ) -}) - -test_that("GitHub tailors precisely `repos_list`", { - gh_repos_by_code <- test_mocker$use("gh_repos_by_code") - gh_repos_by_code_tailored <- - test_host$tailor_repos_response(gh_repos_by_code) - gh_repos_by_code_tailored %>% - expect_type("list") %>% - expect_length(length(gh_repos_by_code)) - expect_list_contains_only( - gh_repos_by_code_tailored[[1]], - c( - "repo_id", "repo_name", "created_at", "last_activity_at", - "forks", "stars", "issues_open", "issues_closed", - "organization" - ) - ) - expect_lt( - length(gh_repos_by_code_tailored[[1]]), - length(gh_repos_by_code[[1]]) - ) - test_mocker$cache(gh_repos_by_code_tailored) -}) - -test_that("`prepare_repos_table()` prepares repos table", { - expect_snapshot( - gh_repos_by_code_table <- test_host$prepare_repos_table_from_rest( - repos_list = test_mocker$use("gh_repos_by_code_tailored") - ) - ) - expect_repos_table( - gh_repos_by_code_table - ) - gh_repos_by_code_table <- test_host$add_repo_api_url(gh_repos_by_code_table) - test_mocker$cache(gh_repos_by_code_table) -}) - -test_that("`prepare_releases_table()` prepares releases table", { - releases_table <- test_host$prepare_releases_table( - releases_response = test_mocker$use("releases_from_repos"), - org = "r-world-devs", - date_from = "2023-05-01", - date_until = "2023-09-30" - ) - expect_releases_table(releases_table) - expect_gt(min(releases_table$published_at), as.POSIXct("2023-05-01")) - expect_lt(max(releases_table$published_at), as.POSIXct("2023-09-30")) - test_mocker$cache(releases_table) -}) - -test_that("`prepare_commits_table()` prepares commits table", { - gh_commits_table <- test_host$prepare_commits_table( - repos_list_with_commits = test_mocker$use("commits_from_repos"), - org = "r-world-devs" - ) - expect_commits_table( - gh_commits_table - ) - test_mocker$cache(gh_commits_table) -}) - -test_that("GitHub prepares user table", { - gh_user_table <- test_host$prepare_user_table( - user_response = test_mocker$use("gh_user_response") - ) - expect_users_table( - gh_user_table, - one_user = TRUE - ) - test_mocker$cache(gh_user_table) -}) - -test_that("get_all_repos_urls prepares api repo_urls vector", { - test_host <- create_github_testhost(orgs = c("r-world-devs", "openpharma"), - mode = "private") - gh_api_repos_urls <- test_host$get_all_repos_urls( - type = "api", - verbose = FALSE - ) - expect_gt(length(gh_api_repos_urls), 0) - expect_true(any(grepl("openpharma", gh_api_repos_urls))) - expect_true(any(grepl("r-world-devs", gh_api_repos_urls))) - expect_true(all(grepl("api", gh_api_repos_urls))) - test_mocker$cache(gh_api_repos_urls) -}) - -test_that("get_all_repos_urls prepares web repo_urls vector", { - test_host <- create_github_testhost(orgs = "r-world-devs", - mode = "private") - mockery::stub( - test_host$get_all_repos_urls, - "rest_engine$pull_repos_urls", - test_mocker$use("gh_repos_urls") - ) - gh_repos_urls <- test_host$get_all_repos_urls( - type = "web", - verbose = FALSE - ) - expect_gt(length(gh_repos_urls), 0) - expect_true(any(grepl("r-world-devs", gh_repos_urls))) - expect_true(all(grepl("https://github.com/", gh_repos_urls))) - test_mocker$cache(gh_repos_urls) -}) - -test_that("`get_repos_with_code_from_orgs()` works", { - mockery::stub( - test_host$get_repos_with_code_from_orgs, - "private$get_repos_response_with_code", - test_mocker$use("gh_repos_by_code") - ) - mockery::stub( - test_host$get_repos_with_code_from_orgs, - "private$prepare_repos_table_from_rest", - test_mocker$use("gh_repos_by_code_table") - ) - repos_with_code <- test_host$get_repos_with_code_from_orgs( - code = "shiny", - verbose = FALSE - ) - expect_repos_table(repos_with_code, with_cols = "api_url") -}) - -test_that("GitHub prepares table from files response", { - gh_files_table <- test_host$prepare_files_table( - files_response = test_mocker$use("github_files_response"), - org = "r-world-devs", - file_path = "LICENSE" - ) - expect_files_table(gh_files_table) - test_mocker$cache(gh_files_table) -}) - -# public methods - -test_host <- create_github_testhost(orgs = "r-world-devs") - - -test_that("`get_commits()` retrieves commits in the table format", { - mockery::stub( - test_host$get_commits, - "private$get_commits_from_host", - test_mocker$use("gh_commits_table") - ) - suppressMessages( - commits_table <- test_host$get_commits( - since = "2023-01-01", - until = "2023-02-28", - settings = test_settings - ) - ) - expect_commits_table( - commits_table - ) -}) - -test_that("`get_files()` pulls files in the table format", { - mockery::stub( - test_host$get_files, - "private$get_files_from_orgs", - test_mocker$use("gh_files_table") - ) - expect_snapshot( - gh_files_table <- test_host$get_files( - file_path = "LICENSE" - ) - ) - expect_files_table(gh_files_table) - test_mocker$cache(gh_files_table) -}) - -test_that("`get_files()` pulls files only for the repositories specified", { - test_host <- create_github_testhost( - repos = c("r-world-devs/GitStats", "openpharma/visR", "openpharma/DataFakeR"), - ) - expect_snapshot( - gh_files_table <- test_host$get_files( - file_path = "renv.lock" - ) - ) - expect_files_table(gh_files_table, with_cols = "api_url") - expect_equal(nrow(gh_files_table), 2) # visR does not have renv.lock -}) - -test_that("`get_release_logs()` pulls release logs in the table format", { - mockery::stub( - test_host$get_release_logs, - "private$prepare_releases_table", - test_mocker$use("releases_table") - ) - releases_table <- test_host$get_release_logs( - since = "2023-05-01", - until = "2023-09-30", - verbose = FALSE, - settings = test_settings - ) - expect_releases_table(releases_table) - expect_gt(min(releases_table$published_at), as.POSIXct("2023-05-01")) - expect_lt(max(releases_table$published_at), as.POSIXct("2023-09-30")) - test_mocker$cache(releases_table) -}) - -# GitLab - private methods - -test_host_gitlab <- create_gitlab_testhost(orgs = "mbtests", mode = "private") - -test_that("`prepare_repos_table()` prepares repos table", { - gl_repos_table <- test_host_gitlab$prepare_repos_table_from_graphql( - repos_list = test_mocker$use("gl_repos_from_org") - ) - expect_repos_table( - gl_repos_table - ) - test_mocker$cache(gl_repos_table) -}) - -test_that("get_all_repos_urls prepares api repo_urls vector", { - mockery::stub( - test_host_gitlab$get_all_repos_urls, - "rest_engine$pull_repos_urls", - test_mocker$use("gl_repos_urls") - ) - gl_api_repos_urls <- test_host_gitlab$get_all_repos_urls( - type = "api", - verbose = FALSE - ) - expect_gt(length(gl_api_repos_urls), 0) - expect_true(all(grepl("api", gl_api_repos_urls))) - test_mocker$cache(gl_api_repos_urls) -}) - -test_that("get_all_repos_urls prepares web repo_urls vector", { - gl_repos_urls <- test_host_gitlab$get_all_repos_urls( - type = "web", - verbose = FALSE - ) - expect_gt(length(gl_repos_urls), 0) - expect_true(all(!grepl("api", gl_repos_urls))) -}) - -test_that("GitLab prepares user table", { - gl_user_table <- test_host_gitlab$prepare_user_table( - user_response = test_mocker$use("gl_user_response") - ) - expect_users_table( - gl_user_table, - one_user = TRUE - ) - test_mocker$cache(gl_user_table) -}) - -test_that("GitLab prepares table from files response", { - gl_files_table <- test_host_gitlab$prepare_files_table( - files_response = test_mocker$use("gitlab_files_response"), - org = "mbtests", - file_path = "meta_data.yaml" - ) - expect_files_table(gl_files_table) - test_mocker$cache(gl_files_table) -}) - -test_that("`tailor_repos_response()` tailors precisely `repos_list`", { - gl_repos_by_code <- test_mocker$use("gl_search_repos_by_code") - - gl_repos_by_code_tailored <- - test_host_gitlab$tailor_repos_response(gl_repos_by_code) - - gl_repos_by_code_tailored %>% - expect_type("list") %>% - expect_length(length(gl_repos_by_code)) - - expect_list_contains_only( - gl_repos_by_code_tailored[[1]], - c( - "repo_id", "repo_name", "created_at", "last_activity_at", - "forks", "stars", "languages", "issues_open", - "issues_closed", "organization" - ) - ) - expect_lt( - length(gl_repos_by_code_tailored[[1]]), - length(gl_repos_by_code[[1]]) - ) - test_mocker$cache(gl_repos_by_code_tailored) -}) - -test_that("GitHost prepares table from GitLab repositories response", { - expect_snapshot( - gl_repos_by_code_table <- test_host_gitlab$prepare_repos_table_from_rest( - repos_list = test_mocker$use("gl_repos_by_code_tailored") - ) - ) - expect_repos_table( - gl_repos_by_code_table - ) - gl_repos_by_code_table <- test_host_gitlab$add_repo_api_url(gl_repos_by_code_table) - test_mocker$cache(gl_repos_by_code_table) -}) - -# public - GitLab - -test_host_gitlab <- create_gitlab_testhost(orgs = "mbtests") - -test_that("`get_files()` pulls files in the table format", { - mockery::stub( - test_host_gitlab$get_files, - "private$get_files_from_orgs", - test_mocker$use("gl_files_table") - ) - expect_snapshot( - gl_files_table <- test_host_gitlab$get_files( - file_path = "README.md" - ) - ) - expect_files_table(gl_files_table) - test_mocker$cache(gl_files_table) -}) - -test_that("`get_files()` pulls two files in the table format", { - expect_snapshot( - gl_files_table <- test_host_gitlab$get_files( - file_path = c("meta_data.yaml", "README.md") - ) - ) - expect_files_table(gl_files_table, with_cols = "api_url") - expect_true( - all(c("meta_data.yaml", "README.md") %in% gl_files_table$file_path) - ) -}) - -test_that("get_users build users table for GitHub", { - users_result <- test_host$get_users( - users = c("maciekbanas", "Cotau", "marcinkowskak") - ) - expect_users_table( - users_result - ) -}) - -test_that("get_users build users table for GitLab", { - users_result <- test_host_gitlab$get_users( - users = c("maciekbanas", "Cotau", "marcinkowskak") - ) - expect_users_table( - users_result - ) -}) diff --git a/tests/testthat/test-05-GitHostGitHub.R b/tests/testthat/test-05-GitHostGitHub.R deleted file mode 100644 index 69609a72..00000000 --- a/tests/testthat/test-05-GitHostGitHub.R +++ /dev/null @@ -1,262 +0,0 @@ -# public methods - -test_host <- create_github_testhost( - orgs = c("openpharma", "r-world-devs") -) - -test_that("GitHost gets users tables", { - users_table <- test_host$get_users( - users = c("maciekbanas", "kalimu", "galachad") - ) - expect_users_table(users_table) - test_mocker$cache(users_table) -}) - -# private methods - -test_host <- create_github_testhost( - orgs = c("openpharma", "r-world-devs"), - mode = "private" -) - -test_that("When token is empty throw error", { - expect_snapshot( - error = TRUE, - test_host$check_token("") - ) -}) - -test_that("`check_token()` prints error when token exists but does not grant access", { - token <- "does_not_grant_access" - expect_snapshot_error( - test_host$check_token(token) - ) -}) - -test_that("when token is proper token is passed", { - expect_equal( - test_host$check_token(Sys.getenv("GITHUB_PAT")), - Sys.getenv("GITHUB_PAT") - ) -}) - -test_that("check_endpoint returns TRUE if they are correct", { - expect_true( - test_host$check_endpoint( - endpoint = "https://api.github.com/repos/r-world-devs/GitStats", - type = "Repository" - ) - ) - expect_true( - test_host$check_endpoint( - endpoint = "https://api.github.com/orgs/openpharma", - ) - ) -}) - -test_that("check_endpoint returns error if they are not correct", { - expect_snapshot_error( - check <- test_host$check_endpoint( - endpoint = "https://api.github.com/repos/r-worlddevs/GitStats", - type = "Repository" - ) - ) -}) - -test_that("`check_if_public` works correctly", { - expect_true( - test_host$check_if_public("api.github.com") - ) - expect_false( - test_host$check_if_public("github.internal.com") - ) -}) - -test_that("`set_default_token` sets default token for public GitHub", { - expect_snapshot( - default_token <- test_host$set_default_token() - ) - test_rest <- create_testrest(token = default_token, - mode = "private") - expect_equal( - test_rest$perform_request( - endpoint = "https://api.github.com", - token = default_token - )$status, - 200 - ) -}) - -test_that("`test_token` works properly", { - expect_true( - test_host$test_token(Sys.getenv("GITHUB_PAT")) - ) - expect_false( - test_host$test_token("false_token") - ) -}) - -test_that("`extract_repos_and_orgs` extracts fullnames vector into a list of GitHub organizations with assigned repositories", { - repos_fullnames <- c( - "r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder", - "openpharma/DataFakeR", "openpharma/GithubMetrics" - ) - expect_equal( - test_host$extract_repos_and_orgs(repos_fullnames), - list( - "r-world-devs" = c("GitStats", "shinyCohortBuilder"), - "openpharma" = c("DataFakeR", "GithubMetrics") - ) - ) -}) - -test_that("GitHub prepares repos table from repositories response", { - gh_repos_table <- test_host$prepare_repos_table_from_graphql( - repos_list = test_mocker$use("gh_repos_from_org") - ) - expect_repos_table( - gh_repos_table - ) - test_mocker$cache(gh_repos_table) -}) - -test_that("GitHost adds `repo_api_url` column to GitHub repos table", { - repos_table <- test_mocker$use("gh_repos_table") - gh_repos_table_with_api_url <- test_host$add_repo_api_url(repos_table) - expect_true(all(grepl("api.github.com", gh_repos_table_with_api_url$api_url))) - test_mocker$cache(gh_repos_table_with_api_url) -}) - -test_that("`get_all_repos()` works as expected", { - mockery::stub( - test_host$get_all_repos, - "private$prepare_repos_table_from_graphql", - test_mocker$use("gh_repos_table_with_api_url") - ) - expect_snapshot( - gh_repos_table <- test_host$get_all_repos() - ) - expect_repos_table( - gh_repos_table, - with_cols = "api_url" - ) - test_mocker$cache(gh_repos_table) -}) - -test_that("get_repo_url_from_response retrieves repositories URLS", { - gh_repo_api_urls <- test_host$get_repo_url_from_response( - search_response = test_mocker$use("gh_search_repos_response"), - type = "api" - ) - expect_type(gh_repo_api_urls, "character") - expect_gt(length(gh_repo_api_urls), 0) - test_mocker$cache(gh_repo_api_urls) - gh_repo_web_urls <- test_host$get_repo_url_from_response( - search_response = test_mocker$use("gh_search_response_in_file"), - type = "web" - ) - expect_type(gh_repo_web_urls, "character") - expect_gt(length(gh_repo_web_urls), 0) - test_mocker$cache(gh_repo_web_urls) -}) - -test_that("get_commits_from_host for GitHub works", { - test_host <- create_github_testhost( - repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder"), - mode = "private" - ) - mockery::stub( - test_host$get_commits_from_host, - "private$prepare_commits_table", - test_mocker$use("gh_commits_table") - ) - suppressMessages( - gh_commits_table <- test_host$get_commits_from_host( - since = "2023-03-01", - until = "2023-04-01", - settings = test_settings_repo - ) - ) - expect_commits_table( - gh_commits_table - ) - test_mocker$cache(gh_commits_table) -}) - -test_that("get_files_from_orgs for GitHub works", { - mockery::stub( - test_host$get_files_from_orgs, - "private$prepare_files_table", - test_mocker$use("gh_files_table") - ) - gh_files_table <- test_host$get_files_from_orgs( - file_path = "DESCRIPTION", - verbose = FALSE - ) - expect_files_table( - gh_files_table, - with_cols = "api_url" - ) - test_mocker$cache(gh_files_table) -}) - -# public methods - -test_host <- create_github_testhost( - orgs = c("openpharma", "r-world-devs") -) - -test_that("get_commits for GitHub works", { - test_host <- create_github_testhost( - repos = c("openpharma/DataFakeR", "r-world-devs/GitStats", "r-world-devs/cohortBuilder") - ) - mockery::stub( - test_host$get_commits, - "private$get_commits_from_host", - test_mocker$use("gh_commits_table") - ) - suppressMessages( - gh_commits_table <- test_host$get_commits( - since = "2023-03-01", - until = "2023-04-01", - settings = test_settings_repo - ) - ) - expect_commits_table( - gh_commits_table - ) -}) - -test_that("get_files for GitHub works", { - mockery::stub( - test_host$get_files, - "private$get_files_from_orgs", - test_mocker$use("gh_files_table") - ) - suppressMessages( - gh_files_table <- test_host$get_files( - file_path = "DESCRIPTION" - ) - ) - expect_files_table( - gh_files_table, - with_cols = "api_url" - ) -}) - -test_that("get_repos_urls returns repositories URLS", { - mockery::stub( - test_host$get_repos_urls, - "private$get_repo_url_from_response", - test_mocker$use("gh_repo_web_urls") - ) - gh_repos_urls_with_code_in_files <- test_host$get_repos_urls( - type = "web", - with_code = "shiny", - in_files = "DESCRIPTION", - verbose = FALSE - ) - expect_type(gh_repos_urls_with_code_in_files, "character") - expect_gt(length(gh_repos_urls_with_code_in_files), 0) - test_mocker$cache(gh_repos_urls_with_code_in_files) -}) diff --git a/tests/testthat/test-05-GitHostGitLab.R b/tests/testthat/test-05-GitHostGitLab.R deleted file mode 100644 index a12b1f5f..00000000 --- a/tests/testthat/test-05-GitHostGitLab.R +++ /dev/null @@ -1,173 +0,0 @@ -# private - -test_host <- create_gitlab_testhost( - orgs = "mbtests", - mode = "private" -) - -test_that("`set_default_token` sets default token for GitLab", { - expect_snapshot( - withr::with_envvar(new = c("GITLAB_PAT" = Sys.getenv("GITLAB_PAT_PUBLIC")), { - default_token <- test_host$set_default_token() - }) - ) - test_rest <- create_testrest(token = default_token, - mode = "private") - expect_equal( - test_rest$perform_request( - endpoint = "https://gitlab.com/api/v4/projects", - token = default_token - )$status, - 200 - ) -}) - -test_that("`set_searching_scope` throws error when both `orgs` and `repos` are defined", { - expect_snapshot_error( - test_host$set_searching_scope( - orgs = "mbtests", - repos = "mbtests/GitStatsTesting" - ) - ) -}) - -test_that("GitHost adds `repo_api_url` column to GitLab repos table", { - repos_table <- test_mocker$use("gl_repos_table") - gl_repos_table_with_api_url <- test_host$add_repo_api_url(repos_table) - expect_true(all(grepl("gitlab.com/api/v4", gl_repos_table_with_api_url$api_url))) - test_mocker$cache(gl_repos_table_with_api_url) -}) - -test_that("`tailor_commits_info()` retrieves only necessary info", { - gl_commits_list <- test_mocker$use("gl_commits_org") - - gl_commits_list_cut <- test_host$tailor_commits_info( - gl_commits_list, - org = "mbtests" - ) - expect_tailored_commits_list( - gl_commits_list_cut[[1]][[1]] - ) - test_mocker$cache(gl_commits_list_cut) -}) - -test_that("`prepare_commits_table()` prepares table of commits properly", { - gl_commits_table <- test_host$prepare_commits_table( - commits_list = test_mocker$use("gl_commits_list_cut") - ) - expect_commits_table( - gl_commits_table, - exp_auth = FALSE - ) - test_mocker$cache(gl_commits_table) -}) - -test_that("`get_repo_url_from_response()` works", { - suppressMessages( - gl_repo_web_urls <- test_host$get_repo_url_from_response( - search_response = test_mocker$use("gl_search_response"), - type = "web" - ) - ) - expect_gt(length(gl_repo_web_urls), 0) - expect_type(gl_repo_web_urls, "character") - test_mocker$cache(gl_repo_web_urls) -}) - -test_that("get_commits_from_host works", { - mockery::stub( - test_host$get_commits_from_host, - "rest_engine$pull_commits_from_repos", - test_mocker$use("gl_commits_org") - ) - suppressMessages( - gl_commits_table <- test_host$get_commits_from_host( - since = "2023-03-01", - until = "2023-04-01", - settings = test_settings - ) - ) - expect_commits_table( - gl_commits_table - ) - test_mocker$cache(gl_commits_table) -}) - -test_that("get_files_from_orgs for GitLab works", { - mockery::stub( - test_host$get_files_from_orgs, - "private$prepare_files_table", - test_mocker$use("gl_files_table") - ) - suppressMessages( - gl_files_table <- test_host$get_files_from_orgs( - file_path = "meta_data.yaml", - verbose = FALSE - ) - ) - expect_files_table( - gl_files_table, with_cols = "api_url" - ) - test_mocker$cache(gl_files_table) -}) - -# public - -test_host <- create_gitlab_testhost( - orgs = c("mbtests") -) - -test_that("get_commits for GitLab works with repos implied", { - test_host <- create_gitlab_testhost( - repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2") - ) - mockery::stub( - test_host$get_commits, - "private$get_commits_from_host", - test_mocker$use("gl_commits_table") - ) - expect_snapshot( - gl_commits_table <- test_host$get_commits( - since = "2023-01-01", - until = "2023-06-01", - settings = test_settings_repo - ) - ) - expect_commits_table( - gl_commits_table - ) -}) - -test_that("get_files for GitLab works", { - mockery::stub( - test_host$get_files, - "private$get_files_from_orgs", - test_mocker$use("gl_files_table") - ) - suppressMessages( - gl_files_table <- test_host$get_files( - file_path = "meta_data.yaml" - ) - ) - expect_files_table( - gl_files_table, with_cols = "api_url" - ) - test_mocker$cache(gl_files_table) -}) - -test_that("get_repos_urls returns repositories URLS", { - mockery::stub( - test_host$get_repos_urls, - "private$get_repo_url_from_response", - test_mocker$use("gl_repo_web_urls") - ) - gl_repos_urls_with_code_in_files <- test_host$get_repos_urls( - type = "web", - with_code = "shiny", - in_files = "DESCRIPTION", - verbose = FALSE - ) - expect_type(gl_repos_urls_with_code_in_files, "character") - expect_gt(length(gl_repos_urls_with_code_in_files), 0) - test_mocker$cache(gl_repos_urls_with_code_in_files) -}) diff --git a/tests/testthat/test-06-EngineRestGitHub-contributors.R b/tests/testthat/test-06-EngineRestGitHub-contributors.R deleted file mode 100644 index 0af13bfb..00000000 --- a/tests/testthat/test-06-EngineRestGitHub-contributors.R +++ /dev/null @@ -1,41 +0,0 @@ -test_rest <- EngineRestGitHub$new( - rest_api_url = "https://api.github.com", - token = Sys.getenv("GITHUB_PAT") -) - -test_that("`pull_repos_issues()` adds issues to repos table", { - - gh_repos_by_code_table <- test_mocker$use("gh_repos_by_code_table") - suppressMessages( - gh_repos_by_code_table <- test_rest$pull_repos_issues( - gh_repos_by_code_table - ) - ) - expect_gt( - length(gh_repos_by_code_table$issues_open), - 0 - ) - expect_gt( - length(gh_repos_by_code_table$issues_closed), - 0 - ) - test_mocker$cache(gh_repos_by_code_table) -}) - -test_that("`pull_repos_contributors()` adds contributors to repos table", { - expect_snapshot( - gh_repos_by_code_table <- test_rest$pull_repos_contributors( - repos_table = test_mocker$use("gh_repos_by_code_table"), - settings = test_settings - ) - ) - expect_repos_table( - gh_repos_by_code_table, - with_cols = c("api_url", "contributors") - ) - expect_gt( - length(gh_repos_by_code_table$contributors), - 0 - ) - test_mocker$cache(gh_repos_by_code_table) -}) diff --git a/tests/testthat/test-06-EngineRestGitLab-contributors.R b/tests/testthat/test-06-EngineRestGitLab-contributors.R deleted file mode 100644 index 147e67e2..00000000 --- a/tests/testthat/test-06-EngineRestGitLab-contributors.R +++ /dev/null @@ -1,54 +0,0 @@ -test_rest <- EngineRestGitLab$new( - rest_api_url = "https://gitlab.com/api/v4", - token = Sys.getenv("GITLAB_PAT_PUBLIC") -) - -test_that("`pull_repos_issues()` adds issues to repos table", { - gl_repos_by_code_table <- test_mocker$use("gl_repos_by_code_table") - suppressMessages( - gl_repos_by_code_table <- test_rest$pull_repos_issues( - gl_repos_by_code_table - ) - ) - expect_gt( - length(gl_repos_by_code_table$issues_open), - 0 - ) - expect_gt( - length(gl_repos_by_code_table$issues_closed), - 0 - ) - test_mocker$cache(gl_repos_by_code_table) -}) - -test_that("`pull_repos_contributors()` adds contributors to repos table", { - expect_snapshot( - gl_repos_table_with_contributors <- test_rest$pull_repos_contributors( - test_mocker$use("gl_repos_table_with_api_url"), - settings = test_settings - ) - ) - expect_repos_table( - gl_repos_table_with_contributors, - with_cols = c("api_url", "contributors") - ) - expect_gt( - length(gl_repos_table_with_contributors$contributors), - 0 - ) - test_mocker$cache(gl_repos_table_with_contributors) -}) - -test_that("`get_commits_authors_handles_and_names()` adds author logis and names to commits table", { - expect_snapshot( - gl_commits_table <- test_rest$get_commits_authors_handles_and_names( - commits_table = test_mocker$use("gl_commits_table"), - verbose = TRUE - ) - ) - expect_commits_table( - gl_commits_table, - exp_auth = TRUE - ) - test_mocker$cache(gl_commits_table) -}) diff --git a/tests/testthat/test-07-GitStats.R b/tests/testthat/test-07-GitStats.R deleted file mode 100644 index ab51636b..00000000 --- a/tests/testthat/test-07-GitStats.R +++ /dev/null @@ -1,421 +0,0 @@ -test_gitstats <- create_gitstats() - -test_that("GitStats object is created", { - expect_s3_class(test_gitstats, "GitStats") -}) - -# print method - -test_that("GitStats prints empty fields.", { - expect_snapshot(test_gitstats) -}) - -test_gitstats <- create_test_gitstats(hosts = 2) - -test_that("GitStats prints the proper info when connections are added.", { - expect_snapshot(test_gitstats) -}) - -test_that("GitStats prints the proper info when repos are passed instead of orgs.", { - suppressMessages( - test_gitstats <- create_gitstats() %>% - set_github_host( - token = Sys.getenv("GITHUB_PAT"), - repos = c("r-world-devs/GitStats", "openpharma/GithubMetrics") - ) %>% - set_gitlab_host( - token = Sys.getenv("GITLAB_PAT_PUBLIC"), - repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2") - ) - ) - expect_snapshot(test_gitstats) -}) - -# private methods -test_gitstats_priv <- create_test_gitstats(hosts = 0, priv_mode = TRUE) - -test_that("check_for_host returns error when no hosts are passed", { - expect_snapshot_error( - test_gitstats_priv$check_for_host() - ) -}) - -test_that("check_params_conflict returns error", { - expect_snapshot_error( - test_gitstats_priv$check_params_conflict( - with_code = NULL, - with_files = NULL, - in_files = "DESCRIPTION" - ) - ) - expect_snapshot_error( - test_gitstats_priv$check_params_conflict( - with_code = "shiny", - with_files = "DESCRIPTION", - in_files = NULL - ) - ) -}) - -test_gitstats_priv <- create_test_gitstats(hosts = 2, priv_mode = TRUE) - -test_that("get_repos_urls_from_hosts gets data from the hosts", { - mockery::stub( - test_gitstats_priv$get_repos_urls_from_hosts, - "host$get_repos_urls", - c(test_mocker$use("gh_api_repos_urls"), test_mocker$use("gl_api_repos_urls")) - ) - repos_urls_from_hosts <- test_gitstats_priv$get_repos_urls_from_hosts( - type = "api", - with_code = NULL, - in_files = NULL, - with_files = NULL, - verbose = FALSE - ) - expect_type(repos_urls_from_hosts, "character") - expect_gt(length(repos_urls_from_hosts), 0) - expect_true(any(grepl("gitlab.com/api", repos_urls_from_hosts))) - expect_true(any(grepl("api.github", repos_urls_from_hosts))) - test_mocker$cache(repos_urls_from_hosts) -}) - -test_that("get_repos_urls_from_hosts gets data with_code in_files from the hosts", { - mockery::stub( - test_gitstats_priv$get_repos_urls_from_hosts, - "private$get_repos_urls_from_host_with_code", - c(test_mocker$use("gh_repos_urls_with_code_in_files"), test_mocker$use("gl_repos_urls_with_code_in_files")) - ) - repos_urls_from_hosts_with_code_in_files <- test_gitstats_priv$get_repos_urls_from_hosts( - type = "api", - with_code = "shiny", - in_files = "DESCRIPTION", - with_files = NULL, - verbose = FALSE - ) - expect_type(repos_urls_from_hosts_with_code_in_files, "character") - expect_gt(length(repos_urls_from_hosts_with_code_in_files), 0) - expect_true(any(grepl("gitlab.com", repos_urls_from_hosts_with_code_in_files))) - expect_true(any(grepl("github.com", repos_urls_from_hosts_with_code_in_files))) - test_mocker$cache(repos_urls_from_hosts_with_code_in_files) -}) - -test_that("set_object_class works correctly", { - repos_urls <- test_gitstats_priv$set_object_class( - object = test_mocker$use("repos_urls_from_hosts_with_code_in_files"), - class = "repos_urls", - attr_list = list( - "type" = "api", - "with_code" = "shiny", - "in_files" = c("NAMESPACE", "DESCRIPTION"), - "with_files" = NULL - ) - ) - expect_s3_class(repos_urls, "repos_urls") - expect_equal(attr(repos_urls, "type"), "api") - expect_equal(attr(repos_urls, "with_code"), "shiny") - expect_equal(attr(repos_urls, "in_files"), c("NAMESPACE", "DESCRIPTION")) -}) - -test_that("get_repos_table works", { - mockery::stub( - test_gitstats_priv$get_repos_table, - "host$get_repos", - purrr::list_rbind(list( - test_mocker$use("gh_repos_table_with_api_url"), - test_mocker$use("gl_repos_table_with_api_url") - )) - ) - repos_table <- test_gitstats_priv$get_repos_table( - with_code = NULL, - in_files = NULL, - with_files = NULL, - verbose = FALSE, - settings = test_settings - ) - expect_repos_table( - repos_table, - repo_cols = repo_gitstats_colnames - ) -}) - -test_that("get_repos_table with_code works", { - mockery::stub( - test_gitstats_priv$get_repos_table, - "private$get_repos_from_host_with_code", - purrr::list_rbind( - list(test_mocker$use("gh_repos_by_code_table"), - test_mocker$use("gl_repos_by_code_table")) - ) - ) - repos_table <- test_gitstats_priv$get_repos_table( - with_code = "shiny", - in_files = "DESCRIPTION", - with_files = NULL, - verbose = FALSE, - settings = test_settings - ) - expect_repos_table( - repos_table, - repo_cols = repo_gitstats_colnames, - with_cols = c("contributors", "contributors_n") - ) - test_mocker$cache(repos_table) -}) - -test_that("set_object_class for repos_table works correctly", { - repos_table <- test_gitstats_priv$set_object_class( - object = test_mocker$use("repos_table"), - class = "repos_table", - attr_list = list( - "with_code" = NULL, - "in_files" = NULL, - "with_files" = "renv.lock" - ) - ) - expect_s3_class(repos_table, "repos_table") - expect_equal(attr(repos_table, "with_files"), "renv.lock") - test_mocker$cache(repos_table) -}) - -test_that("get_R_package_as_dependency work correctly", { - mockery::stub( - test_gitstats_priv$get_R_package_as_dependency, - "private$get_repos_table", - test_mocker$use("repos_table") - ) - R_package_as_dependency <- test_gitstats_priv$get_R_package_as_dependency( - package_name = "shiny", - verbose = FALSE - ) - expect_s3_class( - R_package_as_dependency, - "data.frame" - ) - expect_gt( - nrow(R_package_as_dependency), - 0 - ) - test_mocker$cache(R_package_as_dependency) -}) - -test_that("get_R_package_usage_table works as expected", { - test_gitstats <- create_test_gitstats(hosts = 2, priv_mode = TRUE) - mockery::stub( - test_gitstats$get_R_package_usage_table, - "private$get_R_package_as_dependency", - test_mocker$use("R_package_as_dependency") - ) - mockery::stub( - test_gitstats$get_R_package_usage_table, - "private$get_R_package_loading", - test_mocker$use("R_package_as_dependency") - ) - R_package_usage_table <- test_gitstats$get_R_package_usage_table( - package_name = "shiny", only_loading = FALSE, verbose = FALSE - ) - expect_package_usage_table(R_package_usage_table) - test_mocker$cache(R_package_usage_table) -}) - -test_that("get_release_logs_table works as expected", { - test_gitstats <- create_test_gitstats(hosts = 2, priv_mode = TRUE) - mockery::stub( - test_gitstats$get_release_logs_table, - "host$get_release_logs", - test_mocker$use("releases_table") - ) - release_logs_table <- test_gitstats$get_release_logs_table( - since = "2023-08-01", - until = "2023-09-30", - verbose = FALSE - ) - expect_releases_table(release_logs_table) -}) - -test_that("check_if_args_changed", { - test_gitstats <- create_test_gitstats( - hosts = 2, - priv_mode = TRUE, - inject_repos = "repos_table" - ) - check <- test_gitstats$check_if_args_changed( - storage = "repositories", - args_list = list( - "with_code" = "shiny", - "in_files" = "DESCRIPTION", - "with_files" = NULL - ) - ) - # repos_table from mock should be set to with_files = 'renv.lock' - expect_true( - check - ) - check <- test_gitstats$check_if_args_changed( - storage = "repositories", - args_list = list( - "with_code" = NULL, - "in_files" = NULL, - "with_files" = "renv.lock" - ) - ) - expect_false( - check - ) -}) - -# public methods - -test_that("GitStats get users info", { - test_gitstats <- create_test_gitstats(hosts = 2) - users_result <- test_gitstats$get_users( - c("maciekbanas", "kalimu", "marcinkowskak"), - verbose = FALSE - ) - expect_users_table( - users_result - ) -}) - -test_that("get_repos works properly and for the second time uses cache", { - test_gitstats <- create_test_gitstats(hosts = 2) - mockery::stub( - test_gitstats$get_repos, - "private$get_repos_table", - test_mocker$use("repos_table") - ) - repos_table <- test_gitstats$get_repos(verbose = FALSE) - expect_repos_table_object( - repos_object = repos_table, - with_cols = c("contributors", "contributors_n") - ) - test_mocker$cache(repos_table) - expect_snapshot( - repos_table <- test_gitstats$get_repos() - ) -}) - -test_that("get_repos pulls repositories without contributors", { - test_gitstats <- create_test_gitstats(hosts = 2) - repos_table <- test_gitstats$get_repos(add_contributors = FALSE, verbose = FALSE) - expect_repos_table(repos_table, repo_cols = repo_gitstats_colnames) - expect_false("contributors" %in% names(repos_table)) -}) - -test_that("get_commits works properly", { - test_gitstats <- create_test_gitstats(hosts = 2) - mockery::stub( - test_gitstats$get_commits, - "private$get_commits_table", - purrr::list_rbind( - list( - test_mocker$use("gh_commits_table"), - test_mocker$use("gl_commits_table") - ) - ) - ) - suppressMessages( - commits_table <- test_gitstats$get_commits( - since = "2023-06-15", - until = "2023-06-30", - verbose = FALSE - ) - ) - expect_commits_table( - commits_table - ) - test_mocker$cache(commits_table) -}) - -test_that("get_files works properly", { - test_gitstats <- create_test_gitstats(hosts = 2) - mockery::stub( - test_gitstats$get_files, - "private$get_files_table", - purrr::list_rbind( - list( - test_mocker$use("gh_files_table"), - test_mocker$use("gl_files_table") - ) - ) - ) - files_table <- test_gitstats$get_files( - file_path = "meta_data.yaml", - verbose = FALSE - ) - expect_files_table( - files_table, - with_cols = "api_url" - ) - test_mocker$cache(files_table) -}) - -test_that("show_orgs print orgs properly", { - test_gitstats <- create_test_gitstats(hosts = 2) - expect_equal( - test_gitstats$show_orgs(), - c("r-world-devs", "mbtests") - ) -}) - -suppressMessages( - test_gitstats <- create_gitstats() %>% - set_gitlab_host( - token = Sys.getenv("GITLAB_PAT_PUBLIC"), - orgs = "mbtests/subgroup" - ) -) - -test_that("show_orgs print subgroups properly", { - expect_equal( - test_gitstats$show_orgs(), - "mbtests/subgroup" - ) -}) - -test_that("subgroups are cleanly printed in GitStats", { - expect_snapshot( - test_gitstats - ) -}) - -test_that("get_repos_urls gets vector of repository URLS", { - test_gitstats <- create_test_gitstats(hosts = 2) - mockery::stub( - test_gitstats$get_repos_urls, - "private$get_repos_urls_from_hosts", - test_mocker$use("repos_urls_from_hosts") - ) - repo_urls <- test_gitstats$get_repos_urls( - verbose = FALSE - ) - expect_type( - repo_urls, - "character" - ) - expect_gt( - length(repo_urls), - 1 - ) -}) - -test_that("get_repos_urls gets vector of repository URLS", { - test_gitstats <- create_test_gitstats(hosts = 2) - mockery::stub( - test_gitstats$get_repos_urls, - "private$get_repos_urls_from_hosts", - test_mocker$use("repos_urls_from_hosts_with_code_in_files") - ) - repo_urls <- test_gitstats$get_repos_urls( - with_code = "shiny", - in_files = "DESCRIPTION", - verbose = FALSE - ) - expect_type( - repo_urls, - "character" - ) - expect_gt( - length(repo_urls), - 1 - ) -}) diff --git a/tests/testthat/test-api-requests.R b/tests/testthat/test-api-requests.R new file mode 100644 index 00000000..59db3b48 --- /dev/null +++ b/tests/testthat/test-api-requests.R @@ -0,0 +1,63 @@ +test_that("`perform_request()` returns proper status when token is empty or invalid", { + wrong_tokens <- c("", "bad_token") + purrr::walk( + wrong_tokens, + ~ expect_message( + test_rest_github_priv$perform_request( + endpoint = "https://api.github.com/org/openpharma", + token = . + ), + "HTTP 401 Unauthorized." + ) + ) +}) + +test_that("`perform_request()` throws error on bad host", { + bad_host <- "https://github.bad_host.com" + expect_error( + suppressMessages( + test_rest_github_priv$perform_request( + endpoint = paste0(bad_host), + token = Sys.getenv("GITHUB_PAT") + ) + ), + "Could not resolve host" + ) +}) + +test_that("`perform_request()` returns proper status", { + bad_endpoint <- "https://api.github.com/orgs/everybody_loves_somebody" + expect_error( + test_rest_github_priv$perform_request( + endpoint = bad_endpoint, + token = Sys.getenv("GITHUB_PAT") + ), + "HTTP 404 Not Found" + ) +}) + +test_that("`perform_request()` returns status 200", { + response <- test_rest_github_priv$perform_request( + endpoint = "https://api.github.com/repos/r-world-devs/GitStats", + token = Sys.getenv("GITHUB_PAT") + ) + expect_equal( + response$status_code, + 200 + ) +}) + +test_that("`perform_request()` for GraphQL returns status 200", { + response <- test_graphql_github_priv$perform_request( + gql_query = "{ + viewer { + login + } + }", + vars = NULL + ) + expect_equal( + response$status_code, + 200 + ) +}) diff --git a/tests/testthat/test-get_commits-GitHub.R b/tests/testthat/test-get_commits-GitHub.R new file mode 100644 index 00000000..5c239cb5 --- /dev/null +++ b/tests/testthat/test-get_commits-GitHub.R @@ -0,0 +1,151 @@ +test_that("commits_by_repo GitHub query is built properly", { + gh_commits_by_repo_query <- + test_gqlquery_gh$commits_by_repo( + org = "r-world-devs", + repo = "GitStats", + since = "2023-01-01T00:00:00Z", + until = "2023-02-28T00:00:00Z" + ) + expect_snapshot( + gh_commits_by_repo_query + ) + test_mocker$cache(gh_commits_by_repo_query) +}) + +test_that("GitHub GraphQL API returns commits response", { + gh_commits_by_repo_gql_response <- test_graphql_github$gql_response( + test_mocker$use("gh_commits_by_repo_query") + ) + expect_gh_commit_gql_response( + gh_commits_by_repo_gql_response$data$repository$defaultBranchRef$target$history$edges[[1]] + ) + test_mocker$cache(gh_commits_by_repo_gql_response) +}) + +test_that("`get_commits_page_from_repo()` pulls commits page from repository", { + mockery::stub( + test_graphql_github_priv$get_commits_page_from_repo, + "self$gql_response", + test_mocker$use("gh_commits_by_repo_gql_response") + ) + commits_page <- test_graphql_github_priv$get_commits_page_from_repo( + org = "r-world-devs", + repo = "GitStats", + since = "2023-01-01", + until = "2023-02-28" + ) + expect_gh_commit_gql_response( + commits_page$data$repository$defaultBranchRef$target$history$edges[[1]] + ) + test_mocker$cache(commits_page) +}) + +test_that("`get_commits_from_one_repo()` prepares formatted list", { + # overcome of infinite loop in get_commits_from_repo + commits_page <- test_mocker$use("commits_page") + commits_page$data$repository$defaultBranchRef$target$history$pageInfo$hasNextPage <- FALSE + + mockery::stub( + test_graphql_github_priv$get_commits_from_one_repo, + "private$get_commits_page_from_repo", + commits_page + ) + commits_from_repo <- test_graphql_github_priv$get_commits_from_one_repo( + org = "r-world-devs", + repo = "GitStats", + since = "2023-01-01", + until = "2023-02-28" + ) + expect_gh_commit_gql_response( + commits_from_repo[[1]] + ) + test_mocker$cache(commits_from_repo) +}) + +test_that("`get_commits_from_repos()` pulls commits from repos", { + mockery::stub( + test_graphql_github$get_commits_from_repos, + "private$get_commits_from_one_repo", + test_mocker$use("commits_from_repo") + ) + commits_from_repos <- test_graphql_github$get_commits_from_repos( + org = "r-world-devs", + repo = "GitStats", + since = "2023-01-01", + until = "2023-02-28", + progress = FALSE + ) + expect_gh_commit_gql_response( + commits_from_repos[[1]][[1]] + ) + test_mocker$cache(commits_from_repos) +}) + +test_that("`prepare_commits_table()` prepares commits table", { + gh_commits_table <- github_testhost_priv$prepare_commits_table( + repos_list_with_commits = test_mocker$use("commits_from_repos"), + org = "r-world-devs" + ) + expect_commits_table( + gh_commits_table + ) + test_mocker$cache(gh_commits_table) +}) + +test_that("get_commits_from_orgs for GitHub works", { + mockery::stub( + github_testhost_repos_priv$get_commits_from_orgs, + "private$prepare_commits_table", + test_mocker$use("gh_commits_table") + ) + suppressMessages( + gh_commits_table <- github_testhost_repos_priv$get_commits_from_orgs( + since = "2023-03-01", + until = "2023-04-01", + verbose = FALSE, + progress = FALSE + ) + ) + expect_commits_table( + gh_commits_table + ) + test_mocker$cache(gh_commits_table) +}) + +test_that("`get_commits()` retrieves commits in the table format", { + mockery::stub( + github_testhost$get_commits, + "private$get_commits_from_orgs", + test_mocker$use("gh_commits_table") + ) + suppressMessages( + commits_table <- github_testhost$get_commits( + since = "2023-01-01", + until = "2023-02-28", + verbose = FALSE, + progress = FALSE + ) + ) + expect_commits_table( + commits_table + ) +}) + +test_that("get_commits for GitHub repositories works", { + mockery::stub( + github_testhost_repos$get_commits, + "private$get_commits_from_orgs", + test_mocker$use("gh_commits_table") + ) + suppressMessages( + gh_commits_table <- github_testhost_repos$get_commits( + since = "2023-03-01", + until = "2023-04-01", + verbose = FALSE, + progress = FALSE + ) + ) + expect_commits_table( + gh_commits_table + ) +}) diff --git a/tests/testthat/test-get_commits-GitLab.R b/tests/testthat/test-get_commits-GitLab.R new file mode 100644 index 00000000..32b9fa6f --- /dev/null +++ b/tests/testthat/test-get_commits-GitLab.R @@ -0,0 +1,110 @@ +test_that("GitLab REST API returns commits response", { + gl_commits_rest_response_repo_1 <- test_rest_gitlab$response( + "https://gitlab.com/api/v4/projects/44293594/repository/commits?since='2023-01-01T00:00:00'&until='2023-04-20T00:00:00'&with_stats=true" + ) + expect_gl_commit_rest_response( + gl_commits_rest_response_repo_1 + ) + test_mocker$cache(gl_commits_rest_response_repo_1) + + gl_commits_rest_response_repo_2 <- test_rest_gitlab$response( + "https://gitlab.com/api/v4/projects/44346961/repository/commits?since='2023-01-01T00:00:00'&until='2023-04-20T00:00:00'&with_stats=true" + ) + expect_gl_commit_rest_response( + gl_commits_rest_response_repo_2 + ) + test_mocker$cache(gl_commits_rest_response_repo_2) +}) + +test_that("`get_commits_from_repos()` pulls commits from repo", { + gl_commits_repo_1 <- test_mocker$use("gl_commits_rest_response_repo_1") + + mockery::stub( + test_rest_gitlab$get_commits_from_repos, + "private$get_commits_from_one_repo", + gl_commits_repo_1 + ) + repos_names <- c("mbtests%2Fgitstatstesting", "mbtests%2Fgitstats-testing-2") + gl_commits_org <- test_rest_gitlab$get_commits_from_repos( + repos_names = repos_names, + since = "2023-01-01", + until = "2023-04-20", + progress = FALSE + ) + purrr::walk(gl_commits_org, ~ expect_gl_commit_rest_response(.)) + test_mocker$cache(gl_commits_org) +}) + +test_that("`tailor_commits_info()` retrieves only necessary info", { + gl_commits_list <- test_mocker$use("gl_commits_org") + + gl_commits_list_cut <- gitlab_testhost_priv$tailor_commits_info( + gl_commits_list, + org = "mbtests" + ) + expect_tailored_commits_list( + gl_commits_list_cut[[1]][[1]] + ) + test_mocker$cache(gl_commits_list_cut) +}) + +test_that("`prepare_commits_table()` prepares table of commits properly", { + gl_commits_table <- gitlab_testhost_priv$prepare_commits_table( + commits_list = test_mocker$use("gl_commits_list_cut") + ) + expect_commits_table( + gl_commits_table, + exp_auth = FALSE + ) + test_mocker$cache(gl_commits_table) +}) + +test_that("get_commits_from_orgs works", { + mockery::stub( + gitlab_testhost_priv$get_commits_from_orgs, + "rest_engine$get_commits_from_repos", + test_mocker$use("gl_commits_org") + ) + suppressMessages( + gl_commits_table <- gitlab_testhost_priv$get_commits_from_orgs( + since = "2023-03-01", + until = "2023-04-01", + verbose = FALSE, + progress = FALSE + ) + ) + expect_commits_table( + gl_commits_table + ) + test_mocker$cache(gl_commits_table) +}) + +test_that("get_commits for GitLab works with repos implied", { + mockery::stub( + gitlab_testhost_repos$get_commits, + "private$get_commits_from_orgs", + test_mocker$use("gl_commits_table") + ) + gl_commits_table <- gitlab_testhost_repos$get_commits( + since = "2023-01-01", + until = "2023-06-01", + verbose = FALSE + ) + expect_commits_table( + gl_commits_table + ) +}) + +test_that("`get_commits_authors_handles_and_names()` adds author logis and names to commits table", { + expect_snapshot( + gl_commits_table <- test_rest_gitlab$get_commits_authors_handles_and_names( + commits_table = test_mocker$use("gl_commits_table"), + verbose = TRUE + ) + ) + expect_commits_table( + gl_commits_table, + exp_auth = TRUE + ) + test_mocker$cache(gl_commits_table) +}) diff --git a/tests/testthat/test-09-get_stats.R b/tests/testthat/test-get_commits-GitStats.R similarity index 59% rename from tests/testthat/test-09-get_stats.R rename to tests/testthat/test-get_commits-GitStats.R index be293579..0a3fa461 100644 --- a/tests/testthat/test-09-get_stats.R +++ b/tests/testthat/test-get_commits-GitStats.R @@ -1,3 +1,29 @@ +# GitStats + +test_that("get_commits works properly", { + mockery::stub( + test_gitstats$get_commits, + "private$get_commits_from_hosts", + purrr::list_rbind( + list( + test_mocker$use("gh_commits_table"), + test_mocker$use("gl_commits_table") + ) + ) + ) + suppressMessages( + commits_table <- test_gitstats$get_commits( + since = "2023-06-15", + until = "2023-06-30", + verbose = FALSE + ) + ) + expect_commits_table( + commits_table + ) + test_mocker$cache(commits_table) +}) + test_gitstats <- create_test_gitstats( hosts = 2, inject_commits = "commits_table" diff --git a/tests/testthat/test-get_files_content-GitHub.R b/tests/testthat/test-get_files_content-GitHub.R new file mode 100644 index 00000000..ae530e54 --- /dev/null +++ b/tests/testthat/test-get_files_content-GitHub.R @@ -0,0 +1,192 @@ +test_that("file queries for GitHub are built properly", { + gh_file_blobs_from_repo_query <- + test_gqlquery_gh$file_blob_from_repo() + expect_snapshot( + gh_file_blobs_from_repo_query + ) + test_mocker$cache(gh_file_blobs_from_repo_query) +}) + +test_that("get_repos_data pulls data on repos and branches", { + repos_data <- test_graphql_github_priv$get_repos_data( + org = "r-world-devs", + repos = NULL + ) + expect_equal( + names(repos_data), + c("repositories", "def_branches") + ) + expect_true( + length(repos_data$repositories) > 0 + ) + expect_true( + length(repos_data$def_branches) > 0 + ) +}) + +test_that("GitHub GraphQL Engine pulls file response", { + mockery::stub( + test_graphql_github_priv$get_file_response, + "self$gql_response", + test_fixtures$github_file_response + ) + github_file_response <- test_graphql_github_priv$get_file_response( + org = "r-world-devs", + repo = "GitStats", + def_branch = "master", + file_path = "LICENSE", + files_query = test_mocker$use("gh_file_blobs_from_repo_query") + ) + expect_github_files_response(github_file_response) + test_mocker$cache(github_file_response) +}) + +test_that("GitHub GraphQL Engine pulls png file response", { + mockery::stub( + test_graphql_github_priv$get_file_response, + "self$gql_response", + test_fixtures$github_png_file_response + ) + github_png_file_response <- test_graphql_github_priv$get_file_response( + org = "r-world-devs", + repo = "GitStats", + def_branch = "master", + file_path = "man/figures/logo.png", + files_query = test_mocker$use("gh_file_blobs_from_repo_query") + ) + expect_github_files_response(github_png_file_response) + test_mocker$cache(github_png_file_response) +}) + +test_that("GitHub GraphQL Engine pulls files from organization", { + mockery::stub( + test_graphql_github$get_files_from_org, + "private$get_file_response", + test_mocker$use("github_file_response") + ) + github_files_response <- test_graphql_github$get_files_from_org( + org = "r-world-devs", + repos = NULL, + file_paths = "LICENSE", + only_text_files = TRUE, + host_files_structure = NULL, + verbose = FALSE, + progress = FALSE + ) + expect_github_files_response(github_files_response) + test_mocker$cache(github_files_response) +}) + +test_that("GitHub GraphQL Engine pulls .png files from organization", { + mockery::stub( + test_graphql_github$get_files_from_org, + "private$get_file_response", + test_mocker$use("github_png_file_response") + ) + github_png_files_response <- test_graphql_github$get_files_from_org( + org = "r-world-devs", + repos = NULL, + file_paths = "man/figures/logo.png", + only_text_files = FALSE, + host_files_structure = NULL, + verbose = FALSE, + progress = FALSE + ) + expect_github_files_response(github_png_files_response) + test_mocker$cache(github_png_files_response) +}) + +test_that("GitHub GraphQL Engine pulls files from defined repositories", { + mockery::stub( + test_graphql_github$get_files_from_org, + "private$get_file_response", + test_mocker$use("github_file_response") + ) + github_files_response <- test_graphql_github$get_files_from_org( + org = "openpharma", + repos = c("DataFakeR", "visR"), + file_paths = "README.md", + host_files_structure = NULL, + only_text_files = TRUE, + verbose = FALSE, + progress = FALSE + ) + expect_github_files_response(github_files_response) + expect_equal(length(github_files_response), 2) +}) + +test_that("GitHub GraphQL Engine pulls two files from a group", { + mockery::stub( + test_graphql_github$get_files_from_org, + "private$get_file_response", + test_mocker$use("github_file_response") + ) + github_files_response <- test_graphql_github$get_files_from_org( + org = "r-world-devs", + repos = NULL, + file_paths = c("DESCRIPTION", "NAMESPACE"), + host_files_structure = NULL, + only_text_files = TRUE, + verbose = FALSE, + progress = FALSE + ) + expect_github_files_response(github_files_response) + purrr::walk(github_files_response, ~ {expect_true( + all( + c("DESCRIPTION", "NAMESPACE") %in% + names(.) + ) + ) + }) +}) + +test_that("GitHubHost prepares table from files response", { + gh_files_table <- github_testhost_priv$prepare_files_table( + files_response = test_mocker$use("github_files_response"), + org = "r-world-devs", + file_path = "LICENSE" + ) + expect_files_table(gh_files_table) + test_mocker$cache(gh_files_table) +}) + +test_that("GitHubHost prepares table from png files (with no content) response", { + gh_png_files_table <- github_testhost_priv$prepare_files_table( + files_response = test_mocker$use("github_png_files_response"), + org = "r-world-devs", + file_path = "man/figures/logo.png" + ) + expect_files_table(gh_png_files_table) + expect_true(all(is.na(gh_png_files_table$file_content))) + test_mocker$cache(gh_png_files_table) +}) + +test_that("get_files_content_from_orgs for GitHub works", { + mockery::stub( + github_testhost_priv$get_files_content_from_orgs, + "private$prepare_files_table", + test_mocker$use("gh_files_table") + ) + gh_files_table <- github_testhost_priv$get_files_content_from_orgs( + file_path = "DESCRIPTION", + verbose = FALSE + ) + expect_files_table( + gh_files_table, + with_cols = "api_url" + ) + test_mocker$cache(gh_files_table) +}) + +test_that("`get_files_content()` pulls files in the table format", { + mockery::stub( + github_testhost$get_files_content, + "private$get_files_content_from_orgs", + test_mocker$use("gh_files_table") + ) + gh_files_table <- github_testhost$get_files_content( + file_path = "DESCRIPTION" + ) + expect_files_table(gh_files_table, with_cols = "api_url") + test_mocker$cache(gh_files_table) +}) diff --git a/tests/testthat/test-get_files_content-GitLab.R b/tests/testthat/test-get_files_content-GitLab.R new file mode 100644 index 00000000..e3f5227d --- /dev/null +++ b/tests/testthat/test-get_files_content-GitLab.R @@ -0,0 +1,171 @@ +test_that("file queries for GitLab are built properly", { + gl_files_query <- + test_gqlquery_gl$files_by_org() + expect_snapshot( + gl_files_query + ) + gl_file_blobs_from_repo_query <- + test_gqlquery_gl$file_blob_from_repo() + expect_snapshot( + gl_file_blobs_from_repo_query + ) + test_mocker$cache(gl_file_blobs_from_repo_query) +}) + +test_that("get_file_blobs_response() works", { + mockery::stub( + test_graphql_gitlab_priv$get_file_blobs_response, + "self$gql_response", + test_fixtures$gitlab_file_repo_response + ) + gl_file_blobs_response <- test_graphql_gitlab_priv$get_file_blobs_response( + org = "mbtests", + repo = "graphql_tests", + file_path = c("project_metadata.yaml", "README.md") + ) + expect_gitlab_files_blob_response(gl_file_blobs_response) + test_mocker$cache(gl_file_blobs_response) +}) + + +test_that("get_repos_data pulls data on repositories", { + repositories <- test_graphql_gitlab_priv$get_repos_data( + org = "mbtests", + repos = NULL + ) + expect_true( + length(repositories) > 0 + ) +}) + +test_that("GitLab GraphQL Engine pulls files from a group", { + mockery::stub( + test_graphql_gitlab$get_files_from_org, + "self$gql_response", + test_fixtures$gitlab_file_org_response + ) + gitlab_files_response <- test_graphql_gitlab$get_files_from_org( + org = "mbtests", + repos = NULL, + file_paths = "meta_data.yaml", + only_text_files = TRUE, + host_files_structure = NULL + ) + expect_gitlab_files_from_org_response(gitlab_files_response) + test_mocker$cache(gitlab_files_response) +}) + +test_that("GitLab GraphQL Engine pulls files from org by iterating over repos", { + mockery::stub( + test_graphql_gitlab$get_files_from_org_per_repo, + "private$get_file_blobs_response", + test_mocker$use("gl_file_blobs_response") + ) + gl_files_from_org <- test_graphql_gitlab$get_files_from_org_per_repo( + org = "mbtests", + repos = "graphql_tests", + file_paths = c("project_metadata.yaml", "README.md") + ) + expect_gitlab_files_from_org_by_repos_response( + response = gl_files_from_org, + expected_files = c("project_metadata.yaml", "README.md") + ) +}) + +test_that("is query error is FALSE when response is empty (non query error)", { + expect_false( + test_graphql_gitlab_priv$is_query_error(list()) + ) +}) + +test_that("Gitlab GraphQL switches to pulling files per repositories when query is too complex", { + mockery::stub( + test_graphql_gitlab$get_files_from_org, + "private$is_query_error", + TRUE + ) + mockery::stub( + test_graphql_gitlab$get_files_from_org, + "private$is_complexity_error", + TRUE + ) + expect_snapshot( + gitlab_files_response_by_repos <- test_graphql_gitlab$get_files_from_org( + org = "mbtests", + repos = NULL, + file_paths = c("DESCRIPTION", "project_metadata.yaml", "README.md"), + host_files_structure = NULL, + only_text_files = TRUE, + verbose = TRUE, + progress = FALSE + ) + ) + expect_gitlab_files_from_org_by_repos_response( + response = gitlab_files_response_by_repos, + expected_files = c("DESCRIPTION", "project_metadata.yaml", "README.md") + ) + test_mocker$cache(gitlab_files_response_by_repos) +}) + +test_that("checker properly identifies gitlab files responses", { + expect_false( + gitlab_testhost_priv$response_prepared_by_iteration( + files_response = test_mocker$use("gitlab_files_response") + ) + ) + expect_true( + gitlab_testhost_priv$response_prepared_by_iteration( + files_response = test_mocker$use("gitlab_files_response_by_repos") + ) + ) +}) + +test_that("GitLab prepares table from files response", { + gl_files_table <- gitlab_testhost_priv$prepare_files_table( + files_response = test_mocker$use("gitlab_files_response"), + org = "mbtests", + file_path = "meta_data.yaml" + ) + expect_files_table(gl_files_table) + test_mocker$cache(gl_files_table) +}) + +test_that("GitLab prepares table from files response prepared in alternative way", { + gl_files_table <- gitlab_testhost_priv$prepare_files_table( + files_response = test_mocker$use("gitlab_files_response_by_repos"), + org = "mbtests", + file_path = "meta_data.yaml" + ) + expect_files_table(gl_files_table) +}) + +test_that("get_files_content_from_orgs for GitLab works", { + mockery::stub( + gitlab_testhost_priv$get_files_content_from_orgs, + "private$prepare_files_table", + test_mocker$use("gl_files_table") + ) + suppressMessages( + gl_files_table <- gitlab_testhost_priv$get_files_content_from_orgs( + file_path = "meta_data.yaml", + verbose = FALSE + ) + ) + expect_files_table( + gl_files_table, with_cols = "api_url" + ) + test_mocker$cache(gl_files_table) +}) + +test_that("`get_files_content()` pulls files in the table format", { + mockery::stub( + gitlab_testhost$get_files_content, + "super$get_files_content", + test_mocker$use("gl_files_table") + ) + gl_files_table <- gitlab_testhost$get_files_content( + file_path = "README.md" + ) + expect_files_table(gl_files_table, with_cols = "api_url") + test_mocker$cache(gl_files_table) +}) diff --git a/tests/testthat/test-get_files_content-GitStats.R b/tests/testthat/test-get_files_content-GitStats.R new file mode 100644 index 00000000..8e80af00 --- /dev/null +++ b/tests/testthat/test-get_files_content-GitStats.R @@ -0,0 +1,21 @@ +test_that("get_files_content works properly", { + mockery::stub( + test_gitstats$get_files_content, + "private$get_files_content_from_hosts", + purrr::list_rbind( + list( + test_mocker$use("gh_files_table"), + test_mocker$use("gl_files_table") + ) + ) + ) + files_table <- test_gitstats$get_files_content( + file_path = "meta_data.yaml", + verbose = FALSE + ) + expect_files_table( + files_table, + with_cols = "api_url" + ) + test_mocker$cache(files_table) +}) diff --git a/tests/testthat/test-get_files_structure-GitHub.R b/tests/testthat/test-get_files_structure-GitHub.R new file mode 100644 index 00000000..a7d52af9 --- /dev/null +++ b/tests/testthat/test-get_files_structure-GitHub.R @@ -0,0 +1,246 @@ +test_that("files tree query for GitHub are built properly", { + gh_files_tree_query <- + test_gqlquery_gh$files_tree_from_repo() + expect_snapshot( + gh_files_tree_query + ) + test_mocker$cache(gh_files_tree_query) +}) + +test_that("get_file_response works", { + mockery::stub( + test_graphql_github_priv$get_file_response, + "self$gql_response", + test_fixtures$github_files_tree_response + ) + gh_files_tree_response <- test_graphql_github_priv$get_file_response( + org = "r-world-devs", + repo = "GitStats", + def_branch = "master", + file_path = "", + files_query = test_mocker$use("gh_files_tree_query") + ) + expect_github_files_raw_response( + gh_files_tree_response + ) + test_mocker$cache(gh_files_tree_response) +}) + +test_that("get_dirs_and_files returns list with directories and files", { + files_and_dirs_list <- test_graphql_github_priv$get_files_and_dirs( + files_tree_response = test_mocker$use("gh_files_tree_response") + ) + expect_type( + files_and_dirs_list, + "list" + ) + expect_list_contains( + files_and_dirs_list, + c("files", "dirs") + ) + test_mocker$cache(files_and_dirs_list) +}) + +test_that("get_files_structure_from_repo returns list with files and dirs vectors", { + files_and_dirs <- test_mocker$use("files_and_dirs_list") + files_and_dirs$dirs <- character() + mockery::stub( + test_graphql_github_priv$get_files_structure_from_repo, + "private$get_files_and_dirs", + files_and_dirs + ) + files_structure <- test_graphql_github_priv$get_files_structure_from_repo( + org = "r-world-devs", + repo = "GitStats", + def_branch = "master" + ) + expect_type( + files_structure, + "character" + ) +}) + +test_that("get_files_structure_from_repo returns list of files up to 2 tier of dirs", { + mockery::stub( + test_graphql_github_priv$get_files_structure_from_repo, + "private$get_files_tree_response", + test_mocker$use("gh_files_tree_response") + ) + files_structure_very_shallow <- test_graphql_github_priv$get_files_structure_from_repo( + org = "r-world-devs", + repo = "GitStats", + def_branch = "master", + depth = 1L + ) + files_structure_shallow <- test_graphql_github_priv$get_files_structure_from_repo( + org = "r-world-devs", + repo = "GitStats", + def_branch = "master", + depth = 2L + ) + expect_type( + files_structure_shallow, + "character" + ) + expect_true( + length(files_structure_very_shallow) < length(files_structure_shallow) + ) + files_structure <- files_structure_shallow + test_mocker$cache(files_structure) +}) + +test_that("only files with certain pattern are retrieved", { + md_files_structure <- test_graphql_github_priv$filter_files_by_pattern( + files_structure = test_mocker$use("files_structure"), + pattern = "\\.md|\\.qmd|\\.Rmd" + ) + files_structure <- test_mocker$use("files_structure") + expect_true( + length(md_files_structure) < length(files_structure) + ) + test_mocker$cache(md_files_structure) +}) + +test_that("GitHub GraphQL Engine pulls files structure from repositories", { + mockery::stub( + test_graphql_github$get_files_structure_from_org, + "private$get_files_structure_from_repo", + test_mocker$use("files_structure") + ) + gh_files_structure <- test_graphql_github$get_files_structure_from_org( + org = "r-world-devs", + repos = c("GitStats") + ) + purrr::walk(gh_files_structure, ~ expect_true(length(.) > 0)) + expect_equal( + names(gh_files_structure), + c("GitStats") + ) + test_mocker$cache(gh_files_structure) +}) + +test_that("GitHub GraphQL Engine pulls files structure with pattern from repositories", { + mockery::stub( + test_graphql_github$get_files_structure_from_org, + "private$get_files_structure_from_repo", + test_mocker$use("md_files_structure") + ) + gh_md_files_structure <- test_graphql_github$get_files_structure_from_org( + org = "r-world-devs", + repos = "GitStats", + pattern = "\\.md|\\.qmd|\\.Rmd" + ) + purrr::walk(gh_md_files_structure, ~ expect_true(all(grepl("\\.md|\\.qmd|\\.Rmd", .)))) +}) + +test_that("get_files_structure_from_orgs pulls files structure for repositories in orgs", { + github_testhost_priv <- create_github_testhost( + repos = c("r-world-devs/GitStats", "openpharma/DataFakeR"), + mode = "private" + ) + mockery::stub( + github_testhost_priv$get_files_structure_from_orgs, + "graphql_engine$get_files_structure_from_org", + test_mocker$use("gh_files_structure") + ) + expect_snapshot( + gh_files_structure_from_orgs <- github_testhost_priv$get_files_structure_from_orgs( + pattern = NULL, + depth = 2L, + verbose = TRUE + ) + ) + expect_equal( + names(gh_files_structure_from_orgs), + c("r-world-devs", "openpharma") + ) + expect_true(any(grepl("\\.md|\\.qmd|\\.Rmd", gh_files_structure_from_orgs[[1]]))) + test_mocker$cache(gh_files_structure_from_orgs) +}) + +test_that("get_orgs_and_repos_from_files_structure", { + result <- github_testhost_priv$get_orgs_and_repos_from_files_structure( + host_files_structure = test_mocker$use("gh_files_structure_from_orgs") + ) + expect_equal( + names(result), + c("orgs", "repos") + ) + purrr::walk(result, ~ expect_true(length(.) > 0)) +}) + +test_that("when files_structure is empty, appropriate message is returned", { + github_testhost_priv <- create_github_testhost( + repos = c("r-world-devs/GitStats", "openpharma/DataFakeR", "openpharma/VisR"), + mode = "private" + ) + mockery::stub( + github_testhost_priv$get_files_structure_from_orgs, + "graphql_engine$get_files_structure_from_org", + list() |> + purrr::set_names() + ) + expect_snapshot( + github_testhost_priv$get_files_structure_from_orgs( + pattern = "\\.png", + depth = 1L, + verbose = TRUE + ) + ) +}) + +test_that("get_path_from_files_structure gets file path from files structure", { + test_graphql_github <- EngineGraphQLGitHub$new( + gql_api_url = "https://api.github.com/graphql", + token = Sys.getenv("GITHUB_PAT") + ) + test_graphql_github <- environment(test_graphql_github$initialize)$private + file_path <- test_graphql_github$get_path_from_files_structure( + host_files_structure = test_mocker$use("gh_files_structure_from_orgs"), + only_text_files = FALSE, + org = "r-world-devs", + repo = "GitStats" + ) + + expect_equal(typeof(file_path), "character") + expect_true(length(file_path) > 0) +}) + +test_that("get_files_structure pulls files structure for repositories in orgs", { + mockery::stub( + github_testhost$get_files_structure, + "private$get_files_structure_from_orgs", + test_mocker$use("gh_files_structure_from_orgs") + ) + gh_files_structure_from_orgs <- github_testhost$get_files_structure( + pattern = "\\.md|\\.qmd", + depth = 1L, + verbose = FALSE + ) + expect_equal( + names(gh_files_structure_from_orgs), + c("r-world-devs", "openpharma") + ) + purrr::walk(gh_files_structure_from_orgs[[2]], function(repo_files) { + expect_true(any(grepl("\\.md|\\.Rmd", repo_files))) + }) + test_mocker$cache(gh_files_structure_from_orgs) +}) + +test_that("get_files_content makes use of files_structure", { + mockery::stub( + github_testhost_priv$get_files_content_from_orgs, + "private$add_repo_api_url", + test_mocker$use("gh_files_table") + ) + expect_snapshot( + files_content <- github_testhost_priv$get_files_content_from_orgs( + file_path = NULL, + host_files_structure = test_mocker$use("gh_files_structure_from_orgs") + ) + ) + expect_files_table( + files_content, + with_cols = "api_url" + ) +}) diff --git a/tests/testthat/test-get_files_structure-GitLab.R b/tests/testthat/test-get_files_structure-GitLab.R new file mode 100644 index 00000000..9c5bf19d --- /dev/null +++ b/tests/testthat/test-get_files_structure-GitLab.R @@ -0,0 +1,245 @@ +test_that("files tree query for GitLab are built properly", { + gl_files_tree_query <- + test_gqlquery_gl$files_tree_from_repo() + expect_snapshot( + gl_files_tree_query + ) + test_mocker$cache(gl_files_tree_query) +}) + +test_that("get_files_tree_response() works", { + mockery::stub( + test_graphql_gitlab_priv$get_files_tree_response, + "self$gql_response", + test_fixtures$gitlab_files_tree_response + ) + gl_files_tree_response <- test_graphql_gitlab_priv$get_files_tree_response( + org = "mbtests", + repo = "graphql_tests", + file_path = "" + ) + expect_gitlab_files_tree_response(gl_files_tree_response) + test_mocker$cache(gl_files_tree_response) +}) + +test_that("get_dirs_and_files() returns list with directories and files", { + gl_files_and_dirs_list <- test_graphql_gitlab_priv$get_files_and_dirs( + files_tree_response = test_mocker$use("gl_files_tree_response") + ) + expect_type( + gl_files_and_dirs_list, + "list" + ) + expect_list_contains( + gl_files_and_dirs_list, + c("files", "dirs") + ) + expect_true( + length(gl_files_and_dirs_list$files) > 0 + ) + expect_true( + length(gl_files_and_dirs_list$dirs) > 0 + ) + test_mocker$cache(gl_files_and_dirs_list) +}) + +test_that("get_files_structure_from_repo() pulls files structure from repo", { + mockery::stub( + test_graphql_gitlab_priv$get_files_structure_from_repo, + "private$get_files_tree_response", + test_mocker$use("gl_files_tree_response") + ) + files_and_dirs <- test_mocker$use("gl_files_and_dirs_list") + files_and_dirs$dirs <- character() + mockery::stub( + test_graphql_gitlab_priv$get_files_structure_from_repo, + "private$get_files_and_dirs", + files_and_dirs + ) + gl_files_structure <- test_graphql_gitlab_priv$get_files_structure_from_repo( + org = "mbtests", + repo = "graphql_tests" + ) + expect_type( + gl_files_structure, + "character" + ) + test_mocker$cache(gl_files_structure) +}) + +test_that("only files with certain pattern are retrieved", { + md_files_structure <- test_graphql_gitlab_priv$filter_files_by_pattern( + files_structure = test_mocker$use("gl_files_structure"), + pattern = "\\.md|\\.qmd|\\.Rmd" + ) + files_structure <- test_mocker$use("files_structure") + expect_true( + length(md_files_structure) < length(files_structure) + ) + test_mocker$cache(md_files_structure) +}) + +test_that("get_files_structure_from_repo() pulls files structure (files matching pattern) from repo", { + mockery::stub( + test_graphql_gitlab_priv$get_files_structure_from_repo, + "private$get_files_tree_response", + test_mocker$use("gl_files_tree_response") + ) + files_and_dirs <- test_mocker$use("gl_files_and_dirs_list") + files_and_dirs$dirs <- character() + mockery::stub( + test_graphql_gitlab_priv$get_files_structure_from_repo, + "private$get_files_and_dirs", + files_and_dirs + ) + mockery::stub( + test_graphql_gitlab_priv$get_files_structure_from_repo, + "private$filter_files_by_pattern", + test_mocker$use("md_files_structure") + ) + gl_md_files_structure <- test_graphql_gitlab_priv$get_files_structure_from_repo( + org = "mbtests", + repo = "graphql_tests", + pattern = "\\.md" + ) + expect_type( + gl_md_files_structure, + "character" + ) + test_mocker$cache(gl_md_files_structure) +}) + +test_that("GitLab GraphQL Engine pulls files structure from repositories", { + mockery::stub( + test_graphql_gitlab$get_files_structure_from_org, + "private$get_files_structure_from_repo", + test_mocker$use("gl_files_structure") + ) + gl_files_structure <- test_graphql_gitlab$get_files_structure_from_org( + org = "mbtests", + repos = c("graphql_tests"), + verbose = FALSE, + progress = FALSE + ) + purrr::walk(gl_files_structure, ~ expect_true(length(.) > 0)) + expect_equal( + names(gl_files_structure), + c("graphql_tests") + ) + purrr::walk(gl_files_structure, ~ expect_false(all(grepl("/$", .)))) # no empty dirs + test_mocker$cache(gl_files_structure) +}) + +test_that("GitLab GraphQL Engine pulls files structure from repositories", { + gl_files_structure_shallow <- test_graphql_gitlab$get_files_structure_from_org( + org = "mbtests", + repos = c("gitstatstesting", "graphql_tests"), + depth = 1L, + verbose = FALSE, + progress = FALSE + ) + purrr::walk(gl_files_structure_shallow, ~ expect_true(length(.) > 0)) + expect_equal( + names(gl_files_structure_shallow), + c("graphql_tests", "gitstatstesting") + ) + purrr::walk(gl_files_structure_shallow, ~ expect_false(all(grepl("/", .)))) # no dirs +}) + +test_that("get_files_structure_from_orgs pulls files structure for repositories in orgs", { + gitlab_testhost_priv <- create_gitlab_testhost( + repos = c("mbtests/gitstatstesting", "mbtests/graphql_tests"), + mode = "private" + ) + expect_snapshot( + gl_files_structure_from_orgs <- gitlab_testhost_priv$get_files_structure_from_orgs( + pattern = "\\.md|\\.R", + depth = 1L, + verbose = TRUE, + progress = FALSE + ) + ) + expect_equal( + names(gl_files_structure_from_orgs), + c("mbtests") + ) + purrr::walk(gl_files_structure_from_orgs[[1]], function(repo_files) { + expect_true(all(grepl("\\.md|\\.R", repo_files))) + }) +}) + +test_that("get_files_structure_from_orgs pulls files structure for all repositories in orgs", { + gitlab_testhost_priv <- create_gitlab_testhost( + orgs = c("mbtests", "mbtestapps"), + mode = "private" + ) + gl_files_structure_from_orgs <- gitlab_testhost_priv$get_files_structure_from_orgs( + pattern = "\\.md|\\.R", + depth = 1L, + verbose = FALSE, + progress = FALSE + ) + expect_equal( + names(gl_files_structure_from_orgs), + c("mbtests", "mbtestapps") + ) + purrr::walk(gl_files_structure_from_orgs[[1]], function(repo_files) { + expect_true(all(grepl("\\.md|\\.R", repo_files))) + }) + test_mocker$cache(gl_files_structure_from_orgs) +}) + +test_that("get_path_from_files_structure gets file path from files structure", { + test_graphql_gitlab <- EngineGraphQLGitLab$new( + gql_api_url = "https://gitlab.com/api/v4/graphql", + token = Sys.getenv("GITHLAB_PAT_PUBLIC") + ) + test_graphql_gitlab <- environment(test_graphql_gitlab$initialize)$private + file_path <- test_graphql_gitlab$get_path_from_files_structure( + host_files_structure = test_mocker$use("gl_files_structure_from_orgs"), + only_text_files = TRUE, + org = "mbtests" # this will need fixing and repo parameter must come back + ) + expect_equal(typeof(file_path), "character") + expect_true(length(file_path) > 0) +}) + +test_that("get_files_structure pulls files structure for repositories in orgs", { + mockery::stub( + gitlab_testhost$get_files_structure, + "private$get_files_structure_from_orgs", + test_mocker$use("gl_files_structure_from_orgs") + ) + gl_files_structure_from_orgs <- gitlab_testhost$get_files_structure( + pattern = "\\.md|\\.R", + depth = 2L, + verbose = FALSE, + progress = FALSE + ) + expect_equal( + names(gl_files_structure_from_orgs), + c("mbtests", "mbtestapps") + ) + purrr::walk(gl_files_structure_from_orgs[[1]], function(repo_files) { + expect_true(all(grepl("\\.md|\\.R", repo_files))) + }) + test_mocker$cache(gl_files_structure_from_orgs) +}) + +test_that("get_files_content makes use of files_structure", { + mockery::stub( + gitlab_testhost_priv$get_files_content_from_orgs, + "private$add_repo_api_url", + test_mocker$use("gl_files_table") + ) + expect_snapshot( + files_content <- gitlab_testhost_priv$get_files_content_from_orgs( + file_path = NULL, + host_files_structure = test_mocker$use("gl_files_structure_from_orgs") + ) + ) + expect_files_table( + files_content, + with_cols = "api_url" + ) +}) diff --git a/tests/testthat/test-get_files_structure-GitStats.R b/tests/testthat/test-get_files_structure-GitStats.R new file mode 100644 index 00000000..2b0da258 --- /dev/null +++ b/tests/testthat/test-get_files_structure-GitStats.R @@ -0,0 +1,49 @@ +test_that("get_files_structure_from_hosts works as expected", { + mockery::stub( + test_gitstats_priv$get_files_structure_from_hosts, + "host$get_files_structure", + test_mocker$use("gh_files_structure_from_orgs") + ) + files_structure_from_hosts <- test_gitstats_priv$get_files_structure_from_hosts( + pattern = "\\md", + depth = 1L, + verbose = FALSE + ) + expect_equal(names(files_structure_from_hosts), + c("github.com", "gitlab.com")) + expect_equal(names(files_structure_from_hosts[[1]]), c("r-world-devs", "openpharma")) + files_structure_from_hosts[[2]] <- test_mocker$use("gl_files_structure_from_orgs") + test_mocker$cache(files_structure_from_hosts) +}) + +test_that("if returned files_structure is empty, do not store it and give proper message", { + mockery::stub( + test_gitstats_priv$get_files_structure_from_hosts, + "host$get_files_structure", + list() + ) + expect_snapshot( + files_structure <- test_gitstats_priv$get_files_structure_from_hosts( + pattern = "\\.png", + depth = 1L, + verbose = TRUE + ) + ) +}) + +test_that("get_files_structure works as expected", { + mockery::stub( + test_gitstats$get_files_structure, + "private$get_files_structure_from_hosts", + test_mocker$use("files_structure_from_hosts") + ) + expect_snapshot( + files_structure <- test_gitstats$get_files_structure( + pattern = "\\.md", + depth = 2L, + verbose = TRUE + ) + ) + expect_s3_class(files_structure, "files_structure") + test_mocker$cache(files_structure) +}) diff --git a/tests/testthat/test-get_release-GitHub.R b/tests/testthat/test-get_release-GitHub.R new file mode 100644 index 00000000..72593599 --- /dev/null +++ b/tests/testthat/test-get_release-GitHub.R @@ -0,0 +1,47 @@ +test_that("releases query is built properly", { + gh_releases_query <- + test_gqlquery_gh$releases_from_repo() + expect_snapshot( + gh_releases_query + ) +}) + +test_that("`get_releases_from_org()` pulls releases from the repositories", { + releases_from_repos <- test_graphql_github$get_release_logs_from_org( + repos_names = c("GitStats", "shinyCohortBuilder"), + org = "r-world-devs" + ) + expect_github_releases_response(releases_from_repos) + test_mocker$cache(releases_from_repos) +}) + +test_that("`prepare_releases_table()` prepares releases table", { + releases_table <- github_testhost_priv$prepare_releases_table( + releases_response = test_mocker$use("releases_from_repos"), + org = "r-world-devs", + date_from = "2023-05-01", + date_until = "2023-09-30" + ) + expect_releases_table(releases_table) + expect_gt(min(releases_table$published_at), as.POSIXct("2023-05-01")) + expect_lt(max(releases_table$published_at), as.POSIXct("2023-09-30")) + test_mocker$cache(releases_table) +}) + +test_that("`get_release_logs()` pulls release logs in the table format", { + mockery::stub( + github_testhost$get_release_logs, + "private$prepare_releases_table", + test_mocker$use("releases_table") + ) + releases_table <- github_testhost$get_release_logs( + since = "2023-05-01", + until = "2023-09-30", + verbose = FALSE, + progress = FALSE + ) + expect_releases_table(releases_table) + expect_gt(min(releases_table$published_at), as.POSIXct("2023-05-01")) + expect_lt(max(releases_table$published_at), as.POSIXct("2023-09-30")) + test_mocker$cache(releases_table) +}) diff --git a/tests/testthat/test-get_release-GitLab.R b/tests/testthat/test-get_release-GitLab.R new file mode 100644 index 00000000..ad1d3918 --- /dev/null +++ b/tests/testthat/test-get_release-GitLab.R @@ -0,0 +1,7 @@ +test_that("releases query is built properly", { + gl_releases_query <- + test_gqlquery_gl$releases_from_repo() + expect_snapshot( + gl_releases_query + ) +}) diff --git a/tests/testthat/test-get_release-GitStats.R b/tests/testthat/test-get_release-GitStats.R new file mode 100644 index 00000000..19ff1dcf --- /dev/null +++ b/tests/testthat/test-get_release-GitStats.R @@ -0,0 +1,14 @@ +test_that("get_release_logs_from_hosts works as expected", { + test_gitstats <- create_test_gitstats(hosts = 2, priv_mode = TRUE) + mockery::stub( + test_gitstats$get_release_logs_from_hosts, + "host$get_release_logs", + test_mocker$use("releases_table") + ) + release_logs_table <- test_gitstats$get_release_logs_from_hosts( + since = "2023-08-01", + until = "2023-09-30", + verbose = FALSE + ) + expect_releases_table(release_logs_table) +}) diff --git a/tests/testthat/test-get_repos-GitHub.R b/tests/testthat/test-get_repos-GitHub.R new file mode 100644 index 00000000..b8923055 --- /dev/null +++ b/tests/testthat/test-get_repos-GitHub.R @@ -0,0 +1,251 @@ +test_that("repos_by_org query is built properly", { + gh_repos_by_org_query <- + test_gqlquery_gh$repos_by_org() + expect_snapshot( + gh_repos_by_org_query + ) + test_mocker$cache(gh_repos_by_org_query) +}) + +test_that("GitHub repos response to query works", { + gh_repos_by_org_gql_response <- test_graphql_github$gql_response( + test_mocker$use("gh_repos_by_org_query"), + vars = list(org = "r-world-devs") + ) + expect_gh_repos_gql_response( + gh_repos_by_org_gql_response + ) + test_mocker$cache(gh_repos_by_org_gql_response) +}) + +test_that("`get_repos_page()` pulls repos page from GitHub organization", { + mockery::stub( + test_graphql_github_priv$get_repos_page, + "self$gql_response", + test_mocker$use("gh_repos_by_org_gql_response") + ) + gh_repos_page <- test_graphql_github_priv$get_repos_page( + org = "r-world-devs" + ) + expect_gh_repos_gql_response( + gh_repos_page + ) + test_mocker$cache(gh_repos_page) +}) + +test_that("`get_repos_from_org()` prepares formatted list", { + mockery::stub( + test_graphql_github$get_repos_from_org, + "private$get_repos_page", + test_mocker$use("gh_repos_page") + ) + gh_repos_from_org <- test_graphql_github$get_repos_from_org( + org = "r-world-devs" + ) + expect_list_contains( + gh_repos_from_org[[1]], + c( + "id", "name", "stars", "forks", "created_at", + "last_activity_at", "languages", "issues_open", "issues_closed", + "contributors", "repo_url" + ) + ) + test_mocker$cache(gh_repos_from_org) +}) + +# REST Engine search repos by code + +test_that("`response()` returns search response from GitHub's REST API", { + # search_endpoint <- "https://api.github.com/search/code?q=shiny+user:r-world-devs" + # test_mocker$cache(search_endpoint) + # gh_search_response_raw <- test_rest_github$response(search_endpoint) + gh_search_response_raw <- test_fixtures$github_search_response + expect_gh_search_response(gh_search_response_raw[["items"]]) + test_mocker$cache(gh_search_response_raw) +}) + +test_that("`response()` returns search response from GitHub's REST API", { + search_endpoint <- "https://api.github.com/search/code?q=shiny+user:openpharma+in:file+filename:DESCRIPTION" + gh_search_response_in_file <- test_rest_github$response(search_endpoint)[["items"]] + expect_gh_search_response(gh_search_response_in_file) + test_mocker$cache(gh_search_response_in_file) +}) + +test_that("`search_response()` performs search with limit under 100", { + total_n <- test_mocker$use("gh_search_response_raw")[["total_count"]] + mockery::stub( + test_rest_github_priv$search_response, + "self$response", + test_mocker$use("gh_search_response_raw") + ) + gh_search_repos_response <- test_rest_github_priv$search_response( + search_endpoint = test_mocker$use("search_endpoint"), + total_n = total_n, + byte_max = 384000 + ) + expect_gh_search_response(gh_search_repos_response) + test_mocker$cache(gh_search_repos_response) +}) + +test_that("Mapping search result to repositories works", { + result <- test_rest_github_priv$map_search_into_repos( + search_response = test_mocker$use("gh_search_repos_response"), + progress = FALSE + ) + expect_gh_repos_rest_response(result) +}) + +test_that("`get_repos_by_code()` returns repos output for code search in files", { + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_response", + test_mocker$use("gh_search_response_in_file") + ) + gh_repos_by_code <- test_rest_github$get_repos_by_code( + code = "shiny", + filename = "DESCRIPTION", + org = "openpharma", + verbose = FALSE + ) + expect_gh_repos_rest_response(gh_repos_by_code) + test_mocker$cache(gh_repos_by_code) +}) + +test_that("`get_repos_by_code()` for GitHub prepares a raw (raw_output = TRUE) search response", { + mockery::stub( + test_rest_github$get_repos_by_code, + "private$search_response", + test_mocker$use("gh_search_repos_response") + ) + gh_repos_by_code_raw <- test_rest_github$get_repos_by_code( + code = "shiny", + org = "openpharma", + raw_output = TRUE, + verbose = FALSE + ) + expect_gh_search_response(gh_repos_by_code_raw) + test_mocker$cache(gh_repos_by_code_raw) +}) + +test_that("GitHub tailors precisely `repos_list`", { + gh_repos_by_code <- test_mocker$use("gh_repos_by_code") + gh_repos_by_code_tailored <- + github_testhost_priv$tailor_repos_response(gh_repos_by_code) + gh_repos_by_code_tailored %>% + expect_type("list") %>% + expect_length(length(gh_repos_by_code)) + expect_list_contains_only( + gh_repos_by_code_tailored[[1]], + c( + "repo_id", "repo_name", "created_at", "last_activity_at", + "forks", "stars", "issues_open", "issues_closed", + "organization" + ) + ) + expect_lt( + length(gh_repos_by_code_tailored[[1]]), + length(gh_repos_by_code[[1]]) + ) + test_mocker$cache(gh_repos_by_code_tailored) +}) + +test_that("`prepare_repos_table()` prepares repos table", { + expect_snapshot( + gh_repos_by_code_table <- github_testhost_priv$prepare_repos_table_from_rest( + repos_list = test_mocker$use("gh_repos_by_code_tailored") + ) + ) + expect_repos_table( + gh_repos_by_code_table + ) + gh_repos_by_code_table <- github_testhost_priv$add_repo_api_url(gh_repos_by_code_table) + test_mocker$cache(gh_repos_by_code_table) +}) + +test_that("`get_repos_with_code_from_orgs()` works", { + mockery::stub( + github_testhost_priv$get_repos_with_code_from_orgs, + "private$get_repos_response_with_code", + test_mocker$use("gh_repos_by_code") + ) + mockery::stub( + github_testhost_priv$get_repos_with_code_from_orgs, + "private$prepare_repos_table_from_rest", + test_mocker$use("gh_repos_by_code_table") + ) + repos_with_code <- github_testhost_priv$get_repos_with_code_from_orgs( + code = "shiny", + verbose = FALSE + ) + expect_repos_table(repos_with_code, with_cols = "api_url") +}) + +test_that("GitHub prepares repos table from repositories response", { + gh_repos_table <- github_testhost_priv$prepare_repos_table_from_graphql( + repos_list = test_mocker$use("gh_repos_from_org") + ) + expect_repos_table( + gh_repos_table + ) + test_mocker$cache(gh_repos_table) +}) + +test_that("GitHost adds `repo_api_url` column to GitHub repos table", { + repos_table <- test_mocker$use("gh_repos_table") + gh_repos_table_with_api_url <- github_testhost_priv$add_repo_api_url(repos_table) + expect_true(all(grepl("api.github.com", gh_repos_table_with_api_url$api_url))) + test_mocker$cache(gh_repos_table_with_api_url) +}) + +test_that("`get_all_repos()` works as expected", { + mockery::stub( + github_testhost_priv$get_all_repos, + "private$prepare_repos_table_from_graphql", + test_mocker$use("gh_repos_table_with_api_url") + ) + expect_snapshot( + gh_repos_table <- github_testhost_priv$get_all_repos() + ) + expect_repos_table( + gh_repos_table, + with_cols = "api_url" + ) + test_mocker$cache(gh_repos_table) +}) + +test_that("`get_repos_issues()` adds issues to repos table", { + gh_repos_by_code_table <- test_mocker$use("gh_repos_by_code_table") + suppressMessages( + gh_repos_by_code_table <- test_rest_github$get_repos_issues( + gh_repos_by_code_table, + progress = FALSE + ) + ) + expect_gt( + length(gh_repos_by_code_table$issues_open), + 0 + ) + expect_gt( + length(gh_repos_by_code_table$issues_closed), + 0 + ) + test_mocker$cache(gh_repos_by_code_table) +}) + +test_that("`get_repos_contributors()` adds contributors to repos table", { + expect_snapshot( + gh_repos_by_code_table <- test_rest_github$get_repos_contributors( + repos_table = test_mocker$use("gh_repos_by_code_table"), + progress = FALSE + ) + ) + expect_repos_table( + gh_repos_by_code_table, + with_cols = c("api_url", "contributors") + ) + expect_gt( + length(gh_repos_by_code_table$contributors), + 0 + ) + test_mocker$cache(gh_repos_by_code_table) +}) diff --git a/tests/testthat/test-get_repos-GitLab.R b/tests/testthat/test-get_repos-GitLab.R new file mode 100644 index 00000000..5105cc40 --- /dev/null +++ b/tests/testthat/test-get_repos-GitLab.R @@ -0,0 +1,202 @@ +test_that("repos queries are built properly", { + gl_repos_by_org_query <- + test_gqlquery_gl$repos_by_org() + expect_snapshot( + gl_repos_by_org_query + ) + test_mocker$cache(gl_repos_by_org_query) +}) + +test_that("GitLab repos response to query works", { + gl_repos_by_org_gql_response <- test_graphql_gitlab$gql_response( + gql_query = test_mocker$use("gl_repos_by_org_query"), + vars = list(org = "mbtests") + ) + expect_gl_repos_gql_response( + gl_repos_by_org_gql_response + ) + test_mocker$cache(gl_repos_by_org_gql_response) +}) + +test_that("`get_repos_page()` pulls repos page from GitLab group", { + mockery::stub( + test_graphql_gitlab_priv$get_repos_page, + "self$gql_response", + test_mocker$use("gl_repos_by_org_gql_response") + ) + gl_repos_page <- test_graphql_gitlab_priv$get_repos_page( + org = "mbtests" + ) + expect_gl_repos_gql_response( + gl_repos_page + ) + test_mocker$cache(gl_repos_page) +}) + +test_that("`get_repos_from_org()` prepares formatted list", { + mockery::stub( + test_graphql_gitlab$get_repos_from_org, + "private$get_repos_page", + test_mocker$use("gl_repos_page") + ) + gl_repos_from_org <- test_graphql_gitlab$get_repos_from_org( + org = "mbtests" + ) + expect_equal( + names(gl_repos_from_org[[1]]$node), + c( + "repo_id", "repo_name", "repo_path", "repository", + "stars", "forks", "created_at", "last_activity_at", + "languages", "issues", "group", "repo_url" + ) + ) + test_mocker$cache(gl_repos_from_org) +}) + +test_that("`get_repos_from_org()` does not fail when GraphQL response is not complete", { + mockery::stub( + test_graphql_gitlab$get_repos_from_org, + "private$get_repos_page", + test_fixtures$empty_gql_response + ) + gl_repos_from_org <- test_graphql_gitlab$get_repos_from_org( + org = "mbtests" + ) + expect_type( + gl_repos_from_org, + "list" + ) + expect_length( + gl_repos_from_org, + 0 + ) + mockery::stub( + test_graphql_gitlab$get_repos_from_org, + "private$get_repos_page", + test_fixtures$half_empty_gql_response + ) + gl_repos_from_org <- test_graphql_gitlab$get_repos_from_org( + org = "mbtests" + ) + expect_type( + gl_repos_from_org, + "list" + ) + expect_length( + gl_repos_from_org, + 0 + ) +}) + +test_that("`map_search_into_repos()` works", { + gl_search_response <- test_fixtures$gitlab_search_response + test_mocker$cache(gl_search_response) + gl_search_repos_by_code <- test_rest_gitlab_priv$map_search_into_repos( + gl_search_response, + progress = FALSE + ) + expect_gl_repos_rest_response( + gl_search_repos_by_code + ) + test_mocker$cache(gl_search_repos_by_code) +}) + +test_that("`pull_repos_languages` works", { + repos_list <- test_mocker$use("gl_search_repos_by_code") + repos_list[[1]]$id <- "45300912" + suppressMessages( + repos_list_with_languages <- test_rest_gitlab_priv$pull_repos_languages( + repos_list = repos_list, + progress = FALSE + ) + ) + purrr::walk(repos_list_with_languages, ~ expect_list_contains(., "languages")) +}) + +test_that("`prepare_repos_table()` prepares repos table", { + gl_repos_table <- gitlab_testhost_priv$prepare_repos_table_from_graphql( + repos_list = test_mocker$use("gl_repos_from_org") + ) + expect_repos_table( + gl_repos_table + ) + test_mocker$cache(gl_repos_table) +}) + +test_that("GitHost adds `repo_api_url` column to GitLab repos table", { + repos_table <- test_mocker$use("gl_repos_table") + gl_repos_table_with_api_url <- gitlab_testhost_priv$add_repo_api_url(repos_table) + expect_true(all(grepl("gitlab.com/api/v4", gl_repos_table_with_api_url$api_url))) + test_mocker$cache(gl_repos_table_with_api_url) +}) + +test_that("`tailor_repos_response()` tailors precisely `repos_list`", { + gl_repos_by_code <- test_mocker$use("gl_search_repos_by_code") + + gl_repos_by_code_tailored <- + gitlab_testhost_priv$tailor_repos_response(gl_repos_by_code) + + gl_repos_by_code_tailored %>% + expect_type("list") %>% + expect_length(length(gl_repos_by_code)) + + expect_list_contains_only( + gl_repos_by_code_tailored[[1]], + c( + "repo_id", "repo_name", "created_at", "last_activity_at", + "forks", "stars", "languages", "issues_open", + "issues_closed", "organization" + ) + ) + expect_lt( + length(gl_repos_by_code_tailored[[1]]), + length(gl_repos_by_code[[1]]) + ) + test_mocker$cache(gl_repos_by_code_tailored) +}) + +test_that("GitHost prepares table from GitLab repositories response", { + expect_snapshot( + gl_repos_by_code_table <- gitlab_testhost_priv$prepare_repos_table_from_rest( + repos_list = test_mocker$use("gl_repos_by_code_tailored") + ) + ) + expect_repos_table( + gl_repos_by_code_table + ) + gl_repos_by_code_table <- gitlab_testhost_priv$add_repo_api_url(gl_repos_by_code_table) + test_mocker$cache(gl_repos_by_code_table) +}) + +test_that("`get_repos_issues()` adds issues to repos table", { + gl_repos_by_code_table <- test_mocker$use("gl_repos_by_code_table") + gl_repos_by_code_table <- test_rest_gitlab$get_repos_issues( + gl_repos_by_code_table, + progress = FALSE + ) + expect_gt( + length(gl_repos_by_code_table$issues_open), + 0 + ) + expect_gt( + length(gl_repos_by_code_table$issues_closed), + 0 + ) + test_mocker$cache(gl_repos_by_code_table) +}) + +test_that("`get_repos_contributors()` adds contributors to repos table", { + gl_repos_table_with_contributors <- test_rest_gitlab$get_repos_contributors( + test_mocker$use("gl_repos_table_with_api_url"), + progress = FALSE + ) + expect_repos_table( + gl_repos_table_with_contributors, + with_cols = c("api_url", "contributors") + ) + expect_gt( + length(gl_repos_table_with_contributors$contributors), + 0 + ) + test_mocker$cache(gl_repos_table_with_contributors) +}) diff --git a/tests/testthat/test-get_repos-GitStats.R b/tests/testthat/test-get_repos-GitStats.R new file mode 100644 index 00000000..74d55b7a --- /dev/null +++ b/tests/testthat/test-get_repos-GitStats.R @@ -0,0 +1,91 @@ +test_that("get_repos_from_hosts works", { + mockery::stub( + test_gitstats_priv$get_repos_from_hosts, + "host$get_repos", + purrr::list_rbind(list( + test_mocker$use("gh_repos_table_with_api_url"), + test_mocker$use("gl_repos_table_with_api_url") + )) + ) + repos_table <- test_gitstats_priv$get_repos_from_hosts( + with_code = NULL, + in_files = NULL, + with_files = NULL, + verbose = FALSE, + progress = FALSE + ) + expect_repos_table( + repos_table, + repo_cols = repo_gitstats_colnames + ) +}) + +test_that("get_repos_from_hosts with_code works", { + mockery::stub( + test_gitstats_priv$get_repos_from_hosts, + "private$get_repos_from_host_with_code", + purrr::list_rbind( + list(test_mocker$use("gh_repos_by_code_table"), + test_mocker$use("gl_repos_by_code_table")) + ) + ) + repos_table <- test_gitstats_priv$get_repos_from_hosts( + with_code = "shiny", + in_files = "DESCRIPTION", + with_files = NULL, + verbose = FALSE, + progress = FALSE + ) + expect_repos_table( + repos_table, + repo_cols = repo_gitstats_colnames, + with_cols = c("contributors", "contributors_n") + ) + test_mocker$cache(repos_table) +}) + +test_that("set_object_class for repos_table works correctly", { + repos_table <- test_gitstats_priv$set_object_class( + object = test_mocker$use("repos_table"), + class = "repos_table", + attr_list = list( + "with_code" = NULL, + "in_files" = NULL, + "with_files" = "renv.lock" + ) + ) + expect_s3_class(repos_table, "repos_table") + expect_equal(attr(repos_table, "with_files"), "renv.lock") + test_mocker$cache(repos_table) +}) + +test_that("get_repos works properly and for the second time uses cache", { + test_gitstats <- create_test_gitstats(hosts = 2) + mockery::stub( + test_gitstats$get_repos, + "private$get_repos_from_hosts", + test_mocker$use("repos_table") + ) + repos_table <- test_gitstats$get_repos(verbose = FALSE) + expect_repos_table_object( + repos_object = repos_table, + with_cols = c("contributors", "contributors_n") + ) + expect_snapshot( + repos_table <- test_gitstats$get_repos() + ) + expect_repos_table_object( + repos_object = repos_table, + with_cols = c("contributors", "contributors_n") + ) +}) + +test_that("get_repos pulls repositories without contributors", { + test_gitstats <- create_test_gitstats(hosts = 2) + repos_table <- test_gitstats$get_repos( + add_contributors = FALSE, + verbose = FALSE + ) + expect_repos_table(repos_table, repo_cols = repo_gitstats_colnames) + expect_false("contributors" %in% names(repos_table)) +}) diff --git a/tests/testthat/test-get_urls_repos-GitHub.R b/tests/testthat/test-get_urls_repos-GitHub.R new file mode 100644 index 00000000..49bb1e77 --- /dev/null +++ b/tests/testthat/test-get_urls_repos-GitHub.R @@ -0,0 +1,75 @@ +test_that("get_repos_urls() works", { + gh_repos_urls <- test_rest_github$get_repos_urls( + type = "web", + org = "r-world-devs" + ) + expect_gt( + length(gh_repos_urls), + 0 + ) + test_mocker$cache(gh_repos_urls) +}) + +test_that("get_all_repos_urls prepares api repo_urls vector", { + github_testhost_priv <- create_github_testhost(orgs = c("r-world-devs", "openpharma"), + mode = "private") + gh_api_repos_urls <- github_testhost_priv$get_all_repos_urls( + type = "api", + verbose = FALSE + ) + expect_gt(length(gh_api_repos_urls), 0) + expect_true(any(grepl("openpharma", gh_api_repos_urls))) + expect_true(any(grepl("r-world-devs", gh_api_repos_urls))) + expect_true(all(grepl("api", gh_api_repos_urls))) + test_mocker$cache(gh_api_repos_urls) +}) + +test_that("get_all_repos_urls prepares web repo_urls vector", { + mockery::stub( + github_testhost_priv$get_all_repos_urls, + "rest_engine$get_repos_urls", + test_mocker$use("gh_repos_urls") + ) + gh_repos_urls <- github_testhost_priv$get_all_repos_urls( + type = "web", + verbose = FALSE + ) + expect_gt(length(gh_repos_urls), 0) + expect_true(any(grepl("r-world-devs", gh_repos_urls))) + expect_true(all(grepl("https://github.com/", gh_repos_urls))) + test_mocker$cache(gh_repos_urls) +}) + +test_that("get_repo_url_from_response retrieves repositories URLS", { + gh_repo_api_urls <- github_testhost_priv$get_repo_url_from_response( + search_response = test_mocker$use("gh_search_repos_response"), + type = "api" + ) + expect_type(gh_repo_api_urls, "character") + expect_gt(length(gh_repo_api_urls), 0) + test_mocker$cache(gh_repo_api_urls) + gh_repo_web_urls <- github_testhost_priv$get_repo_url_from_response( + search_response = test_mocker$use("gh_search_response_in_file"), + type = "web" + ) + expect_type(gh_repo_web_urls, "character") + expect_gt(length(gh_repo_web_urls), 0) + test_mocker$cache(gh_repo_web_urls) +}) + +test_that("get_repos_urls returns repositories URLS", { + mockery::stub( + github_testhost$get_repos_urls, + "private$get_repo_url_from_response", + test_mocker$use("gh_repo_web_urls") + ) + gh_repos_urls_with_code_in_files <- github_testhost$get_repos_urls( + type = "web", + with_code = "shiny", + in_files = "DESCRIPTION", + verbose = FALSE + ) + expect_type(gh_repos_urls_with_code_in_files, "character") + expect_gt(length(gh_repos_urls_with_code_in_files), 0) + test_mocker$cache(gh_repos_urls_with_code_in_files) +}) diff --git a/tests/testthat/test-get_urls_repos-GitLab.R b/tests/testthat/test-get_urls_repos-GitLab.R new file mode 100644 index 00000000..f97970c3 --- /dev/null +++ b/tests/testthat/test-get_urls_repos-GitLab.R @@ -0,0 +1,63 @@ +test_that("get_repos_urls() works", { + gl_repos_urls <- test_rest_gitlab$get_repos_urls( + type = "api", + org = "mbtests" + ) + expect_gt( + length(gl_repos_urls), + 0 + ) + test_mocker$cache(gl_repos_urls) +}) + +test_that("get_all_repos_urls prepares api repo_urls vector", { + mockery::stub( + gitlab_testhost_priv$get_all_repos_urls, + "rest_engine$get_repos_urls", + test_mocker$use("gl_repos_urls") + ) + gl_api_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( + type = "api", + verbose = FALSE + ) + expect_gt(length(gl_api_repos_urls), 0) + expect_true(all(grepl("api", gl_api_repos_urls))) + test_mocker$cache(gl_api_repos_urls) +}) + +test_that("get_all_repos_urls prepares web repo_urls vector", { + gl_repos_urls <- gitlab_testhost_priv$get_all_repos_urls( + type = "web", + verbose = FALSE + ) + expect_gt(length(gl_repos_urls), 0) + expect_true(all(!grepl("api", gl_repos_urls))) +}) + +test_that("`get_repo_url_from_response()` works", { + gl_repo_web_urls <- gitlab_testhost_priv$get_repo_url_from_response( + search_response = test_mocker$use("gl_search_response"), + type = "web", + progress = FALSE + ) + expect_gt(length(gl_repo_web_urls), 0) + expect_type(gl_repo_web_urls, "character") + test_mocker$cache(gl_repo_web_urls) +}) + +test_that("get_repos_urls returns repositories URLS", { + mockery::stub( + gitlab_testhost$get_repos_urls, + "private$get_repo_url_from_response", + test_mocker$use("gl_repo_web_urls") + ) + gl_repos_urls_with_code_in_files <- gitlab_testhost$get_repos_urls( + type = "web", + with_code = "shiny", + in_files = "DESCRIPTION", + verbose = FALSE + ) + expect_type(gl_repos_urls_with_code_in_files, "character") + expect_gt(length(gl_repos_urls_with_code_in_files), 0) + test_mocker$cache(gl_repos_urls_with_code_in_files) +}) diff --git a/tests/testthat/test-get_urls_repos-GitStats.R b/tests/testthat/test-get_urls_repos-GitStats.R new file mode 100644 index 00000000..079a3f03 --- /dev/null +++ b/tests/testthat/test-get_urls_repos-GitStats.R @@ -0,0 +1,98 @@ +test_that("get_repos_urls_from_hosts gets data from the hosts", { + mockery::stub( + test_gitstats_priv$get_repos_urls_from_hosts, + "host$get_repos_urls", + c(test_mocker$use("gh_api_repos_urls"), test_mocker$use("gl_api_repos_urls")) + ) + repos_urls_from_hosts <- test_gitstats_priv$get_repos_urls_from_hosts( + type = "api", + with_code = NULL, + in_files = NULL, + with_files = NULL, + verbose = FALSE + ) + expect_type(repos_urls_from_hosts, "character") + expect_gt(length(repos_urls_from_hosts), 0) + expect_true(any(grepl("gitlab.com/api", repos_urls_from_hosts))) + expect_true(any(grepl("api.github", repos_urls_from_hosts))) + test_mocker$cache(repos_urls_from_hosts) +}) + +test_that("get_repos_urls_from_hosts gets data with_code in_files from the hosts", { + mockery::stub( + test_gitstats_priv$get_repos_urls_from_hosts, + "private$get_repos_urls_from_host_with_code", + c(test_mocker$use("gh_repos_urls_with_code_in_files"), test_mocker$use("gl_repos_urls_with_code_in_files")) + ) + repos_urls_from_hosts_with_code_in_files <- test_gitstats_priv$get_repos_urls_from_hosts( + type = "api", + with_code = "shiny", + in_files = "DESCRIPTION", + with_files = NULL, + verbose = FALSE + ) + expect_type(repos_urls_from_hosts_with_code_in_files, "character") + expect_gt(length(repos_urls_from_hosts_with_code_in_files), 0) + expect_true(any(grepl("gitlab.com", repos_urls_from_hosts_with_code_in_files))) + expect_true(any(grepl("github.com", repos_urls_from_hosts_with_code_in_files))) + test_mocker$cache(repos_urls_from_hosts_with_code_in_files) +}) + +test_that("set_object_class works correctly", { + repos_urls <- test_gitstats_priv$set_object_class( + object = test_mocker$use("repos_urls_from_hosts_with_code_in_files"), + class = "repos_urls", + attr_list = list( + "type" = "api", + "with_code" = "shiny", + "in_files" = c("NAMESPACE", "DESCRIPTION"), + "with_files" = NULL + ) + ) + expect_s3_class(repos_urls, "repos_urls") + expect_equal(attr(repos_urls, "type"), "api") + expect_equal(attr(repos_urls, "with_code"), "shiny") + expect_equal(attr(repos_urls, "in_files"), c("NAMESPACE", "DESCRIPTION")) +}) + +test_that("get_repos_urls gets vector of repository URLS", { + test_gitstats <- create_test_gitstats(hosts = 2) + mockery::stub( + test_gitstats$get_repos_urls, + "private$get_repos_urls_from_hosts", + test_mocker$use("repos_urls_from_hosts") + ) + repo_urls <- test_gitstats$get_repos_urls( + verbose = FALSE + ) + expect_type( + repo_urls, + "character" + ) + expect_gt( + length(repo_urls), + 1 + ) +}) + +test_that("get_repos_urls gets vector of repository URLS", { + test_gitstats <- create_test_gitstats(hosts = 2) + mockery::stub( + test_gitstats$get_repos_urls, + "private$get_repos_urls_from_hosts", + test_mocker$use("repos_urls_from_hosts_with_code_in_files") + ) + repo_urls <- test_gitstats$get_repos_urls( + with_code = "shiny", + in_files = "DESCRIPTION", + verbose = FALSE + ) + expect_type( + repo_urls, + "character" + ) + expect_gt( + length(repo_urls), + 1 + ) +}) diff --git a/tests/testthat/test-get_usage_R_package.R b/tests/testthat/test-get_usage_R_package.R new file mode 100644 index 00000000..0e8ac535 --- /dev/null +++ b/tests/testthat/test-get_usage_R_package.R @@ -0,0 +1,39 @@ +test_that("get_R_package_as_dependency work correctly", { + mockery::stub( + test_gitstats_priv$get_R_package_as_dependency, + "private$get_repos_from_hosts", + test_mocker$use("repos_table") + ) + R_package_as_dependency <- test_gitstats_priv$get_R_package_as_dependency( + package_name = "shiny", + verbose = FALSE + ) + expect_s3_class( + R_package_as_dependency, + "data.frame" + ) + expect_gt( + nrow(R_package_as_dependency), + 0 + ) + test_mocker$cache(R_package_as_dependency) +}) + +test_that("get_R_package_usage_from_hosts works as expected", { + test_gitstats <- create_test_gitstats(hosts = 2, priv_mode = TRUE) + mockery::stub( + test_gitstats$get_R_package_usage_from_hosts, + "private$get_R_package_as_dependency", + test_mocker$use("R_package_as_dependency") + ) + mockery::stub( + test_gitstats$get_R_package_usage_from_hosts, + "private$get_R_package_loading", + test_mocker$use("R_package_as_dependency") + ) + R_package_usage_table <- test_gitstats$get_R_package_usage_from_hosts( + package_name = "shiny", only_loading = FALSE, verbose = FALSE + ) + expect_package_usage_table(R_package_usage_table) + test_mocker$cache(R_package_usage_table) +}) diff --git a/tests/testthat/test-get_user-GitHub.R b/tests/testthat/test-get_user-GitHub.R new file mode 100644 index 00000000..a54d6bb6 --- /dev/null +++ b/tests/testthat/test-get_user-GitHub.R @@ -0,0 +1,51 @@ +test_that("user query is built properly", { + gh_user_query <- + test_gqlquery_gh$user() + expect_snapshot( + gh_user_query + ) + test_mocker$cache(gh_user_query) +}) + +test_that("`gql_response()` work as expected for GitHub", { + gh_user_gql_response <- test_graphql_github$gql_response( + test_mocker$use("gh_user_query"), + vars = list(user = "maciekbanas") + ) + expect_user_gql_response( + gh_user_gql_response + ) + test_mocker$cache(gh_user_gql_response) +}) + +test_that("get_user pulls GitHub user response", { + mockery::stub( + test_graphql_github$get_user, + "self$gql_response", + test_mocker$use("gh_user_gql_response") + ) + gh_user_response <- test_graphql_github$get_user(username = "maciekbanas") + expect_user_gql_response( + gh_user_response + ) + test_mocker$cache(gh_user_response) +}) + +test_that("GitHub prepares user table", { + gh_user_table <- github_testhost_priv$prepare_user_table( + user_response = test_mocker$use("gh_user_response") + ) + expect_users_table( + gh_user_table, + one_user = TRUE + ) + test_mocker$cache(gh_user_table) +}) + +test_that("GitHost gets users tables", { + users_table <- github_testhost$get_users( + users = c("maciekbanas", "kalimu", "galachad") + ) + expect_users_table(users_table) + test_mocker$cache(users_table) +}) diff --git a/tests/testthat/test-get_user-GitLab.R b/tests/testthat/test-get_user-GitLab.R new file mode 100644 index 00000000..302cb4db --- /dev/null +++ b/tests/testthat/test-get_user-GitLab.R @@ -0,0 +1,62 @@ +test_that("user query is built properly", { + gl_user_query <- + test_gqlquery_gl$user() + expect_snapshot( + gl_user_query + ) + test_mocker$cache(gl_user_query) +}) + +test_that("`gql_response()` work as expected for GitLab", { + gl_user_gql_response <- test_graphql_gitlab$gql_response( + test_mocker$use("gl_user_query"), + vars = list(user = "maciekbanas") + ) + expect_user_gql_response( + gl_user_gql_response + ) + test_mocker$cache(gl_user_gql_response) +}) + + +test_that("get_user pulls GitLab user response", { + mockery::stub( + test_graphql_gitlab$get_user, + "self$gql_response", + test_mocker$use("gl_user_gql_response") + ) + gl_user_response <- test_graphql_gitlab$get_user(username = "maciekbanas") + expect_user_gql_response( + gl_user_response + ) + test_mocker$cache(gl_user_response) +}) + +test_that("GitLab prepares user table", { + gl_user_table <- gitlab_testhost_priv$prepare_user_table( + user_response = test_mocker$use("gl_user_response") + ) + expect_users_table( + gl_user_table, + one_user = TRUE + ) + test_mocker$cache(gl_user_table) +}) + +test_that("get_users build users table for GitHub", { + users_result <- gitlab_testhost$get_users( + users = c("maciekbanas", "Cotau", "marcinkowskak") + ) + expect_users_table( + users_result + ) +}) + +test_that("get_users build users table for GitLab", { + users_result <- gitlab_testhost$get_users( + users = c("maciekbanas", "Cotau", "marcinkowskak") + ) + expect_users_table( + users_result + ) +}) diff --git a/tests/testthat/test-get_users-GitStats.R b/tests/testthat/test-get_users-GitStats.R new file mode 100644 index 00000000..deca969a --- /dev/null +++ b/tests/testthat/test-get_users-GitStats.R @@ -0,0 +1,10 @@ +test_that("GitStats get users info", { + test_gitstats <- create_test_gitstats(hosts = 2) + users_result <- test_gitstats$get_users( + c("maciekbanas", "kalimu", "marcinkowskak"), + verbose = FALSE + ) + expect_users_table( + users_result + ) +}) diff --git a/tests/testthat/test-helpers.R b/tests/testthat/test-helpers.R new file mode 100644 index 00000000..46ea3660 --- /dev/null +++ b/tests/testthat/test-helpers.R @@ -0,0 +1,146 @@ +test_that("`get_group_id()` gets group's id", { + gl_group_id <- test_rest_gitlab_priv$get_group_id("mbtests") + expect_equal(gl_group_id, 63684059) +}) + +test_that("`set_searching_scope` does not throw error when `orgs` or `repos` are defined", { + expect_snapshot( + gitlab_testhost_priv$set_searching_scope(orgs = "mbtests", repos = NULL, verbose = TRUE) + ) + expect_snapshot( + gitlab_testhost_priv$set_searching_scope(orgs = NULL, repos = "mbtests/GitStatsTesting", verbose = TRUE) + ) +}) + +test_that("`extract_repos_and_orgs` extracts fullnames vector into a list of GitLab organizations with assigned repositories", { + repos_fullnames <- c( + "mbtests/gitstatstesting", "mbtests/gitstats-testing-2", "mbtests/subgroup/test-project-in-subgroup" + ) + expect_equal( + gitlab_testhost_priv$extract_repos_and_orgs(repos_fullnames), + list( + "mbtests" = c("gitstatstesting", "gitstats-testing-2"), + "mbtests/subgroup" = c("test-project-in-subgroup") + ) + ) +}) + +test_that("`extract_repos_and_orgs` extracts fullnames vector into a list of GitHub organizations with assigned repositories", { + repos_fullnames <- c( + "r-world-devs/GitStats", "r-world-devs/shinyCohortBuilder", + "openpharma/DataFakeR", "openpharma/GithubMetrics" + ) + expect_equal( + github_testhost_priv$extract_repos_and_orgs(repos_fullnames), + list( + "r-world-devs" = c("GitStats", "shinyCohortBuilder"), + "openpharma" = c("DataFakeR", "GithubMetrics") + ) + ) +}) + +test_that("When token is empty throw error", { + expect_snapshot( + error = TRUE, + github_testhost_priv$check_token("") + ) +}) + +test_that("`check_token()` prints error when token exists but does not grant access", { + token <- "does_not_grant_access" + expect_snapshot_error( + github_testhost_priv$check_token(token) + ) +}) + +test_that("when token is proper token is passed", { + expect_equal( + github_testhost_priv$check_token(Sys.getenv("GITHUB_PAT")), + Sys.getenv("GITHUB_PAT") + ) +}) + +test_that("check_endpoint returns TRUE if they are correct", { + expect_true( + github_testhost_priv$check_endpoint( + endpoint = "https://api.github.com/repos/r-world-devs/GitStats", + type = "Repository" + ) + ) + expect_true( + github_testhost_priv$check_endpoint( + endpoint = "https://api.github.com/orgs/openpharma", + ) + ) +}) + +test_that("check_endpoint returns error if they are not correct", { + expect_snapshot_error( + check <- github_testhost_priv$check_endpoint( + endpoint = "https://api.github.com/repos/r-worlddevs/GitStats", + type = "Repository" + ) + ) +}) + +test_that("`check_if_public` works correctly", { + expect_true( + github_testhost_priv$check_if_public("api.github.com") + ) + expect_false( + github_testhost_priv$check_if_public("github.internal.com") + ) +}) + +test_that("`set_default_token` sets default token for public GitHub", { + expect_snapshot( + default_token <- github_testhost_priv$set_default_token( + verbose = TRUE + ) + ) + test_rest <- create_testrest(token = default_token, + mode = "private") + expect_equal( + test_rest$perform_request( + endpoint = "https://api.github.com", + token = default_token + )$status, + 200 + ) +}) + +test_that("`test_token` works properly", { + expect_true( + github_testhost_priv$test_token(Sys.getenv("GITHUB_PAT")) + ) + expect_false( + github_testhost_priv$test_token("false_token") + ) +}) + +test_that("`set_default_token` sets default token for GitLab", { + expect_snapshot( + withr::with_envvar(new = c("GITLAB_PAT" = Sys.getenv("GITLAB_PAT_PUBLIC")), { + default_token <- gitlab_testhost_priv$set_default_token(verbose = TRUE) + }) + ) + test_rest <- create_testrest(token = default_token, + mode = "private") + expect_equal( + test_rest$perform_request( + endpoint = "https://gitlab.com/api/v4/projects", + token = default_token + )$status, + 200 + ) +}) + +test_that("`set_searching_scope` throws error when both `orgs` and `repos` are defined", { + expect_snapshot_error( + gitlab_testhost_priv$set_searching_scope( + orgs = "mbtests", + repos = "mbtests/GitStatsTesting", + verbose = TRUE + ) + ) +}) diff --git a/tests/testthat/test-08-show_orgs.R b/tests/testthat/test-show_orgs.R similarity index 50% rename from tests/testthat/test-08-show_orgs.R rename to tests/testthat/test-show_orgs.R index 37af4359..22320700 100644 --- a/tests/testthat/test-08-show_orgs.R +++ b/tests/testthat/test-show_orgs.R @@ -1,9 +1,3 @@ -test_gitstats <- create_test_gitstats( - hosts = 2, - inject_repos = "repos_table", - inject_commits = "commits_table" -) - test_that("show_orgs() returns orgs", { expect_equal( show_orgs(test_gitstats), diff --git a/tests/testthat/test-z-GitStats.R b/tests/testthat/test-z-GitStats.R new file mode 100644 index 00000000..10439310 --- /dev/null +++ b/tests/testthat/test-z-GitStats.R @@ -0,0 +1,114 @@ +test_gitstats <- create_test_gitstats(hosts = 0) + +test_that("GitStats object is created", { + expect_s3_class(test_gitstats, "GitStats") +}) + +test_that("GitStats prints empty fields.", { + expect_snapshot(test_gitstats) +}) + +test_that("GitStats prints the proper info when connections are added.", { + test_gitstats <- create_test_gitstats(hosts = 2) + expect_snapshot(test_gitstats) +}) + +test_that("GitStats prints the proper info when repos are passed instead of orgs.", { + suppressMessages( + test_gitstats <- create_gitstats() %>% + set_github_host( + token = Sys.getenv("GITHUB_PAT"), + repos = c("r-world-devs/GitStats", "openpharma/GithubMetrics") + ) %>% + set_gitlab_host( + token = Sys.getenv("GITLAB_PAT_PUBLIC"), + repos = c("mbtests/gitstatstesting", "mbtests/gitstats-testing-2") + ) + ) + expect_snapshot(test_gitstats) +}) + +test_gitstats_priv <- create_test_gitstats(hosts = 0, priv_mode = TRUE) + +test_that("check_for_host returns error when no hosts are passed", { + expect_snapshot_error( + test_gitstats_priv$check_for_host() + ) +}) + +test_that("check_params_conflict returns error", { + expect_snapshot_error( + test_gitstats_priv$check_params_conflict( + with_code = NULL, + with_files = NULL, + in_files = "DESCRIPTION" + ) + ) + expect_snapshot_error( + test_gitstats_priv$check_params_conflict( + with_code = "shiny", + with_files = "DESCRIPTION", + in_files = NULL + ) + ) +}) + +test_that("check_if_args_changed", { + test_gitstats <- create_test_gitstats( + hosts = 2, + priv_mode = TRUE, + inject_repos = "repos_table" + ) + check <- test_gitstats$check_if_args_changed( + storage = "repositories", + args_list = list( + "with_code" = "shiny", + "in_files" = "DESCRIPTION", + "with_files" = NULL + ) + ) + # repos_table from mock should be set to with_files = 'renv.lock' + expect_true( + check + ) + check <- test_gitstats$check_if_args_changed( + storage = "repositories", + args_list = list( + "with_code" = NULL, + "in_files" = NULL, + "with_files" = "renv.lock" + ) + ) + expect_false( + check + ) +}) + +test_that("show_orgs print orgs properly", { + test_gitstats <- create_test_gitstats(hosts = 2) + expect_equal( + test_gitstats$show_orgs(), + c("r-world-devs", "mbtests") + ) +}) + +suppressMessages( + test_gitstats <- create_gitstats() %>% + set_gitlab_host( + token = Sys.getenv("GITLAB_PAT_PUBLIC"), + orgs = "mbtests/subgroup" + ) +) + +test_that("show_orgs print subgroups properly", { + expect_equal( + test_gitstats$show_orgs(), + "mbtests/subgroup" + ) +}) + +test_that("subgroups are cleanly printed in GitStats", { + expect_snapshot( + test_gitstats + ) +}) diff --git a/vignettes/get_data.Rmd b/vignettes/get_data.Rmd index 7973d616..a6557dfb 100644 --- a/vignettes/get_data.Rmd +++ b/vignettes/get_data.Rmd @@ -37,7 +37,7 @@ git_stats <- create_gitstats() %>% As scanning scope was set to `organizations` (`orgs` parameter in `set_*_host()`), `GitStats` will pull all repositories from these organizations. ```{r} -repos <- get_repos(git_stats) +repos <- get_repos(git_stats, progress = FALSE) dplyr::glimpse(repos) ``` @@ -52,12 +52,13 @@ dplyr::glimpse(repos_urls) If messages overwhelm you, you can switch them off in the function: -```{r, eval = FALSE} +```{r} release_logs <- get_release_logs( - gitstats_object = git_stats, - since = "2024-01-01", + gitstats_object = git_stats, + since = "2024-01-01", verbose = FALSE ) +dplyr::glimpse(release_logs) ``` Or globally: @@ -72,9 +73,10 @@ After pulling, the data is saved to `GitStats`. ```{r} commits <- get_commits( - gitstats_object = git_stats, - since = "2024-06-01", - until = "2024-06-30" + gitstats_object = git_stats, + since = "2024-06-01", + until = "2024-06-30", + progress = FALSE ) dplyr::glimpse(commits) ``` @@ -83,29 +85,33 @@ Caching feature is by default turned on. If you run the `get_*()` function once ```{r} commits <- get_commits( - gitstats_object = git_stats, - since = "2024-06-01", - until = "2024-06-30" + gitstats_object = git_stats, + since = "2024-06-01", + until = "2024-06-30" ) dplyr::glimpse(commits) ``` Unless, you switch off the cache: -```{r, eval = FALSE} +```{r} commits <- get_commits( - gitstats_object = git_stats, - since = "2024-06-01", - until = "2024-06-30", - cache = FALSE + gitstats_object = git_stats, + since = "2024-06-01", + until = "2024-06-30", + cache = FALSE, + progress = FALSE ) +dplyr::glimpse(commits) ``` Or simply change the parameters for the function: -```{r, eval = FALSE} +```{r} commits <- get_commits( - gitstats_object = git_stats, - since = "2024-07-01" + gitstats_object = git_stats, + since = "2024-07-01", + progress = FALSE ) +dplyr::glimpse(commits) ``` diff --git a/vignettes/get_files.Rmd b/vignettes/get_files.Rmd new file mode 100644 index 00000000..30a1226e --- /dev/null +++ b/vignettes/get_files.Rmd @@ -0,0 +1,57 @@ +--- +title: "Get files content" +output: rmarkdown::html_vignette +vignette: > + %\VignetteIndexEntry{Get files content} + %\VignetteEngine{knitr::rmarkdown} + %\VignetteEncoding{UTF-8} +--- + +```{r, include = FALSE} +knitr::opts_chunk$set( + collapse = TRUE, + comment = "#>", + fig.width = 7, + fig.height = 4 +) +``` + +Set connections to hosts. + +> Example workflow makes use of public GitHub and GitLab, but it is plausible, that you will use your internal git platforms, where you need to define `host` parameter. See `vignette("set_hosts")` article on that. + +```{r} +library(GitStats) + +git_stats <- create_gitstats() %>% + set_github_host( + orgs = c("r-world-devs", "openpharma"), + token = Sys.getenv("GITHUB_PAT") + ) %>% + set_gitlab_host( + orgs = c("mbtests"), + token = Sys.getenv("GITLAB_PAT_PUBLIC") + ) +``` + +With `GitStats` you can get the content of all text files in repo that are of your interest. First you need to get the files structure. You can pull specific types of files, by setting `pattern` with regular expression and `depth` with integer, which defines level of directories to look for the files. + +```{r} +files_structure <- get_files_structure( + gitstats_object = git_stats, + pattern = "\\.md", + depth = 1L, + progress = FALSE +) +dplyr::glimpse(files_structure) +``` + +Once you pull the files structure, `GitStats` will store it. If you run then `get_files_content()` function, by default it will make use of this structure (unless you define `file_path`, which will override saved files structure). + +```{r} +files_content <- get_files_content( + gitstats_object = git_stats, + progress = FALSE +) +dplyr::glimpse(files_content) +``` diff --git a/vignettes/get_repos_with_code.Rmd b/vignettes/get_repos_with_code.Rmd index dccfa70c..65c28e0b 100644 --- a/vignettes/get_repos_with_code.Rmd +++ b/vignettes/get_repos_with_code.Rmd @@ -26,11 +26,11 @@ github_stats <- create_gitstats() %>% set_github_host( orgs = c("r-world-devs", "openpharma"), token = Sys.getenv("GITHUB_PAT") - ) %>% + ) %>% verbose_off() repos_urls <- get_repos_urls( - gitstats_object = github_stats, + gitstats_object = github_stats, with_code = "shiny" ) ``` @@ -40,7 +40,7 @@ You can limit your search, as it is allowed with GitLab and GitHub API search en ```{r, eval = FALSE} repos_urls <- get_repos_urls( gitstats_object = github_stats, - with_code = c("purrr", "shiny"), + with_code = c("purrr", "shiny"), in_files = c("DESCRIPTION", "NAMESPACE", "renv.lock") ) ``` @@ -49,7 +49,7 @@ You can also search for repositories with certain files (do not confuse `with_fi ```{r, eval = FALSE} repos_urls <- get_repos_urls( - gitstats_object = github_stats, + gitstats_object = github_stats, with_files = c("renv.lock", "DESCRIPTION") ) ``` @@ -60,7 +60,7 @@ repos_urls <- get_repos_urls( ```{r, eval = FALSE} package_usage <- get_R_package_usage( - gitstats_object = github_stats, + gitstats_object = github_stats, package_name = "shiny" ) ```