Merge pull request #450 from r-world-devs/devel

Release 2.0.2

.github/workflows/R-CMD-check.yaml

@@ -18,6 +18,8 @@ jobs:
       fail-fast: false
       matrix:
         config:
+          - {os: macos-latest,   r: 'release'}
+          - {os: windows-latest, r: 'release'}
           - {os: ubuntu-latest,   r: 'release'}
 
     env:

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: GitStats
 Title: Get Statistics from GitHub and GitLab
-Version: 2.0.1
+Version: 2.0.2
 Authors@R: c(
     person(given = "Maciej", family = "Banas", email = "[email protected]", role = c("aut", "cre")),
     person(given = "Kamil", family = "Koziej", email = "[email protected]", role = "aut"),
@@ -13,23 +13,22 @@ Description: Obtain statistics in a standardized way from multiple Git services:
 License: MIT + file LICENSE
 Encoding: UTF-8
 Roxygen: list(markdown = TRUE, r6 = TRUE)
-RoxygenNote: 7.3.1
+RoxygenNote: 7.3.2
 URL: https://r-world-devs.github.io/GitStats/, https://github.com/r-world-devs/GitStats
 BugReports: https://github.com/r-world-devs/GitStats/issues
 Imports:
     cli,
     dplyr,
     glue,
     httr2,
-    lubridate,
+    lubridate (>= 1.8.0),
     magrittr,
     rlang (>= 1.1.0),
     R6,
     purrr (>= 1.0.0),
     stringr
 Suggests:
-    spelling, 
-    covr,
+    spelling,
     knitr,
     mockery,
     rmarkdown,

NAMESPACE

@@ -8,6 +8,7 @@ export(get_commits_stats)
 export(get_files)
 export(get_release_logs)
 export(get_repos)
+export(get_repos_urls)
 export(get_users)
 export(set_github_host)
 export(set_gitlab_host)

NEWS.md

@@ -1,3 +1,18 @@
+# GitStats 2.0.2
+
+This is a patch release with substantial improvements to some functions (`get_repos()`, `get_files()` and `get_R_package_usage()`), adding `with_files` and `in_files` parameters, fixing `cache` feature and introducing new `get_repos_urls()` function, a minimalist version of `get_repos()`:
+
+- Added new `get_repos_urls()` function to fetch repository URLs (either web or API - choose with `type` parameter). It may return also only these repository URLs that consist of a given file or files (with passing argument to `with_files` parameter) or a text in code blobs (`with_code` parameter). This is a minimalist version of `get_repos()`, which takes out all the process of parsing (search response into repositories one) and adding statistics on repositories. This makes it poorer with content but faster. ([#425](https://github.com/r-world-devs/GitStats/issues/425)).
+- Added `with_files` parameter to `get_repos()` function, which makes it possible to search for repositories with a given file or files and return full output for repositories.
+- It is also possible now to pass multiple code phrases to `with_code` parameter (as a character vector) in `get_repos()` and `get_repos_urls()` ([282](https://github.com/r-world-devs/GitStats/issues/282)).
+- Added `in_files` parameter to `get_repos()` which works with `with_code` parameter. When both are defined, `GitStats` searches code blobs only in given files.
+- Removed `dplyr::glimpse()` from `get_*()` functions, so there is printing to console only if `get_*()` function is not assigned to the object ([#426](https://github.com/r-world-devs/GitStats/issues/426)).
+- Output table of `get_R_package_usage()` consists now also of repository full name ([#438](https://github.com/r-world-devs/GitStats/issues/438)).
+- Improved `get_R_package_usage()` with optimizing search of package names in `DESCRIPTION` and `NAMESPACE` files by removing filtering method and replacing it with `filename:` filter directly in search endpoint query ([#428](https://github.com/r-world-devs/GitStats/issues/428)).
+- Fixed `get_files()` when scanning scope is set to `repositories`. Earlier, it pulled given files from whole organizations, even if scanning scope was set to `repos` with `set_*_host()`. Now it shows only files for the given repositories ([#439](https://github.com/r-world-devs/GitStats/issues/439)).
+- Improved cache feature ([#436](https://github.com/r-world-devs/GitStats/issues/436)).
+- `verbose` parameter controls now showing of the progress bars ([#453](https://github.com/r-world-devs/GitStats/issues/453)).
+
 # GitStats 2.0.1
 
 This is a patch release with some hot issues that needed to be addressed, notably covering `set_*_host()` functions with `verbose` control, tweaking a bit `verbose` feature in general, fixing pulling data for GitLab subgroups and speeding up `get_files()` function.

R/EngineGraphQLGitHub.R

@@ -67,27 +67,31 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
     # Iterator over pulling commits from all repositories.
     pull_commits_from_repos = function(org,
                                        repos_names,
-                                       date_from,
-                                       date_until) {
+                                       since,
+                                       until,
+                                       verbose) {
       repos_list_with_commits <- purrr::map(repos_names, function(repo) {
         private$pull_commits_from_one_repo(
-          org,
-          repo,
-          date_from,
-          date_until
+          org = org,
+          repo = repo,
+          since = since,
+          until = until
         )
-      }, .progress = !private$scan_all)
+      }, .progress = !private$scan_all && verbose)
       names(repos_list_with_commits) <- repos_names
       repos_list_with_commits <- repos_list_with_commits %>%
         purrr::discard(~ length(.) == 0)
       return(repos_list_with_commits)
     },
 
     # Pull all given files from all repositories of an organization.
-    pull_files_from_org = function(org, file_path) {
+    pull_files_from_org = function(org, repos, file_path) {
       repos_list <- self$pull_repos_from_org(
         org = org
       )
+      if (!is.null(repos)) {
+        repos_list <- purrr::keep(repos_list, ~ .$repo_name %in% repos)
+      }
       repositories <- purrr::map(repos_list, ~ .$repo_name)
       def_branches <- purrr::map(repos_list, ~ .$default_branch$name)
       files_list <- purrr::map(file_path, function(file_path) {
@@ -146,17 +150,17 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
       # An iterator over pulling commit pages from one repository.
       pull_commits_from_one_repo = function(org,
                                             repo,
-                                            date_from,
-                                            date_until) {
+                                            since,
+                                            until) {
         next_page <- TRUE
         full_commits_list <- list()
         commits_cursor <- ""
         while (next_page) {
           commits_response <- private$pull_commits_page_from_repo(
             org = org,
             repo = repo,
-            date_from = date_from,
-            date_until = date_until,
+            since = since,
+            until = until,
             commits_cursor = commits_cursor
           )
           commits_list <- commits_response$data$repository$defaultBranchRef$target$history$edges
@@ -176,15 +180,15 @@ EngineGraphQLGitHub <- R6::R6Class("EngineGraphQLGitHub",
       # Wrapper over building GraphQL query and response.
       pull_commits_page_from_repo = function(org,
                                              repo,
-                                             date_from,
-                                             date_until,
+                                             since,
+                                             until,
                                              commits_cursor = "",
                                              author_id = "") {
         commits_by_org_query <- self$gql_query$commits_by_repo(
           org = org,
           repo = repo,
-          since = date_to_gts(date_from),
-          until = date_to_gts(date_until),
+          since = date_to_gts(since),
+          until = date_to_gts(until),
           commits_cursor = commits_cursor,
           author_id = author_id
         )

R/EngineGraphQLGitLab.R

@@ -72,7 +72,7 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab",
      },
 
      # Pull all given files from all repositories of a group.
-     pull_files_from_org = function(org, file_path) {
+     pull_files_from_org = function(org, repos, file_path) {
        org <- URLdecode(org)
        full_files_list <- list()
        next_page <- TRUE
@@ -110,6 +110,12 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab",
          }
          full_files_list <- append(full_files_list, files_list)
        }
+       if (!is.null(repos)) {
+         full_files_list <- purrr::keep(full_files_list, function(project) {
+           repo_name <- private$get_repo_name_from_url(project$webUrl)
+           repo_name %in% repos
+         })
+       }
        return(full_files_list)
      },
 
@@ -144,6 +150,13 @@ EngineGraphQLGitLab <- R6::R6Class("EngineGraphQLGitLab",
          vars = list("org" = org)
        )
        return(response)
+     },
+
+     # Helper
+     get_repo_name_from_url = function(web_url) {
+       url_split <- stringr::str_split(web_url, ":|/")[[1]]
+       repo_name <- url_split[length(url_split)]
+       return(repo_name)
      }
    )
 )

R/EngineRest.R

@@ -50,16 +50,6 @@ EngineRest <- R6::R6Class("EngineRest",
       return(contributors_vec)
     },
 
-    # Filtering handler if files are set for scanning scope
-    limit_search_to_files = function(search_result, files) {
-      if (!is.null(files)) {
-        search_result <- purrr::keep(search_result, function(repository) {
-          any(repository$path %in% files)
-        })
-      }
-      return(search_result)
-    },
-
     # Helper
     paginate_results = function(endpoint, joining_sign = "?") {
       full_response <- list()

R/EngineRestGitHub.R

@@ -24,36 +24,60 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
     },
 
     # Pulling repositories where code appears
-    pull_repos_by_code = function(org = NULL,
-                                  code,
-                                  verbose,
-                                  settings) {
+    pull_repos_by_code = function(code,
+                                  org = NULL,
+                                  filename = NULL,
+                                  in_path = FALSE,
+                                  raw_output = FALSE,
+                                  verbose) {
       private$set_verbose(verbose)
       user_query <- if (!is.null(org)) {
         paste0('+user:', org)
       } else {
         ''
       }
-      query <- paste0('"', code, '"', user_query)
+      query <- if (!in_path) {
+        paste0('"', code, '"', user_query)
+      } else {
+        paste0('"', code, '"+in:path', user_query)
+      }
+      if (!is.null(filename)) {
+        query <- paste0(query, '+in:file+filename:', filename)
+      }
       search_endpoint <- paste0(private$endpoints[["search"]], query)
-      total_n <- self$response(search_endpoint)[["total_count"]]
       if (verbose) cli::cli_alert_info("Searching for code [{code}]...")
+      total_n <- self$response(search_endpoint)[["total_count"]]
       if (length(total_n) > 0) {
         search_result <- private$search_response(
           search_endpoint = search_endpoint,
           total_n = total_n
         )
-        search_result <- private$limit_search_to_files(
-          search_result = search_result,
-          files = settings$files
-        )
-        repos_list <- private$map_search_into_repos(
-          search_response = search_result
-        )
+        if (!raw_output) {
+          search_output <- private$map_search_into_repos(
+            search_response = search_result
+          )
+        } else {
+          search_output <- search_result
+        }
       } else {
-        repos_list <- list()
+        search_output <- list()
       }
-      return(repos_list)
+      return(search_output)
+    },
+
+    #' Pull all repositories URLS from organization
+    pull_repos_urls = function(type, org) {
+      repos_urls <- self$response(
+        endpoint = paste0(private$endpoints[["organizations"]], org, "/repos")
+      ) %>%
+        purrr::map_vec(function(repository) {
+          if (type == "api") {
+            repository$url
+          } else {
+            repository$html_url
+          }
+        })
+      return(repos_urls)
     },
 
     #' A method to add information on open and closed issues of a repository.
@@ -62,11 +86,9 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
         repos_iterator <- paste0(repos_table$organization, "/", repos_table$repo_name)
         issues <- purrr::map_dfr(repos_iterator, function(repo_path) {
           issues_endpoint <- paste0(private$endpoints[["repositories"]], repo_path, "/issues")
-
           issues <- self$response(
             endpoint = issues_endpoint
           )
-
           data.frame(
             "open" = length(purrr::keep(issues, ~ .$state == "open")),
             "closed" = length(purrr::keep(issues, ~ .$state == "closed"))
@@ -124,7 +146,7 @@ EngineRestGitHub <- R6::R6Class("EngineRestGitHub",
         self$rest_api_url,
         '/search/code?q='
       )
-      private$endpoints[["organization"]] <- paste0(
+      private$endpoints[["organizations"]] <- paste0(
         self$rest_api_url,
         "/orgs/"
       )