Merge pull request #12 from BirdStudiesCanada/sandbox

# naturecounts 0.2.0

* Clarified access in `nc_counts()`
* Added extra event columns to `format_zero_fill()`
* No funny dates in Databases and no warnings for data frames when missing 
  `survey_year`, `survey_month` or `survey_day` in `format_dates()`

.Rbuildignore

@@ -6,4 +6,6 @@
 ^\.Rproj\.user$
 ^RELEASE.R$
 ^pkgdown$
+^CODE_DESIGN.md$
+^\.travis\.yml$
 ^appveyor\.yml$

.travis.yml

@@ -0,0 +1,10 @@
+language: R
+cache: packages
+before_install:
+- sudo apt-get -y install libudunits2-dev gdal-bin libgdal1-dev libproj-dev
+env:
+  global:
+    secure: PpldXRYEeEjF0mmR3Kv04WJZiE3w7e48K2Bc5PkP/jK7rrqnswbrvNAe0nuchwEEd8ZruDnU0Ac5ptVOIXTLU2XJ8zF3ZAsIKHcSaGxoHG7Y33+vfcy3Zk4dor3bNC2pyTPCfdaoSU7rotcpmCWDPYBQMGZODBBPT5KkGO1e9vVrCPrTjR9WKORiIf1TkaiPLR4tgNHZFcvZ2yETIoFdyZ6Sjab/TMBrVHuNAK91L5yzgHrnyLl/jHd5Jscr6W+2JBUpQkpQeu8DI8MbHxFi5sMApLeVOMP5EYT5zXHccQJlOR0GpwcvDjAk0UnomyCjO5cLYjLWHItqsiMWkUX8fkQzIw8eQ1c/guLuUYlSVxYoJBsB8QNGI52U3HOCrNE5lPZyJkahc9vgYseClZJtRSUtN+EuQKYOUpiXPSSCu5V4P7zgp2zvfz5FKejYPlGzvtsYe4Yqd4d75nwqisKug9WcLt0ZoQi/v3Zr6e5O6g99yGlb+vUJwdjNLzo7xjWA5yz7pIuFzFyRBWUJKVWkFgggc7Mh856yWxFe6MtsmgF1Qz0TnBBRVl9cu4vKtFp0bNsas2Pe7uGCNbQwArRWXgGpmAjvNJ7FOZD0GZcMF0eoznbExdhq/CQXMcXKU8V4LCP6n80dKB03ID4iZFiYy2T2U4ahfwem0fT2ldOfK5Y=
+
+r_build_args: "--no-build-vignettes"
+r_check_args: "--no-build-vignettes --ignore-vignettes"

CODE_DESIGN.md

@@ -0,0 +1,39 @@
+# naturecounts design principles
+
+## Fields returned by the API
+
+#### Desired behaviour
+Required and extra fields returned by the API are expected and treated, respectively, as follows:
+
+Function           | API Entry Point           | Required fields            | Extra fields
+------------------ |-------------------------- | -------------------------- | ------------
+`nc_count()`       | `data/list_collections`   |  `collection`, `nrecords`  | Ignored
+`nc_count()`       | `data/list_permissions`   |  `collection`, `akn_level` | Ignored
+`nc_count()`       | `metadata/collections`    |  `collection`, `akn_level` | Ignored
+`nc_permissions()` | `data/list_permissions`   |  `collection`, `akn_level` | Ignored
+`srv_auth()`       | `data/authenticate`       |  `token`                   | Ignored
+`nc_data_dl()`     | `data/release_request_id` |  None (close request only) | Ignored
+`nc_data_dl()`     | `data/get_data`           |  Any                       | Added
+`meta_XXX()`       | `metadata/XXX`            |  Any                       | Added
+
+#### What will break the package
+- Missing "Required fields" 
+- Missing/renamed "container"  
+  Some API entry points return data in a container, for example, 
+
+  - `data/get_data` -> `results` (holds main data)
+  - `data/list_collections` -> `results` (holds counts); `request_id` (holds request id)
+  - `data/list_requests` -> `requests` (holds details on individual requests)
+
+  If these "containers" change names, the package will break.
+
+#### Coding Principles
+- In package, after accessing the API, explictly `select()` the fields/columns expected. This way extra fields won't break existing code 
+  - Missing fields **will** the code, but at least they will break the code early!
+  - Do **not** do this for data downloads (i.e. `nc_single_dl()` under `nc_data_dl()`) or metadata downloads (i.e. `meta_XXX()`)
+  - **Unless**, using a `meta_XXX()` download internally. Then always `select()` the fields required
+
+## Testing
+- Tests are run using the user "testuser"
+  - Locally password can be stored in .Renviron as naturecounts_testuser = PASSWORD
+  - For remote testing, password is supplied as encrypted values

DESCRIPTION

@@ -1,7 +1,7 @@
 Package: naturecounts
 Type: Package
 Title: Access and download data on plant and animal populations from NatureCounts
-Version: 0.1.0
+Version: 0.2.0
 Authors@R: c(
     person("Steffi", "LaZerte", email = "[email protected]", role = "aut"),
     person("Denis", "LePage", email = "[email protected]", role = c("aut", "cre")))
@@ -16,24 +16,28 @@ Depends:
 Imports: 
     askpass (>= 1.1),
     DBI (>= 1.0.0),
+    dbplyr (>= 1.4.4),
     dplyr (>= 0.7.8),
     httr (>= 1.4.0),
     jsonlite (>= 1.6),
     lubridate (>= 1.7.4),
     magrittr,
-    purrr(>= 0.3.2),
+    memoise (>= 1.1.0),
+    purrr (>= 0.3.2),
     readr (>= 1.3.1),
     rlang (>= 0.1.2),
     RSQLite (>= 2.1.1),
     stringi (>= 1.2.4),
     stringr (>= 1.3.1),
-    tidyr (>= 0.8.2)
+    tidyr (>= 0.8.2),
+    tidyselect (>= 1.0.0)
 Suggests: 
     ggplot2,
     ggmap,
     here,
     knitr,
     mapview,
+    rgeos,
     rnaturalearth,
     roxygen2,
     rmarkdown,
@@ -42,6 +46,6 @@ Suggests:
     testthat
 Language: en-US
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 6.1.1
+RoxygenNote: 7.1.1
 URL: https://github.com/BirdStudiesCanada/naturecounts, https://naturecounts.ca
 VignetteBuilder: knitr

NAMESPACE

@@ -23,6 +23,7 @@ export(meta_utm_squares)
 export(nc_count)
 export(nc_data_dl)
 export(nc_metadata)
+export(nc_permissions)
 export(nc_requests)
 export(region_search)
 export(search_region)

NEWS.md

@@ -1,3 +1,10 @@
+# naturecounts 0.2.0
+
+* Clarified access in `nc_counts()`
+* Added extra event columns to `format_zero_fill()`
+* No funny dates in Databases and no warnings for data frames when missing 
+  `survey_year`, `survey_month` or `survey_day` in `format_dates()`
+
 # naturecounts 0.1.0
 
 * Initial package development

R/checks.R

@@ -95,7 +95,7 @@ site_type_check <- function(site_type) {
 projects_check <- function(project_ids, collections = NULL) {
   if(!is.null(project_ids)) {
     c1 <- meta_collections() %>%
-      dplyr::filter(project_id %in% project_ids)
+      dplyr::filter(.data$project_id %in% project_ids)
 
     if(nrow(c1) == 0) {
       stop("'project_ids' must be either NULL or a vector of valid ",
@@ -106,14 +106,38 @@ projects_check <- function(project_ids, collections = NULL) {
   c1
 }
 
-collections_check <- function(c) {
-  c1 <- meta_collections() %>%
-    dplyr::filter(collection %in% c)
+collections_check <- function(c, token = NULL) {
+  c1 <- nc_permissions_internal(token) %>%
+    dplyr::pull("collection")
 
-  if(!is.character(c) || nrow(c1) == 0) {
-    stop("'collections' must be either NULL (for all collections) ",
-         "or a character vector of valid 'collection' names specified ",
-         "in 'meta_collections()'.", call. = FALSE)
+  if(!is.null(token)) {
+    c_req <- nc_requests_internal(token = token)
+    if(!is.null(c_req)) {
+      c1 <- c_req %>%
+        dplyr::pull("collection") %>%
+        append(c1) %>%
+        unique()
+    }
+  }
+  c1 <- c[!c %in% c1]
+
+  if(!is.null(c) && (!is.character(c) || length(c1) > 0)) {
+    msg <- paste0("'collections' must be either NULL (return all collections) ",
+                  "or a \ncharacter vector of valid 'collection' names to which ",
+                  "'username' has access.\nSee 'vignette(\"Data Access\", ",
+                  "package = \"naturecounts\")' for details.")
+    if(any(c1 %in% meta_collections()$collection)) {
+      msg <- paste0(msg, "\nNo access to collection(s): ",
+                    paste0(c1[c1 %in% meta_collections()$collection],
+                           collapse = ", "))
+    }
+    if(any(!c1 %in% meta_collections()$collection)) {
+      msg <- paste0(msg, "\nNo access or no collection: ",
+                    paste0(c1[!c1 %in% meta_collections()$collection],
+                           collapse = ", "))
+    }
+
+    stop(msg, call. = FALSE)
   }
   c
 }
@@ -144,7 +168,7 @@ fields_set_check <- function(fields_set) {
 
 fields_check <- function(fields) {
   f <- meta_bmde_fields(version = NULL) %>%
-    dplyr::pull(local_name) %>%
+    dplyr::pull(.data$local_name) %>%
     unique()
 
   w <- fields[!fields %in% f]

R/common_docs.R

@@ -31,18 +31,16 @@
 #' @param verbose Logical. Show messages?
 #'
 #' @section NatureCounts account:
-#'   All public data is available without a username/password. However, to
-#'   access private/semi-public projects/collections you must [sign
-#'   up](https://www.birdscanada.org/birdmon/default/register.jsp) for a free
-#'   NatureCounts account and
-#'   [register](https://www.birdscanada.org/birdmon/default/projects.jsp) for
-#'   the projects you'd like to access. See the `request_id` section for more
-#'   information.
+#'   All public data is available with a username/password
+#'   ([sign up](https://www.birdscanada.org/birdmon/default/register.jsp)
+#'   for a free NatureCounts account). However, to access private/semi-public
+#'   projects/collections you must request access. See the Access and
+#'   `request_id`s section for more information.
 #'
 #' @section Species ids (`species`):
 #'   Numeric species id codes can determined from the functions
-#'   [search_species()] or [search_species_code()]. See
-#'   also the [species codes](https://birdstudiescanada.github.io/naturecounts/articles/species-codes.html)
+#'   [search_species()] or [search_species_code()]. See also the article on
+#'   [species codes](https://birdstudiescanada.github.io/naturecounts/articles/species-codes.html)
 #'   for more information.
 #'
 #' @section Day of Year (`doy`):
@@ -86,26 +84,38 @@
 #'   Note that in all cases there are a set of fields/columns that are *always*
 #'   returned, no matter what `fields_set` is used.
 #'
-#' @section `request_id`'s:
-#' There are two types of data requests: ones made through the api (i.e. this
-#' `naturecounts` R package) and those made through the online [Web Request
-#' Form](https://www.birdscanada.org/birdmon/default/searchquery.jsp). Each
-#' request generates a request id which identifies the filter set and
-#' collections requested.
+#' @section Access and `request_id`s:
 #'
-#' Requests made through this package via the [nc_data_dl()] function only
-#' return data the user has access to. If data was successfully downloaded, the
-#' user can see the request details with the [nc_requests()] function.
+#' Access to a data collection is either available as "full" or "by request".
+#' Use `nc_count(username = "USER", show = "all")`, to see the accessibility of
+#' collections.
 #'
-#' Requests made through the online [Web Request
-#' Form](https://www.birdscanada.org/birdmon/default/searchquery.jsp) may
-#' include collections to which the user does not have access. [nc_requests()]
-#' will list the requests along with the approval status (either 'approved' or
-#' 'pending').
+#' "Full" access means that data can be immediately requested directly through
+#' the `naturecounts` R package. "By request" means that a request must be
+#' [submitted
+#' online](https://www.birdscanada.org/birdmon/default/searchquery.jsp) and
+#' approved before the data can be downloaded through `naturecounts`.
 #'
-#' Any approved request id can be downloaded by supplying the `request_id` to
-#' [nc_data_dl()] (e.g., `nc_data_dl(request_id = 152000, username = "USER")`).
+#' This means that there are two types of data requests: ones made through this
+#' `naturecounts` R package (API requests) and those made through the online
+#' [Web Request
+#' Form](https://www.birdscanada.org/birdmon/default/searchquery.jsp) (Web
+#' requests). Every request (from either method) generates a `request_id` which
+#' identifies the filter set and collections requested. Details of all of
+#' requests can be reviewed with the [nc_requests()] function.
 #'
+#' To download data with "full" access, users can either specify filters, or if
+#' they are repeating a download, can use the `request_id` from [nc_requests()].
+#'
+#' Otherwise, if the user doesn't have "full" access, they must supply an
+#' approved `request_id` to the [nc_data_dl()] function (e.g.,
+#' `nc_data_dl(request_id = 152000, username = "USER")`). Use [nc_requests()] to
+#' see `request_id`s, filters, and approval status.
+#'
+#' Requests for "full" access to additional collections can be made online
+#' through the [Web Request
+#' Form](https://www.birdscanada.org/birdmon/default/searchquery.jsp) by
+#' checking the "Full access?" box in Step 2 of the form.
 #'
 #' @keywords internal
 #' @name args

R/database.R

@@ -12,11 +12,6 @@
 #'
 #' @keywords internal
 #'
-#' @examples
-#'
-#' \donttest{
-#'   db_connect()
-#' }
 
 db_connect <- function(name = paste0("./naturecounts_", Sys.Date()),
                        verbose = TRUE) {
@@ -59,8 +54,8 @@ db_create_primary <- function(con, df, primary_key) {
 # This function is used internally to create a database for use by the db_create function
 db_create_empty <- function(con) {
   # Download and copy empty naturecounts table
-  naturecounts <- nc_data_dl(collections = "RCBIOTABASE", species = 14280,
-                             years = 2010, username = "sample",
+  naturecounts <- nc_data_dl(collections = "SAMPLE1", species = 14280,
+                             username = "sample",
                              info = "nc: create database",
                              verbose = FALSE)[0, ]
   db_create_primary(con, naturecounts, primary_key = keys$data)