From 3dc1ab7c7edda73d1187a2d12967e20ff6d04a08 Mon Sep 17 00:00:00 2001 From: Florian Mayer Date: Fri, 15 Mar 2024 15:32:45 +0800 Subject: [PATCH] Add entitylist_download * Downloads entity lists as `entities.csv` * Does not yet support `$filter` and `ETag` --- NAMESPACE | 1 + R/entitylist_detail.R | 2 + R/entitylist_download.R | 137 ++++++++++++++++++++ R/entitylist_list.R | 2 + R/entitylist_update.R | 2 + man/entitylist_detail.Rd | 19 ++- man/entitylist_download.Rd | 145 ++++++++++++++++++++++ man/entitylist_list.Rd | 9 +- man/entitylist_update.Rd | 16 +-- tests/testthat/test-entitylist_download.R | 50 ++++++++ 10 files changed, 368 insertions(+), 15 deletions(-) create mode 100644 R/entitylist_download.R create mode 100644 man/entitylist_download.Rd create mode 100644 tests/testthat/test-entitylist_download.R diff --git a/NAMESPACE b/NAMESPACE index a62b98fe..4d3257ef 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -12,6 +12,7 @@ export(enexpr) export(enquo) export(ensym) export(entitylist_detail) +export(entitylist_download) export(entitylist_list) export(entitylist_update) export(expr) diff --git a/R/entitylist_detail.R b/R/entitylist_detail.R index a723d49e..b272b320 100644 --- a/R/entitylist_detail.R +++ b/R/entitylist_detail.R @@ -1,5 +1,7 @@ #' Show Entity List details. #' +#' `r lifecycle::badge("maturing")` +#' #' An Entity List is a named collection of Entities that have the same #' properties. #' Entity List can be linked to Forms as Attachments. diff --git a/R/entitylist_download.R b/R/entitylist_download.R new file mode 100644 index 00000000..18261cfc --- /dev/null +++ b/R/entitylist_download.R @@ -0,0 +1,137 @@ +#' Download an Entity List as CSV. +#' +#' `r lifecycle::badge("maturing")` +#' +#' The downloaded file is named "entities.csv". The download location defaults +#' to the current workdir, but can be modified to a folder name. +#' +#' An Entity List is a named collection of Entities that have the same +#' properties. +#' Entity List can be linked to Forms as Attachments. +#' This will make it available to clients as an automatically-updating CSV. +#' +#' Entity Lists can be used as Attachments in other Forms, but they can also be +#' downloaded directly as a CSV file. +#' The CSV format closely matches the OData Dataset (Entity List) Service +#' format, with columns for system properties such as `__id` (the Entity UUID), +#' `__createdAt`, `__creatorName`, etc., the Entity Label label, and the +#' Dataset (Entity List )/Entity Properties themselves. +#' If any Property for an given Entity is blank (e.g. it was not captured by +#' that Form or was left blank), that field of the CSV is blank. +#' +#' The `$filter` querystring parameter can be used to filter on system-level +#' properties, similar to how filtering in the OData Dataset (Entity List) +#' Service works. +#' +#' This endpoint supports `ETag` header, which can be used to avoid downloading +#' the same content more than once. When an API consumer calls this endpoint, +#' the endpoint returns a value in the `ETag` header. +#' If you pass that value in the `If-None-Match` header of a subsequent request, +#' then if the Entity List has not been changed since the previous request, +#' you will receive 304 Not Modified response; otherwise you'll get the new +#' data. +#' +#' `r lifecycle::badge("maturing")` +#' +#' @template param-pid +#' @template param-did +#' @template param-url +#' @template param-auth +#' @param local_dir The local folder to save the downloaded files to, +#' default: \code{here::here}. +#' @param overwrite Whether to overwrite previously downloaded zip files, +#' default: FALSE +#' @template param-retries +#' @template param-odkcv +#' @template param-orders +#' @template param-tz +#' @template param-verbose +#' @return The path to the downloaded CSV. +# nolint start +#' @seealso \url{ https://docs.getodk.org/central-api-dataset-management/#datasets} +# nolint end +#' @family entity-management +#' @export +#' @examples +#' \dontrun{ +#' # See vignette("setup") for setup and authentication options +#' # ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...") +#' +#' ds <- entitylist_list(pid = get_default_pid()) +#' ds1 <- entitylist_download(pid = get_default_pid(), did = ds$name[1]) +#' } +entitylist_download <- function(pid = get_default_pid(), + did = NULL, + url = get_default_url(), + un = get_default_un(), + pw = get_default_pw(), + local_dir = here::here(), + overwrite = TRUE, + retries = get_retries(), + odkc_version = get_default_odkc_version(), + orders = c( + "YmdHMS", + "YmdHMSz", + "Ymd HMS", + "Ymd HMSz", + "Ymd", + "ymd" + ), + tz = get_default_tz(), + verbose = get_ru_verbose()) { + yell_if_missing(url, un, pw, pid = pid) + + if (is.null(did)) { + ru_msg_abort( + "entitylist_download requires the Entity List name as 'did=\"name\"'." + ) + } + + if (odkc_version |> semver_lt("2022.3")) { + ru_msg_warn("entitylist_download is supported from v2022.3") + } + + pth <- fs::path(local_dir, "entities.csv") + + if (fs::file_exists(pth)) { + if (overwrite == TRUE) { + "Overwriting previous entity list: \"{pth}\"" %>% + glue::glue() %>% + ru_msg_success(verbose = verbose) + } else { + "Keeping previous entity list: \"{pth}\"" %>% + glue::glue() %>% + ru_msg_success(verbose = verbose) + + return(pth) + } + } else { + "Downloading entity list \"{did}\" to {pth}" %>% + glue::glue() %>% + ru_msg_success(verbose = verbose) + } + + + httr::RETRY( + "GET", + httr::modify_url(url, + path = glue::glue( + "v1/projects/{pid}/datasets/", + "{URLencode(did, reserved = TRUE)}/entities.csv" + ) + ), + httr::add_headers( + "Accept" = "text/csv" + ), + httr::authenticate(un, pw), + httr::write_disk(pth, overwrite = overwrite), + times = retries + ) |> + yell_if_error(url, un, pw) |> + httr::content(encoding = "utf-8") + + pth +} + + +# usethis::use_test("entitylist_download") # nolint \ No newline at end of file diff --git a/R/entitylist_list.R b/R/entitylist_list.R index e631a29e..f0dcbf67 100644 --- a/R/entitylist_list.R +++ b/R/entitylist_list.R @@ -1,5 +1,7 @@ #' List all Entity Lists of one Project. #' +#' `r lifecycle::badge("maturing")` +#' #' While the API endpoint will return all Entity Lists for one Project, #' \code{\link{entitylist_list}} will fail with incorrect or missing #' authentication. diff --git a/R/entitylist_update.R b/R/entitylist_update.R index 72243db3..128d48c6 100644 --- a/R/entitylist_update.R +++ b/R/entitylist_update.R @@ -1,5 +1,7 @@ #' Update Entity List details. #' +#' `r lifecycle::badge("maturing")` +#' #' You can only update `approvalRequired` using this endpoint. #' The approvalRequired flag controls the Entity creation flow; #' if it is true then the Submission must be approved before an Entity can be diff --git a/man/entitylist_detail.Rd b/man/entitylist_detail.Rd index cf9e6519..7a74df1e 100644 --- a/man/entitylist_detail.Rd +++ b/man/entitylist_detail.Rd @@ -80,12 +80,14 @@ response. Since this nested list is so deeply nested and irregularly shaped it is not trivial to rectangle the result into a tibble. } \description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#maturing}{\figure{lifecycle-maturing.svg}{options: alt='[Maturing]'}}}{\strong{[Maturing]}} +} +\details{ An Entity List is a named collection of Entities that have the same properties. Entity List can be linked to Forms as Attachments. This will make it available to clients as an automatically-updating CSV. -} -\details{ + This function is supported from ODK Central v2022.3 and will warn if the given odkc_version is lower. @@ -100,15 +102,22 @@ ds <- entitylist_list(pid = get_default_pid()) ds1 <- entitylist_detail(pid = get_default_pid(), did = ds$name[1]) ds1 |> listviewer::jsonedit() -ds1$linkedForms |> purrr::list_transpose() |> tibble::as_tibble() -ds1$sourceForms |> purrr::list_transpose() |> tibble::as_tibble() -ds1$properties |> purrr::list_transpose() |> tibble::as_tibble() +ds1$linkedForms |> + purrr::list_transpose() |> + tibble::as_tibble() +ds1$sourceForms |> + purrr::list_transpose() |> + tibble::as_tibble() +ds1$properties |> + purrr::list_transpose() |> + tibble::as_tibble() } } \seealso{ \url{ https://docs.getodk.org/central-api-dataset-management/#datasets} Other entity-management: +\code{\link{entitylist_download}()}, \code{\link{entitylist_list}()} } \concept{entity-management} diff --git a/man/entitylist_download.Rd b/man/entitylist_download.Rd new file mode 100644 index 00000000..5345befb --- /dev/null +++ b/man/entitylist_download.Rd @@ -0,0 +1,145 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/entitylist_download.R +\name{entitylist_download} +\alias{entitylist_download} +\title{Download an Entity List as CSV.} +\usage{ +entitylist_download( + pid = get_default_pid(), + did = NULL, + url = get_default_url(), + un = get_default_un(), + pw = get_default_pw(), + local_dir = here::here(), + overwrite = TRUE, + retries = get_retries(), + odkc_version = get_default_odkc_version(), + orders = c("YmdHMS", "YmdHMSz", "Ymd HMS", "Ymd HMSz", "Ymd", "ymd"), + tz = get_default_tz(), + verbose = get_ru_verbose() +) +} +\arguments{ +\item{pid}{The numeric ID of the project, e.g.: 2. + +Default: \code{\link{get_default_pid}}. + +Set default \code{pid} through \code{ru_setup(pid="...")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{did}{(chr) The name of the Entity List, internally called Dataset.} + +\item{url}{The ODK Central base URL without trailing slash. + +Default: \code{\link{get_default_url}}. + +Set default \code{url} through \code{ru_setup(url="...")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{un}{The ODK Central username (an email address). +Default: \code{\link{get_default_un}}. +Set default \code{un} through \code{ru_setup(un="...")}. +See \code{vignette("Setup", package = "ruODK")}.} + +\item{pw}{The ODK Central password. +Default: \code{\link{get_default_pw}}. +Set default \code{pw} through \code{ru_setup(pw="...")}. +See \code{vignette("Setup", package = "ruODK")}.} + +\item{local_dir}{The local folder to save the downloaded files to, +default: \code{here::here}.} + +\item{overwrite}{Whether to overwrite previously downloaded zip files, +default: FALSE} + +\item{retries}{The number of attempts to retrieve a web resource. + +This parameter is given to \code{\link[httr]{RETRY}(times = retries)}. + +Default: 3.} + +\item{odkc_version}{The ODK Central version as a semantic version string +(year.minor.patch), e.g. "2023.5.1". The version is shown on ODK Central's +version page \verb{/version.txt}. Discard the "v". +\code{ruODK} uses this parameter to adjust for breaking changes in ODK Central. + +Default: \code{\link{get_default_odkc_version}} or "2023.5.1" if unset. + +Set default \code{get_default_odkc_version} through +\code{ru_setup(odkc_version="2023.5.1")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{orders}{(vector of character) Orders of datetime elements for +lubridate. + +Default: +\code{c("YmdHMS", "YmdHMSz", "Ymd HMS", "Ymd HMSz", "Ymd", "ymd")}.} + +\item{tz}{A timezone to convert dates and times to. + +Read \code{vignette("setup", package = "ruODK")} to learn how \code{ruODK}'s +timezone can be set globally or per function.} + +\item{verbose}{Whether to display debug messages or not. + +Read \code{vignette("setup", package = "ruODK")} to learn how \code{ruODK}'s +verbosity can be set globally or per function.} +} +\value{ +The path to the downloaded CSV. +} +\description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#maturing}{\figure{lifecycle-maturing.svg}{options: alt='[Maturing]'}}}{\strong{[Maturing]}} +} +\details{ +The downloaded file is named "entities.csv". The download location defaults +to the current workdir, but can be modified to a folder name. + +An Entity List is a named collection of Entities that have the same +properties. +Entity List can be linked to Forms as Attachments. +This will make it available to clients as an automatically-updating CSV. + +Entity Lists can be used as Attachments in other Forms, but they can also be +downloaded directly as a CSV file. +The CSV format closely matches the OData Dataset (Entity List) Service +format, with columns for system properties such as \verb{__id} (the Entity UUID), +\verb{__createdAt}, \verb{__creatorName}, etc., the Entity Label label, and the +Dataset (Entity List )/Entity Properties themselves. +If any Property for an given Entity is blank (e.g. it was not captured by +that Form or was left blank), that field of the CSV is blank. + +The \verb{$filter} querystring parameter can be used to filter on system-level +properties, similar to how filtering in the OData Dataset (Entity List) +Service works. + +This endpoint supports \code{ETag} header, which can be used to avoid downloading +the same content more than once. When an API consumer calls this endpoint, +the endpoint returns a value in the \code{ETag} header. +If you pass that value in the \code{If-None-Match} header of a subsequent request, +then if the Entity List has not been changed since the previous request, +you will receive 304 Not Modified response; otherwise you'll get the new +data. + +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#maturing}{\figure{lifecycle-maturing.svg}{options: alt='[Maturing]'}}}{\strong{[Maturing]}} +} +\examples{ +\dontrun{ +# See vignette("setup") for setup and authentication options +# ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...") + +ds <- entitylist_list(pid = get_default_pid()) +ds1 <- entitylist_download(pid = get_default_pid(), did = ds$name[1]) +} +} +\seealso{ +\url{ https://docs.getodk.org/central-api-dataset-management/#datasets} + +Other entity-management: +\code{\link{entitylist_detail}()}, +\code{\link{entitylist_list}()} +} +\concept{entity-management} diff --git a/man/entitylist_list.Rd b/man/entitylist_list.Rd index cecf3d35..17681b24 100644 --- a/man/entitylist_list.Rd +++ b/man/entitylist_list.Rd @@ -77,11 +77,13 @@ as per ODK Central API docs. Column names are renamed from ODK's \code{camelCase} to \code{snake_case}. } \description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#maturing}{\figure{lifecycle-maturing.svg}{options: alt='[Maturing]'}}}{\strong{[Maturing]}} +} +\details{ While the API endpoint will return all Entity Lists for one Project, \code{\link{entitylist_list}} will fail with incorrect or missing authentication. -} -\details{ + An Entity List is a named collection of Entities that have the same properties. An Entity List can be linked to Forms as Attachments. This will make it available to clients as an automatically-updating CSV. @@ -109,6 +111,7 @@ ds |> knitr::kable() \url{ https://docs.getodk.org/central-api-dataset-management/#datasets} Other entity-management: -\code{\link{entitylist_detail}()} +\code{\link{entitylist_detail}()}, +\code{\link{entitylist_download}()} } \concept{entity-management} diff --git a/man/entitylist_update.Rd b/man/entitylist_update.Rd index cb12a13f..9ad5662a 100644 --- a/man/entitylist_update.Rd +++ b/man/entitylist_update.Rd @@ -88,6 +88,9 @@ Since this nested list is so deeply nested and irregularly shaped it is not trivial to rectangle the result into a tibble. } \description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#maturing}{\figure{lifecycle-maturing.svg}{options: alt='[Maturing]'}}}{\strong{[Maturing]}} +} +\details{ You can only update \code{approvalRequired} using this endpoint. The approvalRequired flag controls the Entity creation flow; if it is true then the Submission must be approved before an Entity can be @@ -96,8 +99,7 @@ Submission is received by the ODK Central. By default \code{approvalRequired} is false for the Entity Lists created after v2023.3. Entity Lists created prior to that will have approvalRequired set to true. -} -\details{ + An Entity List is a named collection of Entities that have the same properties. An Entity List can be linked to Forms as Attachments. @@ -113,19 +115,19 @@ given odkc_version is lower. # See vignette("setup") for setup and authentication options # ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...") -pid = get_default_pid() +pid <- get_default_pid() ds <- entitylist_list(pid = pid) -did = ds$name[1] +did <- ds$name[1] ds1 <- entitylist_detail(pid = pid, did = did) -ds1$approvalRequired # FALSE +ds1$approvalRequired # FALSE -ds2 <- entitylist_update(pid = pid, did = did, approval_required=TRUE) +ds2 <- entitylist_update(pid = pid, did = did, approval_required = TRUE) ds2$approvalRequired # TRUE -ds3 <- entitylist_update(pid = pid, did = did, approval_required=FALSE) +ds3 <- entitylist_update(pid = pid, did = did, approval_required = FALSE) ds3$approvalRequired # FALSE } } diff --git a/tests/testthat/test-entitylist_download.R b/tests/testthat/test-entitylist_download.R new file mode 100644 index 00000000..c8eb65e4 --- /dev/null +++ b/tests/testthat/test-entitylist_download.R @@ -0,0 +1,50 @@ +test_that("entitylist_download works", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + skip_on_ci() + + tempd <- tempdir() + fs::dir_ls(tempd) %>% fs::file_delete() + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + + ds <- entitylist_list() + + ds1 <- entitylist_download(did = ds$name[1], local_dir = tempd) + + # The file was downloaded + testthat::expect_true( + fs::file_exists(ds1) + ) + + # Download to same location, do not overwrite: emit message, return file path + testthat::expect_message( + ds2 <- entitylist_download( + did = ds$name[1], + local_dir = tempd, + overwrite = FALSE + ) + ) + + # The returned file path is the same as from the first download + testthat::expect_equal(ds1, ds2) +}) + + +test_that("entitylist_download warns if did is missing", { + testthat::expect_error( + entitylist_download() + ) +}) + + +# usethis::use_r("entitylist_download") # nolint