diff --git a/DESCRIPTION b/DESCRIPTION index d7191790..8713a3a3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,11 +1,16 @@ Type: Package Package: ruODK Title: An R Client for the ODK Central API -Version: 1.5.0 -Authors@R:c( - person(c("Florian", "W."), "Mayer", , "Florian.Mayer@dpc.wa.gov.au", role = c("aut", "cre"), - comment = c(ORCID = "0000-0003-4269-4242")), - person("Maëlle", "Salmon", , "maelle.salmon@yahoo.se", role = "rev", +Version: 1.5.1 +Authors@R: c( + person( + given = c("Florian", "W."), + family = "Mayer", + email = "Florian.Mayer@dpc.wa.gov.au", + role = c("aut", "cre"), + comment = c(ORCID = "0000-0003-4269-4242") + ), + person("Maëlle", "Salmon", "maelle.salmon@yahoo.se", role = "rev", comment = c(ORCID = "0000-0002-2815-0399")), person("Karissa", "Whiting", role = "rev", comment = c(ORCID = "0000-0002-4683-1868")), @@ -30,44 +35,46 @@ BugReports: https://github.com/ropensci/ruODK/issues Depends: R (>= 4.1) Imports: + cli (>= 3.6.3), clisymbols (>= 1.2.0), - crayon (>= 1.5.2), - dplyr (>= 1.0.10), - fs (>= 1.6.0), - glue (>= 1.6.2), - httr (>= 1.4.4), - janitor (>= 2.1.0), - lifecycle (>= 1.0.3), - lubridate (>= 1.9.1), + crayon (>= 1.5.3), + dplyr (>= 1.1.4), + fs (>= 1.6.5), + glue (>= 1.8.0), + httr (>= 1.4.7), + janitor (>= 2.2.0), + lifecycle (>= 1.0.4), + lubridate (>= 1.9.3), magrittr (>= 2.0.3), - purrr (>= 1.0.1), - readr (>= 2.1.3), - rlang (>= 1.0.6), + purrr (>= 1.0.2), + readr (>= 2.1.5), + rlang (>= 1.1.4), semver (>= 0.2.0), - stringr (>= 1.5.0), - tibble (>= 3.1.8), - tidyr (>= 1.3.0), - xml2 (>= 1.3.3) + stringr (>= 1.5.1), + tibble (>= 3.2.1), + tidyr (>= 1.3.1), + withr (>= 3.0.2), + xml2 (>= 1.3.6) Suggests: - covr (>= 3.6.1), - DT (>= 0.27), - ggplot2 (>= 3.4.0), + covr (>= 3.6.4), + DT (>= 0.33), + ggplot2 (>= 3.5.1), here (>= 1.0.1), - knitr (>= 1.42), - lattice (>= 0.20-45), - leafem (>= 0.2.0.9012), - leaflet (>= 2.1.1), + knitr (>= 1.48), + lattice (>= 0.22-6), + leafem (>= 0.2.3.9007), + leaflet (>= 2.2.2), leafpop (>= 0.1.0), - listviewer (>= 3.0.0), - mapview (>= 2.11.0.9005), - rmarkdown (>= 2.20), - roxygen2 (>= 7.2.3), - sf (>= 1.0-9), + listviewer (>= 4.0.0), + mapview (>= 2.11.2.9000), + rmarkdown (>= 2.29), + roxygen2 (>= 7.3.2), + sf (>= 1.0-19), skimr (>= 2.1.5), - terra (>= 1.7-3), - testthat (>= 3.1.6), - tmap (>= 3.3-3), - usethis (>= 2.1.6) + terra (>= 1.7-83), + testthat (>= 3.2.1.1), + tmap (>= 3.3-4), + usethis (>= 3.0.0) VignetteBuilder: knitr RdMacros: @@ -81,5 +88,4 @@ LazyData: true Roxygen: list(markdown = TRUE) RoxygenNote: 7.3.2 X-schema.org-applicationCategory: Data Access -X-schema.org-keywords: database, open-data, opendatakit, odk, api, data, - dataset +X-schema.org-keywords: database, open-data, odk, api, data, dataset diff --git a/NAMESPACE b/NAMESPACE index c843baa1..c5ff7a2b 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,8 @@ # Generated by roxygen2: do not edit by hand +S3method(print,odata_entitylist_data_get) +S3method(print,odata_entitylist_metadata_get) +S3method(print,odata_entitylist_service_get) S3method(print,ru_settings) export("%>%") export(attachment_get) @@ -62,6 +65,9 @@ export(handle_ru_datetimes) export(handle_ru_geopoints) export(handle_ru_geoshapes) export(handle_ru_geotraces) +export(odata_entitylist_data_get) +export(odata_entitylist_metadata_get) +export(odata_entitylist_service_get) export(odata_metadata_get) export(odata_service_get) export(odata_submission_get) @@ -94,6 +100,7 @@ export(submission_list) export(sym) export(syms) export(user_list) +import(rlang) import(semver) importFrom(dplyr,all_of) importFrom(dplyr,any_of) @@ -102,11 +109,13 @@ importFrom(dplyr,ends_with) importFrom(dplyr,everything) importFrom(dplyr,one_of) importFrom(dplyr,starts_with) +importFrom(glue,glue) importFrom(httr,GET) importFrom(httr,add_headers) importFrom(httr,authenticate) importFrom(httr,content) importFrom(lifecycle,deprecate_soft) +importFrom(lifecycle,deprecated) importFrom(magrittr,"%>%") importFrom(rlang,"%||%") importFrom(rlang,":=") diff --git a/NEWS.md b/NEWS.md index 81e92096..b3f2e811 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,14 @@ +# ruODK 1.5.1 +## Major changes +* Add support for the OData API endpoints for Entities (#152) +* Bump minimum versions of dependencies + +## Minor changes +* `entity_changes` now returns a tibble instead of nested list. +* `submission_export` gains a new parameter `deleted_fields` to export all + known fields of a form, including fields that were deleted in the latest form + version. (#129, #161) + # ruODK 1.5.0 ## Major changes * Support Entities and Entity Lists (Datasets) (#152) diff --git a/R/attachment_get.R b/R/attachment_get.R index 0ef80e0f..acd11fad 100644 --- a/R/attachment_get.R +++ b/R/attachment_get.R @@ -152,7 +152,7 @@ get_one_attachment <- function(pth, yell_if_missing(url, un, pw) - res <- httr::RETRY( + httr::RETRY( "GET", src, httr::authenticate(un, pw), diff --git a/R/entity_changes.R b/R/entity_changes.R index 0baa7898..f03ad39c 100644 --- a/R/entity_changes.R +++ b/R/entity_changes.R @@ -21,11 +21,12 @@ #' @template param-odkcv #' @template param-orders #' @template param-tz -#' @return A nested list of lists. -#' The first nesting level corresponds to entity updates, the second level -#' lists each updated property. Within the nested list, the names are -#' `old` (old value), `new` (new value), `propertyName` (name of changed -#' entity property). +#' @return A tibble where rows correspond to each Entity update +#' with three columns: +#' +#' - `old` old value +#' - `new` new value +#' - `propertyName` name of changed entity property # nolint start #' @seealso \url{https://docs.getodk.org/central-api-entity-management/#getting-changes-between-versions} # nolint end @@ -105,7 +106,8 @@ entity_changes <- function(pid = get_default_pid(), times = retries ) |> yell_if_error(url, un, pw) |> - httr::content(encoding = "utf-8") + httr::content(encoding = "utf-8") |> + purrr::map_df(~ purrr::map_df(.x, ~ tibble::as_tibble(.x))) } # usethis::use_test("entity_update") # nolint diff --git a/R/entitylist_detail.R b/R/entitylist_detail.R index 89561e21..68103563 100644 --- a/R/entitylist_detail.R +++ b/R/entitylist_detail.R @@ -62,7 +62,7 @@ entitylist_detail <- function(pid = get_default_pid(), ru_msg_warn("entitylist_detail is supported from v2022.3") } - ds <- httr::RETRY( + httr::RETRY( "GET", httr::modify_url(url, path = glue::glue( diff --git a/R/entitylist_update.R b/R/entitylist_update.R index 69441032..44d45e0d 100644 --- a/R/entitylist_update.R +++ b/R/entitylist_update.R @@ -73,7 +73,7 @@ entitylist_update <- function(pid = get_default_pid(), ru_msg_warn("entitylist_update is supported from v2022.3") } - ds <- httr::RETRY( + httr::RETRY( "PATCH", httr::modify_url(url, path = glue::glue( diff --git a/R/odata_entitylist_data_get.R b/R/odata_entitylist_data_get.R new file mode 100644 index 00000000..cca28f1d --- /dev/null +++ b/R/odata_entitylist_data_get.R @@ -0,0 +1,210 @@ +#' Get the Data Document from the OData Dataset Service. +#' +#' `r lifecycle::badge("experimental")` +#' +#' All the Entities in a Dataset. +#' +#' The `$top` and `$skip` querystring parameters, specified by OData, apply limit +#' and offset operations to the data, respectively. +#' +#' The `$count` parameter, also an OData standard, will annotate the response +#' data with the total row count, regardless of the scoping requested by `$top` +#' and `$skip`. +#' If `$top` parameter is provided in the request then the response will include +#' `@odata.nextLink` that you can use as is to fetch the next set of data. +#' As of ODK Central v2023.4, `@odata.nextLink` contains a `$skiptoken` +#' (an opaque cursor) to better paginate around deleted Entities. +#' +#' The `$filter` querystring parameter can be used to filter certain data fields +#' in the system-level schema, but not the Dataset properties. +#' The operators `lt`, `le`, `eq`, `ne`, `ge`, `gt`, `not`, `and`, and `or` +#' and the built-in functions `now`, `year`, `month`, `day`, `hour`, +#' `minute`, `second` are supported. +#' +#' The fields you can query against are as follows: +#' +#' Entity Metadata: `OData Field Name` +#' Entity Creator Actor ID: `__system/creatorId` +#' Entity Timestamp: `__system/createdAt` +#' Entity Update Timestamp: `__system/updatedAt` +#' Entity Conflict: `__system/conflict` +#' +#' Note that `createdAt` and `updatedAt` are time components. +#' This means that any comparisons you make need to account for the full time +#' of the entity. It might seem like `$filter=__system/createdAt le 2020-01-31` +#' would return all results on or before 31 Jan 2020, but in fact only entities +#' made before midnight of that day would be accepted. +#' To include all of the month of January, you need to filter by either +#' `$filter=__system/createdAt le 2020-01-31T23:59:59.999Z` or +#' `$filter=__system/createdAt lt 2020-02-01`. +#' Remember also that you can query by a specific timezone. +#' +#' Please see the OData documentation on `$filter` +# nolint start +#' [operations](http://docs.oasis-open.org/odata/odata/v4.01/cs01/part1-protocol/odata-v4.01-cs01-part1-protocol.html#sec_BuiltinFilterOperations) +#' and [functions](http://docs.oasis-open.org/odata/odata/v4.01/cs01/part1-protocol/odata-v4.01-cs01-part1-protocol.html#sec_BuiltinQueryFunctions) +# nolint end +#' for more information. +#' +#' The `$select` query parameter will return just the fields you specify and is +#' supported on `__id`, `__system`, `__system/creatorId`, `__system/createdAt` +#' and `__system/updatedAt`, as well as on user defined properties. +#' +#' The `$orderby` query parameter will return Entities sorted by different +#' fields, which come from the same list used by `$filter`, as noted above. +#' The order can be specified as ASC (ascending) or DESC (descending), +#' which are case-insensitive. Multiple sort expressions can be used together, +#' separated by commas, +#' e.g. `$orderby=__system/creatorId ASC, __system/conflict DESC`. +#' +#' As the vast majority of clients only support the JSON OData format, +#' that is the only format ODK Central offers. +#' +#' @template tpl-structure-nested +#' @template tpl-names-cleaned-top-level +#' @template tpl-def-entitylist +#' @template tpl-entitylist-dataset +#' @template tpl-auth-missing +#' @template tpl-compat-2022-3 +#' @template param-pid +#' @template param-did +#' @param query An optional named list of query parameters, e.g. +#' ``` +#' list( +#' "$filter" = "__system/createdAt le 2024-11-05", +#' "$orderby" = "__system/creatorId ASC, __system/conflict DESC", +#' "$top" = "100", +#' "$skip" = "3", +#' "$count" = "true" +#' ) +#' ``` +#' No validation is conducted by `ruODK` on the query list prior to +#' passing it to ODK Central. +#' If omitted, no filter query is sent. +#' Note that the behaviour of this parameter differs from the implementation +#' of `odata_submission_get()` in that `query` here accepts a list of all +#' possible OData query parameters and `odata_submission_get()` offers +#' individual function parameters matching supported OData query parameters. +#' Default: `NULL` +#' @template param-url +#' @template param-auth +#' @template param-retries +#' @template param-odkcv +#' @template param-orders +#' @template param-tz +#' @return An S3 class `odata_entitylist_data_get` with two list items: +#' * `context` The URL for the OData metadata document +#' * `value` A tibble of EntitySets available in this EntityList, with names +#' cleaned by `janitor::clean_names()` and unnested list columns +#' (`__system`). +#' @seealso \url{https://docs.getodk.org/central-api-odata-endpoints/#id3} +# nolint start +#' @seealso \url{http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#_Toc31358948} +# nolint end +#' @family entity-management +#' @export +#' @examples +#' \dontrun{ +#' # See vignette("setup") for setup and authentication options +#' # ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...") +#' +#' ds <- entitylist_list(pid = get_default_pid()) +#' +#' ds1 <- odata_entitylist_data_get(pid = get_default_pid(), did = ds$name[1]) +#' +#' ds1 +#' ds1$context +#' ds1$value +#' +#' qry <- list( +#' "$filter" = "__system/createdAt le 2024-11-05", +#' "$orderby" = "__system/creatorId ASC, __system/conflict DESC", +#' "$top" = "100", +#' "$skip" = "3", +#' "$count" = "true" +#' ) +#' ds2 <- odata_entitylist_data_get( +#' pid = get_default_pid(), +#' did = ds$name[1], +#' query = qry +#' ) +#' +#' ds2 +#' } +odata_entitylist_data_get <- function(pid = get_default_pid(), + did = "", + query = NULL, + url = get_default_url(), + un = get_default_un(), + pw = get_default_pw(), + retries = get_retries(), + odkc_version = get_default_odkc_version(), + orders = get_default_orders(), + tz = get_default_tz()) { + yell_if_missing(url, un, pw, pid = pid, did = did) + + if (odkc_version |> semver_lt("2022.3")) { + ru_msg_warn("odata_entitylist_service_get is supported from v2022.3") + } + + ds <- httr::RETRY( + "GET", + httr::modify_url(url, + path = glue::glue( + "v1/projects/{pid}/datasets/", + "{URLencode(did, reserved = TRUE)}.svc/Entities" + ), + query = query + ), + httr::add_headers( + "Accept" = "application/json" + ), + httr::authenticate(un, pw), + times = retries + ) |> + yell_if_error(url, un, pw) |> + httr::content(encoding = "utf-8") |> + janitor::clean_names() + + entities <- ds$value |> + # Replace NULLs with NA in each list + purrr::map(~ purrr::modify_if(.x, is.null, ~NA)) |> + purrr::map_dfr( + ~ { + # Extract main fields excluding __system + main_fields <- .x[!names(.x) %in% "__system"] + + # Extract system fields or empty list if NULL + system_fields <- .x[["__system"]] %||% list() + + # Ensure any NULL fields within system are NA + system_fields <- purrr::modify_if(system_fields, is.null, ~NA) + + # Combine main and system fields into a single tibble row + tibble::as_tibble(c(main_fields, system_fields)) + } + ) |> + janitor::clean_names() |> + # Remove duplicate rows by `id` + dplyr::distinct(id, .keep_all = TRUE) |> + dplyr::mutate( + dplyr::across( + dplyr::matches("created_at|updated_at"), + ~ isodt_to_local(., orders = orders, tz = tz) + ) + ) + + structure( + list(context = ds$odata_context, value = entities), + class = c("odata_entitylist_data_get", "list") + ) +} + +#' @export +print.odata_entitylist_data_get <- function(x, ...) { + cat("", sep = "\n") + cat(" OData Context: ", x$context, "\n") + cat(" OData Entities:", nrow(x$value), "\n") +} + +# usethis::use_test("odata_entitylist_data_get") # nolint diff --git a/R/odata_entitylist_metadata_get.R b/R/odata_entitylist_metadata_get.R new file mode 100644 index 00000000..eb691300 --- /dev/null +++ b/R/odata_entitylist_metadata_get.R @@ -0,0 +1,212 @@ +#' Get the Metadata Document from the OData Dataset Service. +#' +#' `r lifecycle::badge("experimental")` +#' +#' The Metadata Document describes, in +#' [EDMX CSDL](https://docs.oasis-open.org/odata/odata-csdl-xml/v4.01/odata-csdl-xml-v4.01.html), +#' the schema of all the data you can retrieve from the OData Dataset Service +#' in question. Essentially, these are the Dataset properties, or the schema of +#' each Entity, translated into the OData format. +#' +#' @template tpl-structure-nested +#' @template tpl-def-entitylist +#' @template tpl-entitylist-dataset +#' @template tpl-auth-missing +#' @template tpl-compat-2022-3 +#' @template param-pid +#' @template param-did +#' @template param-url +#' @template param-auth +#' @template param-retries +#' @template param-odkcv +#' @template param-orders +#' @template param-tz +#' @return An S3 class `odata_entitylist_metadata_get` and `list` containing +#' the Metadata document following the DDMX CSDL standard +#' +#' * `version` The EDMX version, e.g. "4.0" +#' * `complex_types` +#' * `entity_types` +#' * `containers` +# nolint start +#' @seealso \url{https://docs.getodk.org/central-api-odata-endpoints/#id2} +#' @seealso \url{https://docs.oasis-open.org/odata/odata-csdl-xml/v4.01/odata-csdl-xml-v4.01.html} +# nolint end +#' @family entity-management +#' @export +#' @examples +#' \dontrun{ +#' # See vignette("setup") for setup and authentication options +#' # ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...") +#' +#' ds <- entitylist_list(pid = get_default_pid()) +#' +#' dm1 <- odata_entitylist_metadata_get(pid = get_default_pid(), did = ds$name[1]) +#' +#' # Overview +#' print(dm1) +#' +#' # Get all property names for an entity type +#' names(dm1$entity_types$Entities$properties) +#' +#' # Check what properties are non-filterable +#' dm1$containers$trees$entity_sets$Entities$capabilities +#' +#' # Get complex type definitions +#' dm1$complex_types$metadata$properties +#' } +odata_entitylist_metadata_get <- function(pid = get_default_pid(), + did = "", + url = get_default_url(), + un = get_default_un(), + pw = get_default_pw(), + retries = get_retries(), + odkc_version = get_default_odkc_version(), + orders = get_default_orders(), + tz = get_default_tz()) { + yell_if_missing(url, un, pw, pid = pid, did = did) + + if (odkc_version |> semver_lt("2022.3")) { + ru_msg_warn("odata_entitylist_service_get is supported from v2022.3") + } + + doc <- httr::RETRY( + "GET", + httr::modify_url(url, + path = glue::glue( + "v1/projects/{pid}/datasets/", + "{URLencode(did, reserved = TRUE)}.svc/$metadata" + ) + ), + httr::add_headers( + "Accept" = "application/xml" + ), + httr::authenticate(un, pw), + times = retries + ) |> + yell_if_error(url, un, pw) |> + httr::content(encoding = "utf-8") + + # Convert EDMX XML Document to structured R object + + # Define the namespaces explicitly + ns <- c( + edmx = "http://docs.oasis-open.org/odata/ns/edmx", + edm = "http://docs.oasis-open.org/odata/ns/edm" + ) + + # Helper function to extract property information + extract_properties <- function(type_node) { + props <- xml2::xml_find_all(type_node, "./edm:Property", ns) + property_list <- lapply(props, function(prop) { + list( + name = xml2::xml_attr(prop, "Name"), + type = xml2::xml_attr(prop, "Type") + ) + }) + names(property_list) <- sapply(property_list, `[[`, "name") + property_list + } + + # Extract complex types + complex_types <- xml2::xml_find_all(doc, "//edm:ComplexType", ns) + complex_types_list <- lapply(complex_types, function(type) { + list( + name = xml2::xml_attr(type, "Name"), + properties = extract_properties(type) + ) + }) + names(complex_types_list) <- sapply(complex_types_list, `[[`, "name") + + # Extract entity types + entity_types <- xml2::xml_find_all(doc, "//edm:EntityType", ns) + entity_types_list <- lapply(entity_types, function(type) { + # Extract key properties + keys <- xml2::xml_find_all(type, ".//edm:PropertyRef", ns) + key_names <- sapply(keys, xml2::xml_attr, "Name") + + list( + name = xml2::xml_attr(type, "Name"), + keys = key_names, + properties = extract_properties(type) + ) + }) + names(entity_types_list) <- sapply(entity_types_list, `[[`, "name") + + # Extract entity container information + containers <- xml2::xml_find_all(doc, "//edm:EntityContainer", ns) + container_list <- lapply(containers, function(container) { + entity_sets <- xml2::xml_find_all(container, "./edm:EntitySet", ns) + sets_list <- lapply(entity_sets, function(set) { + # Extract capabilities from annotations + annotations <- xml2::xml_find_all(set, "./edm:Annotation", ns) + capabilities <- lapply(annotations, function(anno) { + term <- xml2::xml_attr(anno, "Term") + if (grepl("ConformanceLevel$", term)) { + list( + capability = "conformance_level", + value = xml2::xml_attr(anno, "EnumMember") + ) + } else if (grepl("BatchSupported$", term)) { + list( + capability = "batch_supported", + value = as.logical(xml2::xml_attr(anno, "Bool")) + ) + } else if (grepl("FilterRestrictions$", term)) { + # Extract non-filterable properties + non_filterable <- xml2::xml_find_all(anno, ".//edm:PropertyPath", ns) + if (length(non_filterable) > 0) { + list( + capability = "filter_restrictions", + non_filterable_properties = xml2::xml_text(non_filterable) + ) + } else { + NULL + } + } else { + NULL + } + }) + capabilities <- Filter(Negate(is.null), capabilities) + + list( + name = xml2::xml_attr(set, "Name"), + type = xml2::xml_attr(set, "EntityType"), + capabilities = capabilities + ) + }) + names(sets_list) <- sapply(sets_list, `[[`, "name") + + list( + name = xml2::xml_attr(container, "Name"), + entity_sets = sets_list + ) + }) + names(container_list) <- sapply(container_list, `[[`, "name") + + # Get the root Edmx node to extract version + edmx_root <- xml2::xml_find_first(doc, "//edmx:Edmx", ns) + version <- xml2::xml_attr(edmx_root, "Version") + + # Create the final structure + structure(list( + version = version, + complex_types = complex_types_list, + entity_types = entity_types_list, + containers = container_list + ), class = c("odata_entitylist_metadata_get", "list")) +} + +#' @export +print.odata_entitylist_metadata_get <- function(x, ...) { + cat("", sep = "\n") + cat(" Complex Types: ") + print(names(x$complex_types)) + cat(" Entity Types: ") + print(names(x$entity_types)) + cat(" Containers: ") + print(names(x$containers)) +} + + +# usethis::use_test("odata_entitylist_metadata_get") # nolint diff --git a/R/odata_entitylist_service_get.R b/R/odata_entitylist_service_get.R new file mode 100644 index 00000000..35c936d9 --- /dev/null +++ b/R/odata_entitylist_service_get.R @@ -0,0 +1,97 @@ +#' Get the Service Document from the OData Dataset Service. +#' +#' `r lifecycle::badge("experimental")` +#' +#' ODK Central presents one OData service for every Dataset (Entity List) +#' as a way to get an OData feed of Entities. +#' To access the OData service, add `.svc` to the resource URL for the given +#' Dataset (Entity List). +#' +#' The Service Document provides a link to the main source of information +#' in this OData service: the list of Entities in this Dataset, +#' as well as the Metadata Document describing the schema of this information. +#' +#' This document is available only in JSON format. +#' +#' @template tpl-structure-nested +#' @template tpl-names-cleaned-top-level +#' @template tpl-def-entitylist +#' @template tpl-entitylist-dataset +#' @template tpl-auth-missing +#' @template tpl-compat-2022-3 +#' @template param-pid +#' @template param-did +#' @template param-url +#' @template param-auth +#' @template param-retries +#' @template param-odkcv +#' @template param-orders +#' @template param-tz +#' @return An S3 class `odata_entitylist_service_get` with two list items: +#' * `context` The URL for the OData metadata document +#' * `value` A tibble of EntitySets available in this EntityList +# nolint start +#' @seealso \url{https://docs.getodk.org/central-api-odata-endpoints/#odata-dataset-service} +# nolint end +#' @family entity-management +#' @export +#' @examples +#' \dontrun{ +#' # See vignette("setup") for setup and authentication options +#' # ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...") +#' +#' ds <- entitylist_list(pid = get_default_pid()) +#' +#' ds1 <- odata_entitylist_service_get(pid = get_default_pid(), did = ds$name[1]) +#' +#' ds1 +#' ds1$context +#' ds1$value +#' } +odata_entitylist_service_get <- function(pid = get_default_pid(), + did = "", + url = get_default_url(), + un = get_default_un(), + pw = get_default_pw(), + retries = get_retries(), + odkc_version = get_default_odkc_version(), + orders = get_default_orders(), + tz = get_default_tz()) { + yell_if_missing(url, un, pw, pid = pid, did = did) + + if (odkc_version |> semver_lt("2022.3")) { + ru_msg_warn("odata_entitylist_service_get is supported from v2022.3") + } + + ds <- httr::RETRY( + "GET", + httr::modify_url(url, + path = glue::glue( + "v1/projects/{pid}/datasets/", + "{URLencode(did, reserved = TRUE)}.svc" + ) + ), + httr::add_headers( + "Accept" = "application/json" + ), + httr::authenticate(un, pw), + times = retries + ) |> + yell_if_error(url, un, pw) |> + httr::content(encoding = "utf-8") |> + janitor::clean_names() + + structure(list( + context = ds$odata_context, + value = purrr::map_df(ds$value, ~ tibble::as_tibble(.x)) + ), class = c("odata_entitylist_service_get", "list")) +} + +#' @export +print.odata_entitylist_service_get <- function(x, ...) { + cat("", sep = "\n") + cat(" OData Context: ", x$context, "\n") + cat(" OData Entities:", nrow(x$value), "\n") +} + +# usethis::use_test("odata_entitylist_service_get") # nolint diff --git a/R/project_detail.R b/R/project_detail.R index ae1280b0..58933a1f 100644 --- a/R/project_detail.R +++ b/R/project_detail.R @@ -48,9 +48,9 @@ project_detail <- function(pid = get_default_pid(), ), httr::authenticate(un, pw), times = retries - ) %>% - yell_if_error(., url, un, pw) %>% - httr::content(.) %>% + ) |> + yell_if_error(url, un, pw) |> + httr::content() %>% { # nolint tibble::tibble( id = .$id, diff --git a/R/ruODK.R b/R/ruODK-package.R similarity index 75% rename from R/ruODK.R rename to R/ruODK-package.R index 664e36a1..ab11ceaa 100644 --- a/R/ruODK.R +++ b/R/ruODK-package.R @@ -13,6 +13,7 @@ utils::globalVariables(c( ".", "archived", "children", + "id", "name", "path", "type", @@ -20,3 +21,10 @@ utils::globalVariables(c( "xx", "xml_form_id" )) + +## usethis namespace: start +#' @import rlang +#' @importFrom glue glue +#' @importFrom lifecycle deprecated +## usethis namespace: end +NULL diff --git a/R/submission_export.R b/R/submission_export.R index 038a2d2e..c6433bbb 100644 --- a/R/submission_export.R +++ b/R/submission_export.R @@ -48,6 +48,10 @@ #' parameter `media`. #' Setting this feature to FALSE with an odkc_version < 1.1 and will display a #' verbose noop message, but still include all repeat data. +#' @param deleted_fields Whether to restore all fields previously deleted +#' from this form for this export (TRUE). +#' All known fields and data for those fields will be merged and exported. +#' default: FALSE #' @template param-pid #' @template param-fid #' @template param-url @@ -88,6 +92,7 @@ submission_export <- function(local_dir = here::here(), overwrite = TRUE, media = TRUE, repeats = TRUE, + deleted_fields = FALSE, pid = get_default_pid(), fid = get_default_fid(), url = get_default_url(), @@ -101,11 +106,13 @@ submission_export <- function(local_dir = here::here(), url_ext <- ".csv.zip" file_ext <- ".zip" + query <- NULL if (semver_gt(odkc_version, "1.0.0")) { # odkc_version >= 1.1 if (media == FALSE) { - url_ext <- ".csv.zip?attachments=false" + url_ext <- ".csv.zip" + query <- list("attachments" = "false") } if (repeats == FALSE) { url_ext <- ".csv" @@ -122,6 +129,12 @@ submission_export <- function(local_dir = here::here(), } } + if (deleted_fields == TRUE) { + query <- c(query, list("deletedFields" = "true")) + } else { + query <- c(query, list("deletedFields" = "false")) + } + url_pth <- glue::glue( "v1/projects/{pid}/forms/", "{URLencode(fid, reserved = TRUE)}/submissions{url_ext}" @@ -180,7 +193,7 @@ submission_export <- function(local_dir = here::here(), # Export form submissions to CSV via POST httr::RETRY( "POST", - httr::modify_url(url, path = url_pth), + httr::modify_url(url, path = url_pth, query = query), body = body, encode = "json", httr::authenticate(un, pw), diff --git a/man/entity_audits.Rd b/man/entity_audits.Rd index 4429f826..733df73a 100644 --- a/man/entity_audits.Rd +++ b/man/entity_audits.Rd @@ -151,6 +151,9 @@ Other entity-management: \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entity_changes.Rd b/man/entity_changes.Rd index 3f5a7cf1..e3278a74 100644 --- a/man/entity_changes.Rd +++ b/man/entity_changes.Rd @@ -83,11 +83,13 @@ Read \code{vignette("setup", package = "ruODK")} to learn how \code{ruODK}'s timezone can be set globally or per function.} } \value{ -A nested list of lists. -The first nesting level corresponds to entity updates, the second level -lists each updated property. Within the nested list, the names are -\code{old} (old value), \code{new} (new value), \code{propertyName} (name of changed -entity property). +A tibble where rows correspond to each Entity update +with three columns: +\itemize{ +\item \code{old} old value +\item \code{new} new value +\item \code{propertyName} name of changed entity property +} } \description{ \ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#maturing}{\figure{lifecycle-maturing.svg}{options: alt='[Maturing]'}}}{\strong{[Maturing]}} @@ -150,6 +152,9 @@ Other entity-management: \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entity_create.Rd b/man/entity_create.Rd index fe655acf..5f9274eb 100644 --- a/man/entity_create.Rd +++ b/man/entity_create.Rd @@ -226,6 +226,9 @@ Other entity-management: \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entity_delete.Rd b/man/entity_delete.Rd index b67be27e..566e1e83 100644 --- a/man/entity_delete.Rd +++ b/man/entity_delete.Rd @@ -127,6 +127,9 @@ Other entity-management: \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entity_detail.Rd b/man/entity_detail.Rd index f7ef608f..01fa0d5c 100644 --- a/man/entity_detail.Rd +++ b/man/entity_detail.Rd @@ -155,6 +155,9 @@ Other entity-management: \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entity_list.Rd b/man/entity_list.Rd index f5a9bf25..b6916357 100644 --- a/man/entity_list.Rd +++ b/man/entity_list.Rd @@ -131,6 +131,9 @@ Other entity-management: \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entity_update.Rd b/man/entity_update.Rd index ebd05b2f..29d365a3 100644 --- a/man/entity_update.Rd +++ b/man/entity_update.Rd @@ -190,6 +190,9 @@ Other entity-management: \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entity_versions.Rd b/man/entity_versions.Rd index 3a211853..20726675 100644 --- a/man/entity_versions.Rd +++ b/man/entity_versions.Rd @@ -150,6 +150,9 @@ Other entity-management: \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entitylist_detail.Rd b/man/entitylist_detail.Rd index fcd553a4..84b17046 100644 --- a/man/entitylist_detail.Rd +++ b/man/entitylist_detail.Rd @@ -122,6 +122,9 @@ Other entity-management: \code{\link{entity_versions}()}, \code{\link{entitylist_download}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entitylist_download.Rd b/man/entitylist_download.Rd index 493e58ab..6a0aaddc 100644 --- a/man/entitylist_download.Rd +++ b/man/entitylist_download.Rd @@ -206,6 +206,9 @@ Other entity-management: \code{\link{entity_versions}()}, \code{\link{entitylist_detail}()}, \code{\link{entitylist_list}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entitylist_list.Rd b/man/entitylist_list.Rd index c487433d..9bc6bff0 100644 --- a/man/entitylist_list.Rd +++ b/man/entitylist_list.Rd @@ -107,6 +107,9 @@ Other entity-management: \code{\link{entity_versions}()}, \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, -\code{\link{entitylist_update}()} +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/entitylist_update.Rd b/man/entitylist_update.Rd index 9b109b20..e9535c82 100644 --- a/man/entitylist_update.Rd +++ b/man/entitylist_update.Rd @@ -137,6 +137,9 @@ Other entity-management: \code{\link{entity_versions}()}, \code{\link{entitylist_detail}()}, \code{\link{entitylist_download}()}, -\code{\link{entitylist_list}()} +\code{\link{entitylist_list}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} } \concept{entity-management} diff --git a/man/figures/lifecycle-deprecated.svg b/man/figures/lifecycle-deprecated.svg index 02aee08b..b61c57c3 100644 --- a/man/figures/lifecycle-deprecated.svg +++ b/man/figures/lifecycle-deprecated.svg @@ -1 +1,21 @@ -lifecyclelifecycledeprecateddeprecated + + lifecycle: deprecated + + + + + + + + + + + + + + + lifecycle + + deprecated + + diff --git a/man/figures/lifecycle-experimental.svg b/man/figures/lifecycle-experimental.svg index 5df89276..5d88fc2c 100644 --- a/man/figures/lifecycle-experimental.svg +++ b/man/figures/lifecycle-experimental.svg @@ -1 +1,21 @@ -lifecyclelifecycleexperimentalexperimental + + lifecycle: experimental + + + + + + + + + + + + + + + lifecycle + + experimental + + diff --git a/man/figures/lifecycle-stable.svg b/man/figures/lifecycle-stable.svg index 89a2b8e8..9bf21e76 100644 --- a/man/figures/lifecycle-stable.svg +++ b/man/figures/lifecycle-stable.svg @@ -1 +1,29 @@ -lifecyclelifecyclestablestable + + lifecycle: stable + + + + + + + + + + + + + + + + lifecycle + + + + stable + + + diff --git a/man/figures/lifecycle-superseded.svg b/man/figures/lifecycle-superseded.svg index a9118b6f..db8d757f 100644 --- a/man/figures/lifecycle-superseded.svg +++ b/man/figures/lifecycle-superseded.svg @@ -1 +1,21 @@ - lifecyclelifecyclesupersededsuperseded + + lifecycle: superseded + + + + + + + + + + + + + + + lifecycle + + superseded + + diff --git a/man/odata_entitylist_data_get.Rd b/man/odata_entitylist_data_get.Rd new file mode 100644 index 00000000..888e6787 --- /dev/null +++ b/man/odata_entitylist_data_get.Rd @@ -0,0 +1,219 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/odata_entitylist_data_get.R +\name{odata_entitylist_data_get} +\alias{odata_entitylist_data_get} +\title{Get the Data Document from the OData Dataset Service.} +\usage{ +odata_entitylist_data_get( + pid = get_default_pid(), + did = "", + query = NULL, + url = get_default_url(), + un = get_default_un(), + pw = get_default_pw(), + retries = get_retries(), + odkc_version = get_default_odkc_version(), + orders = get_default_orders(), + tz = get_default_tz() +) +} +\arguments{ +\item{pid}{The numeric ID of the project, e.g.: 2. + +Default: \code{\link{get_default_pid}}. + +Set default \code{pid} through \code{ru_setup(pid="...")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{did}{(chr) The name of the Entity List, internally called Dataset. +The function will error if this parameter is not given. +Default: "".} + +\item{query}{An optional named list of query parameters, e.g. + +\if{html}{\out{
}}\preformatted{list( + "$filter" = "__system/createdAt le 2024-11-05", + "$orderby" = "__system/creatorId ASC, __system/conflict DESC", + "$top" = "100", + "$skip" = "3", + "$count" = "true" +) +}\if{html}{\out{
}} + +No validation is conducted by \code{ruODK} on the query list prior to +passing it to ODK Central. +If omitted, no filter query is sent. +Note that the behaviour of this parameter differs from the implementation +of \code{odata_submission_get()} in that \code{query} here accepts a list of all +possible OData query parameters and \code{odata_submission_get()} offers +individual function parameters matching supported OData query parameters. +Default: \code{NULL}} + +\item{url}{The ODK Central base URL without trailing slash. + +Default: \code{\link{get_default_url}}. + +Set default \code{url} through \code{ru_setup(url="...")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{un}{The ODK Central username (an email address). +Default: \code{\link{get_default_un}}. +Set default \code{un} through \code{ru_setup(un="...")}. +See \code{vignette("Setup", package = "ruODK")}.} + +\item{pw}{The ODK Central password. +Default: \code{\link{get_default_pw}}. +Set default \code{pw} through \code{ru_setup(pw="...")}. +See \code{vignette("Setup", package = "ruODK")}.} + +\item{retries}{The number of attempts to retrieve a web resource. + +This parameter is given to \code{\link[httr]{RETRY}(times = retries)}. + +Default: 3.} + +\item{odkc_version}{The ODK Central version as a semantic version string +(year.minor.patch), e.g. "2023.5.1". The version is shown on ODK Central's +version page \verb{/version.txt}. Discard the "v". +\code{ruODK} uses this parameter to adjust for breaking changes in ODK Central. + +Default: \code{\link{get_default_odkc_version}} or "2023.5.1" if unset. + +Set default \code{get_default_odkc_version} through +\code{ru_setup(odkc_version="2023.5.1")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{orders}{(vector of character) Orders of datetime elements for +lubridate. + +Default: +\code{c("YmdHMS", "YmdHMSz", "Ymd HMS", "Ymd HMSz", "Ymd", "ymd")}.} + +\item{tz}{A timezone to convert dates and times to. + +Read \code{vignette("setup", package = "ruODK")} to learn how \code{ruODK}'s +timezone can be set globally or per function.} +} +\value{ +An S3 class \code{odata_entitylist_data_get} with two list items: +\itemize{ +\item \code{context} The URL for the OData metadata document +\item \code{value} A tibble of EntitySets available in this EntityList, with names +cleaned by \code{janitor::clean_names()} and unnested list columns +(\verb{__system}). +} +} +\description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} +} +\details{ +All the Entities in a Dataset. + +The \verb{$top} and \verb{$skip} querystring parameters, specified by OData, apply limit +and offset operations to the data, respectively. + +The \verb{$count} parameter, also an OData standard, will annotate the response +data with the total row count, regardless of the scoping requested by \verb{$top} +and \verb{$skip}. +If \verb{$top} parameter is provided in the request then the response will include +\verb{@odata.nextLink} that you can use as is to fetch the next set of data. +As of ODK Central v2023.4, \verb{@odata.nextLink} contains a \verb{$skiptoken} +(an opaque cursor) to better paginate around deleted Entities. + +The \verb{$filter} querystring parameter can be used to filter certain data fields +in the system-level schema, but not the Dataset properties. +The operators \code{lt}, \code{le}, \code{eq}, \code{ne}, \code{ge}, \code{gt}, \code{not}, \code{and}, and \code{or} +and the built-in functions \code{now}, \code{year}, \code{month}, \code{day}, \code{hour}, +\code{minute}, \code{second} are supported. + +The fields you can query against are as follows: + +Entity Metadata: \verb{OData Field Name} +Entity Creator Actor ID: \verb{__system/creatorId} +Entity Timestamp: \verb{__system/createdAt} +Entity Update Timestamp: \verb{__system/updatedAt} +Entity Conflict: \verb{__system/conflict} + +Note that \code{createdAt} and \code{updatedAt} are time components. +This means that any comparisons you make need to account for the full time +of the entity. It might seem like \verb{$filter=__system/createdAt le 2020-01-31} +would return all results on or before 31 Jan 2020, but in fact only entities +made before midnight of that day would be accepted. +To include all of the month of January, you need to filter by either +\verb{$filter=__system/createdAt le 2020-01-31T23:59:59.999Z} or +\verb{$filter=__system/createdAt lt 2020-02-01}. +Remember also that you can query by a specific timezone. + +Please see the OData documentation on \verb{$filter} +\href{http://docs.oasis-open.org/odata/odata/v4.01/cs01/part1-protocol/odata-v4.01-cs01-part1-protocol.html#sec_BuiltinFilterOperations}{operations} +and \href{http://docs.oasis-open.org/odata/odata/v4.01/cs01/part1-protocol/odata-v4.01-cs01-part1-protocol.html#sec_BuiltinQueryFunctions}{functions} +for more information. + +The \verb{$select} query parameter will return just the fields you specify and is +supported on \verb{__id}, \verb{__system}, \verb{__system/creatorId}, \verb{__system/createdAt} +and \verb{__system/updatedAt}, as well as on user defined properties. + +The \verb{$orderby} query parameter will return Entities sorted by different +fields, which come from the same list used by \verb{$filter}, as noted above. +The order can be specified as ASC (ascending) or DESC (descending), +which are case-insensitive. Multiple sort expressions can be used together, +separated by commas, +e.g. \verb{$orderby=__system/creatorId ASC, __system/conflict DESC}. + +As the vast majority of clients only support the JSON OData format, +that is the only format ODK Central offers. +} +\examples{ +\dontrun{ +# See vignette("setup") for setup and authentication options +# ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...") + +ds <- entitylist_list(pid = get_default_pid()) + +ds1 <- odata_entitylist_data_get(pid = get_default_pid(), did = ds$name[1]) + +ds1 +ds1$context +ds1$value + +qry <- list( + "$filter" = "__system/createdAt le 2024-11-05", + "$orderby" = "__system/creatorId ASC, __system/conflict DESC", + "$top" = "100", + "$skip" = "3", + "$count" = "true" +) +ds2 <- odata_entitylist_data_get( + pid = get_default_pid(), + did = ds$name[1], + query = qry +) + +ds2 +} +} +\seealso{ +\url{https://docs.getodk.org/central-api-odata-endpoints/#id3} + +\url{http://docs.oasis-open.org/odata/odata/v4.01/odata-v4.01-part1-protocol.html#_Toc31358948} + +Other entity-management: +\code{\link{entity_audits}()}, +\code{\link{entity_changes}()}, +\code{\link{entity_create}()}, +\code{\link{entity_delete}()}, +\code{\link{entity_detail}()}, +\code{\link{entity_list}()}, +\code{\link{entity_update}()}, +\code{\link{entity_versions}()}, +\code{\link{entitylist_detail}()}, +\code{\link{entitylist_download}()}, +\code{\link{entitylist_list}()}, +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_metadata_get}()}, +\code{\link{odata_entitylist_service_get}()} +} +\concept{entity-management} diff --git a/man/odata_entitylist_metadata_get.Rd b/man/odata_entitylist_metadata_get.Rd new file mode 100644 index 00000000..3d1522ef --- /dev/null +++ b/man/odata_entitylist_metadata_get.Rd @@ -0,0 +1,142 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/odata_entitylist_metadata_get.R +\name{odata_entitylist_metadata_get} +\alias{odata_entitylist_metadata_get} +\title{Get the Metadata Document from the OData Dataset Service.} +\usage{ +odata_entitylist_metadata_get( + pid = get_default_pid(), + did = "", + url = get_default_url(), + un = get_default_un(), + pw = get_default_pw(), + retries = get_retries(), + odkc_version = get_default_odkc_version(), + orders = get_default_orders(), + tz = get_default_tz() +) +} +\arguments{ +\item{pid}{The numeric ID of the project, e.g.: 2. + +Default: \code{\link{get_default_pid}}. + +Set default \code{pid} through \code{ru_setup(pid="...")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{did}{(chr) The name of the Entity List, internally called Dataset. +The function will error if this parameter is not given. +Default: "".} + +\item{url}{The ODK Central base URL without trailing slash. + +Default: \code{\link{get_default_url}}. + +Set default \code{url} through \code{ru_setup(url="...")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{un}{The ODK Central username (an email address). +Default: \code{\link{get_default_un}}. +Set default \code{un} through \code{ru_setup(un="...")}. +See \code{vignette("Setup", package = "ruODK")}.} + +\item{pw}{The ODK Central password. +Default: \code{\link{get_default_pw}}. +Set default \code{pw} through \code{ru_setup(pw="...")}. +See \code{vignette("Setup", package = "ruODK")}.} + +\item{retries}{The number of attempts to retrieve a web resource. + +This parameter is given to \code{\link[httr]{RETRY}(times = retries)}. + +Default: 3.} + +\item{odkc_version}{The ODK Central version as a semantic version string +(year.minor.patch), e.g. "2023.5.1". The version is shown on ODK Central's +version page \verb{/version.txt}. Discard the "v". +\code{ruODK} uses this parameter to adjust for breaking changes in ODK Central. + +Default: \code{\link{get_default_odkc_version}} or "2023.5.1" if unset. + +Set default \code{get_default_odkc_version} through +\code{ru_setup(odkc_version="2023.5.1")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{orders}{(vector of character) Orders of datetime elements for +lubridate. + +Default: +\code{c("YmdHMS", "YmdHMSz", "Ymd HMS", "Ymd HMSz", "Ymd", "ymd")}.} + +\item{tz}{A timezone to convert dates and times to. + +Read \code{vignette("setup", package = "ruODK")} to learn how \code{ruODK}'s +timezone can be set globally or per function.} +} +\value{ +An S3 class \code{odata_entitylist_metadata_get} and \code{list} containing +the Metadata document following the DDMX CSDL standard +\itemize{ +\item \code{version} The EDMX version, e.g. "4.0" +\item \code{complex_types} +\item \code{entity_types} +\item \code{containers} +} +} +\description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} +} +\details{ +The Metadata Document describes, in +\href{https://docs.oasis-open.org/odata/odata-csdl-xml/v4.01/odata-csdl-xml-v4.01.html}{EDMX CSDL}, +the schema of all the data you can retrieve from the OData Dataset Service +in question. Essentially, these are the Dataset properties, or the schema of +each Entity, translated into the OData format. +} +\examples{ +\dontrun{ +# See vignette("setup") for setup and authentication options +# ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...") + +ds <- entitylist_list(pid = get_default_pid()) + +dm1 <- odata_entitylist_metadata_get(pid = get_default_pid(), did = ds$name[1]) + +# Overview +print(dm1) + +# Get all property names for an entity type +names(dm1$entity_types$Entities$properties) + +# Check what properties are non-filterable +dm1$containers$trees$entity_sets$Entities$capabilities + +# Get complex type definitions +dm1$complex_types$metadata$properties +} +} +\seealso{ +\url{https://docs.getodk.org/central-api-odata-endpoints/#id2} + +\url{https://docs.oasis-open.org/odata/odata-csdl-xml/v4.01/odata-csdl-xml-v4.01.html} + +Other entity-management: +\code{\link{entity_audits}()}, +\code{\link{entity_changes}()}, +\code{\link{entity_create}()}, +\code{\link{entity_delete}()}, +\code{\link{entity_detail}()}, +\code{\link{entity_list}()}, +\code{\link{entity_update}()}, +\code{\link{entity_versions}()}, +\code{\link{entitylist_detail}()}, +\code{\link{entitylist_download}()}, +\code{\link{entitylist_list}()}, +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_service_get}()} +} +\concept{entity-management} diff --git a/man/odata_entitylist_service_get.Rd b/man/odata_entitylist_service_get.Rd new file mode 100644 index 00000000..7e76a6bd --- /dev/null +++ b/man/odata_entitylist_service_get.Rd @@ -0,0 +1,134 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/odata_entitylist_service_get.R +\name{odata_entitylist_service_get} +\alias{odata_entitylist_service_get} +\title{Get the Service Document from the OData Dataset Service.} +\usage{ +odata_entitylist_service_get( + pid = get_default_pid(), + did = "", + url = get_default_url(), + un = get_default_un(), + pw = get_default_pw(), + retries = get_retries(), + odkc_version = get_default_odkc_version(), + orders = get_default_orders(), + tz = get_default_tz() +) +} +\arguments{ +\item{pid}{The numeric ID of the project, e.g.: 2. + +Default: \code{\link{get_default_pid}}. + +Set default \code{pid} through \code{ru_setup(pid="...")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{did}{(chr) The name of the Entity List, internally called Dataset. +The function will error if this parameter is not given. +Default: "".} + +\item{url}{The ODK Central base URL without trailing slash. + +Default: \code{\link{get_default_url}}. + +Set default \code{url} through \code{ru_setup(url="...")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{un}{The ODK Central username (an email address). +Default: \code{\link{get_default_un}}. +Set default \code{un} through \code{ru_setup(un="...")}. +See \code{vignette("Setup", package = "ruODK")}.} + +\item{pw}{The ODK Central password. +Default: \code{\link{get_default_pw}}. +Set default \code{pw} through \code{ru_setup(pw="...")}. +See \code{vignette("Setup", package = "ruODK")}.} + +\item{retries}{The number of attempts to retrieve a web resource. + +This parameter is given to \code{\link[httr]{RETRY}(times = retries)}. + +Default: 3.} + +\item{odkc_version}{The ODK Central version as a semantic version string +(year.minor.patch), e.g. "2023.5.1". The version is shown on ODK Central's +version page \verb{/version.txt}. Discard the "v". +\code{ruODK} uses this parameter to adjust for breaking changes in ODK Central. + +Default: \code{\link{get_default_odkc_version}} or "2023.5.1" if unset. + +Set default \code{get_default_odkc_version} through +\code{ru_setup(odkc_version="2023.5.1")}. + +See \code{vignette("Setup", package = "ruODK")}.} + +\item{orders}{(vector of character) Orders of datetime elements for +lubridate. + +Default: +\code{c("YmdHMS", "YmdHMSz", "Ymd HMS", "Ymd HMSz", "Ymd", "ymd")}.} + +\item{tz}{A timezone to convert dates and times to. + +Read \code{vignette("setup", package = "ruODK")} to learn how \code{ruODK}'s +timezone can be set globally or per function.} +} +\value{ +An S3 class \code{odata_entitylist_service_get} with two list items: +\itemize{ +\item \code{context} The URL for the OData metadata document +\item \code{value} A tibble of EntitySets available in this EntityList +} +} +\description{ +\ifelse{html}{\href{https://lifecycle.r-lib.org/articles/stages.html#experimental}{\figure{lifecycle-experimental.svg}{options: alt='[Experimental]'}}}{\strong{[Experimental]}} +} +\details{ +ODK Central presents one OData service for every Dataset (Entity List) +as a way to get an OData feed of Entities. +To access the OData service, add \code{.svc} to the resource URL for the given +Dataset (Entity List). + +The Service Document provides a link to the main source of information +in this OData service: the list of Entities in this Dataset, +as well as the Metadata Document describing the schema of this information. + +This document is available only in JSON format. +} +\examples{ +\dontrun{ +# See vignette("setup") for setup and authentication options +# ruODK::ru_setup(svc = "....svc", un = "me@email.com", pw = "...") + +ds <- entitylist_list(pid = get_default_pid()) + +ds1 <- odata_entitylist_service_get(pid = get_default_pid(), did = ds$name[1]) + +ds1 +ds1$context +ds1$value +} +} +\seealso{ +\url{https://docs.getodk.org/central-api-odata-endpoints/#odata-dataset-service} + +Other entity-management: +\code{\link{entity_audits}()}, +\code{\link{entity_changes}()}, +\code{\link{entity_create}()}, +\code{\link{entity_delete}()}, +\code{\link{entity_detail}()}, +\code{\link{entity_list}()}, +\code{\link{entity_update}()}, +\code{\link{entity_versions}()}, +\code{\link{entitylist_detail}()}, +\code{\link{entitylist_download}()}, +\code{\link{entitylist_list}()}, +\code{\link{entitylist_update}()}, +\code{\link{odata_entitylist_data_get}()}, +\code{\link{odata_entitylist_metadata_get}()} +} +\concept{entity-management} diff --git a/man/ruODK-package.Rd b/man/ruODK-package.Rd index e14ebdd8..c672a8e7 100644 --- a/man/ruODK-package.Rd +++ b/man/ruODK-package.Rd @@ -1,5 +1,5 @@ % Generated by roxygen2: do not edit by hand -% Please edit documentation in R/ruODK.R +% Please edit documentation in R/ruODK-package.R \docType{package} \name{ruODK-package} \alias{ruODK} @@ -28,7 +28,7 @@ Useful links: Other contributors: \itemize{ - \item Maëlle Salmon \email{maelle.salmon@yahoo.se} (\href{https://orcid.org/0000-0002-2815-0399}{ORCID}) [reviewer] + \item Maëlle maelle.salmon@yahoo.se Salmon (\href{https://orcid.org/0000-0002-2815-0399}{ORCID}) [reviewer] \item Karissa Whiting (\href{https://orcid.org/0000-0002-4683-1868}{ORCID}) [reviewer] \item Jason Taylor [reviewer] \item Marcelo Tyszler (\href{https://orcid.org/0000-0002-4573-0002}{ORCID}) [contributor] diff --git a/man/submission_export.Rd b/man/submission_export.Rd index 1683d059..d90e7471 100644 --- a/man/submission_export.Rd +++ b/man/submission_export.Rd @@ -9,6 +9,7 @@ submission_export( overwrite = TRUE, media = TRUE, repeats = TRUE, + deleted_fields = FALSE, pid = get_default_pid(), fid = get_default_fid(), url = get_default_url(), @@ -39,6 +40,11 @@ parameter \code{media}. Setting this feature to FALSE with an odkc_version < 1.1 and will display a verbose noop message, but still include all repeat data.} +\item{deleted_fields}{Whether to restore all fields previously deleted +from this form for this export (TRUE). +All known fields and data for those fields will be merged and exported. +default: FALSE} + \item{pid}{The numeric ID of the project, e.g.: 2. Default: \code{\link{get_default_pid}}. diff --git a/tests/testthat/test-entity_update.R b/tests/testthat/test-entity_update.R index f0faaa6c..02b308dc 100644 --- a/tests/testthat/test-entity_update.R +++ b/tests/testthat/test-entity_update.R @@ -56,7 +56,7 @@ test_that("entity_update works", { # Interlude: entity_changes after update 2 ec_2 <- entity_changes(did = did, eid = en$uuid[1]) - testthat::expect_gt(length(ec_2), length(ec_1)) + testthat::expect_gt(nrow(ec_2), nrow(ec_1)) # Test entity_versions without conflicts flag ev <- entity_versions(did = did, eid = en$uuid[1]) diff --git a/tests/testthat/test-odata_entitylist_data_get.R b/tests/testthat/test-odata_entitylist_data_get.R new file mode 100644 index 00000000..ee388149 --- /dev/null +++ b/tests/testthat/test-odata_entitylist_data_get.R @@ -0,0 +1,124 @@ +test_that("odata_entitylist_data_get works correctly with valid inputs", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + + ds1 <- odata_entitylist_data_get(did = ds$name[1]) + + # Check the structure of the result + testthat::expect_s3_class(ds1, "odata_entitylist_data_get") + testthat::expect_is(ds1$context, "character") +}) + + +test_that("odata_entitylist_data_get filters work", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + + qry <- list( + "$filter" = "__system/createdAt le 2024-11-05", + "$orderby" = "__system/creatorId ASC, __system/conflict DESC", + "$top" = "100", + "$skip" = "3", + "$count" = "true" + ) + ds1 <- odata_entitylist_data_get(did = ds$name[1], query = qry) + + # Check the structure of the result + testthat::expect_s3_class(ds1, "odata_entitylist_data_get") + testthat::expect_is(ds1$context, "character") +}) + +test_that("odata_entitylist_data_get print works", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + + ds1 <- odata_entitylist_data_get(did = ds$name[1]) + + # Test print + out <- testthat::capture_output(print(ds1)) + testthat::expect_true(any(grepl("", out))) + testthat::expect_true(any(grepl(glue::glue("OData Context"), out))) + testthat::expect_true(any(grepl(glue::glue("OData Entities"), out))) +}) + + +test_that("odata_entitylist_data_get warns on missing arguments", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + testthat::expect_error( + odata_entitylist_data_get(pid = "", did = "") + ) + + testthat::expect_error( + odata_entitylist_data_get(pid = get_test_pid(), did = "") + ) +}) + + +test_that("odata_entitylist_data_get warns if odkc_version too low", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + did <- ds$name[1] + + ds1 <- odata_entitylist_data_get(did = did) + + testthat::expect_warning( + ds1 <- odata_entitylist_data_get(did = did, odkc_version = "1.5.3") + ) +}) + +# usethis::use_r("odata_entitylist_data_get") # nolint diff --git a/tests/testthat/test-odata_entitylist_metadata_get.R b/tests/testthat/test-odata_entitylist_metadata_get.R new file mode 100644 index 00000000..637e65b4 --- /dev/null +++ b/tests/testthat/test-odata_entitylist_metadata_get.R @@ -0,0 +1,97 @@ +# Test the odata_entitylist_metadata_get function +test_that("odata_entitylist_metadata_get works correctly with valid inputs", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + + ds1 <- odata_entitylist_metadata_get(did = ds$name[1]) + + # Check the structure of the result + testthat::expect_s3_class(ds1, "odata_entitylist_metadata_get") + # testthat::expect_equal(nrow(ds1$value), 1) + # testthat::expect_equal(ds1$value$name[1], "Entities") +}) + +test_that("odata_entitylist_service_get print works", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + + ds1 <- odata_entitylist_metadata_get(did = ds$name[1]) + + # Test print + out <- testthat::capture_output(print(ds1)) + testthat::expect_true(any(grepl("", out))) + testthat::expect_true(any(grepl(glue::glue("Complex Types"), out))) + testthat::expect_true(any(grepl(glue::glue("Entity Types"), out))) + testthat::expect_true(any(grepl(glue::glue("Containers"), out))) +}) + + +test_that("odata_entitylist_metadata_get warns on missing arguments", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + testthat::expect_error( + odata_entitylist_metadata_get(pid = "", did = "") + ) + + testthat::expect_error( + odata_entitylist_metadata_get(pid = get_test_pid(), did = "") + ) +}) + + +test_that("odata_entitylist_metadata_get warns if odkc_version too low", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + did <- ds$name[1] + + ds1 <- odata_entitylist_metadata_get(did = did) + + testthat::expect_warning( + ds1 <- odata_entitylist_metadata_get(did = did, odkc_version = "1.5.3") + ) +}) + +# usethis::use_r("odata_entitylist_metadata_get") # nolint diff --git a/tests/testthat/test-odata_entitylist_service_get.R b/tests/testthat/test-odata_entitylist_service_get.R new file mode 100644 index 00000000..3643a0bd --- /dev/null +++ b/tests/testthat/test-odata_entitylist_service_get.R @@ -0,0 +1,105 @@ +# Test the odata_entitylist_service_get function +test_that("odata_entitylist_service_get works correctly with valid inputs", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + + ds1 <- odata_entitylist_service_get(did = ds$name[1]) + + ctx <- as.character(glue::glue( + "{get_test_url()}/v1/projects/{get_test_pid()}/", + "datasets/{ds$name[1]}.svc/$metadata" + )) + + # Check the structure of the result + testthat::expect_s3_class(ds1, "odata_entitylist_service_get") + expect_equal( + ds1$context, + ctx + ) + testthat::expect_equal(nrow(ds1$value), 1) + testthat::expect_equal(ds1$value$name[1], "Entities") +}) + +test_that("odata_entitylist_service_get print works", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + + ds1 <- odata_entitylist_service_get(did = ds$name[1]) + + # Test print + out <- testthat::capture_output(print(ds1)) + testthat::expect_true(any(grepl("", out))) + testthat::expect_true(any(grepl(glue::glue("OData Context"), out))) + testthat::expect_true(any(grepl(glue::glue("OData Entities"), out))) +}) + + +test_that("odata_entitylist_service_get warns on missing arguments", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + testthat::expect_error( + odata_entitylist_service_get(pid = "", did = "") + ) + + testthat::expect_error( + odata_entitylist_service_get(pid = get_test_pid(), did = "") + ) +}) + + +test_that("odata_entitylist_data_get warns if odkc_version too low", { + skip_if(Sys.getenv("ODKC_TEST_URL") == "", + message = "Test server not configured" + ) + + ru_setup( + pid = get_test_pid(), + url = get_test_url(), + un = get_test_un(), + pw = get_test_pw(), + odkc_version = get_test_odkc_version() + ) + + ds <- entitylist_list() + did <- ds$name[1] + + ds1 <- odata_entitylist_data_get(did = did) + + testthat::expect_warning( + ds1 <- odata_entitylist_data_get(did = did, odkc_version = "1.5.3") + ) +}) + +# usethis::use_r("odata_entitylist_service_get") # nolint