Translate data sqids

DESCRIPTION

@@ -21,4 +21,7 @@ Imports:
     httr,
     jsonlite,
     dplyr,
-    stringr
+    stringr,
+    data.table,
+    magrittr,
+    rlang

NAMESPACE

@@ -3,6 +3,7 @@
 export(api_url)
 export(api_url_pages)
 export(api_url_query)
+export(example_data_raw)
 export(example_id)
 export(get_data_catalogue)
 export(get_dataset)
@@ -16,7 +17,10 @@ export(parse_meta_filter_columns)
 export(parse_meta_filter_item_ids)
 export(parse_meta_location_ids)
 export(parse_meta_time_periods)
+export(parse_sqids_filters)
+export(parse_sqids_indicators)
 export(query_dataset)
 export(validate_ees_id)
 export(validate_page_size)
 export(warning_max_pages)
+importFrom(data.table,":=")

R/eesyapi-package.R

@@ -0,0 +1,7 @@
+#' @keywords internal
+"_PACKAGE"
+
+## usethis namespace: start
+#' @importFrom data.table :=
+## usethis namespace: end
+NULL

R/example_id.R → R/examples.R

@@ -21,12 +21,14 @@
       "dataset",
       "location_id",
       "location_code",
+      "filter",
       "filter_item",
       "indicator",
       "publication",
       "dataset",
       "location_id",
       "location_code",
+      "filter",
       "filter_item",
       "indicator"
     ),
@@ -42,6 +44,8 @@
       "dev",
       "dev",
       "dev",
+      "dev",
+      "dev",
       "dev"
     ),
     example_group = c(
@@ -51,6 +55,8 @@
       "attendance",
       "attendance",
       "attendance",
+      "attendance",
+      "public-api-testing",
       "public-api-testing",
       "public-api-testing",
       "public-api-testing",
@@ -63,12 +69,14 @@
       "7c0e9201-c7c0-ff73-bee4-304e731ec0e6",
       "NAT|id|dP0Zw",
       "NAT|code|E92000001",
-      "hl2Gy",
+      "4kdUZ",
+      "5UNdi",
       "bqZtT",
       "d823e4df-626f-4450-9b21-08dc8b95fc02",
       "830f9201-9e11-ad75-8dcd-d2efe2834457",
       "LA|id|ml79K",
       "NAT|code|E92000001",
+      "5mvdi",
       "HsQzL",
       "h8fyW"
     )
@@ -96,3 +104,28 @@
     )
   }
 }
+
+#' Title
+#'
+#' @inheritParams example_id
+#' @param size Number of rows to return (max = 1000)
+#'
+#' @return Nested list form of example data from the API
+#' @export
+#'
+#' @examples
+#' example_data_raw()
+example_data_raw <- function(
+    group = "attendance",
+    size = 32) {
+  api_url(
+    "get-data",
+    dataset_id = example_id(group = group),
+    indicators = example_id("indicator", group = group),
+    page = 1, page_size = size
+  ) |>
+    httr::GET() |>
+    httr::content("text") |>
+    jsonlite::fromJSON() |>
+    magrittr::use_series("results")
+}

R/get_dataset.R

@@ -57,7 +57,7 @@ get_dataset <- function(
     message(paste("Total number of pages: ", response_json$paging$totalPages))
   }
   dfresults <- response_json$results |>
-    eesyapi::parse_api_dataset(verbose = verbose)
+    eesyapi::parse_api_dataset(dataset_id = dataset_id, verbose = verbose)
   # Unless the user has requested a specific page, then assume they'd like all pages collated and
   # recursively run the query.
   if (is.null(page)) {

R/parse_api_dataset.R

@@ -11,24 +11,19 @@
 #'       parse_api_dataset()
 #'
 #' @param api_data_result A json data result list as returned from the API
-#' @param dataset_id ID of data set to be connected to.
-#' @param verbose Run in verbose mode, logical, default = FALSE
+#' @inheritParams api_url
 #'
 #' @return Data frame containing API data results
 #' @export
 #'
 #' @examples
-#' api_url(
-#'   "get-data",
-#'   dataset_id = example_id(), indicators = example_id("indicator"), page_size = 10
-#' ) |>
-#'   httr::GET() |>
-#'   httr::content("text") |>
-#'   jsonlite::fromJSON() |>
-#'   parse_api_dataset()
+#' example_data_raw(group = "attendance") |>
+#'   parse_api_dataset(example_id(group = "attendance"))
 parse_api_dataset <- function(
     api_data_result,
-    dataset_id = NULL,
+    dataset_id,
+    dataset_version = NULL,
+    api_version = NULL,
     verbose = FALSE) {
   if (!is.null(dataset_id)) {
     eesyapi::validate_ees_id(dataset_id, level = "dataset")
@@ -41,12 +36,19 @@
     print(names(api_data_result$locations))
     print(names(api_data_result$filters))
   }
+  meta <- get_meta(
+    dataset_id,
+    dataset_version = dataset_version,
+    api_version = api_version
+  )
   dplyr::bind_cols(
     api_data_result$timePeriod,
     data.frame(geographic_level = api_data_result$geographicLevel),
     api_data_result$locations,
-    api_data_result$filters,
-    api_data_result$values
+    api_data_result$filters |>
+      eesyapi::parse_sqids_filters(meta),
+    api_data_result$values |>
+      eesyapi::parse_sqids_indicators(meta),
   )
   # Next aim here is to pull in the meta data automatically at this point to translate
   # all the API codes...

R/parse_sqids.R

@@ -0,0 +1,77 @@
+#' Parse IDs in a set of filters
+#'
+#' @description
+#' The API uses unique IDs (sqids) to identify each filter column and its contents (filter items).
+#' This function parses those into the data creators' id and item labels based on the meta data
+#' stored on the API for the data set.
+#'
+#' @param filters A set of filter item columns as taken from a data set downloaded from the API
+#' @param meta Meta data for the data set as provided by `get_meta()`
+#' @param verbose Run in verbose mode with debugging messages
+#'
+#' @return Data frame
+#' @export
+#'
+#' @examples
+#' example_data_raw() |>
+#'   magrittr::use_series("filters") |>
+#'   parse_sqids_filters(get_meta(example_id(group = "attendance")))
+parse_sqids_filters <- function(filters, meta, verbose = FALSE) {
+  filter_ids <- meta |>
+    magrittr::use_series("filter_columns") |>
+    dplyr::filter(!!rlang::sym("col_id") %in% colnames(filters)) |>
+    dplyr::pull("col_id")
+  if (verbose) {
+    print(filter_ids)
+  }
+  for (column_sqid in filter_ids) {
+    col_name <- meta |>
+      magrittr::use_series("filter_columns") |>
+      dplyr::filter(!!rlang::sym("col_id") == column_sqid) |>
+      dplyr::pull("col_name")
+    if (verbose) {
+      message("Matched ", column_sqid, " to ", col_name)
+    }
+    lookup <- meta |>
+      magrittr::use_series("filter_items") |>
+      dplyr::filter(!!rlang::sym("col_id") == column_sqid) |>
+      dplyr::select("item_label", "item_id") |>
+      dplyr::rename(
+        !!rlang::sym(col_name) := "item_label",
+        !!rlang::sym(column_sqid) := "item_id"
+      )
+    filters <- filters |>
+      dplyr::left_join(lookup, by = column_sqid) |>
+      dplyr::select(-column_sqid)
+  }
+  return(filters)
+}
+
+#' Parse IDs in a set of indicators
+#'
+#' @description
+#' The API uses unique IDs (sqids) to identify each filter column and its contents (filter items).
+#' This function parses those into the data creators' id and item labels based on the meta data
+#' stored on the API for the data set.
+#'
+#' @inheritParams parse_sqids_filters
+#' @param indicators A set of indicator columns as taken from a data set downloaded from the API
+#'
+#' @return Data frame
+#' @export
+#'
+#' @examples
+#' example_data_raw(group = "attendance") |>
+#'   magrittr::use_series("values") |>
+#'   parse_sqids_indicators(get_meta(example_id(group = "attendance")))
+parse_sqids_indicators <- function(indicators, meta, verbose = FALSE) {
+  indicator_ids <- meta |>
+    magrittr::use_series("indicators") |>
+    dplyr::filter(!!rlang::sym("col_id") %in% colnames(indicators))
+  indicator_lookup <- indicator_ids |>
+    dplyr::pull("col_id")
+  names(indicator_lookup) <- indicator_ids |> dplyr::pull("col_name")
+  indicators <- indicators |>
+    dplyr::rename(dplyr::all_of(indicator_lookup))
+  return(indicators)
+}

_pkgdown.yml

@@ -17,7 +17,6 @@ reference:
 - title: Support for generating API URLs and interpreting responses
   desc: These functions are helpful for deriving urls and handling HTTP responses and are used widely by the API workflow functions.
   contents:
-  - example_id
   - starts_with("api_url")
   - http_request_error
   - parse_filter_in
@@ -33,6 +32,12 @@ reference:
   contents:
   - get_dataset
   - parse_api_dataset
+  - starts_with("parse_sqid")
+
+- title: Generate example IDs and data
+  desc: These functions are used widely to create working example code and tests
+  contents:
+  - starts_with("example")
 
 - title: Validation functions
   desc: These functions are used across the package to validate elements being passed as part of an API url or query.