Features/post data query (#28)

* Created initial layout for post_dataset()

* Added basic read-some-json functionality to post_dataset

* Updated post_dataset with some basic results extraction from the http response

* Created placeholder parse_params_to_json function

* Updating pkgdown listings

* Resolving conflicts with main

* Updating docs and tests

* Adding POST into query_dataset() and creating first working post_dataset example

* Switching filter-parsing based querying to GET whilst I develop that functionality in POST

* Added parsing functions for creating geographic levels and time periods json query strings

* Quick typo fix

* Developed json creation to include time periods, geographic levels, locations and filter item querying

* Updated post/query_dataset examples

* Added locations processing to json parsing

* Overhauled parse geographies to combine geographic level and location selection in a (hopefully) sensible way

* Tidy up of post and query data set examples and param defs

* Adding geographies param definiteion

* Fixing some extra lintr comments from the PR

* Added more complex filter querying for post_datatset

* Added indicator parsing for post_dataset()

* Added basic description for parse_tojson_params()

* Added test of post_dataset collecting the required geographies

* Improved examples on query_dataset and added some tests

* Trying to fix some odd behaviour with the tests

* Test snapshot updates

* Code styling

* Bit of clean up on tests and adding the debug mode flag for posting queries

* Cleaning up some roxygen text

* Extra documentation prompted by PR comments

* Expanding examples for query_datatset and post_dataset

* Improved examples around query_dataset()

* Added some extra tests on query_dataset

* Renamed to_api_filter_type

* Adding explicit package refs in seed_tests

* Added extra documentation to query_dataset()

* Added time period definition to query_dataset docs

* Added more explicit function calls to seed_tests

* Fixing bug in get_meta function when only one geographic level is available

* Fixed bug in location validation and updated some example sqids

* Playing around with code/id validation

* Added additional validation and updated meta request to handle school level locations

* Added time_periods validation

* Updated testing of validation scripts

* Added testing of time period query in post data set

* Added extra tests around filter queries using post data set

* Fixed bug in validate ees id

* Updated get data set geography levels in line with data set

* Updating testing and get-meta functionality for http_request_error

* Updated example_id function to handle a wider range of objects of differing structures

* Updated version to 0.3.0 and updated NEWS.md

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: eesyapi
 Title: EES-y API
-Version: 0.2.1
+Version: 0.3.0
 Authors@R: 
     c(
     person("Rich", "Bielby", , "[email protected]", role = c("aut", "cre"),
@@ -21,4 +21,6 @@ Imports:
     httr,
     jsonlite,
     dplyr,
-    stringr
+    stringr,
+    rlang,
+    magrittr

NAMESPACE

@@ -3,20 +3,34 @@
 export(api_url)
 export(api_url_pages)
 export(api_url_query)
+export(convert_api_filter_type)
+export(example_geography_query)
 export(example_id)
+export(example_json_query)
 export(get_data_catalogue)
 export(get_dataset)
 export(get_meta)
 export(get_meta_response)
 export(get_publications)
 export(http_request_error)
 export(parse_api_dataset)
-export(parse_filter_in)
 export(parse_meta_filter_columns)
 export(parse_meta_filter_item_ids)
 export(parse_meta_location_ids)
 export(parse_meta_time_periods)
+export(parse_tojson_filter)
+export(parse_tojson_filter_eq)
+export(parse_tojson_filter_in)
+export(parse_tojson_geographies)
+export(parse_tojson_indicators)
+export(parse_tojson_location)
+export(parse_tojson_params)
+export(parse_tojson_time_periods)
+export(parse_tourl_filter_in)
+export(post_dataset)
 export(query_dataset)
+export(validate_ees_filter_type)
 export(validate_ees_id)
 export(validate_page_size)
+export(validate_time_periods)
 export(warning_max_pages)

NEWS.md

@@ -1,6 +1,17 @@
+# eesyapi 0.3.0
+
+* Created capacity to query data using POST:
+  - `query_dataset()`: Now defaults to using POST instead of GET
+  - `post_dataset()`: Sends a query of a data set, either using a json file, json string or 
+  parameters
+* Updated how `example_id()` works to allow more complex examples
+
 # eesyapi 0.2.1
 
-* Updated `get_meta()` to work with new API meta output (addition of id alongside col_name and label)
+* Created initial `query_dataset()` function that queries a data set using `get_dataset()`
+* Created `get_dataset()` function that queries a data set using GET and URL parameters
+* Updated `get_meta()` to work with new API meta output (addition of id alongside col_name and 
+label)
 * Removed redundant function: `parse_meta_filter_columns()`
 * Hex logo added for documentation
 

R/api_url.R

@@ -24,8 +24,7 @@
 #' @param page Page number of query results to return
 #' @param api_version EES API version
 #' @param environment EES environment to connect to: "dev", "test", "preprod" or "prod"
-#' @param verbose Add extra contextual information whilst running
-#'
+#' @param verbose Run with additional contextual messaging, logical, default = FALSE
 #' @return A string containing the URL for connecting to the EES API
 #' @export
 #'

R/api_url_query.R

@@ -27,17 +27,17 @@ api_url_query <- function(
   eesyapi::validate_ees_id(indicators, level = "indicator")
   # Create the appropriate query strings for each level provided
   if (!is.null(time_periods)) {
-    query_time_periods <- eesyapi::parse_filter_in(time_periods, "time_periods")
+    query_time_periods <- eesyapi::parse_tourl_filter_in(time_periods, "time_periods")
   }
   if (!is.null(geographic_levels)) {
-    query_geographic_levels <- eesyapi::parse_filter_in(
+    query_geographic_levels <- eesyapi::parse_tourl_filter_in(
       geographic_levels,
-      type = "geographic_levels"
+      filter_type = "geographic_levels"
     )
   }
   if (!is.null(locations)) {
     eesyapi::validate_ees_id(locations, level = "location")
-    query_locations <- eesyapi::parse_filter_in(locations, type = "locations")
+    query_locations <- eesyapi::parse_tourl_filter_in(locations, filter_type = "locations")
   }
   if (!is.null(filter_items)) {
     # Note the idea below was to differentiate the logic between AND / OR based on whether
@@ -50,11 +50,14 @@ api_url_query <- function(
       for (filter_set in filter_items) {
         query_filter_items <- paste0(
           query_filter_items,
-          eesyapi::parse_filter_in(filter_set, type = "filter_items")
+          eesyapi::parse_tourl_filter_in(filter_set, filter_type = "filter_items")
         )
       }
     } else {
-      query_filter_items <- eesyapi::parse_filter_in(filter_items, type = "filter_items")
+      query_filter_items <- eesyapi::parse_tourl_filter_in(
+        filter_items,
+        filter_type = "filter_items"
+      )
     }
   }
   query_indicators <- paste0(

R/api_url_query_utils.R

@@ -5,25 +5,21 @@
 #' use with querying a data set via GET.
 #'
 #' @param items items to be included in the "in" statement
-#' @param type type of filter items being queried: "time_periods", "geographic_levels",
+#' @param filter_type type of filter being queried: "time_periods", "geographic_levels",
 #' "locations" or "filter_items"
 #'
 #' @return Query string for use in URL based API queries
 #' @export
 #'
 #' @examples
-#' parse_filter_in(c("2024|W11", "2024|W12"), type = "time_periods")
-parse_filter_in <- function(
+#' parse_tourl_filter_in(c("2024|W11", "2024|W12"), filter_type = "time_periods")
+parse_tourl_filter_in <- function(
     items,
-    type) {
-  if (!(type %in% c("time_periods", "geographic_levels", "locations", "filter_items"))) {
-    stop("type keyword should be one of time_periods, geographic_levels, locations or filter_items")
-  }
-  type_string <- type |>
-    stringr::str_replace("_item", "")
-  type_string <- gsub("_(\\w?)", "\\U\\1", type_string, perl = TRUE)
+    filter_type) {
+  eesyapi::validate_ees_filter_type(filter_type)
+  type_string <- eesyapi::convert_api_filter_type(filter_type)
   if (!is.null(items)) {
-    if (type %in% c("time_period", "locations")) {
+    if (filter_type %in% c("time_period", "locations")) {
       items <- gsub("\\|", "%7C", items)
     }
     paste0(

R/convert_api_filter_type.R

@@ -0,0 +1,23 @@
+#' Convert filter type to API filter type
+#'
+#' @description
+#' The API uses a slightly different naming convention for the different types of
+#' filters to what is used by analysts within data files. The function just converts
+#' from the file versions to the API versions.
+#'
+#' @inheritParams parse_tourl_filter_in
+#'
+#' @return String containing API friendly filter type descriptor
+#' @export
+#'
+#' @examples
+#' convert_api_filter_type("filter_items")
+#' convert_api_filter_type("geographic_levels")
+#' convert_api_filter_type("locations")
+#' convert_api_filter_type("filter_items")
+convert_api_filter_type <- function(filter_type) {
+  eesyapi::validate_ees_filter_type(filter_type)
+  filter_type <- filter_type |>
+    stringr::str_replace("_item", "")
+  gsub("_(\\w?)", "\\U\\1", filter_type, perl = TRUE)
+}