Merge pull request #275 from rOpenGov/dev

Version 0.17
rOpenGov · Jan 28, 2024 · 31e5795 · 31e5795
2 parents b08f38a + d7e4424
commit 31e5795
Show file tree

Hide file tree

Showing 21 changed files with 386 additions and 204 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,8 +1,8 @@
 Type: Package
 Package: pxweb
 Title: R Interface to PXWEB APIs
-Version: 0.16.3
-Date: 2024-01-13
+Version: 0.17.0
+Date: 2024-01-28
 Authors@R: c(
     person("Mans", "Magnusson", , "[email protected]", role = c("aut", "cre"),
            comment = c(ORCID = "0000-0002-0296-2719")),
@@ -35,6 +35,7 @@ Depends:
     R (>= 3.5.0)
 Imports: 
     checkmate,
+    curl,
     httr (>= 1.1),
     jsonlite
 Suggests: 

diff --git a/R/pxweb.R b/R/pxweb.R
@@ -34,6 +34,10 @@ pxweb <- function(url) {
   }
   checkmate::assert_string(url)
   url_parsed <- parse_url_or_fail(x = url)
+  if(!has_internet(url_parsed$hostname)){
+    message(no_internet_msg(url_parsed$hostname))
+    return(NULL)
+  }
 
   obj <- list(
     url = url_parsed,
@@ -119,3 +123,11 @@ pxweb_tempdir <- function(to = "apis") {
     return(tmp_dir_logs)
   }
 }
+
+no_internet_msg <- function(host){
+  return(paste0("No internet access to '", host, "'."))
+}
+
+has_internet <- function(host){
+  !is.null(curl::nslookup(host, error = FALSE))
+}
diff --git a/R/pxweb_get.R b/R/pxweb_get.R
@@ -100,6 +100,13 @@ pxweb_advanced_get <- function(url, query = NULL, verbose = TRUE, log_http_calls
   if (log_http_calls) {
     pxweb_http_log_on()
   }
+  if(!is.pxweb(url)){
+    url_parsed <- parse_url_or_fail(x = url)
+    if(!has_internet(url_parsed$hostname)){
+      message(no_internet_msg(url_parsed$hostname))
+      return(NULL)
+    }
+  }
 
   px <- pxweb(url)
   if (!is.null(query)) {

diff --git a/R/pxweb_interactive.R b/R/pxweb_interactive.R
@@ -23,6 +23,17 @@
 #' ##  x <- pxweb_interactive(x = "https://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0101/BE0101A/")
 #'
 pxweb_interactive <- function(x = NULL) {
+  # Check internet access if x is an url
+  if(!is.null(x)){
+    url_parsed <- parse_url(x)
+    if(parsed_url_has_hostname(url_parsed)){
+      if(!has_internet(url_parsed$hostname)){
+        message(no_internet_msg(url_parsed$hostname))
+        return(NULL)
+      }
+    }
+  }
+
   # Setup structure
   pxe <- pxweb_explorer(x)
 
@@ -145,7 +156,11 @@ pxweb_explorer.pxweb <- function(x) {
 #' @rdname pxweb_explorer
 #' @keywords internal
 pxweb_explorer.pxweb_api_catalogue_entry <- function(x) {
-  pxe <- list(pxweb = pxweb(build_pxweb_url(x)))
+  suppressMessages(pxe <- list(pxweb = pxweb(build_pxweb_url(x))))
+  if(is.null(pxe$pxweb)) {
+    stop(no_internet_msg(parse_url(x$url)$hostname),
+         call. = FALSE)
+  }
   tot_pos <- pxweb_api_path(pxe$pxweb, as_vector = TRUE)
   pxe$root <- character(0)
   pxe$position <- character(0)

diff --git a/R/pxweb_parse_url_or_fail.R b/R/pxweb_parse_url_or_fail.R
@@ -4,11 +4,22 @@
 #'
 #' @keywords internal
 parse_url_or_fail <- function(x) {
+  pu <- parse_url(x)
+  if (!parsed_url_has_hostname(x)) {
+    stop("Cannot parse url (hostname). Please check url for potential errors (?parse_url): '", x, "'", call. = FALSE)
+  }
+  pu
+}
+
+parse_url <- function(x) {
   checkmate::assert_string(x)
   x <- pxweb_fix_url(x)
-  parsed_url <- httr::parse_url(x)
-  if (is.null(parsed_url$hostname)) {
-    stop("Cannot parse url (hostname). Please check url for potential errors (?parse_url): '", x, "'", call. = FALSE)
+  httr::parse_url(x)
+}
+
+parsed_url_has_hostname <- function(x){
+  if(!checkmate::test_class(x, "url")){
+    x <- parse_url(x)
   }
-  parsed_url
+  !is.null(x$hostname)
 }
diff --git a/tests/testthat/test-pxweb_api_paths.R b/tests/testthat/test-pxweb_api_paths.R
@@ -5,6 +5,7 @@ context("pxweb_api_paths")
 test_that(desc = "Access api paths", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "https://api.scb.se/OV0104/v1/doris/sv/ssd/START/AM/AM0301/"
   expect_silent(scb <- pxweb(url))

diff --git a/tests/testthat/test-pxweb_as_dataframe.R b/tests/testthat/test-pxweb_as_dataframe.R
@@ -5,6 +5,7 @@ context("pxweb conversions")
 test_that(desc = "Converting pxweb data to matrices and data.frames", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   # Move to
   url <- "https://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy"

diff --git a/tests/testthat/test-pxweb_constructor.R b/tests/testthat/test-pxweb_constructor.R
@@ -5,6 +5,7 @@ context("pxweb")
 test_that(desc = "Constructor works as it should with Statistics Sweden", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   expect_silent(pxapi1 <- pxweb(url = "https://api.scb.se/OV0104/v1/doris/sv/ssd/START/ME/ME0104/ME0104C/ME0104T24"))
   expect_true(file.exists(pxapi1$paths$rda_file_path))
@@ -30,6 +31,7 @@ test_that(desc = "Constructor works as it should with Statistics Sweden", {
 test_that(desc = "Constructor works for erroneous urls", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   expect_silent(pxapi1 <- pxweb(url = "https://api.scb.se/OV0104/v1/doris/sv/ssd/START/ME/ME0104/ME0104C/ME0104T24"))
 
@@ -50,6 +52,7 @@ test_that(desc = "Constructor works for erroneous urls", {
 test_that(desc = "Cache cleaner and print", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   expect_silent(pxapi1 <- pxweb(url = "https://api.scb.se/OV0104/v1/doris/sv/ssd/START/ME/ME0104/ME0104C/ME0104T24"))
   expect_silent(pxapi2 <- pxweb("https://statfin.stat.fi/PXWeb/api/v1/fi/StatFin/tym/tyonv/statfin_tyonv_pxt_001.px"))

diff --git a/tests/testthat/test-pxweb_data_comments.R b/tests/testthat/test-pxweb_data_comments.R
@@ -5,6 +5,7 @@ context("pxweb_data_comments")
 test_that(desc = "test data comment structure", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "https://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy"
   json_query <- file.path(system.file(package = "pxweb"), "extdata", "examples", "json_query_example.json")

diff --git a/tests/testthat/test-pxweb_get.R b/tests/testthat/test-pxweb_get.R
@@ -5,6 +5,7 @@ context("pxweb_get")
 test_that(desc = "Test to download px and sdmx", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
   json_px_query <- readLines(test_path("test_data/test_query_px.json"))
 
   expect_silent(px_file_path1 <-
@@ -37,6 +38,7 @@ test_that(desc = "Test to download px and sdmx", {
 test_that(desc = "Constructor works as it should with Statistics Sweden", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "https://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy"
   expect_silent(px_meta_data <- pxweb_get(url))
@@ -89,6 +91,7 @@ test_that(desc = "Constructor works as it should with Statistics Sweden", {
 test_that(desc = "Previous bugs", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   # This is a bug in the previous implementation of pxweb
   url <- "https://bank.stat.gl/api/v1/en/Greenland/BE/BE01"
@@ -104,6 +107,7 @@ test_that(desc = "Previous bugs", {
 test_that(desc = "Test to download json-stat objects", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   # Test json-stat
   url <- "https://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy"
@@ -126,6 +130,7 @@ test_that(desc = "Test to download json-stat objects", {
 test_that(desc = "Test pxweb_get_data", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "https://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy"
   json_query <- file.path(system.file(package = "pxweb"), "extdata", "examples", "json_query_example.json")
@@ -138,6 +143,7 @@ test_that(desc = "Test pxweb_get_data", {
 test_that(desc = "Test http logger", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "https://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy"
   expect_silent(px <- pxweb(url))
@@ -154,6 +160,8 @@ test_that(desc = "Test http logger", {
 test_that(desc = "large variable call", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
+
   url <- "https://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0001/BE0001G/BE0001ENamn10"
   json_query <- file.path(system.file(package = "pxweb"), "extdata", "examples", "json_query_last_names.json")
   expect_silent(px <- pxweb_get(url, query = pxweb_query(json_query)))
@@ -163,6 +171,7 @@ test_that(desc = "large variable call", {
 test_that(desc = "Cite data", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "https://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0101/BE0101A/BefolkningNy"
   json_query <- file.path(system.file(package = "pxweb"), "extdata", "examples", "json_query_example.json")
@@ -175,6 +184,7 @@ test_that(desc = "Cite data", {
 test_that(desc = "Filter query error bug", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "http://data.ssb.no/api/v0/en/table/04861"
   json_query <- readLines(test_path("test_data/filter_query.json"))
@@ -198,6 +208,7 @@ test_that(desc = "Filter query error bug", {
 test_that(desc = "a small big query", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   pxweb_query_list <-
     list(
@@ -225,6 +236,7 @@ test_that(desc = "a small big query", {
 test_that(desc = "manually supplying a pxmdo", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   pxweb_query_list <-
     list(
@@ -249,6 +261,7 @@ test_that(desc = "manually supplying a pxmdo", {
 test_that(desc = "return clear error message when missing values", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   pql <- list(
     "Tilltalsnamn" = c("20Agnes"),
@@ -263,6 +276,7 @@ test_that(desc = "return clear error message when missing values", {
 test_that(desc = "Query with non-ascii characters work as well", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   pxweb_query_list <-
     list(

diff --git a/tests/testthat/test-pxweb_interactive.R b/tests/testthat/test-pxweb_interactive.R
@@ -5,6 +5,7 @@ context("pxweb_interactive")
 test_that(desc = "Basic usage", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   expect_silent(pxe <- pxweb:::pxweb_explorer.character("https://api.scb.se/OV0104/v1/doris/sv/ssd/START/ME/ME0104/ME0104C/ME0104T24"))
   expect_output(pxweb:::print.pxweb_explorer(pxe), "/OV0104/v1/doris/sv/ssd/START/ME/ME0104/ME0104C/ME0104T24")
@@ -29,6 +30,7 @@ test_that(desc = "Basic usage", {
 test_that(desc = "API catalogue usage", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   expect_silent(pxe_scb <- pxweb:::pxweb_explorer.character("api.scb.se"))
   expect_output(pxweb:::print.pxweb_explorer(pxe_scb), "v1")
@@ -57,6 +59,7 @@ test_that(desc = "API catalogue usage", {
 test_that(desc = "Select all and eliminate", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   expect_silent(pxe <- pxweb:::pxweb_explorer.character("https://api.scb.se/OV0104/v1/doris/sv/ssd/START/ME/ME0104/ME0104C/ME0104T24"))
   expect_output(pxweb:::print.pxweb_explorer(pxe), "/OV0104/v1/doris/sv/ssd/START/ME/ME0104/ME0104C/ME0104T24")
@@ -70,6 +73,7 @@ test_that(desc = "Select all and eliminate", {
 test_that(desc = "Select all and eliminate", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   expect_silent(pxe <- pxweb:::pxweb_explorer.character("https://api.scb.se/OV0104/v1/doris/sv/ssd/START/ME/ME0104/ME0104C/ME0104T24"))
   expect_output(pxe <- pxweb:::pxweb_interactive_input(pxe, test_input = "e"))
@@ -108,6 +112,7 @@ test_that(desc = "Select all and eliminate", {
 test_that(desc = "Stat Iceland structure", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   expect_silent(pxe <- pxweb:::pxweb_explorer.NULL())
   expect_output(pxe <- pxweb:::pxweb_interactive_input(pxe, test_input = "10"))
@@ -124,8 +129,19 @@ test_that(desc = "Stat Iceland structure", {
 test_that(desc = "No value bug", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "http://px.hagstofa.is/pxen/api/v1/en/Efnahagur/utanrikisverslun/1_voruvidskipti/02_uttollskra/UTA02801.px"
   expect_silent(pxe <- pxweb:::pxweb_explorer.character(url))
   expect_output(pxweb:::print.pxweb_explorer(pxe), regexp = "\\[\\[HS-Number\\]\\]")
 })
+
+
+test_that(desc = "Fail on incorrect", {
+  # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
+  skip_on_cran()
+  skip_if_offline()
+
+  expect_error(pxweb_interactive("incorrect url"))
+})
+
diff --git a/tests/testthat/test-pxweb_query.R b/tests/testthat/test-pxweb_query.R
@@ -30,6 +30,7 @@ test_that(desc = "pxweb_query object", {
 test_that(desc = "split pxweb_query object", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "https://api.scb.se/OV0104/v1/doris/sv/ssd/BE/BE0101/BE0101A/BefolkningNy"
   dims <- list(
@@ -85,6 +86,7 @@ test_that(desc = "split pxweb_query object", {
 test_that(desc = "split pxweb_query bug", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "http://px.hagstofa.is/pxis/api/v1/is/Efnahagur/utanrikisverslun/1_voruvidskipti/03_inntollskra/UTA03801.px"
   pxweb_query_list <- list(
@@ -120,6 +122,8 @@ test_that(desc = "pxweb_query JSON parse error message", {
 
 
 test_that(desc = "mandatory variables are included automatically", {
+  skip_if_offline()
+  skip_on_cran()
   fp <- test_path(file.path("test_data", "pxm1_test.rda"))
   url <- "https://api.scb.se/OV0104/v1/doris/en/ssd/BE/BE0101/BE0101A/BefolkningNy"
 

diff --git a/tests/testthat/test-pxweb_test_api.R b/tests/testthat/test-pxweb_test_api.R
@@ -5,6 +5,7 @@ context("pxweb_test_api")
 test_that(desc = "Mixed node levels object", {
   # CRAN seem to run tests in parallel, hence API tests cannot be run on CRAN.
   skip_on_cran()
+  skip_if_offline()
 
   url <- "https://api.scb.se/OV0104/v1/doris/sv/ssd/START/AM/AM0301/"
   expect_output(res <- suppressWarnings(pxweb_test_api(url)), regexp = "4 node.+and 14 table")

diff --git a/vignettes/px_levels_example.rda b/vignettes/px_levels_example.rda
diff --git a/vignettes/px_meta_example.rda b/vignettes/px_meta_example.rda
diff --git a/vignettes/pxd_example.rda b/vignettes/pxd_example.rda
diff --git a/vignettes/pxfp_example.rda b/vignettes/pxfp_example.rda
diff --git a/vignettes/pxjstat_example.rda b/vignettes/pxjstat_example.rda