From f6c8086d9cc04aed5779b71ed2c98e94bdba7559 Mon Sep 17 00:00:00 2001 From: s915 Date: Tue, 24 Sep 2019 01:31:43 +0200 Subject: [PATCH] Small modifications (docs, README) --- README.md | 51 ++++++++++++++++++++++------------------- src/DBnomics.jl | 7 ++++-- src/rdb.jl | 14 +++++++---- src/rdb_by_api_link.jl | 13 +++++++---- src/rdb_last_updates.jl | 10 ++++---- src/rdb_providers.jl | 10 ++++---- src/utils.jl | 6 ----- 7 files changed, 60 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index d421f67..1c5edd9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # DBnomics.jl -## DBnomics Julia client (Julia version ≥ 0.7) +## DBnomics Julia client This package provides you access to DBnomics data series. DBnomics is an open-source project with the goal of aggregating the world's economic data in one location, free of charge to the public. DBnomics covers hundreds of millions of series from international and national institutions (Eurostat, World Bank, IMF, ...). @@ -17,7 +17,7 @@ or install the github version with : add https://github.com/s915/DBnomics.jl ``` -All the functions, and their names, are derived from the R package rdbnomics. +All the functions, and their names, are derived from the R package rdbnomics which I also maintain. ## Examples Fetch time series by `ids` : @@ -92,13 +92,14 @@ When using the functions `rdb` or `rdb_...`, if you come across an error concern 2. using the functions `readlines` and `download` if you have problem with `HTTP.get`. ### Configure **curl** to use a specific and authorized proxy -In **DBnomics**, by default the function `HTTP.get` or `HTTP.post` is used to fetch the data. If a specific proxy must be used, it is possible to define it permanently with the package global variable `curl_config` or on the fly through the argument `curl_config`. In that way the object is passed to the keyword arguments of the `HTTP.get` or `HTTP.post` function. -To see the available parameters, visit the website https://curl.haxx.se/libcurl/c/curl_easy_setopt.html. Once they are chosen, you define the curl object as follows : +In **DBnomics.jl**, by default the function `HTTP.get` or `HTTP.post` are used to fetch the data. If a specific proxy must be used, it is possible to define it permanently with the package global variable `curl_config` or on the fly through the argument `curl_config`. In that way the object is passed to the keyword arguments of the function `HTTP.get` or `HTTP.post`. +To see the available parameters, visit the website https://curl.haxx.se/libcurl/c/curl_easy_setopt.html. +Once they are chosen, you define the curl object as follows : ```julia h = Dict(:proxy => "http://:"); ``` -Regarding the functioning of **HTTP.jl**, you might need to modify another option to change the db/editor.nomics.world url from *https://* to *http://* (see https://github.com/JuliaWeb/HTTP.jl/pull/390) : +Regarding the functioning of **HTTP.jl**, you might need to modify another option to change the *db/editor.nomics.world* url from *https://* to *http://* (see https://github.com/JuliaWeb/HTTP.jl/pull/390) : ```julia DBnomics.options("secure", false); ``` @@ -108,7 +109,7 @@ The curl connection can be set up for a session by modifying the following packa ```julia DBnomics.options("curl_config", h); ``` -After configuration, just use the standard functions of **DBnomics** e.g. : +After configuration, just use the standard functions of **DBnomics.jl** e.g. : ```julia df1 = rdb(ids = "AMECO/ZUTN/EA19.1.0.0.0.ZUTN"); ``` @@ -124,7 +125,7 @@ df1 = rdb(ids = "AMECO/ZUTN/EA19.1.0.0.0.ZUTN", curl_config = h); ``` ### Use the standard functions `readlines` and `download` -To retrieve the data **DBnomics** can also use the standard functions `readlines` and `download`. +To retrieve the data **DBnomics.jl** can also use the standard functions `readlines` and `download`. #### Set the connection up for a session To activate this feature for a session, you need to enable an option of the package : @@ -158,9 +159,11 @@ filters = Dict(:code => "interpolate", :parameters => Dict(:frequency => "monthl df = rdb(ids = ["AMECO/ZUTN/EA19.1.0.0.0.ZUTN", "AMECO/ZUTN/DNK.1.0.0.0.ZUTN"], filters = filters); ``` -If you want to apply more than one filter, the `filters` argument will be a list of valid filters: +If you want to apply more than one filter, the `filters` argument will be a Tuple of valid filters: ```julia -filters = (Dict(:code => "interpolate", :parameters => Dict(:frequency => "monthly", :method => "spline")), Dict(:code => "aggregate", :parameters => Dict(:frequency => "bi-annual", :method => "end_of_period"))); +filter1 = Dict(:code => "interpolate", :parameters => Dict(:frequency => "monthly", :method => "spline")); +filter2 = Dict(:code => "aggregate", :parameters => Dict(:frequency => "bi-annual", :method => "end_of_period")); +filters = (filter1, filter2); df = rdb(ids = ["AMECO/ZUTN/EA19.1.0.0.0.ZUTN", "AMECO/ZUTN/DNK.1.0.0.0.ZUTN"], filters = filters); ``` @@ -175,7 +178,7 @@ The content of two columns are modified: - `series_code`: same as before for original series, but the suffix `_filtered` is added for filtered series. - `series_name`: same as before for original series, but the suffix ` (filtered)` is added for filtered series. -## Transform the `DataFrame` object into a `TimeArray` object (Julia ≥ 0.7) +## Transform the `DataFrame` object into a `TimeArray` object For some analysis, it is more convenient to have a `TimeArray` object instead of a `DataFrame` object. To transform it, you can use the following functions : ```julia @@ -186,9 +189,9 @@ using TimeSeries function to_namedtuples(x::DataFrames.DataFrame) nm = names(x) try - vl = [x[:, col] for col in names(x)] - catch vl = [x[!, col] for col in names(x)] + catch + vl = [x[:, col] for col in names(x)] end nm = tuple(nm...) vl = tuple(vl...) @@ -207,25 +210,25 @@ function to_timeseries( end rdb("IMF", "CPI", mask = "M.DE+FR.PCPIEC_WT") -#> 570×13 DataFrame. Omitted printing of 9 columns -#> │ Row │ @frequency │ dataset_code │ dataset_name │ indexed_at │ -#> │ │ String │ String │ String │ TimeZones.ZonedDateTime │ -#> ├─────┼────────────┼──────────────┼────────────────────────────┼───────────────────────────────┤ -#> │ 1 │ monthly │ CPI │ Consumer Price Index (CPI) │ 2019-05-18T02:48:55.708+00:00 │ -#> │ 2 │ monthly │ CPI │ Consumer Price Index (CPI) │ 2019-05-18T02:48:55.708+00:00 │ -#> │ ... │ ... │ ... │ ... │ ... │ -#> │ 569 │ monthly │ CPI │ Consumer Price Index (CPI) │ 2019-05-18T02:48:55.708+00:00 │ -#> │ 570 │ monthly │ CPI │ Consumer Price Index (CPI) │ 2019-05-18T02:48:55.708+00:00 │ +#> 580×17 DataFrame. Omitted printing of 12 columns +#> │ Row │ @frequency │ dataset_code │ dataset_name │ FREQ │ Frequency │ +#> │ │ String │ String │ String │ String │ String │ +#> ├─────┼────────────┼──────────────┼────────────────────────────┼────────┼───────────┼ +#> │ 1 │ monthly │ CPI │ Consumer Price Index (CPI) │ M │ Monthly │ +#> │ 2 │ monthly │ CPI │ Consumer Price Index (CPI) │ M │ Monthly │ +#> │ ... │ ... │ ... │ ... │ ... │ ... │ +#> │ 579 │ monthly │ CPI │ Consumer Price Index (CPI) │ M │ Monthly │ +#> │ 580 │ monthly │ CPI │ Consumer Price Index (CPI) │ M │ Monthly │ to_timeseries(rdb("IMF", "CPI", mask = "M.DE+FR.PCPIEC_WT")) -#> 291×2 TimeArray{Union{Missing, Float64},2,Date,Array{Union{Missing, Float64},2}} 1995-01-01 to 2019-03-01 +#> 296×2 TimeArray{Union{Missing, Float64},2,Date,Array{Union{Missing, Float64},2}} 1995-01-01 to 2019-08-01 #> │ │ M.DE.PCPIEC_WT │ M.FR.PCPIEC_WT │ #> ├────────────┼────────────────┼────────────────┤ #> │ 1995-01-01 │ missing │ 20.0 │ #> │ 1995-02-01 │ missing │ 20.0 │ #> │ ... │ ... │ ... │ -#> │ 2019-02-01 │ 30.1 │ 25.8 │ -#> │ 2019-03-01 │ 30.1 │ 25.8 │ +#> │ 2019-07-01 │ 30.1 │ 25.8 │ +#> │ 2019-08-01 │ 30.1 │ 25.8 │ ``` ## P.S. diff --git a/src/DBnomics.jl b/src/DBnomics.jl index 25efd96..e701212 100644 --- a/src/DBnomics.jl +++ b/src/DBnomics.jl @@ -1,4 +1,7 @@ module DBnomics + # Visit + println("Visit .") + #--------------------------------------------------------------------------- # Julia version smaller than 1.2.0 version12 = (VERSION >= VersionNumber("0.7.0")) & (VERSION < VersionNumber("1.2.0")); @@ -10,8 +13,8 @@ module DBnomics JSON_version = Pkg.installed()["JSON"] >= VersionNumber("0.21.0") if Parsers_version & JSON_version error( - "The package version of JSON must be 0.20.0 because of the " * - "version of Parsers." * + "The package version of JSON.jl must be 0.20.0 because of the " * + "version of Parsers.jl." * "\n" * """Please run "add JSON@0.20.0" in the package manager.""" ) diff --git a/src/rdb.jl b/src/rdb.jl index 3444d1b..500fdca 100644 --- a/src/rdb.jl +++ b/src/rdb.jl @@ -32,7 +32,7 @@ the arguments names can be dropped. The `mask` will be passed through `mask_arg` - `ids::Union{Array, String, Nothing} = nothing`: DBnomics code of one or several series. - `dimensions::Union{Dict, NamedTuple, String, Nothing} = nothing`: DBnomics code of one or several dimensions in the specified provider and dataset. If it is a `Dict` or a - `NamedTuple`, then then function `json` (from the package **JSON**) is applied to + `NamedTuple`, then then function `json` (from the package **JSON.jl**) is applied to generate the json object. - `mask::Union{String, Nothing} = nothing`: DBnomics code of one or several masks in the specified provider and dataset. @@ -41,10 +41,10 @@ the arguments names can be dropped. The `mask` will be passed through `mask_arg` - `curl_config::Union{Nothing, Dict, NamedTuple} = DBnomics.curl_config`: (default `nothing`) If not `nothing`, it is used to configure a proxy connection. This configuration is passed to the keyword arguments of the function `HTTP.get` or `HTTP.post` of - the package *HTTP*. + the package **HTTP.jl**. - `filters::Union{Nothing, Dict, Tuple} = DBnomics.filters`: (default `nothing`) This argument must be a `Dict` for one filter because the function `json` of the - package *JSON* is used before sending the request to the server. For multiple + package **JSON.jl** is used before sending the request to the server. For multiple filters, you have to provide a `Tuple` of valid filters (see examples). A valid filter is a `Dict` with a key `code` which value is a character string, and a key `parameters` which value is a `Dict` with keys `frequency` @@ -124,10 +124,14 @@ julia> filters = Dict(:code => "interpolate", :parameters => Dict(:frequency => julia> df1 = rdb(ids = ["IMF/WEO/ABW.BCA", "IMF/WEO/ABW.BCA_NGDPD"], filters = filters); # For two filters -julia> filters = (Dict(:code => "interpolate", :parameters => Dict(:frequency => "quarterly", :method => "spline")), Dict(:code => "aggregate", :parameters => Dict(:frequency => "annual", :method => "average"))); +julia> filter1 = Dict(:code => "interpolate", :parameters => Dict(:frequency => "quarterly", :method => "spline")); +julia> filter2 = Dict(:code => "aggregate", :parameters => Dict(:frequency => "annual", :method => "average")); +julia> filters = (filter1, filter2); julia> df1 = rdb(ids = ["IMF/WEO/ABW.BCA", "IMF/WEO/ABW.BCA_NGDPD"], filters = filters); -julia> filters = (Dict(:code => "interpolate", :parameters => Dict(:frequency => "monthly", :method => "linear")), Dict(:code => "x13", :parameters => nothing)); +julia> filter1 = Dict(:code => "interpolate", :parameters => Dict(:frequency => "monthly", :method => "linear")); +julia> filter2 = Dict(:code => "x13", :parameters => nothing); +julia> filters = (filter1, filter2); julia> df1 = rdb("ECB/EXR/A.AUD.EUR.SP00.A", filters = filters); ``` """ diff --git a/src/rdb_by_api_link.jl b/src/rdb_by_api_link.jl index 3ef9e53..a97abe9 100644 --- a/src/rdb_by_api_link.jl +++ b/src/rdb_by_api_link.jl @@ -19,10 +19,10 @@ The code of each series is given on the [DBnomics](https://db.nomics.world/) web - `curl_config::Union{Nothing, Dict, NamedTuple} = DBnomics.curl_config`: (default `nothing`) If not `nothing`, it is used to configure a proxy connection. This configuration is passed to the keyword arguments of the function `HTTP.get` or `HTTP.post` of - the package *HTTP*. + the package **HTTP.jl**. - `filters::Union{Nothing, Dict, Tuple} = DBnomics.filters`: (default `nothing`) This argument must be a `Dict` for one filter because the function `json` of the - package *JSON* is used before sending the request to the server. For multiple + package **JSON.jl** is used before sending the request to the server. For multiple filters, you have to provide a `Tuple` of valid filters (see examples). A valid filter is a `Dict` with a key `code` which value is a character string, and a key `parameters` which value is a `Dict` with keys `frequency` @@ -63,10 +63,14 @@ julia> filters = Dict(:code => "interpolate", :parameters => Dict(:frequency => julia> df1 = rdb_by_api_link("https://api.db.nomics.world/v22/series/IMF/WEO/ABW.BCA?observations=1", filters = filters); # Two filters -julia> filters = (Dict(:code => "interpolate", :parameters => Dict(:frequency => "quarterly", :method => "spline")), Dict(:code => "aggregate", :parameters => Dict(:frequency => "annual", :method => "average"))); +julia> filter1 = Dict(:code => "interpolate", :parameters => Dict(:frequency => "quarterly", :method => "spline")); +julia> filter2 = Dict(:code => "aggregate", :parameters => Dict(:frequency => "annual", :method => "average")); +julia> filters = (filter1, filter2); julia> df1 = rdb_by_api_link("https://api.db.nomics.world/v22/series/IMF/WEO/ABW.BCA?observations=1", filters = filters); -julia> filters = (Dict(:code => "interpolate", :parameters => Dict(:frequency => "monthly", :method => "linear")), Dict(:code => "x13", :parameters => nothing)); +julia> filter1 = Dict(:code => "interpolate", :parameters => Dict(:frequency => "monthly", :method => "linear")); +julia> filter2 = Dict(:code => "x13", :parameters => nothing); +julia> filters = (filter1, filter2); julia> df1 = rdb_by_api_link("https://api.db.nomics.world/v22/series?observations=1&series_ids=ECB/EXR/A.AUD.EUR.SP00.A", filters = filters); ``` """ @@ -229,6 +233,7 @@ function rdb_by_api_link( # POST request body body = JSON.json((filters = filters, series = (series,))) + # null are not valid in json body = replace(body, "parameters\":null" => "parameters\":{}") body = replace(body, "null" => "\"NA\"") diff --git a/src/rdb_last_updates.jl b/src/rdb_last_updates.jl index 3aac5ed..4d90b42 100644 --- a/src/rdb_last_updates.jl +++ b/src/rdb_last_updates.jl @@ -21,18 +21,18 @@ containing the last 100 updates from - `curl_config::Union{Nothing, Dict, NamedTuple} = DBnomics.curl_config`: (default `nothing`) If not `nothing`, it is used to configure a proxy connection. This configuration is passed to the keyword arguments of the function `HTTP.get` of - the package *HTTP*. + the package **HTTP.jl**. - `kwargs...`: Keyword arguments to be passed to `HTTP.get`. # Examples ```jldoctest -julia> rdb_last_updates() +julia> rdb_last_updates(); -julia> rdb_last_updates(true) +julia> rdb_last_updates(true); -julia> rdb_last_updates(use_readlines = true) +julia> rdb_last_updates(use_readlines = true); -julia> rdb_last_updates(curl_config = Dict(:proxy => "http://:")) +julia> rdb_last_updates(curl_config = Dict(:proxy => "http://:")); # Regarding the functioning of HTTP.jl, you might need to modify another option # It will change the url from https:// to http:// diff --git a/src/rdb_providers.jl b/src/rdb_providers.jl index 9e538c0..f63d40e 100644 --- a/src/rdb_providers.jl +++ b/src/rdb_providers.jl @@ -21,18 +21,18 @@ the region, the website, etc. - `curl_config::Union{Nothing, Dict, NamedTuple} = DBnomics.curl_config`: (default `nothing`) If not `nothing`, it is used to configure a proxy connection. This configuration is passed to the keyword arguments of the function `HTTP.get` of - the package *HTTP*. + the package **HTTP.jl**. - `kwargs...`: Keyword arguments to be passed to `HTTP.get`. # Examples ```jldoctest -julia> rdb_providers() +julia> rdb_providers(); -julia> rdb_providers(true) +julia> rdb_providers(true); -julia> rdb_providers(use_readlines = true) +julia> rdb_providers(use_readlines = true); -julia> rdb_providers(curl_config = Dict(:proxy => "http://:")) +julia> rdb_providers(curl_config = Dict(:proxy => "http://:")); # Regarding the functioning of HTTP.jl, you might need to modify another option # It will change the url from https:// to http:// diff --git a/src/utils.jl b/src/utils.jl index dc88659..d6d4c74 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -688,9 +688,3 @@ function original_value_to_string!(x::DataFrames.DataFrame, y) x[selectop, :original_value] = y nothing end - -#------------------------------------------------------------------------------- -# JuliaDBtable -# function JuliaDBtable(x::Dict) -# table((; value_to_array(key_to_symbol(x))...)) -# end