Skip to content

Commit

Permalink
Add optional column mapping (#51)
Browse files Browse the repository at this point in the history
  • Loading branch information
fivegrant committed Jun 15, 2023
1 parent 4246273 commit 46f92d7
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 4 deletions.
2 changes: 2 additions & 0 deletions paths.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@
filename:
type: string
example: "dataset.csv"
mappings:
type: object
timespan:
type: object
properties:
Expand Down
2 changes: 1 addition & 1 deletion src/service/ArgIO.jl
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ function prepare_input(args; context...)
args[:model] = fetch_model(args[:model_config_id])
end
if in(:dataset, keys(args)) && !isa(args[:dataset], String)
args[:dataset] = fetch_dataset(args[:dataset]["id"], args[:dataset]["filename"])
args[:dataset] = fetch_dataset(args[:dataset]["id"], args[:dataset]["filename"], get(args[:dataset], "mappings", Dict()))
end
if in(:model_config_ids, keys(args))
args[:models] = fetch_model.(map(string, args[:model_ids]))
Expand Down
8 changes: 5 additions & 3 deletions src/service/AssetManager.jl
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ Asset fetching from TDS
"""
module AssetManager

import DataFrames: DataFrame
import DataFrames: rename!, DataFrame
import CSV, Downloads, HTTP
import OpenAPI.Clients: Client
import JSON3 as JSON
Expand Down Expand Up @@ -31,15 +31,17 @@ end
"""
Return csv from TDS by ID
"""
function fetch_dataset(dataset_id::String, filename::String)
function fetch_dataset(dataset_id::String, filename::String, mappings::Dict=Dict())
# TODO(five): Select name dynamicially
url = "$(settings["TDS_URL"])/datasets/$dataset_id/download-url?filename=$filename"
response = HTTP.get(url, ["Content-Type" => "application/json"])
body = response.body |> JSON.read String
io = IOBuffer()
Downloads.download(body.url, io)
seekstart(io)
CSV.read(io, DataFrame)
dataframe = CSV.read(io, DataFrame)
for (from, to) in mappings rename!(dataframe, Symbol(from)=>Symbol(to)) end
dataframe
end

"""
Expand Down

0 comments on commit 46f92d7

Please sign in to comment.