-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Cli param overrides with an embedded jq (#3)
* chore: bump deps * feat: added jq script to the query transformation pipeline * feat: transforms for impact support jq scripts * feat: support for jq config overrides
- Loading branch information
1 parent
e902af4
commit 2a6116f
Showing
10 changed files
with
233 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| (ns polyglot.jq | ||
| (:require [clojure.string :as str] | ||
| [core.json :as json] | ||
| [jq.core :as jq])) | ||
|
|
||
| (defn script->transform-fn | ||
| "Given a JQ script snippet creates a function that expects a JSON | ||
| string that is going to be passed into that function as an argument. | ||
| Returns a string." | ||
| [^String script] | ||
| (jq/processor script)) | ||
|
|
||
| (defn script->transform-fn-vals | ||
| "Given a JQ script snippet creates a function that accepts any number | ||
| of arguments where first argument must be a JSON string and the rest | ||
| of the arguments can be of any type. The output of is a JSON string." | ||
| [^String script] | ||
| (let [jq-fn (jq/processor script)] | ||
| (fn [& args] | ||
| (jq-fn (str "[" (str/join "," (concat [(first args)] (map json/encode-vanilla (rest args)))) "]"))))) | ||
|
|
||
| (comment | ||
| ;; Add an foo attribute with value bar to a map | ||
| (let [tf (polyglot.jq/script->transform-fn ".foo = \"bar\"")] | ||
| (tf "{}")) | ||
| ;; => {"foo":"bar"} | ||
|
|
||
| ;; String interpolation | ||
| (let [tf (polyglot.jq/script->transform-fn ".a = \">>\\(.a+10)<<\"")] | ||
| (tf "{\"a\": 12}")) | ||
| ;; => {"a":">>22<<"} | ||
|
|
||
| ;; Generate an array | ||
| (let [tf (polyglot.jq/script->transform-fn "[(.a,.b)=range(3)]")] | ||
| (tf "{\"a\": 12}")) | ||
| ;; => [{"a":0,"b":0},{"a":1,"b":1},{"a":2,"b":2}] | ||
|
|
||
|
|
||
| ;; Collect all the keys from an array of maps to an array | ||
| (let [tf (polyglot.jq/script->transform-fn-vals "[.[] | keys] | add")] | ||
| (tf "{\"a\": 1}" {:b 2})) | ||
| ;; => ["a","b"] | ||
|
|
||
| ;; Assoc attribute e with the value of second arg to the first argument | ||
| (let [tf (polyglot.jq/script->transform-fn-vals ". as [$first_arg, $second_arg] | $first_arg | .e = $second_arg")] | ||
| (tf "{\"a\": 1}" {:b 2})) | ||
| ;; => {"a":1,"e":{"b":2}} | ||
|
|
||
| ;; Work with an array of elements | ||
| (let [tf (polyglot.jq/script->transform-fn-vals | ||
| ".[0] as $first_arg | .[1] as $second_arg | {} | .\"1st\" = $first_arg | .\"2nd\" = $second_arg")] | ||
| (tf "{\"a\": 1}" "{\"b\": 2}")) | ||
| ;; => {"1st":{"a":1},"2nd":{"b":2}} | ||
|
|
||
| ;; Example of destructuring an array | ||
| (let [tf (polyglot.jq/script->transform-fn-vals | ||
| ". as [$first_arg, $second_arg] | {} | .fa = $first_arg | .sa = $second_arg")] | ||
| (tf "{\"a\": 1}" "{\"b\": 2}")) | ||
| ;; => {"fa":{"a":1},"sa":{"b":2}} | ||
|
|
||
| ;; Recursively collect all the 'a' attributes from an array of maps | ||
| ;; and remove those records that value is null | ||
| (let [tf (polyglot.jq/script->transform-fn-vals | ||
| "[.. | .a?] | map(select(. != null))")] | ||
| (tf "{\"a\": 1}" "{\"b\": 2}")) | ||
| ;; => [1] | ||
|
|
||
| ; Collect al; the paths from a nested map | ||
| (let [tf (polyglot.jq/script->transform-fn-vals | ||
| "select(objects)|=[.] | map( paths(scalars) ) | map( map(select(numbers)=\"[]\") | join(\".\")) | unique")] | ||
| (tf "{\"a\": 1, \"b\": {\"c\": 3}}")) | ||
| ;; => ["a","b.c"] | ||
|
|
||
| ;; Collect all paths from an array | ||
| (let [tf (polyglot.jq/script->transform-fn-vals | ||
| "select(objects)|=[.] | map( paths(scalars) ) | map( map(select(numbers)=\"[]\") | join(\".\")) | unique")] | ||
| (tf "[{\"a\": 1, \"b\": {\"c\": 3}}]")) | ||
| ;; => ["[].a","[].b.c"] | ||
|
|
||
| ;; With plain values | ||
| (let [tf (polyglot.jq/script->transform-fn-vals | ||
| ". as [$first_arg, $second_arg, $third_arg] | {} | .fa = $first_arg | .sa = $second_arg | .ta = $third_arg")] | ||
| (tf "{\"a\": 1}" 11 "\"string\"")) | ||
| ;; => {"fa":{"a":1},"sa":11,"ta":"string"} | ||
|
|
||
| ;; All types of values can be passed to the function as args | ||
| (let [tf (polyglot.jq/script->transform-fn-vals | ||
| ". as [$first_arg, $second_arg, $third_arg, $fourth_arg, $fifth_arg, $sixth_arg] | ||
| | {} | ||
| | .\"1st\" = $first_arg | ||
| | .\"2dn\" = $second_arg | ||
| | .\"3rd\" = $third_arg | ||
| | .\"4th\" = $fourth_arg | ||
| | .\"5th\" = $fifth_arg | ||
| | .\"6th\" = $sixth_arg")] | ||
| (tf "{\"a\": 1}" "my_string" 12 false nil {:b 10})) | ||
| ;; => {"1st":{"a":1},"2dn":"my_string","3rd":12,"4th":false,"5th":null,"6th":{"b":10}} | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,24 +1,38 @@ | ||
| (ns replay.transform.query | ||
| (:require [polyglot.js :as js] | ||
| [polyglot.jq :as jq] | ||
| [polyglot.sci :as sci])) | ||
|
|
||
| (defn compile-transform [{:keys [lang script]}] | ||
| (case (keyword lang) | ||
| :sci (sci/script->transform-fn script) | ||
| :js (js/script->transform-fn script) | ||
| :jq (jq/script->transform-fn script) | ||
| (throw (Exception. (format "No such language supported: '%s'" (name lang)))))) | ||
|
|
||
| ;; TODO: optimize consecutive JQ scripts to be executed in one pass | ||
| (defn transform-fn [transforms] | ||
| (let [tf-fn (apply comp (map compile-transform (reverse transforms)))] | ||
| (fn [query] (tf-fn query)))) | ||
|
|
||
| (comment | ||
| (time | ||
| (let [data "{}" | ||
| tfs [{:lang :js | ||
| :script "(request) => request"} | ||
| {:lang :sci | ||
| :script "(fn [q] (assoc q :_explain true))"}] | ||
| tf (transform-fn tfs)] | ||
| (dotimes [i 1000] | ||
| (tf data))))) | ||
| ;; Applies transforms on the input string in order | ||
| (let [data "{}" | ||
| tfs [{:lang :js | ||
| :script "(request) => { request['_source'] = false; return request; }"} | ||
| {:lang :sci | ||
| :script "(fn [q] (assoc q :_explain true))"} | ||
| {:lang :jq | ||
| :script ".size = 15"}] | ||
| tf (transform-fn tfs)] | ||
| (tf data)) | ||
| ;; => {"_explain":true,"_source":false,"size":15} | ||
|
|
||
| ;; JQ scripts are the fastest option to transform | ||
| (let [data "{\"a\": 12}" | ||
| tfs [{:lang :jq | ||
| :script ".foo |= \"bar\""}] | ||
| tf (transform-fn tfs)] | ||
| (tf data)) | ||
| ;; => {"a":12,"foo":"bar"} | ||
| ) |
Oops, something went wrong.