From 20393014b149e416debe5a013c9a13e950ed06f1 Mon Sep 17 00:00:00 2001 From: pat Date: Sun, 21 May 2023 14:36:12 -0400 Subject: [PATCH 01/17] bump cljs-node-io for advanced compilation safety --- README.org | 7 ++++--- deps.edn | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/README.org b/README.org index b5da303..7d85236 100644 --- a/README.org +++ b/README.org @@ -197,7 +197,8 @@ Usage: #+BEGIN_SRC clojure (ns test-db - (:require [konserve.memory :refer [connect-fs-store]] + (:require [#?(:clj konserve.filestore + :cljs konserve.node-filestore) :refer [connect-fs-store]] [konserve.core :as k])) (def my-folder "path/to/folder") @@ -210,13 +211,13 @@ Usage: :END: [[https://developer.mozilla.org/en-US/docs/IndexedDB][IndexedDB]] is provided as reference implementation for -ClojureScript browser backends. +ClojureScript browser backends. The IndexedDB store is restricted to the async api only. Usage: #+BEGIN_SRC clojure (ns test-db - (:require [konserve.memory :refer [connect-idb-store]] + (:require [konserve.indexeddb :refer [connect-idb-store]] [konserve.core :as k]) (:require-macros [cljs.core.async.macros :refer [go]])) diff --git a/deps.edn b/deps.edn index c82abed..e48926d 100644 --- a/deps.edn +++ b/deps.edn @@ -9,7 +9,7 @@ org.lz4/lz4-java {:mvn/version "1.8.0"} com.taoensso/timbre {:mvn/version "6.0.1"} ;; cljs - com.github.pkpkpk/cljs-node-io {:mvn/version "2.0.332"} + com.github.pkpkpk/cljs-node-io {:mvn/version "2.0.339"} fress/fress {:mvn/version "0.4.0"} org.clojars.mmb90/cljs-cache {:mvn/version "0.1.4"}} :aliases {:cljs {:extra-deps {org.clojure/clojurescript {:mvn/version "1.11.60"} From 5405b083e4a41d3b8cd7357d0433d87446d79c0f Mon Sep 17 00:00:00 2001 From: pat killean Date: Sun, 22 Oct 2023 19:46:51 -0400 Subject: [PATCH 02/17] tweak macro referencing in cljc paths to make shadow-cljs happy --- deps.edn | 10 ++++----- shadow-cljs.edn | 17 +++++++++++++++- src/konserve/cache.cljc | 3 ++- src/konserve/core.cljc | 3 ++- src/konserve/indexeddb.cljs | 2 +- src/konserve/node_filestore.cljs | 2 +- src/konserve/utils.cljc | 35 ++++++++++++++++---------------- 7 files changed, 44 insertions(+), 28 deletions(-) diff --git a/deps.edn b/deps.edn index e48926d..45658c3 100644 --- a/deps.edn +++ b/deps.edn @@ -13,7 +13,7 @@ fress/fress {:mvn/version "0.4.0"} org.clojars.mmb90/cljs-cache {:mvn/version "0.1.4"}} :aliases {:cljs {:extra-deps {org.clojure/clojurescript {:mvn/version "1.11.60"} - thheller/shadow-cljs {:mvn/version "2.22.0"} + thheller/shadow-cljs {:mvn/version "2.25.9"} binaryage/devtools {:mvn/version "1.0.6"}} :extra-paths ["test"]} :dev {:extra-deps {criterium/criterium {:mvn/version "0.4.6"} @@ -28,12 +28,12 @@ lambdaisland/kaocha-cljs {:mvn/version "1.4.130"} org.clojure/test.check {:mvn/version "1.1.1"}} :extra-paths ["test"] - :main-opts ["-e" "(set! *warn-on-reflection* true)"]} + :main-opts ["-e" "(set! *warn-on-reflection* true)"]} :run-cljs-tests {:extra-deps {olical/cljs-test-runner {:mvn/version "3.8.0"}} :extra-paths ["test"] - :main-opts ["-m" "cljs-test-runner.main" - "-o" "target/cljs" - "--exclude" "browser" + :main-opts ["-m" "cljs-test-runner.main" + "-o" "target/cljs" + "--exclude" "browser" "--env" "node"]} :build {:deps {io.github.clojure/tools.build {:mvn/version "0.9.3"} slipset/deps-deploy {:mvn/version "0.2.0"} diff --git a/shadow-cljs.edn b/shadow-cljs.edn index b1ea626..af476fb 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -1,7 +1,22 @@ {:deps {:aliases [:cljs]} :source-paths ["src"] + :dev-http {8021 "out/browser-tests"} :builds {:app {:target :browser :output-dir "public/js" - :modules {:main {:entries [konserve.core]}}}}} + :modules {:main {:entries [konserve.core]}}} + :node-tests ; shadow-cljs compile node-tests && node out/node-tests.js + {:target :node-test + :output-to "out/node-tests.js" + :ns-regexp "^(?!konserve.indexeddb-test)" + :compiler-options {:infer-externs true} + :autorun true} + :browser-tests + ; shadow-cljs watch :browser-tests + ; http://localhost:8021/ + {:target :browser-test + :test-dir "out/browser-tests" + :ns-regexp "konserve.indexeddb-test" ;; TODO add in-mem for other tests + :compiler-options {:infer-externs true} + :autorun true}}} diff --git a/src/konserve/cache.cljc b/src/konserve/cache.cljc index 8fdaaa3..c173191 100644 --- a/src/konserve/cache.cljc +++ b/src/konserve/cache.cljc @@ -9,7 +9,8 @@ #?(:clj [clojure.core.cache :as cache] :cljs [cljs.cache :as cache]) [konserve.core #?@(:clj (:refer [go-locked locked])) :as core] - [konserve.utils :refer [meta-update async+sync *default-sync-translation*]] + [konserve.utils :refer [meta-update #?(:clj async+sync) *default-sync-translation*] + #?@(:cljs [:refer-macros [async+sync]])] [taoensso.timbre :refer [trace]] [superv.async :refer [go-try- sync async-code] - (let [async->sync (if (symbol? async->sync) - (or (resolve async->sync) - (when-let [_ns (or (get-in &env [:ns :use-macros async->sync]) - (get-in &env [:ns :uses async->sync]))] - (resolve (symbol (str _ns) (str async->sync))))) - async->sync)] - (assert (some? async->sync)) - `(if ~sync? - ~(clojure.walk/postwalk (fn [n] - (if-not (meta n) - (async->sync n n) ;; primitives have no metadata - (with-meta (async->sync n n) - (update (meta n) :tag (fn [t] (async->sync t t)))))) - async-code) - ~async-code)))) +(defmacro async+sync + [sync? async->sync async-code] + (let [async->sync (if (symbol? async->sync) + (or (resolve async->sync) + (when-let [_ns (or (get-in &env [:ns :use-macros async->sync]) + (get-in &env [:ns :uses async->sync]))] + (resolve (symbol (str _ns) (str async->sync))))) + async->sync)] + (assert (some? async->sync)) + `(if ~sync? + ~(clojure.walk/postwalk (fn [n] + (if-not (meta n) + (async->sync n n) ;; primitives have no metadata + (with-meta (async->sync n n) + (update (meta n) :tag (fn [t] (async->sync t t)))))) + async-code) + ~async-code))) (def ^:dynamic *default-sync-translation* '{go-try try From 8370b15ebc1496f206a2e0a9df880aa995853d38 Mon Sep 17 00:00:00 2001 From: pat killean Date: Sat, 28 Oct 2023 08:29:38 -0400 Subject: [PATCH 03/17] minor fress bump to silence abs warning --- deps.edn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 45658c3..5c963dd 100644 --- a/deps.edn +++ b/deps.edn @@ -10,7 +10,7 @@ com.taoensso/timbre {:mvn/version "6.0.1"} ;; cljs com.github.pkpkpk/cljs-node-io {:mvn/version "2.0.339"} - fress/fress {:mvn/version "0.4.0"} + com.github.pkpkpk/fress {:mvn/version "0.4.307"} org.clojars.mmb90/cljs-cache {:mvn/version "0.1.4"}} :aliases {:cljs {:extra-deps {org.clojure/clojurescript {:mvn/version "1.11.60"} thheller/shadow-cljs {:mvn/version "2.25.9"} From 947a5379a8cf02f695a0872dfa09f85299425427 Mon Sep 17 00:00:00 2001 From: pat killean Date: Sat, 28 Oct 2023 20:06:34 -0400 Subject: [PATCH 04/17] WIP cache tests --- deps.edn | 2 +- src/konserve/indexeddb.cljs | 51 ++++++++++++++++--- test/konserve/cache_test.clj | 19 ++++++- test/konserve/cache_test.cljs | 70 -------------------------- test/konserve/cache_test_common.cljc | 66 ++++++++++++++++++++++++ test/konserve/indexeddb_test.cljs | 37 +++++++++++++- test/konserve/node_filestore_test.cljs | 37 +++++++++++++- 7 files changed, 199 insertions(+), 83 deletions(-) delete mode 100644 test/konserve/cache_test.cljs create mode 100644 test/konserve/cache_test_common.cljc diff --git a/deps.edn b/deps.edn index 5c963dd..332077d 100644 --- a/deps.edn +++ b/deps.edn @@ -19,7 +19,7 @@ :dev {:extra-deps {criterium/criterium {:mvn/version "0.4.6"} metasoarous/oz {:mvn/version "2.0.0-alpha5"} org.clojure/tools.cli {:mvn/version "1.0.214"}} - :extra-paths ["benchmark/src"]} + :extra-paths ["benchmark/src" "test"]} :benchmark {:extra-deps {metasoarous/oz {:mvn/version "2.0.0-alpha5"} org.clojure/tools.cli {:mvn/version "1.0.214"}} :extra-paths ["benchmark/src"] diff --git a/src/konserve/indexeddb.cljs b/src/konserve/indexeddb.cljs index d17fa8c..42a01bd 100644 --- a/src/konserve/indexeddb.cljs +++ b/src/konserve/indexeddb.cljs @@ -1,6 +1,5 @@ (ns konserve.indexeddb - (:require-macros [cljs.core.async.macros :refer [go]]) - (:require [cljs.core.async :refer [take! put! close!]] + (:require [clojure.core.async :refer [go take! put! close!]] [konserve.compressor] [konserve.encryptor] [konserve.impl.defaults :as defaults] @@ -267,6 +266,39 @@ (db-exists? db-name)) (-sync-store [_this env] (when-not (:sync? env) (go)))) +(defn read-web-stream + "Accepts the bget locked callback arg and returns a promise-chan containing + a concatenated byte array with the first offset bytes dropped: + (k/bget store + :key + (fn [{:keys [offset input-stream] :as m}] <-- locked-cb + (-> (read-web-stream m) + (handle-promise-chan))))" + [{:keys [input-stream offset]}] + (let [reader (.getReader input-stream) + chunks #js[]] + (with-promise out + (let [read-chunk (fn read-chunk [] + (.then (.read reader) + (fn [result] + (let [done (.-done result) + value (.-value result)] + (if done + (if (== 1 (alength chunks)) + (put! out (.slice (aget chunks 0) offset)) + (let [total-length (reduce + (map count chunks)) + final-array (js/Uint8Array. (inc total-length)) + _i (atom 0)] + (doseq [chunk (array-seq chunks)] + (.set final-array chunk @_i) + (swap! _i + (alength chunk))) + (put! out (.slice final-array offset)))) + (do + (.push chunks value) + (read-chunk))))) + (fn [err] (put! out err))))] + (read-chunk))))) + (defn connect-idb-store "Connect to a IndexedDB backed KV store with the given db name. Optional serializer, read-handlers, write-handlers. @@ -288,15 +320,20 @@ db and core.async gets into a weird state due to an unhandled error, you will be unable to delete the database until the vm is restarted - + As of November 2022 firefox does not support IDBFactory.databases() so + + As of November 2023 firefox does not support IDBFactory.databases() so expect list-dbs, db-exists?, & PBackingStore/-store-exists? to all throw. You must work around this by keeping track of each db name you intend to delete https://developer.mozilla.org/en-US/docs/Web/API/IDBFactory/databases#browser_compatibility - + PBackingBlob/-read-binary returns a webstream that is *not* queued to the - value offset in the same way that the filestore implementations are. Consumers - must discard the amount of bytes found in the :offset key of the locked-cb - arg map. See: https://developer.mozilla.org/en-US/docs/Web/API/Blob/stream" + + `konserve.core/bget`locked-cb arg is given a webstream that is *not* queued + to the value offset in the same way that the filestore implementations are. + See: https://developer.mozilla.org/en-US/docs/Web/API/Blob/stream + - consumers must discard the amount of bytes found in the :offset key of + the locked-cb arg map. These bytes are meta data for konserve and not + part of the value you are retrieving + - `konserve.indexeddb/read-web-stream` will accept the argument to the + locked-cb and return a promise-chan receiving err|bytes at the cost of + allocating a larger array for the chunks to be copied into" [db-name & {:as params}] (let [store-config (merge {:default-serializer :FressianSerializer :compressor konserve.compressor/null-compressor diff --git a/test/konserve/cache_test.clj b/test/konserve/cache_test.clj index 8cb94c1..2ed5b5d 100644 --- a/test/konserve/cache_test.clj +++ b/test/konserve/cache_test.clj @@ -1,7 +1,8 @@ (ns konserve.cache-test (:require [konserve.cache :as k] + [konserve.cache-test-common :as ktc] [konserve.filestore :as fstore] - [clojure.core.async :refer [promise] + (let [opts {:sync? false} + data [:this/is + 'some/fressian + "data 😀😀😀" + #?(:cljs (js/Date.) :clj (java.util.Date.)) + #{true false nil}] + bytes #?(:cljs (fress/write data) + :clj (.array (fress/write data))) ;; TODO add HeapByteArrayBuffer to nio protocols + bytes-ch (promise-chan) + test-ch (promise-chan) + locked-cb (fn [locked] + (locked->promise locked bytes-ch) + (take! bytes-ch + (fn [bytes] + (put! test-ch (fress/read bytes)))))] + (go + (and + (is (true? ( Date: Sun, 29 Oct 2023 21:11:44 -0400 Subject: [PATCH 05/17] WIP cache/gc, fixes for bget semantics --- shadow-cljs.edn | 2 +- src/konserve/core.cljc | 4 +- src/konserve/indexeddb.cljs | 12 +-- src/konserve/node_filestore.cljs | 37 +++++--- test/konserve/cache_test.clj | 84 ------------------- test/konserve/core_test.clj | 9 -- test/konserve/filestore_test.clj | 34 +++++++- test/konserve/gc_test.clj | 45 ---------- test/konserve/gc_test.cljs | 47 ----------- test/konserve/indexeddb_test.cljs | 47 ++++++----- test/konserve/node_filestore_test.cljs | 74 ++++++++-------- .../cache.cljc} | 33 ++++---- test/konserve/tests/gc.cljc | 32 +++++++ 13 files changed, 183 insertions(+), 277 deletions(-) delete mode 100644 test/konserve/cache_test.clj delete mode 100644 test/konserve/core_test.clj delete mode 100644 test/konserve/gc_test.clj delete mode 100644 test/konserve/gc_test.cljs rename test/konserve/{cache_test_common.cljc => tests/cache.cljc} (76%) create mode 100644 test/konserve/tests/gc.cljc diff --git a/shadow-cljs.edn b/shadow-cljs.edn index af476fb..c756e54 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -17,6 +17,6 @@ ; http://localhost:8021/ {:target :browser-test :test-dir "out/browser-tests" - :ns-regexp "konserve.indexeddb-test" ;; TODO add in-mem for other tests + :ns-regexp "^(?!konserve.node-filestore-test|konserve.node-filestore)" :compiler-options {:infer-externs true} :autorun true}}} diff --git a/src/konserve/core.cljc b/src/konserve/core.cljc index 6035d49..069fb3f 100644 --- a/src/konserve/core.cljc +++ b/src/konserve/core.cljc @@ -247,7 +247,9 @@ (fn [{is :input-stream}] (let [tmp-file (io/file \"/tmp/my-private-copy\")] (io/copy is tmp-file))) - " + + When called asynchronously (by default or w/ {:sync? false}), the locked-cb + must synchronously return a channel." ([store key locked-cb] (bget store key locked-cb {:sync? false})) ([store key locked-cb opts] diff --git a/src/konserve/indexeddb.cljs b/src/konserve/indexeddb.cljs index 42a01bd..720cc77 100644 --- a/src/konserve/indexeddb.cljs +++ b/src/konserve/indexeddb.cljs @@ -134,11 +134,11 @@ (put! out (ex-info "error reading blob from objectStore" {:cause res :caller 'konserve.indexeddb/read-binary})) - (do + (take! (locked-cb {:input-stream (.stream res) :size (.-size res) :offset (+ meta-size storage-layout/header-size)}) - (close! out))))))) + #(put! out %))))))) (defrecord ^{:doc "buf is cached data that has been read from the db, & {header metadata value} are bin data to be written. @@ -269,11 +269,7 @@ (defn read-web-stream "Accepts the bget locked callback arg and returns a promise-chan containing a concatenated byte array with the first offset bytes dropped: - (k/bget store - :key - (fn [{:keys [offset input-stream] :as m}] <-- locked-cb - (-> (read-web-stream m) - (handle-promise-chan))))" + (k/bget store :key (read-web-stream m))" [{:keys [input-stream offset]}] (let [reader (.getReader input-stream) chunks #js[]] @@ -325,7 +321,7 @@ must work around this by keeping track of each db name you intend to delete https://developer.mozilla.org/en-US/docs/Web/API/IDBFactory/databases#browser_compatibility - + `konserve.core/bget`locked-cb arg is given a webstream that is *not* queued + + `konserve.core/bget` locked-cb arg is given a webstream that is *not* queued to the value offset in the same way that the filestore implementations are. See: https://developer.mozilla.org/en-US/docs/Web/API/Blob/stream - consumers must discard the amount of bytes found in the :offset key of diff --git a/src/konserve/node_filestore.cljs b/src/konserve/node_filestore.cljs index c53f1f2..2e3ba05 100644 --- a/src/konserve/node_filestore.cljs +++ b/src/konserve/node_filestore.cljs @@ -65,10 +65,11 @@ buf)) (-read-binary [this meta-size locked-cb _env] (let [blob-size ^number (.-size (fs.fstatSync fd)) - pos (+ meta-size storage-layout/header-size) - rstream (fs.createReadStream nil #js{:fd (.-fd this) :start pos})] - (.on rstream "readable" - #(locked-cb {:input-stream rstream :size blob-size})))) + offset (+ meta-size storage-layout/header-size) + value-len (- blob-size offset) + blob (js/Buffer. value-len)] + (iofs/read (.-fd this) blob offset value-len 0) + (locked-cb {:blob blob}))) (-write-header [_this header _env] (let [buffer (js/Buffer.from header) bytes-written (iofs/write fd buffer {:position 0})] @@ -195,17 +196,33 @@ (defn- afread-binary ;=> ch [fd meta-size locked-cb _env] - (go - (let [[?err total-size] ( ch [fd meta-size blob _env] diff --git a/test/konserve/cache_test.clj b/test/konserve/cache_test.clj deleted file mode 100644 index 2ed5b5d..0000000 --- a/test/konserve/cache_test.clj +++ /dev/null @@ -1,84 +0,0 @@ -(ns konserve.cache-test - (:require [konserve.cache :as k] - [konserve.cache-test-common :as ktc] - [konserve.filestore :as fstore] - [clojure.core.async :refer [> list-keys (map #(clojure.core/dissoc % :last-write)) set) - true - (every? - (fn [{:keys [:last-write]}] - (= (type (java.util.Date.)) (type last-write))) - list-keys))) - - (doseq [to-delete [:baz :binbar :foolog]] - ( e :konserve.core/timestamp .getTime)]) (store-key key))] (testing "no lock, writes ok" @@ -178,26 +175,35 @@ (filestore/delete-store "/tmp/cache-store") (async done (go - (let [store (promise] - (let [opts {:sync? false} +(defn test-cached-PBin-async [store locked-cb] + (let [store (kc/ensure-cache store) data [:this/is 'some/fressian "data 😀😀😀" #?(:cljs (js/Date.) :clj (java.util.Date.)) #{true false nil}] bytes #?(:cljs (fress/write data) - :clj (.array (fress/write data))) ;; TODO add HeapByteArrayBuffer to nio protocols - bytes-ch (promise-chan) - test-ch (promise-chan) - locked-cb (fn [locked] - (locked->promise locked bytes-ch) - (take! bytes-ch - (fn [bytes] - (put! test-ch (fress/read bytes)))))] + ;; TODO add HeapByteArrayBuffer to nio protocols + :clj (.array (fress/write data))) + bytes-ch (promise-chan)] (go (and (is (true? ( this can be tested by abstract out timestamps where there are written +;; and then using redef in tests. CLJS impls missing entirely + +;it seems clock of threads are not sync. +;(prn (.getTime ts) (map (fn [e] [(:key e) (-> e :konserve.core/timestamp .getTime)]) ( Date: Mon, 6 Nov 2023 14:39:45 -0400 Subject: [PATCH 06/17] CLJC serializers tests --- deps.edn | 2 +- src/konserve/indexeddb.cljs | 18 +-- src/konserve/node_filestore.cljs | 32 ++--- test/konserve/filestore_test.clj | 18 ++- test/konserve/indexeddb_test.cljs | 41 ++++-- test/konserve/node_filestore_test.cljs | 19 ++- test/konserve/serializers_test.clj | 51 -------- test/konserve/serializers_test.cljs | 39 ------ test/konserve/tests/cache.cljc | 4 - test/konserve/tests/serializers.cljc | 166 +++++++++++++++++++++++++ 10 files changed, 257 insertions(+), 133 deletions(-) delete mode 100644 test/konserve/serializers_test.clj delete mode 100644 test/konserve/serializers_test.cljs create mode 100644 test/konserve/tests/serializers.cljc diff --git a/deps.edn b/deps.edn index 332077d..32f27fc 100644 --- a/deps.edn +++ b/deps.edn @@ -10,7 +10,7 @@ com.taoensso/timbre {:mvn/version "6.0.1"} ;; cljs com.github.pkpkpk/cljs-node-io {:mvn/version "2.0.339"} - com.github.pkpkpk/fress {:mvn/version "0.4.307"} + com.github.pkpkpk/fress {:mvn/version "0.4.312"} org.clojars.mmb90/cljs-cache {:mvn/version "0.1.4"}} :aliases {:cljs {:extra-deps {org.clojure/clojurescript {:mvn/version "1.11.60"} thheller/shadow-cljs {:mvn/version "2.25.9"} diff --git a/src/konserve/indexeddb.cljs b/src/konserve/indexeddb.cljs index 720cc77..87452ad 100644 --- a/src/konserve/indexeddb.cljs +++ b/src/konserve/indexeddb.cljs @@ -268,8 +268,8 @@ (defn read-web-stream "Accepts the bget locked callback arg and returns a promise-chan containing - a concatenated byte array with the first offset bytes dropped: - (k/bget store :key (read-web-stream m))" + a concatenated byte array with the first offset bytes dropped + `(k/bget store :key read-web-stream)`" [{:keys [input-stream offset]}] (let [reader (.getReader input-stream) chunks #js[]] @@ -277,9 +277,9 @@ (let [read-chunk (fn read-chunk [] (.then (.read reader) (fn [result] - (let [done (.-done result) - value (.-value result)] - (if done + (if (.-done result) + (do + (some->> (.-value result) (.push chunks)) (if (== 1 (alength chunks)) (put! out (.slice (aget chunks 0) offset)) (let [total-length (reduce + (map count chunks)) @@ -288,10 +288,10 @@ (doseq [chunk (array-seq chunks)] (.set final-array chunk @_i) (swap! _i + (alength chunk))) - (put! out (.slice final-array offset)))) - (do - (.push chunks value) - (read-chunk))))) + (put! out (.slice final-array offset))))) + (do + (.push chunks (.-value result)) + (read-chunk)))) (fn [err] (put! out err))))] (read-chunk))))) diff --git a/src/konserve/node_filestore.cljs b/src/konserve/node_filestore.cljs index 2e3ba05..2b8102c 100644 --- a/src/konserve/node_filestore.cljs +++ b/src/konserve/node_filestore.cljs @@ -214,13 +214,9 @@ (reset! _readable-fired? true) (let [ret (locked-cb {:input-stream rstream :size total-size})] (take! ret (fn [res] (put! out res))))))) - (.on rstream "close" - (fn [& args] - (println "CLOSE" args))) (.on rstream "error" (fn [err] - (put! out (ex-info "error reading from stream" {:cause err})))) - ) + (put! out (ex-info "error reading from stream" {:cause err}))))) (catch js/Error err (put! out (ex-info "error creating readstream" {:cause err}))))))))) @@ -367,6 +363,7 @@ (.delete f) (try (sync-base parent-base) + nil (catch js/Error e e)))) @@ -515,19 +512,24 @@ (sync-base-async base)))) (defn detect-old-file-schema [& _args] (throw (js/Error "TODO detect-old-file-schema"))) -;; get-file-channel -;; migration (defn connect-fs-store "Create Filestore in given path. - Optional serializer, read-handlers, write-handlers, buffer-size and config (for fsync) can be changed. - Defaults are - {:base path - :serializer fressian-serializer - :read-handlers empty - :write-handlers empty - :buffer-size 1 MB - :config config} " + Optional serializer, read-handlers, write-handlers, buffer-size and config (for fsync) can be changed. + + + the `k/bget` callback gets different args depending on `:sync?` + - async bget callbacks recieve `{:input-stream }` akin to + the same call on the JVM filestore impl. These streams are opened to the + same fd that konserve is managing for the blob, so users should not call + destroy() or it will raise an error + - sync bget callbacks are called with `{:blob }` + + {:base path + :serializer fressian-serializer + :read-handlers empty + :write-handlers empty + :buffer-size 1 MB + :config config} " [path & {:keys [detect-old-file-schema? ephemeral? config] :or {detect-old-file-schema? false ephemeral? (fn [pathstr] diff --git a/test/konserve/filestore_test.clj b/test/konserve/filestore_test.clj index 6e46ea4..1d71b9d 100644 --- a/test/konserve/filestore_test.clj +++ b/test/konserve/filestore_test.clj @@ -6,7 +6,8 @@ [konserve.compliance-test :refer [compliance-test]] [konserve.filestore :refer [connect-fs-store delete-store]] [konserve.tests.cache :as ct] - [konserve.tests.gc :as gct])) + [konserve.tests.gc :as gct] + [konserve.tests.serializers :as st])) (deftest filestore-compliance-test (let [folder "/tmp/konserve-fs-comp-test" @@ -119,3 +120,18 @@ (delete-store "/tmp/gc-store") (let [store (connect-fs-store "/tmp/gc-store" :opts {:sync? true})] (MyRecord {:a 0 :b 1}) + res (and + (is [nil 42] (MyRecord {:a 0 :b 1}) + res (and + (is (= [nil my-record] (MyRecord} + store (MyRecord {:a 0 :b 1})] + (and + (is (nil? ( Date: Fri, 10 Nov 2023 14:08:54 -0400 Subject: [PATCH 07/17] CLJS encryptor tests --- test/konserve/encryptor_test.cljc | 24 ---------------------- test/konserve/filestore_test.clj | 14 +++++++++++++ test/konserve/indexeddb_test.cljs | 12 +++++++++++ test/konserve/node_filestore_test.cljs | 13 ++++++++++++ test/konserve/tests/encryptor.cljc | 28 ++++++++++++++++++++++++++ 5 files changed, 67 insertions(+), 24 deletions(-) delete mode 100644 test/konserve/encryptor_test.cljc create mode 100644 test/konserve/tests/encryptor.cljc diff --git a/test/konserve/encryptor_test.cljc b/test/konserve/encryptor_test.cljc deleted file mode 100644 index 7b8db4d..0000000 --- a/test/konserve/encryptor_test.cljc +++ /dev/null @@ -1,24 +0,0 @@ -(ns konserve.encryptor-test - (:require [clojure.test :refer [deftest]] - #?(:cljs [clojure.core.async :refer [go Date: Fri, 10 Nov 2023 22:05:21 -0400 Subject: [PATCH 08/17] tweak in-mem impl to align with other stores --- src/konserve/memory.cljc | 24 +++++++------ test/konserve/memory_test.cljc | 53 ++++++++++++++++++++++++++++ test/konserve/tests/gc.cljc | 29 ++++++++------- test/konserve/tests/serializers.cljc | 13 +++---- 4 files changed, 87 insertions(+), 32 deletions(-) create mode 100644 test/konserve/memory_test.cljc diff --git a/src/konserve/memory.cljc b/src/konserve/memory.cljc index d31f577..f5e358f 100644 --- a/src/konserve/memory.cljc +++ b/src/konserve/memory.cljc @@ -1,5 +1,6 @@ (ns konserve.memory - "Address globally aggregated immutable key-value store(s)." + "Address globally aggregated immutable key-value store(s). + Does not support serialization." (:require [clojure.core.async :as async :refer [go MemoryStore]] + [konserve.tests.cache :as ct] + [konserve.tests.encryptor :as et] + [konserve.tests.gc :as gct] + [konserve.tests.serializers :as st])) + +(defn connect-mem-store + [init-atom & {:as params opts :opts}] + (let [store-config (merge {:state init-atom + :read-handlers (atom {}) + :write-handlers (atom {}) + :locks (atom {})} + (dissoc params :config)) + store (map->MemoryStore store-config)] + (if (:sync? opts) store (go store)))) + +#?(:clj + (deftest memory-store-compliance-test + (compliance-test ( this can be tested by abstract out timestamps where there are written -;; and then using redef in tests. CLJS impls missing entirely - -;it seems clock of threads are not sync. -;(prn (.getTime ts) (map (fn [e] [(:key e) (-> e :konserve.core/timestamp .getTime)]) ( Date: Sun, 12 Nov 2023 14:36:17 -0400 Subject: [PATCH 09/17] =?UTF-8?q?node=20:advanced=20=E2=9C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- shadow-cljs.edn | 15 +++++---- src/konserve/core.cljc | 7 ++-- src/konserve/node_filestore.cljs | 29 +++++++++++------ src/konserve/serializers.cljc | 12 ++++++- test/konserve/memory_test.cljc | 4 +-- test/konserve/tests/serializers.cljc | 48 ++++++++++++++++++++-------- 6 files changed, 79 insertions(+), 36 deletions(-) diff --git a/shadow-cljs.edn b/shadow-cljs.edn index c756e54..286dc5b 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -6,17 +6,20 @@ {:target :browser :output-dir "public/js" :modules {:main {:entries [konserve.core]}}} - :node-tests ; shadow-cljs compile node-tests && node out/node-tests.js + + :node-tests ; shadow-cljs release node-tests && node out/node-tests.js {:target :node-test :output-to "out/node-tests.js" :ns-regexp "^(?!konserve.indexeddb-test)" - :compiler-options {:infer-externs true} - :autorun true} + :compiler-options {:infer-externs true + :externs ["cljs_node_io/externs.js"] + :closure-warnings {:useless-code :off}}} + :browser-tests ; shadow-cljs watch :browser-tests ; http://localhost:8021/ - {:target :browser-test + {:target :browser-test :test-dir "out/browser-tests" :ns-regexp "^(?!konserve.node-filestore-test|konserve.node-filestore)" - :compiler-options {:infer-externs true} - :autorun true}}} + :compiler-options {:infer-externs true + :closure-warnings {:useless-code :off}}}}} diff --git a/src/konserve/core.cljc b/src/konserve/core.cljc index 069fb3f..7be4c23 100644 --- a/src/konserve/core.cljc +++ b/src/konserve/core.cljc @@ -1,6 +1,6 @@ (ns konserve.core (:refer-clojure :exclude [get get-in update update-in assoc assoc-in exists? dissoc keys]) - (:require [clojure.core.async :refer [chan put! #?(:clj poll!)]] + (:require [clojure.core.async :refer [chan put! poll!]] [hasch.core :as hasch] [konserve.protocols :refer [-exists? -get-meta -get-in -assoc-in -update-in -dissoc -bget -bassoc @@ -28,10 +28,11 @@ (clojure.core/assoc old key c)))) key)))) -(defn wait [#?(:clj lock :cljs _)] +(defn wait [lock] #?(:clj (while (not (poll! lock)) (Thread/sleep (long (rand-int 20)))) - :cljs (debug "WARNING: konserve lock is not active. Only use the synchronous variant with the memory store in JavaScript."))) + :cljs (when (nil? (cljs.core.async/poll! lock)) + (debug "WARNING: konserve lock is not active. Only use the synchronous variant with the memory store in JavaScript.")))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} (defmacro locked [store key & code] diff --git a/src/konserve/node_filestore.cljs b/src/konserve/node_filestore.cljs index 2b8102c..7c80c0e 100644 --- a/src/konserve/node_filestore.cljs +++ b/src/konserve/node_filestore.cljs @@ -40,6 +40,16 @@ (set! (.-locked this) lock) lock)) +(defn verify-read-size + [expected actual] + (when-not (== expected actual) + (throw (js/Error. (str "Expected " expected " bytes, but read " actual))))) + +(defn verify-write-size + [expected actual] + (when-not (== expected actual) + (throw (js/Error. (str "Expected to write " expected " bytes, but wrote " actual))))) + (deftype FileChannel [path fd open? locked] storage-layout/PBackingBlob (-sync [this _env] (.force this true)) @@ -48,12 +58,12 @@ (-read-header [_this _env] (let [buf (js/Buffer.alloc storage-layout/header-size) bytes-read (iofs/read fd buf {:position 0})] - (assert (== bytes-read storage-layout/header-size)) + (verify-read-size storage-layout/header-size bytes-read) buf)) (-read-meta [_this meta-size _env] (let [buf (js/Buffer.alloc meta-size) bytes-read (iofs/read fd buf {:position storage-layout/header-size})] - (assert (== bytes-read meta-size)) + (verify-read-size meta-size bytes-read) buf)) (-read-value [_this meta-size _env] (let [blob-size ^number (.-size (fs.fstatSync fd)) @@ -61,34 +71,35 @@ buf (js/Buffer.alloc value-size) pos (+ meta-size storage-layout/header-size) bytes-read (iofs/read fd buf {:position pos :length (alength buf)})] - (assert (== bytes-read value-size)) + (verify-read-size value-size bytes-read) buf)) (-read-binary [this meta-size locked-cb _env] (let [blob-size ^number (.-size (fs.fstatSync fd)) offset (+ meta-size storage-layout/header-size) value-len (- blob-size offset) - blob (js/Buffer. value-len)] - (iofs/read (.-fd this) blob offset value-len 0) + blob (js/Buffer. value-len) + bytes-read (iofs/read (.-fd this) blob offset value-len 0)] + (verify-read-size value-len bytes-read) (locked-cb {:blob blob}))) (-write-header [_this header _env] (let [buffer (js/Buffer.from header) bytes-written (iofs/write fd buffer {:position 0})] - (assert (== bytes-written (alength header))))) + (verify-write-size (alength header) bytes-written))) (-write-meta [_this meta-arr _env] (let [buffer (js/Buffer.from meta-arr) pos storage-layout/header-size bytes-written (iofs/write fd buffer {:position pos})] - (assert (== bytes-written (alength buffer))))) + (verify-write-size (alength buffer) bytes-written))) (-write-value [_this value-arr meta-size _env] (let [buffer (js/Buffer.from value-arr) pos (+ storage-layout/header-size meta-size) bytes-written (iofs/write fd buffer {:position pos})] - (assert (== bytes-written (alength buffer))))) + (verify-write-size (alength buffer) bytes-written))) (-write-binary [_this meta-size blob _env] (let [buffer (js/Buffer.from blob) pos (+ storage-layout/header-size meta-size) bytes-written (iofs/write fd buffer {:position pos})] - (assert (== bytes-written (alength buffer))))) + (verify-write-size (alength buffer) bytes-written))) Object (force [_this _] (fs.fsyncSync fd)) (close [this] diff --git a/src/konserve/serializers.cljc b/src/konserve/serializers.cljc index 0ba22cd..495efba 100644 --- a/src/konserve/serializers.cljc +++ b/src/konserve/serializers.cljc @@ -32,7 +32,17 @@ (-namespace [_] "konserve.serializers"))) PStoreSerializer (-deserialize [_ read-handlers bytes] - (let [handlers #?(:cljs (merge custom-read-handlers (incognito-read-handlers read-handlers)) + (let [[irecord-handler + incognito-handlers] ((juxt #(get % "irecord") + #(dissoc % "irecord")) + (incognito-read-handlers read-handlers)) + handlers #?(:cljs (merge (assoc custom-read-handlers + "irecord" (fn [rdr tag] + (let [{:keys [tag value] :as itl} (irecord-handler rdr tag)] + (if-let [user-handler (get @read-handlers tag)] + (user-handler value) + itl)))) + incognito-handlers) :clj (-> (merge fress/clojure-read-handlers custom-read-handlers (incognito-read-handlers read-handlers)) diff --git a/test/konserve/memory_test.cljc b/test/konserve/memory_test.cljc index ae7752b..7e5559f 100644 --- a/test/konserve/memory_test.cljc +++ b/test/konserve/memory_test.cljc @@ -5,9 +5,7 @@ async-compliance-test]] [konserve.memory :refer [new-mem-store map->MemoryStore]] [konserve.tests.cache :as ct] - [konserve.tests.encryptor :as et] - [konserve.tests.gc :as gct] - [konserve.tests.serializers :as st])) + [konserve.tests.gc :as gct])) (defn connect-mem-store [init-atom & {:as params opts :opts}] diff --git a/test/konserve/tests/serializers.cljc b/test/konserve/tests/serializers.cljc index 3f3ce4b..e06ba2f 100644 --- a/test/konserve/tests/serializers.cljc +++ b/test/konserve/tests/serializers.cljc @@ -95,6 +95,27 @@ (fress/write-object writer (.-a o)) (fress/write-object writer (.-b o))))}}) +(defn- test-fressian-incognito-record-recovery + [store-name connect-store delete-store-async locked-cb] + (go + (testing ":read-handlers arg to connect-store let's us recover records" + (let [read-handlers {'konserve.tests.serializers.MyRecord map->MyRecord} + _(MyRecord {:a 0 :b 1})] + (and + (is (nil? (MyRecord {:a 0 :b 1}) res (and (is (= [nil my-record] (MyRecord} - store (MyRecord {:a 0 :b 1})] - (and - (is (nil? ( Date: Sun, 12 Nov 2023 19:22:52 -0400 Subject: [PATCH 10/17] karma for browser tests, update bin scripts for CI --- bin/install | 9 +++++++++ bin/kaocha | 3 --- bin/run-all | 4 ++-- bin/run-cljs-tests | 17 +++++++++++++++++ bin/run-cljstests | 19 ------------------- bin/run-jvm-tests | 5 +++++ bin/run-unittests | 5 ----- deps.edn | 25 +++++++++---------------- karma.conf.js | 15 +++++++++++++++ package.json | 13 +++++++++++++ shadow-cljs.edn | 9 ++++++++- tests.edn | 13 ------------- 12 files changed, 78 insertions(+), 59 deletions(-) create mode 100644 bin/install delete mode 100755 bin/kaocha create mode 100755 bin/run-cljs-tests delete mode 100755 bin/run-cljstests create mode 100755 bin/run-jvm-tests delete mode 100755 bin/run-unittests create mode 100644 karma.conf.js create mode 100644 package.json delete mode 100644 tests.edn diff --git a/bin/install b/bin/install new file mode 100644 index 0000000..d015ba2 --- /dev/null +++ b/bin/install @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +npm install +npm install karma +npm install karma-cljs-test +npm install karma-chrome-launcher diff --git a/bin/kaocha b/bin/kaocha deleted file mode 100755 index 30ff6d7..0000000 --- a/bin/kaocha +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash - -clojure -M:test -m kaocha.runner "$@" diff --git a/bin/run-all b/bin/run-all index 4c549c8..4c8d363 100755 --- a/bin/run-all +++ b/bin/run-all @@ -3,9 +3,9 @@ set -o errexit set -o pipefail -./bin/run-cljstests +./bin/run-jvm-tests echo echo -./bin/run-unittests +./bin/run-cljs-tests diff --git a/bin/run-cljs-tests b/bin/run-cljs-tests new file mode 100755 index 0000000..b6ca8a6 --- /dev/null +++ b/bin/run-cljs-tests @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail + +echo "Running tests for node" +rm out/node-tests.js +shadow-cljs release node-tests +node out/node-tests.js + +echo +echo + +echo "Running tests for browser" +rm target/*.js target/*.map +shadow-cljs release ci +karma start --single-run diff --git a/bin/run-cljstests b/bin/run-cljstests deleted file mode 100755 index dd3cc6e..0000000 --- a/bin/run-cljstests +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit -set -o pipefail - -echo "Running tests for node" -# olical version: -clojure -M:run-cljs-tests -# kaocha version: -#[ -d "node_modules/ws" ] || npm install ws -#clojure -M:test:cljs -m kaocha.runner unit-node - -echo -echo - -echo "Running tests for browser" -# kaocha version: -clojure -M:test:cljs -m kaocha.runner unit-browser -# olical version requires karma diff --git a/bin/run-jvm-tests b/bin/run-jvm-tests new file mode 100755 index 0000000..790af09 --- /dev/null +++ b/bin/run-jvm-tests @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +echo "Running JVM tests" + +TIMBRE_LEVEL=':warn' clojure -X:test diff --git a/bin/run-unittests b/bin/run-unittests deleted file mode 100755 index 7978b84..0000000 --- a/bin/run-unittests +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -echo "Running unittests" - -TIMBRE_LEVEL=':warn' clojure -M:test -m kaocha.runner unit diff --git a/deps.edn b/deps.edn index 32f27fc..d344d32 100644 --- a/deps.edn +++ b/deps.edn @@ -12,29 +12,22 @@ com.github.pkpkpk/cljs-node-io {:mvn/version "2.0.339"} com.github.pkpkpk/fress {:mvn/version "0.4.312"} org.clojars.mmb90/cljs-cache {:mvn/version "0.1.4"}} - :aliases {:cljs {:extra-deps {org.clojure/clojurescript {:mvn/version "1.11.60"} - thheller/shadow-cljs {:mvn/version "2.25.9"} - binaryage/devtools {:mvn/version "1.0.6"}} - :extra-paths ["test"]} - :dev {:extra-deps {criterium/criterium {:mvn/version "0.4.6"} + :aliases {:dev {:extra-deps {criterium/criterium {:mvn/version "0.4.6"} metasoarous/oz {:mvn/version "2.0.0-alpha5"} org.clojure/tools.cli {:mvn/version "1.0.214"}} :extra-paths ["benchmark/src" "test"]} + :cljs {:extra-deps {org.clojure/clojurescript {:mvn/version "1.11.60"} + thheller/shadow-cljs {:mvn/version "2.26.0"}} + :extra-paths ["test"]} :benchmark {:extra-deps {metasoarous/oz {:mvn/version "2.0.0-alpha5"} org.clojure/tools.cli {:mvn/version "1.0.214"}} :extra-paths ["benchmark/src"] :main-opts ["-m" "benchmark.core"]} - :test {:extra-deps {lambdaisland/kaocha {:mvn/version "1.80.1274"} - lambdaisland/kaocha-cljs {:mvn/version "1.4.130"} - org.clojure/test.check {:mvn/version "1.1.1"}} - :extra-paths ["test"] - :main-opts ["-e" "(set! *warn-on-reflection* true)"]} - :run-cljs-tests {:extra-deps {olical/cljs-test-runner {:mvn/version "3.8.0"}} - :extra-paths ["test"] - :main-opts ["-m" "cljs-test-runner.main" - "-o" "target/cljs" - "--exclude" "browser" - "--env" "node"]} + :test {:extra-paths ["test"] + :extra-deps {io.github.cognitect-labs/test-runner + {:git/tag "v0.5.1" :git/sha "dfb30dd"}} + :main-opts ["-m" "cognitect.test-runner"] + :exec-fn cognitect.test-runner.api/test} :build {:deps {io.github.clojure/tools.build {:mvn/version "0.9.3"} slipset/deps-deploy {:mvn/version "0.2.0"} io.github.borkdude/gh-release-artifact {:git/sha "b946558225a7839f6a0f644834e838e190dc2262"} diff --git a/karma.conf.js b/karma.conf.js new file mode 100644 index 0000000..6314a8b --- /dev/null +++ b/karma.conf.js @@ -0,0 +1,15 @@ +module.exports = function (config) { + config.set({ + browsers: ['ChromeHeadless'], + basePath: 'target', + files: ['ci.js'], + frameworks: ['cljs-test'], + plugins: ['karma-cljs-test', 'karma-chrome-launcher'], + colors: true, + logLevel: config.LOG_INFO, + client: { + args: ["shadow.test.karma.init"], + singleRun: true + } + }) +}; \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..3c35ba0 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "CITests", + "version": "1.0.0", + "description": "Testing", + "devDependencies": { + "karma": "^6.4.2", + "karma-chrome-launcher": "^2.2.0", + "karma-cljs-test": "^0.1.0", + "shadow-cljs": "^2.26.0" + }, + "author": "", + "license": "MIT" +} diff --git a/shadow-cljs.edn b/shadow-cljs.edn index 286dc5b..8ae3849 100644 --- a/shadow-cljs.edn +++ b/shadow-cljs.edn @@ -1,5 +1,4 @@ {:deps {:aliases [:cljs]} - :source-paths ["src"] :dev-http {8021 "out/browser-tests"} :builds {:app @@ -22,4 +21,12 @@ :test-dir "out/browser-tests" :ns-regexp "^(?!konserve.node-filestore-test|konserve.node-filestore)" :compiler-options {:infer-externs true + :closure-warnings {:useless-code :off}}} + + :ci + {:target :karma + :output-to "target/ci.js" + :ns-regexp "^(?!konserve.node-filestore-test|konserve.node-filestore)" + :compiler-options {:infer-externs true + :compiler-options {:optimizations :advanced} :closure-warnings {:useless-code :off}}}}} diff --git a/tests.edn b/tests.edn deleted file mode 100644 index c4d5de6..0000000 --- a/tests.edn +++ /dev/null @@ -1,13 +0,0 @@ -#kaocha/v1 {:tests [{:id :unit} - #_{:id :unit-node ;; Throws known kaocha error - :type :kaocha.type/cljs - :kaocha.filter/skip-meta [:browser] - :cljs/repl-env cljs.repl.node/repl-env - :timeout 30000} - {:id :unit-browser - :type :kaocha.type/cljs - :kaocha.filter/focus-meta [:browser] - :cljs/repl-env cljs.repl.browser/repl-env - :timeout 30000}] - :bindings {kaocha.type.cljs/*debug* true} - :reporter kaocha.report/documentation} From 467b50a57940a57aa7ae1db64ae980f30f1c4466 Mon Sep 17 00:00:00 2001 From: pat killean Date: Sun, 12 Nov 2023 19:37:06 -0400 Subject: [PATCH 11/17] kondo pass --- deps.edn | 2 +- src/konserve/core.cljc | 2 +- src/konserve/memory.cljc | 4 ++-- src/konserve/serializers.cljc | 12 +----------- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/deps.edn b/deps.edn index d344d32..9890f3b 100644 --- a/deps.edn +++ b/deps.edn @@ -39,7 +39,7 @@ :main-opts ["-m" "cljfmt.main" "check"]} :ffix {:extra-deps {cljfmt/cljfmt {:mvn/version "0.9.2"}} :main-opts ["-m" "cljfmt.main" "fix"]} - :lint {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.02.17"}} + :lint {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.10.20"}} :main-opts ["-m" "clj-kondo.main" "--lint" "src"]} :outdated {:extra-deps {com.github.liquidz/antq {:mvn/version "2.2.983"}} :main-opts ["-m" "antq.core"]}}} diff --git a/src/konserve/core.cljc b/src/konserve/core.cljc index 7be4c23..83a032f 100644 --- a/src/konserve/core.cljc +++ b/src/konserve/core.cljc @@ -31,7 +31,7 @@ (defn wait [lock] #?(:clj (while (not (poll! lock)) (Thread/sleep (long (rand-int 20)))) - :cljs (when (nil? (cljs.core.async/poll! lock)) + :cljs (when (nil? (poll! lock)) (debug "WARNING: konserve lock is not active. Only use the synchronous variant with the memory store in JavaScript.")))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} diff --git a/src/konserve/memory.cljc b/src/konserve/memory.cljc index f5e358f..0cf1383 100644 --- a/src/konserve/memory.cljc +++ b/src/konserve/memory.cljc @@ -43,8 +43,8 @@ (if rkey (update-in data rkey up-fn) (up-fn data))]))))) - [old-meta old-val] (get @state fkey) - {[new-meta new-val] fkey} (update-atom state)] + [_ old-val] (get @state fkey) + {[_ new-val] fkey} (update-atom state)] (if overwrite? [nil new-val] [old-val new-val])))))) diff --git a/src/konserve/serializers.cljc b/src/konserve/serializers.cljc index 495efba..0ba22cd 100644 --- a/src/konserve/serializers.cljc +++ b/src/konserve/serializers.cljc @@ -32,17 +32,7 @@ (-namespace [_] "konserve.serializers"))) PStoreSerializer (-deserialize [_ read-handlers bytes] - (let [[irecord-handler - incognito-handlers] ((juxt #(get % "irecord") - #(dissoc % "irecord")) - (incognito-read-handlers read-handlers)) - handlers #?(:cljs (merge (assoc custom-read-handlers - "irecord" (fn [rdr tag] - (let [{:keys [tag value] :as itl} (irecord-handler rdr tag)] - (if-let [user-handler (get @read-handlers tag)] - (user-handler value) - itl)))) - incognito-handlers) + (let [handlers #?(:cljs (merge custom-read-handlers (incognito-read-handlers read-handlers)) :clj (-> (merge fress/clojure-read-handlers custom-read-handlers (incognito-read-handlers read-handlers)) From 52361edd52c7d8163ac22e2d9e31395bf9abe188 Mon Sep 17 00:00:00 2001 From: pat killean Date: Sun, 12 Nov 2023 19:47:15 -0400 Subject: [PATCH 12/17] cljfmt pass --- src/konserve/core.cljc | 2 +- src/konserve/indexeddb.cljs | 8 +- src/konserve/node_filestore.cljs | 42 +++--- src/konserve/utils.cljc | 32 ++--- test/konserve/indexeddb_test.cljs | 72 +++++------ test/konserve/node_filestore_test.cljs | 56 ++++---- test/konserve/tests/cache.cljc | 74 +++++------ test/konserve/tests/encryptor.cljc | 12 +- test/konserve/tests/gc.cljc | 40 +++--- test/konserve/tests/serializers.cljc | 170 ++++++++++++------------- 10 files changed, 254 insertions(+), 254 deletions(-) diff --git a/src/konserve/core.cljc b/src/konserve/core.cljc index 83a032f..c1bd5c2 100644 --- a/src/konserve/core.cljc +++ b/src/konserve/core.cljc @@ -6,7 +6,7 @@ -update-in -dissoc -bget -bassoc -keys]] [konserve.utils :refer [meta-update #?(:clj async+sync) *default-sync-translation*] - #?@(:cljs [:refer-macros [async+sync]])] + #?@(:cljs [:refer-macros [async+sync]])] [superv.async :refer [go-try- ch [fd meta-size blob _env] diff --git a/src/konserve/utils.cljc b/src/konserve/utils.cljc index d11bff8..246928d 100644 --- a/src/konserve/utils.cljc +++ b/src/konserve/utils.cljc @@ -22,22 +22,22 @@ (clojure.core/assoc old :last-write (now)))) (defmacro async+sync - [sync? async->sync async-code] - (let [async->sync (if (symbol? async->sync) - (or (resolve async->sync) - (when-let [_ns (or (get-in &env [:ns :use-macros async->sync]) - (get-in &env [:ns :uses async->sync]))] - (resolve (symbol (str _ns) (str async->sync))))) - async->sync)] - (assert (some? async->sync)) - `(if ~sync? - ~(clojure.walk/postwalk (fn [n] - (if-not (meta n) - (async->sync n n) ;; primitives have no metadata - (with-meta (async->sync n n) - (update (meta n) :tag (fn [t] (async->sync t t)))))) - async-code) - ~async-code))) + [sync? async->sync async-code] + (let [async->sync (if (symbol? async->sync) + (or (resolve async->sync) + (when-let [_ns (or (get-in &env [:ns :use-macros async->sync]) + (get-in &env [:ns :uses async->sync]))] + (resolve (symbol (str _ns) (str async->sync))))) + async->sync)] + (assert (some? async->sync)) + `(if ~sync? + ~(clojure.walk/postwalk (fn [n] + (if-not (meta n) + (async->sync n n) ;; primitives have no metadata + (with-meta (async->sync n n) + (update (meta n) :tag (fn [t] (async->sync t t)))))) + async-code) + ~async-code))) (def ^:dynamic *default-sync-translation* '{go-try try diff --git a/test/konserve/indexeddb_test.cljs b/test/konserve/indexeddb_test.cljs index 4117f97..967e96d 100644 --- a/test/konserve/indexeddb_test.cljs +++ b/test/konserve/indexeddb_test.cljs @@ -98,33 +98,33 @@ (deftest cache-PEDNKeyValueStore-test (async done - (go - (MyRecord} - _(MyRecord {:a 0 :b 1})] - (and - (is (nil? (MyRecord} + _ (MyRecord {:a 0 :b 1})] + (and + (is (nil? (MyRecord {:a 0 :b 1}) - res (and - (is [nil 42] (MyRecord {:a 0 :b 1}) + res (and + (is [nil 42] (MyRecord {:a 0 :b 1}) - res (and - (is (= [nil my-record] (MyRecord {:a 0 :b 1}) + res (and + (is (= [nil my-record] ( Date: Mon, 13 Nov 2023 22:09:46 -0400 Subject: [PATCH 13/17] doc --- README.org | 9 +- doc/api-walkthrough.md | 179 ++++++++++++++++++++++++++++++++++++ src/konserve/core.cljc | 2 +- src/konserve/indexeddb.cljs | 1 - 4 files changed, 184 insertions(+), 7 deletions(-) create mode 100644 doc/api-walkthrough.md diff --git a/README.org b/README.org index 7d85236..d06866b 100644 --- a/README.org +++ b/README.org @@ -217,12 +217,11 @@ Usage: #+BEGIN_SRC clojure (ns test-db - (:require [konserve.indexeddb :refer [connect-idb-store]] - [konserve.core :as k]) - (:require-macros [cljs.core.async.macros :refer [go]])) + (:require [clojure.core.async :refer [go]] + [konserve.indexeddb :refer [connect-idb-store]] + [konserve.core :as k])) - (def dbname "example-db") - (go (def my-db ( + +## Connecting Stores + +```clojure +(require '[konserve.filestore :refer [connect-fs-store]]) + +(def store ( `{:sync? }` + - This is an env map passed around by most functions internally within konserve. The only entry you should typically need to concern yourself is `:sync?` which is used to control whether functions return channels or values + - an opts map is the last parameter accepted by `konserve.core` functions, but for creating stores, it must be identified by the keyword `:opts` ++ `:config` => map + - this map includes options for manipulating blobs in store specific ways. Very rarely should you ever need to alter the defaults ++ `:buffer-size` => number + - in clj this lets you control the chunk size used for writing blobs. the default is 1mb ++ `:default-serializer` => keyword + - the default serializer is `:FressianSerializer`, but you can override + - `(connect-store store-name :default-serializer :StringSerializer)` => writes string edn + - jvm also supports `:CBORSerializer` + - you can provide your own serializer byte giving a map of `{:MySerializer PStoreSerializerImpl}` to `:serializers` (..see next bullet) and then referencing it via `:default-serializer :MySerializer` + ++ `:serializers` => Map + - this is where you can provide your own serializer to reference via `:default-serializer` + - `konserve.serializers/fressian-serializer` is a convenience function that accepts 2 maps: a map of read-handlers and a map of write-handlers and returns a fressian serializer supporting your custom types + - in clj the handlers are reified `org.fressian.ReadHandlers` & `org.fressian.WriteHandlers` + - see [https://github.com/clojure/data.fressian/wiki/Creating-custom-handlers](https://github.com/clojure/data.fressian/wiki/Creating-custom-handlers) + - in cljs handlers are just functions + - see [https://github.com/pkpkpk/fress](https://github.om/pkpkpk/fress) + ++ `:encryptor` => `{:type :aes :key "s3cr3t"}` + - currently only supports `:aes` in default stores + ++ `:compressor` => `{:type :lz4}` + - currently LZ4 compression is only supported on the jvm + +### Incognito & Records +Konserve intercepts records and writes them as [incognito](https://github.com/replikativ/incognito) tagged literals such that the details of serialization formats are abstracted away and allowing easier interop between different formats. The `:read-handlers` and `:write-handlers` varg args are explicitly meant for working with incognito's tagged literals. + - `:read-handlers` expects an atom wrapping `{'symbol.for.MyRecord map->MyRecord}` for recovering records from incognito tagged literals + - `:write-handlers` expects an atom wrapping `{'symbol.for.MyRecord (fn [record] ..)}` for writing records as incognito tagged literals + - the symbols used in these maps **are not safe for clojurescript** so you should avoid using them + +
+ + +## Working with data +Once you have a store you can use it using `konserve.core` functions. By default functions are asynchronous and return channels yielding values or errors. You can override this by passing an opts map with `:sync? true` + +```clojure +(require '[konserve.core :as k]) + +(k/exists? store :false) ;=> channel + +( true +``` + +You can `get` `get-in` `update` `update-in` and `dissoc` just like a clojure map. + +```clojure +(k/assoc-in store [:fruits :parcha] {:color "yellow" :taste "sour" :quantity 0}) + +(k/update-in store [:fruits :parcha :quantity] inc) + +(k/get-in store [:fruits :parcha :quantity]) ;=> channel<1> + +(k/dissoc store :fruits) +``` + +In the fruits example a simple keyword is the store key, but keys themselves can be arbitrary edn: + +```clojure +(defn memoize-to-store-sync [f] + (fn [& args] + (if-let [result ( channel +``` + +## Working with binary data + +```clojure +(k/bassoc store :blob blob) + +(k/bget store :blob + (fn locked-cb [{is :input-stream}] + (go (input-stream->byte-buffer is)))) ;=> ch +``` +With `bassoc` binary data is written as-is without passing through serialization/encryption/compression. + +`bget` is probably the trickiest function in konserve. It accepts a callback function that is passed a map of `{:input-stream }`. While `locked-cb` is running, konserve locks & holds onto underyling resources (ie file descriptors) until the function exits. You can choose to read from the input stream however you choose, but you need to return your desired value else the lock will remain held. + ++ when called async, the `locked-cb` should return a channel yielding the desired value that will be read from and yielded by `bget`'s channel ++ in both clojurescript stores, synchronous input streams are not possible. + - On nodejs you can called `bget` synchronously but in this instance it will be called with `{:blob js/Buffer}` + - In the browser, `bget` the (async) locked-cb is given `{:input-stream :offset }` where offset indicates the amount of bytes to drop before reaching the desired blob start. `konserve.indexeddb/read-web-stream` can serve as a locked-cb that will yield a `Uint8Array`. + +## Metadata + +Konserve does some bookkeeping for values by storing them with metadata + +```clojure +(k/get-meta store :key {:sync? true}) +;=> +; {:key :key +; :type +; :last-write } +``` + +## The append log +Konserve provides an append log for writing values quickly. These entries are a special case managed by konserve, where the sequence is stored as a linked list of blobs where each blob is a cons cell of the list. You can name the log with any key, but that key should only be written to or read from using the `append`, `log`, and `reduce-log` functions. + +```clojure +(dotimes [n 6] + ( channel<(0 1 2 3 4 5)> + +(k/reduce-log store :log + (fn [acc n] + (if (even? n) + (conj acc n) + acc)) + []) ;=> channel<[0 2 4]> +``` + +## konserve.gc + +`konserve.gc/sweep!` lets you prune the store based on a whitelist set of keys to keep and a timestamp cutoff before which un whitelisted entries should be deleted + +## konserve.cache + +`konserve.cache/ensure-cache` wraps a store with a lru-cache to avoid hitting external memory for frequently accessed keys diff --git a/src/konserve/core.cljc b/src/konserve/core.cljc index c1bd5c2..49c1672 100644 --- a/src/konserve/core.cljc +++ b/src/konserve/core.cljc @@ -31,7 +31,7 @@ (defn wait [lock] #?(:clj (while (not (poll! lock)) (Thread/sleep (long (rand-int 20)))) - :cljs (when (nil? (poll! lock)) + :cljs (when-not (some-> lock poll!) (debug "WARNING: konserve lock is not active. Only use the synchronous variant with the memory store in JavaScript.")))) #_{:clj-kondo/ignore [:clojure-lsp/unused-public-var]} diff --git a/src/konserve/indexeddb.cljs b/src/konserve/indexeddb.cljs index c968df1..1f47682 100644 --- a/src/konserve/indexeddb.cljs +++ b/src/konserve/indexeddb.cljs @@ -297,7 +297,6 @@ (defn connect-idb-store "Connect to a IndexedDB backed KV store with the given db name. - Optional serializer, read-handlers, write-handlers. This implementation stores all values as js/Blobs in an IndexedDB object store instance. The object store itself is nameless, and there From c90ac63f38e9cd3d1e0444ea9be1820d3313eeb3 Mon Sep 17 00:00:00 2001 From: pat killean Date: Mon, 13 Nov 2023 23:34:20 -0400 Subject: [PATCH 14/17] chmod +x bin/install --- bin/install | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 bin/install diff --git a/bin/install b/bin/install old mode 100644 new mode 100755 From 9f2ae16a6bd4b24f19f06dff099b1049d675737b Mon Sep 17 00:00:00 2001 From: pkpkpk Date: Tue, 14 Nov 2023 00:39:53 -0400 Subject: [PATCH 15/17] Update api-walkthrough.md --- doc/api-walkthrough.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api-walkthrough.md b/doc/api-walkthrough.md index 9d98e9f..c8cd809 100644 --- a/doc/api-walkthrough.md +++ b/doc/api-walkthrough.md @@ -83,7 +83,7 @@ Konserve intercepts records and writes them as [incognito](https://github.com/re ## Working with data -Once you have a store you can use it using `konserve.core` functions. By default functions are asynchronous and return channels yielding values or errors. You can override this by passing an opts map with `:sync? true` +Once you have a store you can access it using `konserve.core` functions. By default functions are asynchronous and return channels yielding values or errors. You can override this by passing an opts map with `:sync? true` ```clojure (require '[konserve.core :as k]) From ff7dd719af196fea0eb1b300ea9c1c6968474643 Mon Sep 17 00:00:00 2001 From: pkpkpk Date: Tue, 14 Nov 2023 11:30:02 -0400 Subject: [PATCH 16/17] Update api-walkthrough.md --- doc/api-walkthrough.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/api-walkthrough.md b/doc/api-walkthrough.md index c8cd809..1919533 100644 --- a/doc/api-walkthrough.md +++ b/doc/api-walkthrough.md @@ -47,17 +47,17 @@ Stores also accept var-args. Support for each entry varies per implementation + `:opts` => `{:sync? }` - - This is an env map passed around by most functions internally within konserve. The only entry you should typically need to concern yourself is `:sync?` which is used to control whether functions return channels or values + - This is an env map passed around by most functions internally within konserve. The only entry you should typically need to concern yourself with is `:sync?` which is used to control whether functions return channels or values - an opts map is the last parameter accepted by `konserve.core` functions, but for creating stores, it must be identified by the keyword `:opts` + `:config` => map - this map includes options for manipulating blobs in store specific ways. Very rarely should you ever need to alter the defaults + `:buffer-size` => number - in clj this lets you control the chunk size used for writing blobs. the default is 1mb + `:default-serializer` => keyword - - the default serializer is `:FressianSerializer`, but you can override + - the default serializer is `:FressianSerializer`, but you can override with a keyword identifying a different serializer implementation - `(connect-store store-name :default-serializer :StringSerializer)` => writes string edn - jvm also supports `:CBORSerializer` - - you can provide your own serializer byte giving a map of `{:MySerializer PStoreSerializerImpl}` to `:serializers` (..see next bullet) and then referencing it via `:default-serializer :MySerializer` + - you can provide your own serializer by giving a map of `{:MySerializer PStoreSerializerImpl}` to `:serializers` (..see next bullet) and then referencing it via `:default-serializer :MySerializer` + `:serializers` => Map - this is where you can provide your own serializer to reference via `:default-serializer` @@ -134,12 +134,13 @@ In the fruits example a simple keyword is the store key, but keys themselves can ``` With `bassoc` binary data is written as-is without passing through serialization/encryption/compression. -`bget` is probably the trickiest function in konserve. It accepts a callback function that is passed a map of `{:input-stream }`. While `locked-cb` is running, konserve locks & holds onto underyling resources (ie file descriptors) until the function exits. You can choose to read from the input stream however you choose, but you need to return your desired value else the lock will remain held. +`bget` is probably the trickiest function in konserve. It accepts a callback function that is passed a map of `{:input-stream }`. While `locked-cb` is running, konserve locks & holds onto underyling resources (ie file descriptors) until the function exits. You can choose to read from the input stream however you like, but rather than running side-effects within the callback, you should instead return your desired value else the lock will remain held. + when called async, the `locked-cb` should return a channel yielding the desired value that will be read from and yielded by `bget`'s channel + in both clojurescript stores, synchronous input streams are not possible. - - On nodejs you can called `bget` synchronously but in this instance it will be called with `{:blob js/Buffer}` - - In the browser, `bget` the (async) locked-cb is given `{:input-stream :offset }` where offset indicates the amount of bytes to drop before reaching the desired blob start. `konserve.indexeddb/read-web-stream` can serve as a locked-cb that will yield a `Uint8Array`. ++ On nodejs you can called `bget` synchronously but in this instance it will be called with `{:blob js/Buffer}` ++ In the browser with indexedDB, the async only `bget` calls its locked-cb with `{:input-stream :offset }` where offset indicates the amount of bytes to drop before reaching the desired blob start. + - `konserve.indexeddb/read-web-stream` can serve as a locked-cb that will yield a `Uint8Array`. ## Metadata From 318952709c8dc5419d8535a2ca44a01386f83469 Mon Sep 17 00:00:00 2001 From: pkpkpk Date: Wed, 15 Nov 2023 13:17:32 -0400 Subject: [PATCH 17/17] Update api-walkthrough.md --- doc/api-walkthrough.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api-walkthrough.md b/doc/api-walkthrough.md index 1919533..c123465 100644 --- a/doc/api-walkthrough.md +++ b/doc/api-walkthrough.md @@ -138,7 +138,7 @@ With `bassoc` binary data is written as-is without passing through serialization + when called async, the `locked-cb` should return a channel yielding the desired value that will be read from and yielded by `bget`'s channel + in both clojurescript stores, synchronous input streams are not possible. -+ On nodejs you can called `bget` synchronously but in this instance it will be called with `{:blob js/Buffer}` ++ On nodejs you can call `bget` synchronously but the locked-cb will be called with `{:blob js/Buffer}` + In the browser with indexedDB, the async only `bget` calls its locked-cb with `{:input-stream :offset }` where offset indicates the amount of bytes to drop before reaching the desired blob start. - `konserve.indexeddb/read-web-stream` can serve as a locked-cb that will yield a `Uint8Array`.