Enh: withPersistentMap writes only changed items

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.4.0 (2022-12-13)
+
+- **BREAKING**: `withPersistentMap()` now requires the given Driver to be
+  `StoreDriver` rather then `StoreDriverMapped` as before. This is how it should
+  to work, but it wasn't before.
+- Fix: `withPersistentMap()` now tracks changed elements in underlying `Map` and
+  writes to Driver only what was changed. Before this a whole `Map` was sent to
+  overwrite on every update.
+
 ## 0.3.0 (2022-12-12)
 
 - **BREAKING**: Bump `effector` in `peerDependencies` to `~20.4.0 || >=21` for

README.md

@@ -68,12 +68,15 @@ inline.
 ```ts
 function withPersistentMap<Key, Value, Serialized = Value>(
   store:    Store<ReadonlyMap<Key, Value>>,
-  driver:   StoreDriverMapped<Key, Serialized>
-          | Promise<StoreDriverMapped<Key, Serialized>>,
+  driver:   StoreDriver<Key, Serialized>
+          | Promise<StoreDriver<Key, Serialized>>,
   options?: WithPersistentOptions<ReadonlyMap<Key, Value>, Value, Serialized>
 ): typeof store
 ```
 
+Under the hood on every `store` change it will detect changes in underlying
+`Map` entries and will send to `driver` only those was changed.
+
 **Notice:** Serialization when used with `options` will be applied to individual
 values `Value` rather than to whole `Map<Key, Value>`.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cubux/effector-persistent",
-  "version": "0.3.0",
+  "version": "0.4.0",
   "description": "Persist data in effector store.",
   "keywords": [
     "effector",

src/lib/addFlush.ts

@@ -1,8 +1,8 @@
-import { Event, Store } from 'effector';
+import { Event } from 'effector';
 import { flushDelayed } from '../flushDelayed';
 
 export function addFlush<V>(
-  source: Store<V> | Event<V>,
+  source: Event<V>,
   flushDelay: number | undefined,
   flush: (value: V) => void
 ) {

src/lib/initialize.ts

@@ -1,18 +1,59 @@
 import {
-  combine,
   createEvent,
   createStore,
   Event,
   guard,
   is,
+  merge,
+  sample,
   Store,
 } from 'effector';
-import { addFlush } from './addFlush';
 import { WithPersistentOptions } from '../types';
+import { addFlush } from './addFlush';
+import { noopSerialize } from './noopSerialize';
 
-const noop = (v: any) => v;
 const isStore = <S, U>(unit: Store<S> | U): unit is Store<S> => is.store(unit);
 
+function initWakeUp<Driver, Value, Serialized>(
+  driver: Driver,
+  unserialize: (output: Serialized) => Promise<Value> | Value,
+  wakeUp: Store<Value> | ((state: Value) => void),
+  read: (driver: Driver) => Promise<Serialized | undefined>
+) {
+  const setWakingUp = createEvent<boolean>();
+  const $isWritable = createStore(true).on(setWakingUp, (_, b) => !b);
+
+  read(driver).then(
+    (s) => {
+      if (s !== undefined) {
+        return Promise.resolve(unserialize(s)).then(
+          (v) => {
+            setWakingUp(true);
+            try {
+              if (isStore(wakeUp)) {
+                const init = createEvent<Value>();
+                wakeUp.on(init, (_, v) => v);
+                init(v);
+              } else {
+                wakeUp(v);
+              }
+            } finally {
+              setWakingUp(false);
+            }
+          },
+          (e) =>
+            console.error(
+              'Failed to unserialize output from persistent driver',
+              e
+            )
+        );
+      }
+    },
+    (e) => console.error('Failed to read value from persistent driver', e)
+  );
+  return $isWritable;
+}
+
 const safeFire = <V>(event: Event<V> | undefined, payload: V) => {
   if (event) {
     try {
@@ -23,6 +64,47 @@ const safeFire = <V>(event: Event<V> | undefined, payload: V) => {
   }
 };
 
+function initFlush<Value>(
+  store: Store<Value>,
+  readOnly: Store<boolean> | undefined,
+  isWritable: Store<boolean>
+) {
+  const setPrev = createEvent<Value>();
+  const $prev = createStore(store.defaultState).on(setPrev, (_, p) => p);
+
+  const didUpdate = readOnly
+    ? guard({
+        source: store.updates,
+        filter: readOnly.map((ro) => !ro),
+      })
+    : store.updates;
+
+  const toWrite = guard({
+    source: sample({
+      clock: didUpdate,
+      source: $prev,
+      fn: (prev, next) => ({ next, prev }),
+    }),
+    filter: isWritable,
+  });
+
+  (readOnly
+    ? merge([
+        didUpdate,
+        sample({
+          clock: guard({
+            source: readOnly,
+            filter: (ro) => !ro,
+          }),
+          source: store,
+        }),
+      ])
+    : didUpdate
+  ).watch(setPrev);
+
+  return toWrite;
+}
+
 export function initialize<Driver, Value, Serialized = Value>(
   driver: Driver | Promise<Driver>,
   store: Store<Value>,
@@ -33,80 +115,29 @@ export function initialize<Driver, Value, Serialized = Value>(
     onFlushFail,
     onFlushFinally,
     readOnly,
-    serialize = noop,
-    unserialize = noop,
+    unserialize = noopSerialize,
     wakeUp = store,
-  }: WithPersistentOptions<Value, Value, Serialized> = {},
+  }: Omit<WithPersistentOptions<Value, Value, Serialized>, 'serialize'>,
   read: (driver: Driver) => Promise<Serialized | undefined>,
-  write: (driver: Driver, value: Serialized) => Promise<void>
+  write: (driver: Driver, value: Value, prev: Value) => Promise<void>
 ) {
   function setup(driver: Driver) {
-    const setWakingUp = createEvent<boolean>();
-    const $isWritable = createStore(true).on(setWakingUp, (_, b) => !b);
-
-    read(driver).then(
-      (s) => {
-        if (s !== undefined) {
-          return Promise.resolve(unserialize(s)).then(
-            (v) => {
-              setWakingUp(true);
-              try {
-                if (isStore(wakeUp)) {
-                  const init = createEvent<Value>();
-                  wakeUp.on(init, (_, v) => v);
-                  init(v);
-                } else {
-                  wakeUp(v);
-                }
-              } finally {
-                setWakingUp(false);
-              }
-            },
-            (e) =>
-              console.error(
-                'Failed to unserialize output from persistent driver',
-                e
-              )
-          );
-        }
-      },
-      (e) => console.error('Failed to read value from persistent driver', e)
-    );
-
     addFlush(
-      guard({
-        source: store.updates,
-        filter: readOnly
-          ? combine($isWritable, readOnly, (w, ro) => w && !ro)
-          : $isWritable,
-      }),
+      initFlush(store, readOnly, initWakeUp(driver, unserialize, wakeUp, read)),
       flushDelay,
-      (v) => {
+      ({ next, prev }) => {
         const id = Symbol();
         safeFire(onFlushStart, { id });
         // TODO: returned promise should be used to queue parallel write attempts
         //   Now it does not matter because `localStorage` is not async,
         //   and `indexedDB` uses transactions with exclusive lock.
-        Promise.resolve(serialize(v))
-          .then(
-            (s) =>
-              write(driver, s).catch((e) => {
-                console.error('Failed to write data to persistent driver', e);
-                return Promise.reject(e);
-              }),
-            (e) => {
-              console.error(
-                'Failed to serialize input before write to persistent driver',
-                e
-              );
-              return Promise.reject(e);
-            }
-          )
+        write(driver, next, prev)
           .then(
             () => {
               safeFire(onFlushDone, { id });
             },
             (error) => {
+              console.error('Failed to write data to persistent driver', error);
               safeFire(onFlushFail, { id, error });
             }
           )

src/lib/noopSerialize.ts

@@ -0,0 +1,5 @@
+/**
+ * Internal no-op serializer when `U` is `T`
+ * @param v
+ */
+export const noopSerialize = <T, U>(v: T) => v as any as U;

src/withPersistent.ts

@@ -17,14 +17,18 @@ export function withPersistent<Key, Value, Serialized = Value>(
     | StoreDriverSingle<Key, Serialized>
     | Promise<StoreDriverSingle<Key, Serialized>>,
   key: Key,
-  options?: WithPersistentOptions<Value, Value, Serialized>
+  options: WithPersistentOptions<Value, Value, Serialized> = {}
 ): typeof store {
+  const { serialize } = options;
   initialize(
     driver,
     store,
     options,
     (driver) => driver.getItem(key),
-    (driver, value) => driver.setItem(key, value)
+    serialize
+      ? (driver, value) =>
+          Promise.resolve(serialize(value)).then((s) => driver.setItem(key, s))
+      : (driver, value) => driver.setItem(key, value as any as Serialized)
   );
 
   return store;

src/withPersistentMap.ts

@@ -1,6 +1,7 @@
 import { Store } from 'effector';
-import { StoreDriverMapped } from '@cubux/storage-driver';
+import { StoreDriver } from '@cubux/storage-driver';
 import { initialize } from './lib/initialize';
+import { noopSerialize } from './lib/noopSerialize';
 import { WithPersistentOptions } from './types';
 
 const containsNoPromises = <T>(
@@ -41,8 +42,8 @@ interface WithPersistentMapFn {
   <Key, Value, Serialized>(
     store: Store<ReadonlyMap<Key, Value>>,
     driver:
-      | StoreDriverMapped<Key, Serialized>
-      | Promise<StoreDriverMapped<Key, Serialized>>,
+      | StoreDriver<Key, Serialized>
+      | Promise<StoreDriver<Key, Serialized>>,
     options: WithPersistentOptions<ReadonlyMap<Key, Value>, Value, Serialized>
   ): typeof store;
 
@@ -55,9 +56,7 @@ interface WithPersistentMapFn {
    */
   <Key, Value>(
     store: Store<ReadonlyMap<Key, Value>>,
-    driver:
-      | StoreDriverMapped<Key, Value>
-      | Promise<StoreDriverMapped<Key, Value>>,
+    driver: StoreDriver<Key, Value> | Promise<StoreDriver<Key, Value>>,
     options?: WithPersistentOptions<ReadonlyMap<Key, Value>, Value, Value>
   ): typeof store;
 }
@@ -75,26 +74,37 @@ export const withPersistentMap: WithPersistentMapFn = <
   Serialized = Value
 >(
   store: Store<ReadonlyMap<Key, Value>>,
-  driver:
-    | StoreDriverMapped<Key, Serialized>
-    | Promise<StoreDriverMapped<Key, Serialized>>,
-  options?: WithPersistentOptions<ReadonlyMap<Key, Value>, Value, Serialized>
+  driver: StoreDriver<Key, Serialized> | Promise<StoreDriver<Key, Serialized>>,
+  options: WithPersistentOptions<
+    ReadonlyMap<Key, Value>,
+    Value,
+    Serialized
+  > = {}
 ): typeof store => {
-  const { serialize, unserialize } = options || {};
+  const { serialize = noopSerialize, unserialize } = options;
   initialize<
-    StoreDriverMapped<Key, Serialized>,
+    StoreDriver<Key, Serialized>,
     ReadonlyMap<Key, Value>,
     ReadonlyMap<Key, Serialized>
   >(
     driver,
     store,
-    options && {
+    {
       ...options,
-      serialize: buildMapMapper(serialize),
       unserialize: buildMapMapper(unserialize),
     },
     (driver) => driver.getAll(),
-    (driver, value) => driver.setAll(value)
+    (driver, value, prev) =>
+      Promise.all([
+        ...Array.from(value)
+          .filter(([k, v]) => v !== prev.get(k))
+          .map(([k, v]) =>
+            Promise.resolve(serialize(v)).then((s) => driver.setItem(k, s))
+          ),
+        ...Array.from(prev)
+          .filter(([k]) => !value.has(k))
+          .map(([k]) => driver.removeItem(k)),
+      ]).then(() => {})
   );
 
   return store;