feat: compress MemoryStore persistence and handle .gz in FileUtils.sa…

…ve/load

packages/core/src/stores/memory.spec.ts

@@ -1,10 +1,10 @@
 import { suite, test } from "@testdeck/mocha";
 import * as assert from "assert";
-import { existsSync, unlinkSync } from "fs";
+import { existsSync } from "fs";
 import sinon from "sinon";
 import { AggregatorService, CoreModel, Ident, MemoryStore, Store, User, WebdaError } from "../index";
 import { HttpContext } from "../utils/httpcontext";
-import { JSONUtils } from "../utils/serializers";
+import { FileUtils } from "../utils/serializers";
 import { StoreNotFoundError } from "./store";
 import { PermissionModel, StoreTest } from "./store.spec";
 import { WebdaQL } from "./webdaql/query";
@@ -260,26 +260,24 @@ class MemoryStoreTest extends StoreTest {
 
   @test
   async persistence() {
-    // Remove the path if exists
-    if (existsSync(".test.json")) {
-      unlinkSync(".test.json");
-    }
+    this.cleanFiles.push(".test.json.gz");
+
     let identStore: MemoryStore<CoreModel> = <MemoryStore<CoreModel>>this.getIdentStore();
     identStore.getParameters().persistence = {
-      path: ".test.json",
+      path: ".test.json.gz",
       delay: 10
     };
     await identStore.init();
     await identStore.put("test", {});
     await this.sleep(10);
     // Check basic persistence
-    assert.ok(existsSync(".test.json"));
-    assert.notStrictEqual(JSONUtils.loadFile(".test.json").test, undefined);
+    assert.ok(existsSync(".test.json.gz"));
+    assert.notStrictEqual(FileUtils.load(".test.json.gz").test, undefined);
     identStore.storage = {};
     // Check basic load of persistence
     await identStore.init();
     assert.notStrictEqual(identStore.storage.test, undefined);
-
+    FileUtils.save({ test: "ok" }, ".test.json");
     // Check encryption
     identStore.getParameters().persistence = {
       path: ".test.json",

packages/core/src/stores/memory.ts

@@ -1,5 +1,6 @@
 import * as crypto from "crypto";
 import { existsSync, readFileSync, writeFileSync } from "fs";
+import { gunzipSync, gzipSync } from "zlib";
 import { CoreModel } from "../models/coremodel";
 import { Store, StoreFindResult, StoreNotFoundError, StoreParameters, UpdateConditionFailError } from "./store";
 import { WebdaQL } from "./webdaql/query";
@@ -82,30 +83,43 @@ class MemoryStore<
    * Encrypt data with provided key
    * @returns
    */
-  encrypt() {
-    let data = JSON.stringify(this.storage, undefined, 2);
+  encrypt(): Buffer {
+    let data = Buffer.from(JSON.stringify(this.storage, undefined, 2));
     if (!this.key) {
-      return data;
+      return gzipSync(data);
     }
     // Initialization Vector
     let iv = crypto.randomBytes(16);
     let cipher = crypto.createCipheriv(this.parameters.persistence.cipher, this.key, iv);
-    return Buffer.concat([iv, cipher.update(Buffer.from(data)), cipher.final()]).toString("base64");
+    return Buffer.concat([iv, cipher.update(gzipSync(data)), cipher.final()]);
+  }
+
+  /**
+   * Decompress data if compressed
+   * @param data
+   * @returns
+   */
+  uncompress(data: Buffer): string {
+    // gzip header
+    if (data[0] === 0x1f && data[1] === 0x8b) {
+      return gunzipSync(data).toString();
+    }
+    return data.toString();
   }
 
   /**
    * Decrypt data with provided key
    * @param data
    * @returns
    */
-  decrypt(data: string) {
+  decrypt(input: Buffer): string {
     if (!this.key) {
-      return data;
+      return this.uncompress(input);
     }
-    let input = Buffer.from(data, "base64");
     let iv = input.slice(0, 16);
     let decipher = crypto.createDecipheriv(this.parameters.persistence.cipher, this.key, iv);
-    return decipher.update(input.slice(16)).toString() + decipher.final().toString();
+    let decrypted = Buffer.concat([decipher.update(input.slice(16)), decipher.final()]);
+    return this.uncompress(decrypted);
   }
 
   /**
@@ -125,7 +139,7 @@ class MemoryStore<
       }
       try {
         if (existsSync(this.parameters.persistence.path)) {
-          this.storage = JSON.parse(this.decrypt(readFileSync(this.parameters.persistence.path).toString()));
+          this.storage = JSON.parse(this.decrypt(readFileSync(this.parameters.persistence.path)));
         }
       } catch (err) {
         this.log("INFO", "Cannot loaded persisted memory data", err);
@@ -318,4 +332,4 @@ class MemoryStore<
   }
 }
 
-export { MemoryStore, StorageMap, MemoryStore as Plop };
+export { MemoryStore, MemoryStore as Plop, StorageMap };

packages/core/src/utils/serializers.spec.ts

@@ -3,6 +3,7 @@ import * as assert from "assert";
 import { existsSync, readFileSync, symlinkSync } from "fs";
 import * as path from "path";
 import { fileURLToPath } from "url";
+import { gunzipSync } from "zlib";
 import { FileUtils, JSONUtils, YAMLUtils } from "./serializers";
 
 const TEST_FOLDER = path.dirname(fileURLToPath(import.meta.url)) + "/../../test/jsonutils/";
@@ -118,9 +119,22 @@ class UtilsTest {
       file = path.join(TEST_FOLDER, "writeTest.json");
       FileUtils.save({ test: "plop" }, file);
       assert.strictEqual(readFileSync(file).toString(), '{\n  "test": "plop"\n}');
+
+      // Gzip
+      file = path.join(TEST_FOLDER, "writeTest.json.gz");
+      FileUtils.save({ test: "plop" }, file);
+      const buf = readFileSync(file);
+      assert.strictEqual(buf[0], 0x1f);
+      assert.strictEqual(buf[1], 0x8b);
+      assert.deepStrictEqual(JSON.parse(gunzipSync(buf).toString()), { test: "plop" });
+
       assert.throws(() => FileUtils.save({}, "./Dockerfile.zzz"), /Unknown format/);
     } finally {
-      FileUtils.clean("test/jsonutils/writeTest.json", "test/jsonutils/writeTest.yaml");
+      FileUtils.clean(
+        "test/jsonutils/writeTest.json.gz",
+        "test/jsonutils/writeTest.json",
+        "test/jsonutils/writeTest.yaml"
+      );
     }
   }
 

packages/core/src/utils/serializers.ts

@@ -13,6 +13,7 @@ import * as jsonc from "jsonc-parser";
 import { join } from "path";
 import { Readable, Writable } from "stream";
 import * as yaml from "yaml";
+import { gunzipSync, gzipSync } from "zlib";
 import { Core } from "../core";
 
 type WalkerOptionsType = {
@@ -182,7 +183,13 @@ export const FileUtils: StorageFinder & {
     if (!existsSync(filename)) {
       throw new Error(`File '${filename}' does not exist.`);
     }
-    let content = readFileSync(filename, "utf-8");
+    let content;
+    if (filename.endsWith(".gz")) {
+      content = gunzipSync(readFileSync(filename)).toString();
+      filename = filename.slice(0, -3);
+    } else {
+      content = readFileSync(filename, "utf-8");
+    }
     format ??= getFormatFromFilename(filename);
     if (format === "yaml") {
       let res = yaml.parseAllDocuments(content);
@@ -205,15 +212,21 @@ export const FileUtils: StorageFinder & {
    * @returns
    */
   save: (object, filename = "", publicAudience: boolean = false, format?: Format) => {
-    format ??= getFormatFromFilename(filename);
+    if (filename.endsWith(".gz")) {
+      format ??= getFormatFromFilename(filename.slice(0, -3));
+    } else {
+      format ??= getFormatFromFilename(filename);
+    }
+    let res;
     if (format === "yaml") {
-      return writeFileSync(
-        filename,
-        yaml.stringify(JSON.parse(JSONUtils.stringify(object, undefined, 0, publicAudience)))
-      );
+      res = yaml.stringify(JSON.parse(JSONUtils.stringify(object, undefined, 0, publicAudience)));
     } else if (format === "json") {
-      return writeFileSync(filename, JSONUtils.stringify(object, undefined, 2, publicAudience));
+      res = JSONUtils.stringify(object, undefined, 2, publicAudience);
+    }
+    if (filename.endsWith(".gz")) {
+      res = gzipSync(res);
     }
+    writeFileSync(filename, res);
   },
   /**
    * Delete files if exists