From 997e11b04661f3dd14ab68c568ff6d103df7572f Mon Sep 17 00:00:00 2001 From: Remi Cattiau Date: Wed, 29 Nov 2023 11:27:35 -0800 Subject: [PATCH] feat: compress MemoryStore persistence and handle .gz in FileUtils.save/load --- packages/core/src/stores/memory.spec.ts | 18 +++++------ packages/core/src/stores/memory.ts | 34 +++++++++++++++------ packages/core/src/utils/serializers.spec.ts | 16 +++++++++- packages/core/src/utils/serializers.ts | 27 +++++++++++----- 4 files changed, 67 insertions(+), 28 deletions(-) diff --git a/packages/core/src/stores/memory.spec.ts b/packages/core/src/stores/memory.spec.ts index f6b6b76b7..c7be9cd25 100644 --- a/packages/core/src/stores/memory.spec.ts +++ b/packages/core/src/stores/memory.spec.ts @@ -1,10 +1,10 @@ import { suite, test } from "@testdeck/mocha"; import * as assert from "assert"; -import { existsSync, unlinkSync } from "fs"; +import { existsSync } from "fs"; import sinon from "sinon"; import { AggregatorService, CoreModel, Ident, MemoryStore, Store, User, WebdaError } from "../index"; import { HttpContext } from "../utils/httpcontext"; -import { JSONUtils } from "../utils/serializers"; +import { FileUtils } from "../utils/serializers"; import { StoreNotFoundError } from "./store"; import { PermissionModel, StoreTest } from "./store.spec"; import { WebdaQL } from "./webdaql/query"; @@ -260,26 +260,24 @@ class MemoryStoreTest extends StoreTest { @test async persistence() { - // Remove the path if exists - if (existsSync(".test.json")) { - unlinkSync(".test.json"); - } + this.cleanFiles.push(".test.json.gz"); + let identStore: MemoryStore = >this.getIdentStore(); identStore.getParameters().persistence = { - path: ".test.json", + path: ".test.json.gz", delay: 10 }; await identStore.init(); await identStore.put("test", {}); await this.sleep(10); // Check basic persistence - assert.ok(existsSync(".test.json")); - assert.notStrictEqual(JSONUtils.loadFile(".test.json").test, undefined); + assert.ok(existsSync(".test.json.gz")); + assert.notStrictEqual(FileUtils.load(".test.json.gz").test, undefined); identStore.storage = {}; // Check basic load of persistence await identStore.init(); assert.notStrictEqual(identStore.storage.test, undefined); - + FileUtils.save({ test: "ok" }, ".test.json"); // Check encryption identStore.getParameters().persistence = { path: ".test.json", diff --git a/packages/core/src/stores/memory.ts b/packages/core/src/stores/memory.ts index ac306d245..b4e456dc6 100644 --- a/packages/core/src/stores/memory.ts +++ b/packages/core/src/stores/memory.ts @@ -1,5 +1,6 @@ import * as crypto from "crypto"; import { existsSync, readFileSync, writeFileSync } from "fs"; +import { gunzipSync, gzipSync } from "zlib"; import { CoreModel } from "../models/coremodel"; import { Store, StoreFindResult, StoreNotFoundError, StoreParameters, UpdateConditionFailError } from "./store"; import { WebdaQL } from "./webdaql/query"; @@ -82,15 +83,28 @@ class MemoryStore< * Encrypt data with provided key * @returns */ - encrypt() { - let data = JSON.stringify(this.storage, undefined, 2); + encrypt(): Buffer { + let data = Buffer.from(JSON.stringify(this.storage, undefined, 2)); if (!this.key) { - return data; + return gzipSync(data); } // Initialization Vector let iv = crypto.randomBytes(16); let cipher = crypto.createCipheriv(this.parameters.persistence.cipher, this.key, iv); - return Buffer.concat([iv, cipher.update(Buffer.from(data)), cipher.final()]).toString("base64"); + return Buffer.concat([iv, cipher.update(gzipSync(data)), cipher.final()]); + } + + /** + * Decompress data if compressed + * @param data + * @returns + */ + uncompress(data: Buffer): string { + // gzip header + if (data[0] === 0x1f && data[1] === 0x8b) { + return gunzipSync(data).toString(); + } + return data.toString(); } /** @@ -98,14 +112,14 @@ class MemoryStore< * @param data * @returns */ - decrypt(data: string) { + decrypt(input: Buffer): string { if (!this.key) { - return data; + return this.uncompress(input); } - let input = Buffer.from(data, "base64"); let iv = input.slice(0, 16); let decipher = crypto.createDecipheriv(this.parameters.persistence.cipher, this.key, iv); - return decipher.update(input.slice(16)).toString() + decipher.final().toString(); + let decrypted = Buffer.concat([decipher.update(input.slice(16)), decipher.final()]); + return this.uncompress(decrypted); } /** @@ -125,7 +139,7 @@ class MemoryStore< } try { if (existsSync(this.parameters.persistence.path)) { - this.storage = JSON.parse(this.decrypt(readFileSync(this.parameters.persistence.path).toString())); + this.storage = JSON.parse(this.decrypt(readFileSync(this.parameters.persistence.path))); } } catch (err) { this.log("INFO", "Cannot loaded persisted memory data", err); @@ -318,4 +332,4 @@ class MemoryStore< } } -export { MemoryStore, StorageMap, MemoryStore as Plop }; +export { MemoryStore, MemoryStore as Plop, StorageMap }; diff --git a/packages/core/src/utils/serializers.spec.ts b/packages/core/src/utils/serializers.spec.ts index abd27619b..9e5ac5198 100644 --- a/packages/core/src/utils/serializers.spec.ts +++ b/packages/core/src/utils/serializers.spec.ts @@ -3,6 +3,7 @@ import * as assert from "assert"; import { existsSync, readFileSync, symlinkSync } from "fs"; import * as path from "path"; import { fileURLToPath } from "url"; +import { gunzipSync } from "zlib"; import { FileUtils, JSONUtils, YAMLUtils } from "./serializers"; const TEST_FOLDER = path.dirname(fileURLToPath(import.meta.url)) + "/../../test/jsonutils/"; @@ -118,9 +119,22 @@ class UtilsTest { file = path.join(TEST_FOLDER, "writeTest.json"); FileUtils.save({ test: "plop" }, file); assert.strictEqual(readFileSync(file).toString(), '{\n "test": "plop"\n}'); + + // Gzip + file = path.join(TEST_FOLDER, "writeTest.json.gz"); + FileUtils.save({ test: "plop" }, file); + const buf = readFileSync(file); + assert.strictEqual(buf[0], 0x1f); + assert.strictEqual(buf[1], 0x8b); + assert.deepStrictEqual(JSON.parse(gunzipSync(buf).toString()), { test: "plop" }); + assert.throws(() => FileUtils.save({}, "./Dockerfile.zzz"), /Unknown format/); } finally { - FileUtils.clean("test/jsonutils/writeTest.json", "test/jsonutils/writeTest.yaml"); + FileUtils.clean( + "test/jsonutils/writeTest.json.gz", + "test/jsonutils/writeTest.json", + "test/jsonutils/writeTest.yaml" + ); } } diff --git a/packages/core/src/utils/serializers.ts b/packages/core/src/utils/serializers.ts index 0462ff2cc..dcec04a94 100644 --- a/packages/core/src/utils/serializers.ts +++ b/packages/core/src/utils/serializers.ts @@ -13,6 +13,7 @@ import * as jsonc from "jsonc-parser"; import { join } from "path"; import { Readable, Writable } from "stream"; import * as yaml from "yaml"; +import { gunzipSync, gzipSync } from "zlib"; import { Core } from "../core"; type WalkerOptionsType = { @@ -182,7 +183,13 @@ export const FileUtils: StorageFinder & { if (!existsSync(filename)) { throw new Error(`File '${filename}' does not exist.`); } - let content = readFileSync(filename, "utf-8"); + let content; + if (filename.endsWith(".gz")) { + content = gunzipSync(readFileSync(filename)).toString(); + filename = filename.slice(0, -3); + } else { + content = readFileSync(filename, "utf-8"); + } format ??= getFormatFromFilename(filename); if (format === "yaml") { let res = yaml.parseAllDocuments(content); @@ -205,15 +212,21 @@ export const FileUtils: StorageFinder & { * @returns */ save: (object, filename = "", publicAudience: boolean = false, format?: Format) => { - format ??= getFormatFromFilename(filename); + if (filename.endsWith(".gz")) { + format ??= getFormatFromFilename(filename.slice(0, -3)); + } else { + format ??= getFormatFromFilename(filename); + } + let res; if (format === "yaml") { - return writeFileSync( - filename, - yaml.stringify(JSON.parse(JSONUtils.stringify(object, undefined, 0, publicAudience))) - ); + res = yaml.stringify(JSON.parse(JSONUtils.stringify(object, undefined, 0, publicAudience))); } else if (format === "json") { - return writeFileSync(filename, JSONUtils.stringify(object, undefined, 2, publicAudience)); + res = JSONUtils.stringify(object, undefined, 2, publicAudience); + } + if (filename.endsWith(".gz")) { + res = gzipSync(res); } + writeFileSync(filename, res); }, /** * Delete files if exists