From db313c5bfc902369ee0ffc12e8efc05518da3a5a Mon Sep 17 00:00:00 2001 From: Michael Vlasov Date: Fri, 29 Sep 2023 15:07:25 +0300 Subject: [PATCH] NEW: - `config.blockBocs.s3.numBuckets` field --- CHANGELOG.md | 6 +++ npm-shrinkwrap.json | 77 ++++++++++++++++++++------------- package.json | 5 ++- src/__tests__/cloud15.ts | 27 ++++++++++++ src/server/config-param.ts | 9 ++++ src/server/config.ts | 1 + src/server/data/boc-provider.ts | 17 +++++++- 7 files changed, 109 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18b0662..9938909 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ All notable changes to this project will be documented in this file. +## [0.65.4] - 2023-09-29 + +### New + +- `config.blockBocs.s3.numBuckets` field + ## [0.65.3] - 2023-09-14 ### New diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index a82faff..5efe3b9 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,12 +1,12 @@ { "name": "ton-q-server", - "version": "0.65.2", + "version": "0.65.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ton-q-server", - "version": "0.65.2", + "version": "0.65.3", "license": "ISC", "dependencies": { "@aws-sdk/client-s3": "^3.347.1", @@ -26,6 +26,7 @@ "arangochair": "git+https://github.com/tonlabs/arangochair.git", "arangojs": "6.14.1", "commander": "6.0.0", + "crc": "^4.3.2", "express": "4.17.1", "graphql": "15.7.2", "graphql-redis-subscriptions": "^2.4.2", @@ -7109,29 +7110,6 @@ "ws": "^8.8.1" } }, - "node_modules/adnl/node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, "node_modules/adnl/node_modules/isomorphic-ws": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-5.0.0.tgz", @@ -8436,10 +8414,9 @@ } }, "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", "funding": [ { "type": "github", @@ -8456,7 +8433,7 @@ ], "dependencies": { "base64-js": "^1.3.1", - "ieee754": "^1.1.13" + "ieee754": "^1.2.1" } }, "node_modules/buffer-equal-constant-time": { @@ -9083,6 +9060,22 @@ "@iarna/toml": "^2.2.5" } }, + "node_modules/crc": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/crc/-/crc-4.3.2.tgz", + "integrity": "sha512-uGDHf4KLLh2zsHa8D8hIQ1H/HtFQhyHrc0uhHBcoKGol/Xnb+MPYfUMw7cvON6ze/GUESTudKayDcJC5HnJv1A==", + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "buffer": ">=6.0.3" + }, + "peerDependenciesMeta": { + "buffer": { + "optional": true + } + } + }, "node_modules/crc-32": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", @@ -18471,6 +18464,30 @@ "node": ">=8" } }, + "node_modules/sync-fetch/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/table": { "version": "6.7.1", "resolved": "https://registry.npmjs.org/table/-/table-6.7.1.tgz", diff --git a/package.json b/package.json index 534e670..07ce973 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ton-q-server", - "version": "0.65.3", + "version": "0.65.4", "description": "TON Q Server – realtime queries over TON blockchain.", "main": "index.js", "repository": "git@github.com:tonlabs/ton-q-server.git", @@ -65,7 +65,8 @@ "ton-crypto": "^3.2.0", "ton-lite-client": "^2.0.1", "js-yaml": "^4.1.0", - "ws": "7.4.6" + "ws": "7.4.6", + "crc": "^4.3.2" }, "devDependencies": { "@babel/preset-env": "^7.14.7", diff --git a/src/__tests__/cloud15.ts b/src/__tests__/cloud15.ts index 7dfe65f..7c82161 100644 --- a/src/__tests__/cloud15.ts +++ b/src/__tests__/cloud15.ts @@ -9,6 +9,7 @@ import { accountDbFields, convertDocFromDb, } from "../server/graphql/blockchain/fetchers" +import { bucketName } from "../server/data/boc-provider" beforeAll(async () => { await createTestData() @@ -370,3 +371,29 @@ test("cloud15.blocks_signatures", async () => { await test.close() }) + +test("cloud15.s3-bucket-names", () => { + const buckets = [ + "some-file-name", + "7cd2c97c7fd81f3498d3cad64c8d751a95edbf03ce80eeaf9e8f7b3e2ecd59c2", + "6926e5e954b1541943832c77b736d45365cb882d98c1926c0c2edd532811d09f", + "b4f8b57574509b0303f8845c1465153ec96868a162e6357605fe6b88323ffbc1", + "46d58e9c208265fc11d538b7ebe090f1a13f685b1eb408e24fdb0ea8b1250e28", + "834968f0d418400b9f8b215903d26e2eae01a4b86daf81c84a6ffad997f766d6", + "be907549f71edd3239b034088aadd281a4b87e03782c2d98df51a95454f7dee1", + "11b38729f15d4dbaa3814f66921def45bc9da0ebef0f466a497cf885efce4d53", + "2ef17fc66d177cfb39f85c2fbee75a664c19a6ec7d856beb7a19ee4ff4e91113", + "b43dfb8edd8f79b486bf928cf67bd21f9939d3e1661603f741e874ab4af58cc9", + "bb1265e2db2faa5ae9a99af3a8baaa9950eaa3bec7778bef8c582898e0d36631", + "cb5f507ab331c322de68a1f6a8f2fe58ac5f0061dc9a452b27a4f159b909af10", + "bc61269bbb253c9e13f441c36430517c059d04037905cbe89eec536803fa69f5", + "2537dc69bbd74c5ec0da26b20dad952e9daa6d5d54918dbc2faf328af09bb78d", + "869079f2c936dc26122be9bd30659525a72819ddd18d3b5277cfe7bf7934878b", + "03f2765eb08ba81e1a598e1b7a50e247a261f589e664ca456b1700ffe2fc80f0", + ].map(x => bucketName(x, "foo", 10)) + const expected = [7, 3, 5, 4, 5, 3, 1, 7, 0, 6, 1, 6, 5, 0, 6, 1].map( + x => `foo-${x}`, + ) + expect(buckets).toStrictEqual(expected) + expect(bucketName("file", "foo", 0)).toBe("foo") +}) diff --git a/src/server/config-param.ts b/src/server/config-param.ts index 14a0d2a..3fcb346 100644 --- a/src/server/config-param.ts +++ b/src/server/config-param.ts @@ -88,6 +88,7 @@ type BocResolverParams = { endpoint: ConfigParam region: ConfigParam bucket: ConfigParam + numBuckets: ConfigParam accessKey: ConfigParam secretKey: ConfigParam timeout: ConfigParam @@ -257,6 +258,14 @@ export class ConfigParam { endpoint: opt("s3-endpoint", "S3 endpoint"), region: opt("s3-region", "S3 region"), bucket: opt("s3-bucket", "S3 bucket", "everblocks"), + numBuckets: ConfigParam.integer( + prefixedOption(prefix, "s3-num-buckets"), + 0, + withPrefix( + descriptionPrefix ?? prefix, + "S3 number of buckets (if specified and > 0, then bucket name will be equal to `{bucket}-{crc32(file) % numBuckets}`)", + ), + ), accessKey: opt("s3-access-key", "S3 access key"), secretKey: opt("s3-secret-key", "S3 secret key"), timeout: ConfigParam.integer( diff --git a/src/server/config.ts b/src/server/config.ts index a60389a..4c11c0d 100644 --- a/src/server/config.ts +++ b/src/server/config.ts @@ -102,6 +102,7 @@ export type QBocResolverConfig = { endpoint: string region: string bucket: string + numBuckets?: number accessKey?: string secretKey?: string timeout?: number diff --git a/src/server/data/boc-provider.ts b/src/server/data/boc-provider.ts index aac4dd4..02bb136 100644 --- a/src/server/data/boc-provider.ts +++ b/src/server/data/boc-provider.ts @@ -2,6 +2,7 @@ import { parseArangoConfig, QBocResolverConfig } from "../config" import { S3 } from "@aws-sdk/client-s3" import { Database } from "arangojs" import { createDatabase } from "./database-provider" +import { crc32 } from "crc" export interface IBocProvider { getBocs( @@ -32,6 +33,7 @@ class S3Provider implements IBocProvider { endpoint: string region: string bucket: string + numBuckets: number accessKey: string secretKey: string timeout?: number @@ -57,7 +59,11 @@ class S3Provider implements IBocProvider { for (const { hash, boc } of bocHashes) { const getObjectResult = await this.client.getObject( { - Bucket: this.config.bucket, + Bucket: bucketName( + hash, + this.config.bucket, + this.config.numBuckets, + ), Key: hash, }, { @@ -72,6 +78,14 @@ class S3Provider implements IBocProvider { } } +export function bucketName( + file: string, + bucket: string, + numBuckets: number, +): string { + return numBuckets > 0 ? `${bucket}-${crc32(file) % numBuckets}` : bucket +} + class ArangoProvider implements IBocProvider { private readonly database: Database constructor( @@ -119,6 +133,7 @@ export function createBocProvider( return new S3Provider({ endpoint: s3endpoint, bucket: config.s3?.bucket ?? "", + numBuckets: config.s3?.numBuckets ?? 0, region: config.s3?.region ?? "", accessKey: config.s3?.accessKey ?? "", secretKey: config.s3?.secretKey ?? "",