Merge pull request #380 from tonlabs/0.65.4-rc

Version 0.65.4

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.65.4] - 2023-09-29
+
+### New
+
+- `config.blockBocs.s3.numBuckets` field
+
 ## [0.65.3] - 2023-09-14
 
 ### New

package.json

@@ -1,6 +1,6 @@
 {
     "name": "ton-q-server",
-    "version": "0.65.3",
+    "version": "0.65.4",
     "description": "TON Q Server – realtime queries over TON blockchain.",
     "main": "index.js",
     "repository": "[email protected]:tonlabs/ton-q-server.git",
@@ -65,7 +65,8 @@
         "ton-crypto": "^3.2.0",
         "ton-lite-client": "^2.0.1",
         "js-yaml": "^4.1.0",
-        "ws": "7.4.6"
+        "ws": "7.4.6",
+        "crc": "^4.3.2"
     },
     "devDependencies": {
         "@babel/preset-env": "^7.14.7",

src/__tests__/cloud15.ts

@@ -9,6 +9,7 @@ import {
     accountDbFields,
     convertDocFromDb,
 } from "../server/graphql/blockchain/fetchers"
+import { bucketName } from "../server/data/boc-provider"
 
 beforeAll(async () => {
     await createTestData()
@@ -370,3 +371,29 @@ test("cloud15.blocks_signatures", async () => {
 
     await test.close()
 })
+
+test("cloud15.s3-bucket-names", () => {
+    const buckets = [
+        "some-file-name",
+        "7cd2c97c7fd81f3498d3cad64c8d751a95edbf03ce80eeaf9e8f7b3e2ecd59c2",
+        "6926e5e954b1541943832c77b736d45365cb882d98c1926c0c2edd532811d09f",
+        "b4f8b57574509b0303f8845c1465153ec96868a162e6357605fe6b88323ffbc1",
+        "46d58e9c208265fc11d538b7ebe090f1a13f685b1eb408e24fdb0ea8b1250e28",
+        "834968f0d418400b9f8b215903d26e2eae01a4b86daf81c84a6ffad997f766d6",
+        "be907549f71edd3239b034088aadd281a4b87e03782c2d98df51a95454f7dee1",
+        "11b38729f15d4dbaa3814f66921def45bc9da0ebef0f466a497cf885efce4d53",
+        "2ef17fc66d177cfb39f85c2fbee75a664c19a6ec7d856beb7a19ee4ff4e91113",
+        "b43dfb8edd8f79b486bf928cf67bd21f9939d3e1661603f741e874ab4af58cc9",
+        "bb1265e2db2faa5ae9a99af3a8baaa9950eaa3bec7778bef8c582898e0d36631",
+        "cb5f507ab331c322de68a1f6a8f2fe58ac5f0061dc9a452b27a4f159b909af10",
+        "bc61269bbb253c9e13f441c36430517c059d04037905cbe89eec536803fa69f5",
+        "2537dc69bbd74c5ec0da26b20dad952e9daa6d5d54918dbc2faf328af09bb78d",
+        "869079f2c936dc26122be9bd30659525a72819ddd18d3b5277cfe7bf7934878b",
+        "03f2765eb08ba81e1a598e1b7a50e247a261f589e664ca456b1700ffe2fc80f0",
+    ].map(x => bucketName(x, "foo", 10))
+    const expected = [7, 3, 5, 4, 5, 3, 1, 7, 0, 6, 1, 6, 5, 0, 6, 1].map(
+        x => `foo-${x}`,
+    )
+    expect(buckets).toStrictEqual(expected)
+    expect(bucketName("file", "foo", 0)).toBe("foo")
+})

src/server/config-param.ts

@@ -88,6 +88,7 @@ type BocResolverParams = {
         endpoint: ConfigParam<string>
         region: ConfigParam<string>
         bucket: ConfigParam<string>
+        numBuckets: ConfigParam<number>
         accessKey: ConfigParam<string>
         secretKey: ConfigParam<string>
         timeout: ConfigParam<number>
@@ -257,6 +258,14 @@ export class ConfigParam<T extends ConfigValue> {
                 endpoint: opt("s3-endpoint", "S3 endpoint"),
                 region: opt("s3-region", "S3 region"),
                 bucket: opt("s3-bucket", "S3 bucket", "everblocks"),
+                numBuckets: ConfigParam.integer(
+                    prefixedOption(prefix, "s3-num-buckets"),
+                    0,
+                    withPrefix(
+                        descriptionPrefix ?? prefix,
+                        "S3 number of buckets (if specified and > 0, then bucket name will be equal to `{bucket}-{crc32(file) % numBuckets}`)",
+                    ),
+                ),
                 accessKey: opt("s3-access-key", "S3 access key"),
                 secretKey: opt("s3-secret-key", "S3 secret key"),
                 timeout: ConfigParam.integer(

src/server/config.ts

@@ -102,6 +102,7 @@ export type QBocResolverConfig = {
         endpoint: string
         region: string
         bucket: string
+        numBuckets?: number
         accessKey?: string
         secretKey?: string
         timeout?: number

src/server/data/boc-provider.ts

@@ -2,6 +2,7 @@ import { parseArangoConfig, QBocResolverConfig } from "../config"
 import { S3 } from "@aws-sdk/client-s3"
 import { Database } from "arangojs"
 import { createDatabase } from "./database-provider"
+import { crc32 } from "crc"
 
 export interface IBocProvider {
     getBocs(
@@ -32,6 +33,7 @@ class S3Provider implements IBocProvider {
             endpoint: string
             region: string
             bucket: string
+            numBuckets: number
             accessKey: string
             secretKey: string
             timeout?: number
@@ -57,7 +59,11 @@ class S3Provider implements IBocProvider {
         for (const { hash, boc } of bocHashes) {
             const getObjectResult = await this.client.getObject(
                 {
-                    Bucket: this.config.bucket,
+                    Bucket: bucketName(
+                        hash,
+                        this.config.bucket,
+                        this.config.numBuckets,
+                    ),
                     Key: hash,
                 },
                 {
@@ -72,6 +78,14 @@ class S3Provider implements IBocProvider {
     }
 }
 
+export function bucketName(
+    file: string,
+    bucket: string,
+    numBuckets: number,
+): string {
+    return numBuckets > 0 ? `${bucket}-${crc32(file) % numBuckets}` : bucket
+}
+
 class ArangoProvider implements IBocProvider {
     private readonly database: Database
     constructor(
@@ -119,6 +133,7 @@ export function createBocProvider(
         return new S3Provider({
             endpoint: s3endpoint,
             bucket: config.s3?.bucket ?? "",
+            numBuckets: config.s3?.numBuckets ?? 0,
             region: config.s3?.region ?? "",
             accessKey: config.s3?.accessKey ?? "",
             secretKey: config.s3?.secretKey ?? "",