Skip to content

Commit

Permalink
feat: add config to allow return if found key in S3
Browse files Browse the repository at this point in the history
  • Loading branch information
fibonacci998 committed Feb 5, 2024
1 parent 27316a3 commit 3bcfd0b
Show file tree
Hide file tree
Showing 4 changed files with 15 additions and 5 deletions.
3 changes: 2 additions & 1 deletion ci/config.json.ci
Original file line number Diff line number Diff line change
Expand Up @@ -301,7 +301,8 @@
"key": "uploadBlockRawLogToS3",
"millisecondCrawl": 1000,
"blocksPerCall": 100,
"overwriteS3IfFound": true
"overwriteS3IfFound": false,
"returnIfFound": true
},
"uploadTransactionRawLogToS3": {
"key": "uploadTransactionRawLogToS3",
Expand Down
3 changes: 2 additions & 1 deletion config.json
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,8 @@
"key": "uploadBlockRawLogToS3",
"millisecondCrawl": 1000,
"blocksPerCall": 100,
"overwriteS3IfFound": true
"overwriteS3IfFound": false,
"returnIfFound": true
},
"uploadTransactionRawLogToS3": {
"key": "uploadTransactionRawLogToS3",
Expand Down
11 changes: 9 additions & 2 deletions src/common/utils/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,8 @@ export default class Utils {
data: Buffer,
bucketName: string,
s3Gateway: string,
overwrite = false
overwriteS3IfFound = false,
returnIfFound = false
) {
const foundS3Object = await s3Client
.headObject({
Expand All @@ -243,7 +244,13 @@ export default class Utils {
if (foundS3Object) {
const err = `This S3 key is found in S3: ${fileName}`;
console.warn(err);
if (!overwrite) {
if (!overwriteS3IfFound) {
if (returnIfFound) {
return {
key: s3Gateway ? s3Gateway + fileName : fileName,
id,
};
}
throw new Error(err);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ export default class UploadBlockRawLogToS3 extends BullableService {
Buffer.from(JSON.stringify(block.data)),
Config.BUCKET,
Config.S3_GATEWAY,
config.uploadBlockRawLogToS3.overwriteS3IfFound
config.uploadBlockRawLogToS3.overwriteS3IfFound,
config.uploadBlockRawLogToS3.returnIfFound
)
)
).catch((err) => {
Expand Down

0 comments on commit 3bcfd0b

Please sign in to comment.