Skip to content

Commit

Permalink
throw error if wrong file is used for the repair command
Browse files Browse the repository at this point in the history
  • Loading branch information
@igorls
igorls committed May 2, 2024
1 parent 3c24fea commit e3dc0e1
Showing 1 changed file with 95 additions and 91 deletions.
186 changes: 95 additions & 91 deletions scripts/hyp-repair.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
import { Command } from 'commander';
import { readFileSync } from 'node:fs';
import { Client, estypes } from '@elastic/elasticsearch';
import {Command} from 'commander';
import {readFileSync} from 'node:fs';
import {Client, estypes} from '@elastic/elasticsearch';
// @ts-ignore
import cliProgress from 'cli-progress';
import { JsonRpc } from 'eosjs';
import {JsonRpc} from 'eosjs';
import fetch from 'cross-fetch';
import { existsSync, mkdirSync, writeFileSync } from 'fs';
import { HyperionBlock } from './repair-cli/interfaces.js';
import {existsSync, mkdirSync, writeFileSync} from 'fs';
import {HyperionBlock} from './repair-cli/interfaces.js';
import {
getBlocks,
getFirstIndexedBlock,
Expand All @@ -15,10 +15,10 @@ import {
readChainConfig,
readConnectionConfig,
} from './repair-cli/functions.js';
import { SearchResponse } from '@elastic/elasticsearch/api/types';
import {SearchResponse} from '@elastic/elasticsearch/api/types';

import { WebSocket } from 'ws';
import { block } from '../definitions/index-templates';
import {WebSocket} from 'ws';
import {block} from '../definitions/index-templates';

const progressBar = new cliProgress.SingleBar(
{},
Expand Down Expand Up @@ -51,15 +51,15 @@ async function run(
blockFinal = firstBlock;
}
try {
let { body } = await getBlocks(
let {body} = await getBlocks(
client,
indexName,
blockInitial,
blockFinal,
qtdTotal
);
let {
hits: { hits },
hits: {hits},
} = body;
const blocks = hits.map((obj: any) => obj._source);
await findForksOnRange(blocks, rpc);
Expand Down Expand Up @@ -133,7 +133,7 @@ async function findForksOnRange(blocks: HyperionBlock[], rpc: JsonRpc) {
removals.add(currentBlock.block_id);
} else {
end = currentBlockNumber + 1;
const range = { start, end, ids: [...removals] };
const range = {start, end, ids: [...removals]};
errorRanges.push(range);
removals.clear();
// console.log(`\n ⚠️⚠️ Forked at ${range.start} to ${range.end}`);
Expand All @@ -155,7 +155,7 @@ async function scanChain(chain: string, args: any) {

const client = initESClient(config);

const jsonRpc = new JsonRpc(config.chains[chain].http, { fetch });
const jsonRpc = new JsonRpc(config.chains[chain].http, {fetch});

const ping = await client.ping();

Expand Down Expand Up @@ -404,7 +404,7 @@ async function repairChain(chain: string, file: string, args: any) {
must: [
{
range: {
block: { lte: range.start, gte: range.end },
block: {lte: range.start, gte: range.end},
},
},
],
Expand All @@ -422,7 +422,7 @@ async function repairChain(chain: string, file: string, args: any) {
deleteAbis += (
resultAbis.body.hits.total as estypes.SearchTotalHits
)?.value;
console.log('ABIs', { lte: range.start, gte: range.end });
console.log('ABIs', {lte: range.start, gte: range.end});
}
} else {
const indexExists = await client.indices.exists({
Expand Down Expand Up @@ -493,23 +493,23 @@ async function repairChain(chain: string, file: string, args: any) {
});

if (indexExists.body) {
delete searchAccounts.size;
const deletedAccountsResult = await client.deleteByQuery(
searchAccounts
);
if (
deletedAccountsResult &&
deletedAccountsResult.body.deleted &&
deletedAccountsResult.body.deleted > 0
) {
deleteAccounts += deletedAccountsResult.body.deleted;
delete searchAccounts.size;
const deletedAccountsResult = await client.deleteByQuery(
searchAccounts
);
if (
deletedAccountsResult &&
deletedAccountsResult.body.deleted &&
deletedAccountsResult.body.deleted > 0
) {
deleteAccounts += deletedAccountsResult.body.deleted;
}
} else {
console.log(
`Index ${searchAccounts.index} doesn't exist. Unable to delete.`
);
}
}else {
console.log(
`Index ${searchAccounts.index} doesn't exist. Unable to delete.`
);
}
}

// VOTERS
const searchVoters = {
Expand Down Expand Up @@ -557,23 +557,23 @@ async function repairChain(chain: string, file: string, args: any) {

if (indexExists.body) {

delete searchVoters.size;
const deletedVotersResult = await client.deleteByQuery(
searchVoters
);
if (
deletedVotersResult &&
deletedVotersResult.body.deleted &&
deletedVotersResult.body.deleted > 0
) {
deleteVoters += deletedVotersResult.body.deleted;
delete searchVoters.size;
const deletedVotersResult = await client.deleteByQuery(
searchVoters
);
if (
deletedVotersResult &&
deletedVotersResult.body.deleted &&
deletedVotersResult.body.deleted > 0
) {
deleteVoters += deletedVotersResult.body.deleted;
}
} else {
console.log(
`Index ${searchVoters.index} doesn't exist. Unable to delete.`
);
}
}else {
console.log(
`Index ${searchVoters.index} doesn't exist. Unable to delete.`
);
}
}

// PROPOSALS
const searchProposals = {
Expand Down Expand Up @@ -620,19 +620,21 @@ async function repairChain(chain: string, file: string, args: any) {
const indexExists = await client.indices.exists({index: searchProposals.index});

if (indexExists.body) {
delete searchProposals.size;
const deletedProposalsResult = await client.deleteByQuery(
searchProposals
);
if (
deletedProposalsResult &&
deletedProposalsResult.body.deleted &&
deletedProposalsResult.body.deleted > 0
) {
deleteProposals += deletedProposalsResult.body.deleted;
delete searchProposals.size;
const deletedProposalsResult = await client.deleteByQuery(
searchProposals
);
if (
deletedProposalsResult &&
deletedProposalsResult.body.deleted &&
deletedProposalsResult.body.deleted > 0
) {
deleteProposals += deletedProposalsResult.body.deleted;
}
} else {
console.log(`Index ${searchProposals.index} doesn't exist. Unable to delete.`);
}
}else { console.log(`Index ${searchProposals.index} doesn't exist. Unable to delete.`);}
}
}

// LINKS
const searchLinks = {
Expand Down Expand Up @@ -677,17 +679,19 @@ async function repairChain(chain: string, file: string, args: any) {
const indexExists = await client.indices.exists({index: searchLinks.index});

if (indexExists.body) {
delete searchLinks.size;
const deletedLinksResult = await client.deleteByQuery(searchLinks);
if (
deletedLinksResult &&
deletedLinksResult.body.deleted &&
deletedLinksResult.body.deleted > 0
) {
deleteLinks += deletedLinksResult.body.deleted;
delete searchLinks.size;
const deletedLinksResult = await client.deleteByQuery(searchLinks);
if (
deletedLinksResult &&
deletedLinksResult.body.deleted &&
deletedLinksResult.body.deleted > 0
) {
deleteLinks += deletedLinksResult.body.deleted;
}
} else {
console.log(`Index ${searchLinks.index} doesn't exist. Unable to delete.`);
}
}else { console.log(`Index ${searchLinks.index} doesn't exist. Unable to delete.`);}
}
}

// PERMISSIONS
const searchPermissions = {
Expand Down Expand Up @@ -728,7 +732,7 @@ async function repairChain(chain: string, file: string, args: any) {
)?.value,
'permissions needs to be updated'
);
console.log({ lte: range.start, gte: range.end });
console.log({lte: range.start, gte: range.end});
deletePermissions += (
resultPermissions.body.hits.total as estypes.SearchTotalHits
)?.value;
Expand All @@ -737,26 +741,33 @@ async function repairChain(chain: string, file: string, args: any) {

const indexExists = await client.indices.exists({index: searchPermissions.index});
if (indexExists.body) {
delete searchPermissions.size;
const deletedPermissionsResult = await client.deleteByQuery(
searchPermissions
);
if (
deletedPermissionsResult &&
deletedPermissionsResult.body.deleted &&
deletedPermissionsResult.body.deleted > 0
) {
deletePermissions += deletedPermissionsResult.body.deleted;
delete searchPermissions.size;
const deletedPermissionsResult = await client.deleteByQuery(
searchPermissions
);
if (
deletedPermissionsResult &&
deletedPermissionsResult.body.deleted &&
deletedPermissionsResult.body.deleted > 0
) {
deletePermissions += deletedPermissionsResult.body.deleted;
}
} else {
console.log(`Index ${searchPermissions.index} not present!`);
}
}else { console.log(`Index ${searchPermissions.index} doens't exist. Não foi possível realizar a exclusão.`);}
}
}

if (!range.ids) {
console.log("Invalid file format! Please check that you are calling the repair tool with the right file.");
process.exit(1);
}

for (const id of range.ids) {
const searchBlocks = {
index: blockIndex,
body: {
query: {
bool: { must: [{ term: { block_id: { value: id } } }] },
bool: {must: [{term: {block_id: {value: id}}}]},
},
},
};
Expand Down Expand Up @@ -813,20 +824,16 @@ async function fillMissingBlocksFromFile(host, chain, file, dryRun) {
}
const controller = new WebSocket(hyperionIndexer + '/local');

// Function to send Chunk
async function sendChunk(chunk): Promise<void> {
return new Promise((resolve, reject) => {
const payload = {
event: 'fill_missing_blocks',
data: chunk,
};
controller.send(JSON.stringify(payload));

// Wait repair_completed confirmation
controller.once('message', (data) => {
const parsed = JSON.parse(data.toString());
if (parsed.event === 'repair_completed') {
// console.log(`Hyperion repair completed for chunk!`);
resolve();
}
});
Expand All @@ -841,24 +848,21 @@ async function fillMissingBlocksFromFile(host, chain, file, dryRun) {

if (!dryRun) {
let completedLines = 0;

for (let i = 0; i < parsedFile.length; i += chunkSize) {
const chunk = parsedFile.slice(i, i + chunkSize);
await sendChunk(chunk);

// Atualizar o progresso com base no número total de linhas
completedLines += chunk.length;
const progress = (completedLines / totalLines) * 100;
const progressBar = Array(Math.round(progress / 2))
.fill('#')
.join('');
process.stdout.clearLine(0); // Limpar a linha anterior
process.stdout.cursorTo(0); // Mover o cursor para o início da linha
process.stdout.clearLine(0);
process.stdout.cursorTo(0);
process.stdout.write(
`Progress: [${progressBar}] ${progress.toFixed(2)}%`
);
}
console.log(); // Pule para a próxima linha após a conclusão
console.log();
controller.close();
} else {
console.log('Dry run, skipping repair');
Expand Down

0 comments on commit e3dc0e1

Please sign in to comment.