diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index 19f32e2a..3e1891b1 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -45,8 +45,8 @@ interface Config { } const defaultConfig: Config = { - hasuraAdminSecret: process.env.HASURA_ADMIN_SECRET = '', - hasuraEndpoint: process.env.HASURA_ENDPOINT = '', + hasuraAdminSecret: process.env.HASURA_ADMIN_SECRET ?? '', + hasuraEndpoint: process.env.HASURA_ENDPOINT ?? '', }; export default class Indexer { diff --git a/runner/src/provisioner/provisioner.test.ts b/runner/src/provisioner/provisioner.test.ts index 90d93218..8ef12371 100644 --- a/runner/src/provisioner/provisioner.test.ts +++ b/runner/src/provisioner/provisioner.test.ts @@ -17,6 +17,10 @@ describe('Provisioner', () => { const functionName = 'test-function'; const databaseSchema = 'CREATE TABLE blocks (height numeric)'; indexerConfig = new IndexerConfig('', accountId, functionName, 0, '', databaseSchema, LogLevel.INFO); + const testingRetryConfig = { + maxRetries: 5, + baseDelay: 10 + }; const setProvisioningStatusQuery = `INSERT INTO ${indexerConfig.schemaName()}.sys_metadata (attribute, value) VALUES ('STATUS', 'PROVISIONING') ON CONFLICT (attribute) DO UPDATE SET value = EXCLUDED.value RETURNING *`; const logsDDL = expect.any(String); const metadataDDL = expect.any(String); @@ -68,7 +72,7 @@ describe('Provisioner', () => { }; }); - provisioner = new Provisioner(hasuraClient, adminPgClient, cronPgClient, undefined, crypto, pgFormat, PgClient as any); + provisioner = new Provisioner(hasuraClient, adminPgClient, cronPgClient, undefined, crypto, pgFormat, PgClient as any, testingRetryConfig); indexerConfig = new IndexerConfig('', accountId, functionName, 0, '', databaseSchema, LogLevel.INFO); }); @@ -318,12 +322,14 @@ describe('Provisioner', () => { hasuraClient.trackForeignKeyRelationships = jest.fn().mockRejectedValue(error); await expect(provisioner.provisionUserApi(indexerConfig)).rejects.toThrow('Failed to provision endpoint: Failed to track foreign key relationships: some error'); + expect(hasuraClient.trackForeignKeyRelationships).toHaveBeenCalledTimes(testingRetryConfig.maxRetries); }); it('throws an error when it fails to add permissions to tables', async () => { hasuraClient.addPermissionsToTables = jest.fn().mockRejectedValue(error); await expect(provisioner.provisionUserApi(indexerConfig)).rejects.toThrow('Failed to provision endpoint: Failed to add permissions to tables: some error'); + expect(hasuraClient.addPermissionsToTables).toHaveBeenCalledTimes(testingRetryConfig.maxRetries); }); it('throws when grant cron access fails', async () => { diff --git a/runner/src/provisioner/provisioner.ts b/runner/src/provisioner/provisioner.ts index 5533c677..f9d4e511 100644 --- a/runner/src/provisioner/provisioner.ts +++ b/runner/src/provisioner/provisioner.ts @@ -39,6 +39,11 @@ interface Config { postgresPort: number } +interface RetryConfig { + maxRetries: number + baseDelay: number +} + const defaultConfig: Config = { cronDatabase: process.env.CRON_DATABASE, pgBouncerHost: process.env.PGHOST_PGBOUNCER ?? process.env.PGHOST, @@ -47,6 +52,11 @@ const defaultConfig: Config = { postgresPort: Number(process.env.PGPORT) }; +const defaultRetryConfig: RetryConfig = { + maxRetries: 5, + baseDelay: 1000 +}; + export default class Provisioner { tracer: Tracer = trace.getTracer('queryapi-runner-provisioner'); @@ -57,7 +67,8 @@ export default class Provisioner { private readonly config: Config = defaultConfig, private readonly crypto: typeof cryptoModule = cryptoModule, private readonly pgFormat: typeof pgFormatLib = pgFormatLib, - private readonly PgClient: typeof PgClientClass = PgClientClass + private readonly PgClient: typeof PgClientClass = PgClientClass, + private readonly retryConfig: RetryConfig = defaultRetryConfig, ) {} generatePassword (length: number = DEFAULT_PASSWORD_LENGTH): string { @@ -336,15 +347,33 @@ export default class Provisioner { await this.trackTables(schemaName, updatedTableNames, databaseName); - await this.trackForeignKeyRelationships(schemaName, databaseName); + await this.exponentialRetry(async () => { + await this.trackForeignKeyRelationships(schemaName, databaseName); + }); - await this.addPermissionsToTables(indexerConfig, updatedTableNames, ['select', 'insert', 'update', 'delete']); + await this.exponentialRetry(async () => { + await this.addPermissionsToTables(indexerConfig, updatedTableNames, ['select', 'insert', 'update', 'delete']); + }); }, 'Failed to provision endpoint' ); }, this.tracer, 'provision indexer resources'); } + async exponentialRetry (fn: () => Promise): Promise { + let lastError = null; + for (let i = 0; i < this.retryConfig.maxRetries; i++) { + try { + await fn(); + return; + } catch (e) { + lastError = e; + await new Promise((resolve) => setTimeout(resolve, this.retryConfig.baseDelay * (2 ** i))); + } + } + throw lastError; + } + async getPostgresConnectionParameters (userName: string): Promise { const userDbConnectionParameters: HasuraDatabaseConnectionParameters = await this.hasuraClient.getDbConnectionParameters(userName); return {