fix: Implement retry on tracking foreign keys and resolve hasura env …

…issue (#919)

This PR adds exponential retry for the tracking of foreign keys. For now
I'm keeping the implementation of expo retry in provisioner light for
two reasons:
1. I want to confirm that expo retry actually solves the issue.
2. I plan to refactor the provisioning process entirely. 

I also noticed a bug where the env variables were being set to empty in
all cases instead of only if null. I fix this bug as well.

runner/src/indexer/indexer.ts

@@ -45,8 +45,8 @@ interface Config {
 }
 
 const defaultConfig: Config = {
-  hasuraAdminSecret: process.env.HASURA_ADMIN_SECRET = '',
-  hasuraEndpoint: process.env.HASURA_ENDPOINT = '',
+  hasuraAdminSecret: process.env.HASURA_ADMIN_SECRET ?? '',
+  hasuraEndpoint: process.env.HASURA_ENDPOINT ?? '',
 };
 
 export default class Indexer {

runner/src/provisioner/provisioner.test.ts

@@ -17,6 +17,10 @@ describe('Provisioner', () => {
   const functionName = 'test-function';
   const databaseSchema = 'CREATE TABLE blocks (height numeric)';
   indexerConfig = new IndexerConfig('', accountId, functionName, 0, '', databaseSchema, LogLevel.INFO);
+  const testingRetryConfig = {
+    maxRetries: 5,
+    baseDelay: 10
+  };
   const setProvisioningStatusQuery = `INSERT INTO ${indexerConfig.schemaName()}.sys_metadata (attribute, value) VALUES ('STATUS', 'PROVISIONING') ON CONFLICT (attribute) DO UPDATE SET value = EXCLUDED.value RETURNING *`;
   const logsDDL = expect.any(String);
   const metadataDDL = expect.any(String);
@@ -68,7 +72,7 @@ describe('Provisioner', () => {
       };
     });
 
-    provisioner = new Provisioner(hasuraClient, adminPgClient, cronPgClient, undefined, crypto, pgFormat, PgClient as any);
+    provisioner = new Provisioner(hasuraClient, adminPgClient, cronPgClient, undefined, crypto, pgFormat, PgClient as any, testingRetryConfig);
 
     indexerConfig = new IndexerConfig('', accountId, functionName, 0, '', databaseSchema, LogLevel.INFO);
   });
@@ -318,12 +322,14 @@ describe('Provisioner', () => {
       hasuraClient.trackForeignKeyRelationships = jest.fn().mockRejectedValue(error);
 
       await expect(provisioner.provisionUserApi(indexerConfig)).rejects.toThrow('Failed to provision endpoint: Failed to track foreign key relationships: some error');
+      expect(hasuraClient.trackForeignKeyRelationships).toHaveBeenCalledTimes(testingRetryConfig.maxRetries);
     });
 
     it('throws an error when it fails to add permissions to tables', async () => {
       hasuraClient.addPermissionsToTables = jest.fn().mockRejectedValue(error);
 
       await expect(provisioner.provisionUserApi(indexerConfig)).rejects.toThrow('Failed to provision endpoint: Failed to add permissions to tables: some error');
+      expect(hasuraClient.addPermissionsToTables).toHaveBeenCalledTimes(testingRetryConfig.maxRetries);
     });
 
     it('throws when grant cron access fails', async () => {

runner/src/provisioner/provisioner.ts

@@ -39,6 +39,11 @@ interface Config {
   postgresPort: number
 }
 
+interface RetryConfig {
+  maxRetries: number
+  baseDelay: number
+}
+
 const defaultConfig: Config = {
   cronDatabase: process.env.CRON_DATABASE,
   pgBouncerHost: process.env.PGHOST_PGBOUNCER ?? process.env.PGHOST,
@@ -47,6 +52,11 @@ const defaultConfig: Config = {
   postgresPort: Number(process.env.PGPORT)
 };
 
+const defaultRetryConfig: RetryConfig = {
+  maxRetries: 5,
+  baseDelay: 1000
+};
+
 export default class Provisioner {
   tracer: Tracer = trace.getTracer('queryapi-runner-provisioner');
 
@@ -57,7 +67,8 @@ export default class Provisioner {
     private readonly config: Config = defaultConfig,
     private readonly crypto: typeof cryptoModule = cryptoModule,
     private readonly pgFormat: typeof pgFormatLib = pgFormatLib,
-    private readonly PgClient: typeof PgClientClass = PgClientClass
+    private readonly PgClient: typeof PgClientClass = PgClientClass,
+    private readonly retryConfig: RetryConfig = defaultRetryConfig,
   ) {}
 
   generatePassword (length: number = DEFAULT_PASSWORD_LENGTH): string {
@@ -336,15 +347,33 @@ export default class Provisioner {
 
           await this.trackTables(schemaName, updatedTableNames, databaseName);
 
-          await this.trackForeignKeyRelationships(schemaName, databaseName);
+          await this.exponentialRetry(async () => {
+            await this.trackForeignKeyRelationships(schemaName, databaseName);
+          });
 
-          await this.addPermissionsToTables(indexerConfig, updatedTableNames, ['select', 'insert', 'update', 'delete']);
+          await this.exponentialRetry(async () => {
+            await this.addPermissionsToTables(indexerConfig, updatedTableNames, ['select', 'insert', 'update', 'delete']);
+          });
         },
         'Failed to provision endpoint'
       );
     }, this.tracer, 'provision indexer resources');
   }
 
+  async exponentialRetry (fn: () => Promise<void>): Promise<void> {
+    let lastError = null;
+    for (let i = 0; i < this.retryConfig.maxRetries; i++) {
+      try {
+        await fn();
+        return;
+      } catch (e) {
+        lastError = e;
+        await new Promise((resolve) => setTimeout(resolve, this.retryConfig.baseDelay * (2 ** i)));
+      }
+    }
+    throw lastError;
+  }
+
   async getPostgresConnectionParameters (userName: string): Promise<PostgresConnectionParams> {
     const userDbConnectionParameters: HasuraDatabaseConnectionParameters = await this.hasuraClient.getDbConnectionParameters(userName);
     return {