From 67908d3ea2beaf48aec4cd946da59ac1752ed0c9 Mon Sep 17 00:00:00 2001 From: Darun Seethammagari Date: Tue, 29 Aug 2023 15:45:33 -0700 Subject: [PATCH] Address comments in PR --- frontend/src/utils/indexerRunner.js | 102 ++++++++++++----------- runner/src/indexer/indexer.test.ts | 34 +++++--- runner/src/indexer/indexer.ts | 122 ++++++++++++++++------------ 3 files changed, 146 insertions(+), 112 deletions(-) diff --git a/frontend/src/utils/indexerRunner.js b/frontend/src/utils/indexerRunner.js index 1ec587ad4..6ca8dfa88 100644 --- a/frontend/src/utils/indexerRunner.js +++ b/frontend/src/utils/indexerRunner.js @@ -50,39 +50,8 @@ export default class IndexerRunner { return new Promise((resolve) => setTimeout(resolve, ms)); } - validateTableNames(tableNames) { - if (!(Array.isArray(tableNames) && tableNames.length > 0)) { - throw new Error("Schema does not have any tables. There should be at least one table."); - } - const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - - tableNames.forEach(name => { - if (!name.includes("\"") && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes - throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); - } - }); - } - - getTableNames (schema) { - const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS)?\s+"?(.+?)"?\s*\(/g; - const tableNames = Array.from(schema.matchAll(tableRegex), match => { - let tableName; - if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName - tableName = match[1].split('.')[1]; - tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName; - } else { - tableName = match[1]; - } - return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes - }); - this.validateTableNames(tableNames); - console.log('Retrieved the following table names from schema: ', tableNames); - return tableNames; - } - async executeIndexerFunction(height, blockDetails, indexingCode, schema, schemaName) { let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1]; - const tableNames = this.getTableNames(schema); if (blockDetails) { const block = Block.fromStreamerMessage(blockDetails); @@ -91,7 +60,7 @@ export default class IndexerRunner { block.events() console.log(block) - await this.runFunction(blockDetails, height, innerCode, schemaName, tableNames); + await this.runFunction(blockDetails, height, innerCode, schemaName, schema); } } @@ -119,7 +88,7 @@ export default class IndexerRunner { console.groupEnd() } - async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, tableNames) { + async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, schema) { const innerCodeWithBlockHelper = ` const block = Block.fromStreamerMessage(streamerMessage); @@ -178,30 +147,65 @@ export default class IndexerRunner { log: async (message) => { this.handleLog(blockHeight, message); }, - db: this.buildDatabaseContext(blockHeight, schemaName, tableNames) + db: this.buildDatabaseContext(blockHeight, schemaName, schema) }; wrappedFunction(Block, streamerMessage, context); } - buildDatabaseContext (blockHeight, schemaName, tables) { + validateTableNames(tableNames) { + if (!(Array.isArray(tableNames) && tableNames.length > 0)) { + throw new Error("Schema does not have any tables. There should be at least one table."); + } + const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + + tableNames.forEach(name => { + if (!name.includes("\"") && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes + throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); + } + }); + } + + getTableNames (schema) { + const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g; + const tableNames = Array.from(schema.matchAll(tableRegex), match => { + let tableName; + if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName + tableName = match[1].split('.')[1]; + tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName; + } else { + tableName = match[1]; + } + return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes + }); + this.validateTableNames(tableNames); + console.log('Retrieved the following table names from schema: ', tableNames); + return tableNames; + } + + sanitizeTableName (tableName) { + tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName; + return tableName.replace(/[^a-zA-Z0-9_]/g, '_'); + } + + buildDatabaseContext (blockHeight, schemaName, schema) { try { - const result = tables.reduce((prev, tableName) => ({ - ...prev, - [`insert_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects), - [`select_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit), - }), {}); + const tables = this.getTableNames(schema); + const result = tables.reduce((prev, tableName) => { + const sanitizedTableName = this.sanitizeTableName(tableName); + const funcForTable = { + [`insert_${sanitizedTableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects), + [`select_${sanitizedTableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit) + }; + + return { + ...prev, + ...funcForTable + }; + }, {}); return result; } catch (error) { - console.error('Caught error when generating DB methods. Falling back to generic methods.', error); - return { - insert: async (tableName, objects) => { - this.insert(blockHeight, schemaName, tableName, objects); - }, - select: async (tableName, object, limit = 0) => { - this.select(blockHeight, schemaName, tableName, object, limit); - } - }; + console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error); } } diff --git a/runner/src/indexer/indexer.test.ts b/runner/src/indexer/indexer.test.ts index d51da2709..f2c16b9e8 100644 --- a/runner/src/indexer/indexer.test.ts +++ b/runner/src/indexer/indexer.test.ts @@ -65,8 +65,7 @@ describe('Indexer unit tests', () => { );`; const STRESS_TEST_SCHEMA = ` -CREATE TABLE - creator_quest ( +CREATE TABLE creator_quest ( account_id VARCHAR PRIMARY KEY, num_components_created INTEGER NOT NULL DEFAULT 0, completed BOOLEAN NOT NULL DEFAULT FALSE @@ -519,30 +518,43 @@ CREATE TABLE const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler }); const context = indexer.buildContext(STRESS_TEST_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres'); - // These calls would fail on a real database, but we are merely checking to ensure they exist expect(Object.keys(context.db)).toStrictEqual( ['insert_creator_quest', 'select_creator_quest', 'insert_composer_quest', 'select_composer_quest', - 'insert__contractor___quest_', - 'select__contractor___quest_', + 'insert_contractor___quest', + 'select_contractor___quest', 'insert_posts', 'select_posts', 'insert_comments', 'select_comments', 'insert_post_likes', 'select_post_likes', - 'insert__My_Table1_', - 'select__My_Table1_', - 'insert__Another_Table_', - 'select__Another_Table_', - 'insert__Third_Table_', - 'select__Third_Table_', + 'insert_My_Table1', + 'select_My_Table1', + 'insert_Another_Table', + 'select_Another_Table', + 'insert_Third_Table', + 'select_Third_Table', 'insert_yet_another_table', 'select_yet_another_table']); }); + test('indexer builds context and returns empty array if failed to generate db methods', async () => { + const mockDmlHandler: any = jest.fn().mockImplementation(() => { + return { + insert: jest.fn().mockReturnValue(true), + select: jest.fn().mockReturnValue(true) + }; + }); + + const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler }); + const context = indexer.buildContext('', 'morgs.near/social_feed1', 1, 'postgres'); + + expect(Object.keys(context.db)).toStrictEqual([]); + }); + test('Indexer.runFunctions() allows imperative execution of GraphQL operations', async () => { const postId = 1; const commentId = 2; diff --git a/runner/src/indexer/indexer.ts b/runner/src/indexer/indexer.ts index 36f1bca4a..fe825bcd3 100644 --- a/runner/src/indexer/indexer.ts +++ b/runner/src/indexer/indexer.ts @@ -186,38 +186,7 @@ export default class Indexer { ].reduce((acc, val) => val(acc), indexerFunction); } - validateTableNames (tableNames: string[]): void { - if (!(Array.isArray(tableNames) && tableNames.length > 0)) { - throw new Error('Schema does not have any tables. There should be at least one table.'); - } - const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; - - tableNames.forEach(name => { - if (!name.includes('"') && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes - throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); - } - }); - } - - getTableNames (schema: string): string[] { - const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS)?\s+"?(.+?)"?\s*\(/g; - const tableNames = Array.from(schema.matchAll(tableRegex), match => { - let tableName; - if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName - tableName = match[1].split('.')[1]; - tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName; - } else { - tableName = match[1]; - } - return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes - }); - this.validateTableNames(tableNames); - console.log('Retrieved the following table names from schema: ', tableNames); - return tableNames; - } - buildContext (schema: string, functionName: string, blockHeight: number, hasuraRoleName: string): Context { - const tables = this.getTableNames(schema); const account = functionName.split('/')[0].replace(/[.-]/g, '_'); const functionNameWithoutAccount = functionName.split('/')[1].replace(/[.-]/g, '_'); const schemaName = functionName.replace(/[^a-zA-Z0-9]/g, '_'); @@ -246,30 +215,79 @@ export default class Indexer { fetchFromSocialApi: async (path, options) => { return await this.deps.fetch(`https://api.near.social${path}`, options); }, - db: this.buildDatabaseContext(account, schemaName, tables, blockHeight) + db: this.buildDatabaseContext(account, schemaName, schema, blockHeight) }; } - buildDatabaseContext (account: string, schemaName: string, tables: string[], blockHeight: number): Record any> { - let dmlHandler: DmlHandler | null = null; - const result = tables.reduce((prev, tableName) => ({ - ...prev, - [`insert_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`]: async (objects: any) => { - await this.writeLog(`context.db.insert_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`, blockHeight, - `Calling context.db.insert_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}.`, - `Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`); - dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); - return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]); - }, - [`select_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`]: async (object: any, limit = null) => { - await this.writeLog(`context.db.select_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`, blockHeight, - `Calling context.db.select_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}.`, - `Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`); - dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); - return await dmlHandler.select(schemaName, tableName, object, limit); - }, - }), {}); - return result; + validateTableNames (tableNames: string[]): void { + if (!(Array.isArray(tableNames) && tableNames.length > 0)) { + throw new Error('Schema does not have any tables. There should be at least one table.'); + } + const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/; + + tableNames.forEach(name => { + if (!name.includes('"') && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes + throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`); + } + }); + } + + getTableNames (schema: string): string[] { + const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g; + const tableNames = Array.from(schema.matchAll(tableRegex), match => { + let tableName; + if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName + tableName = match[1].split('.')[1]; + tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName; + } else { + tableName = match[1]; + } + return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes + }); + this.validateTableNames(tableNames); + console.log('Retrieved the following table names from schema: ', tableNames); + return tableNames; + } + + sanitizeTableName (tableName: string): string { + tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName; + return tableName.replace(/[^a-zA-Z0-9_]/g, '_'); + } + + buildDatabaseContext (account: string, schemaName: string, schema: string, blockHeight: number): Record any> { + try { + const tables = this.getTableNames(schema); + let dmlHandler: DmlHandler | null = null; + const result = tables.reduce((prev, tableName) => { + const sanitizedTableName = this.sanitizeTableName(tableName); + const funcForTable = { + [`insert_${sanitizedTableName}`]: async (objects: any) => { + await this.writeLog(`context.db.insert_${sanitizedTableName}`, blockHeight, + `Calling context.db.insert_${sanitizedTableName}.`, + `Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`); + dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); + return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]); + }, + [`select_${sanitizedTableName}`]: async (object: any, limit = null) => { + await this.writeLog(`context.db.select_${sanitizedTableName}`, blockHeight, + `Calling context.db.select_${sanitizedTableName}.`, + `Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`); + dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account); + return await dmlHandler.select(schemaName, tableName, object, limit); + } + }; + + return { + ...prev, + ...funcForTable + }; + }, {}); + return result; + } catch (error) { + console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error); + } + + return {}; // Default to empty object if error } async setStatus (functionName: string, blockHeight: number, status: string): Promise {