Skip to content

Commit

Permalink
Address comments in PR
Browse files Browse the repository at this point in the history
  • Loading branch information
@darunrs
darunrs committed Aug 29, 2023
1 parent 6ec6489 commit 67908d3
Show file tree
Hide file tree
Showing 3 changed files with 146 additions and 112 deletions.
102 changes: 53 additions & 49 deletions frontend/src/utils/indexerRunner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,39 +50,8 @@ export default class IndexerRunner {
return new Promise((resolve) => setTimeout(resolve, ms));
}

validateTableNames(tableNames) {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error("Schema does not have any tables. There should be at least one table.");
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!name.includes("\"") && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema) {
const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS)?\s+"?(.+?)"?\s*\(/g;
const tableNames = Array.from(schema.matchAll(tableRegex), match => {
let tableName;
if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName
tableName = match[1].split('.')[1];
tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName;
} else {
tableName = match[1];
}
return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes
});
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

async executeIndexerFunction(height, blockDetails, indexingCode, schema, schemaName) {
let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1];
const tableNames = this.getTableNames(schema);

if (blockDetails) {
const block = Block.fromStreamerMessage(blockDetails);
Expand All @@ -91,7 +60,7 @@ export default class IndexerRunner {
block.events()

console.log(block)
await this.runFunction(blockDetails, height, innerCode, schemaName, tableNames);
await this.runFunction(blockDetails, height, innerCode, schemaName, schema);
}
}

Expand Down Expand Up @@ -119,7 +88,7 @@ export default class IndexerRunner {
console.groupEnd()
}

async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, tableNames) {
async runFunction(streamerMessage, blockHeight, indexerCode, schemaName, schema) {
const innerCodeWithBlockHelper =
`
const block = Block.fromStreamerMessage(streamerMessage);
Expand Down Expand Up @@ -178,30 +147,65 @@ export default class IndexerRunner {
log: async (message) => {
this.handleLog(blockHeight, message);
},
db: this.buildDatabaseContext(blockHeight, schemaName, tableNames)
db: this.buildDatabaseContext(blockHeight, schemaName, schema)
};

wrappedFunction(Block, streamerMessage, context);
}

buildDatabaseContext (blockHeight, schemaName, tables) {
validateTableNames(tableNames) {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error("Schema does not have any tables. There should be at least one table.");
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!name.includes("\"") && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema) {
const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g;
const tableNames = Array.from(schema.matchAll(tableRegex), match => {
let tableName;
if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName
tableName = match[1].split('.')[1];
tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName;
} else {
tableName = match[1];
}
return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes
});
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

sanitizeTableName (tableName) {
tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName;
return tableName.replace(/[^a-zA-Z0-9_]/g, '_');
}

buildDatabaseContext (blockHeight, schemaName, schema) {
try {
const result = tables.reduce((prev, tableName) => ({
...prev,
[`insert_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects),
[`select_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit),
}), {});
const tables = this.getTableNames(schema);
const result = tables.reduce((prev, tableName) => {
const sanitizedTableName = this.sanitizeTableName(tableName);
const funcForTable = {
[`insert_${sanitizedTableName}`]: async (objects) => await this.insert(blockHeight, schemaName, tableName, objects),
[`select_${sanitizedTableName}`]: async (object, limit = 0) => await this.select(blockHeight, schemaName, tableName, object, limit)
};

return {
...prev,
...funcForTable
};
}, {});
return result;
} catch (error) {
console.error('Caught error when generating DB methods. Falling back to generic methods.', error);
return {
insert: async (tableName, objects) => {
this.insert(blockHeight, schemaName, tableName, objects);
},
select: async (tableName, object, limit = 0) => {
this.select(blockHeight, schemaName, tableName, object, limit);
}
};
console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error);
}
}

Expand Down
34 changes: 23 additions & 11 deletions runner/src/indexer/indexer.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,7 @@ describe('Indexer unit tests', () => {
);`;

const STRESS_TEST_SCHEMA = `
CREATE TABLE
creator_quest (
CREATE TABLE creator_quest (
account_id VARCHAR PRIMARY KEY,
num_components_created INTEGER NOT NULL DEFAULT 0,
completed BOOLEAN NOT NULL DEFAULT FALSE
Expand Down Expand Up @@ -519,30 +518,43 @@ CREATE TABLE
const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler });
const context = indexer.buildContext(STRESS_TEST_SCHEMA, 'morgs.near/social_feed1', 1, 'postgres');

// These calls would fail on a real database, but we are merely checking to ensure they exist
expect(Object.keys(context.db)).toStrictEqual(
['insert_creator_quest',
'select_creator_quest',
'insert_composer_quest',
'select_composer_quest',
'insert__contractor___quest_',
'select__contractor___quest_',
'insert_contractor___quest',
'select_contractor___quest',
'insert_posts',
'select_posts',
'insert_comments',
'select_comments',
'insert_post_likes',
'select_post_likes',
'insert__My_Table1_',
'select__My_Table1_',
'insert__Another_Table_',
'select__Another_Table_',
'insert__Third_Table_',
'select__Third_Table_',
'insert_My_Table1',
'select_My_Table1',
'insert_Another_Table',
'select_Another_Table',
'insert_Third_Table',
'select_Third_Table',
'insert_yet_another_table',
'select_yet_another_table']);
});

test('indexer builds context and returns empty array if failed to generate db methods', async () => {
const mockDmlHandler: any = jest.fn().mockImplementation(() => {
return {
insert: jest.fn().mockReturnValue(true),
select: jest.fn().mockReturnValue(true)
};
});

const indexer = new Indexer('mainnet', { DmlHandler: mockDmlHandler });
const context = indexer.buildContext('', 'morgs.near/social_feed1', 1, 'postgres');

expect(Object.keys(context.db)).toStrictEqual([]);
});

test('Indexer.runFunctions() allows imperative execution of GraphQL operations', async () => {
const postId = 1;
const commentId = 2;
Expand Down
122 changes: 70 additions & 52 deletions runner/src/indexer/indexer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -186,38 +186,7 @@ export default class Indexer {
].reduce((acc, val) => val(acc), indexerFunction);
}

validateTableNames (tableNames: string[]): void {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error('Schema does not have any tables. There should be at least one table.');
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!name.includes('"') && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema: string): string[] {
const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS)?\s+"?(.+?)"?\s*\(/g;
const tableNames = Array.from(schema.matchAll(tableRegex), match => {
let tableName;
if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName
tableName = match[1].split('.')[1];
tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName;
} else {
tableName = match[1];
}
return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes
});
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

buildContext (schema: string, functionName: string, blockHeight: number, hasuraRoleName: string): Context {
const tables = this.getTableNames(schema);
const account = functionName.split('/')[0].replace(/[.-]/g, '_');
const functionNameWithoutAccount = functionName.split('/')[1].replace(/[.-]/g, '_');
const schemaName = functionName.replace(/[^a-zA-Z0-9]/g, '_');
Expand Down Expand Up @@ -246,30 +215,79 @@ export default class Indexer {
fetchFromSocialApi: async (path, options) => {
return await this.deps.fetch(`https://api.near.social${path}`, options);
},
db: this.buildDatabaseContext(account, schemaName, tables, blockHeight)
db: this.buildDatabaseContext(account, schemaName, schema, blockHeight)
};
}

buildDatabaseContext (account: string, schemaName: string, tables: string[], blockHeight: number): Record<string, (...args: any[]) => any> {
let dmlHandler: DmlHandler | null = null;
const result = tables.reduce((prev, tableName) => ({
...prev,
[`insert_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`]: async (objects: any) => {
await this.writeLog(`context.db.insert_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`, blockHeight,
`Calling context.db.insert_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}.`,
`Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]);
},
[`select_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`]: async (object: any, limit = null) => {
await this.writeLog(`context.db.select_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}`, blockHeight,
`Calling context.db.select_${tableName.replace(/[^a-zA-Z0-9_]/g, '_')}.`,
`Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.select(schemaName, tableName, object, limit);
},
}), {});
return result;
validateTableNames (tableNames: string[]): void {
if (!(Array.isArray(tableNames) && tableNames.length > 0)) {
throw new Error('Schema does not have any tables. There should be at least one table.');
}
const correctTableNameFormat = /^[a-zA-Z_][a-zA-Z0-9_]*$/;

tableNames.forEach(name => {
if (!name.includes('"') && !correctTableNameFormat.test(name)) { // Only test if table name doesn't have quotes
throw new Error(`Table name ${name} is not formatted correctly. Table names must not start with a number and only contain alphanumerics or underscores.`);
}
});
}

getTableNames (schema: string): string[] {
const tableRegex = /CREATE TABLE\s+(?:IF NOT EXISTS\s+)?"?(.+?)"?\s*\(/g;
const tableNames = Array.from(schema.matchAll(tableRegex), match => {
let tableName;
if (match[1].includes('.')) { // If expression after create has schemaName.tableName, return only tableName
tableName = match[1].split('.')[1];
tableName = tableName.startsWith('"') ? tableName.substring(1) : tableName;
} else {
tableName = match[1];
}
return /^\w+$/.test(tableName) ? tableName : `"${tableName}"`; // If table name has special characters, it must be inside double quotes
});
this.validateTableNames(tableNames);
console.log('Retrieved the following table names from schema: ', tableNames);
return tableNames;
}

sanitizeTableName (tableName: string): string {
tableName = tableName.startsWith('"') && tableName.endsWith('"') ? tableName.substring(1, tableName.length - 1) : tableName;
return tableName.replace(/[^a-zA-Z0-9_]/g, '_');
}

buildDatabaseContext (account: string, schemaName: string, schema: string, blockHeight: number): Record<string, (...args: any[]) => any> {
try {
const tables = this.getTableNames(schema);
let dmlHandler: DmlHandler | null = null;
const result = tables.reduce((prev, tableName) => {
const sanitizedTableName = this.sanitizeTableName(tableName);
const funcForTable = {
[`insert_${sanitizedTableName}`]: async (objects: any) => {
await this.writeLog(`context.db.insert_${sanitizedTableName}`, blockHeight,
`Calling context.db.insert_${sanitizedTableName}.`,
`Inserting object ${JSON.stringify(objects)} into table ${tableName} on schema ${schemaName}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.insert(schemaName, tableName, Array.isArray(objects) ? objects : [objects]);
},
[`select_${sanitizedTableName}`]: async (object: any, limit = null) => {
await this.writeLog(`context.db.select_${sanitizedTableName}`, blockHeight,
`Calling context.db.select_${sanitizedTableName}.`,
`Selecting objects with values ${JSON.stringify(object)} from table ${tableName} on schema ${schemaName} with limit ${limit === null ? 'no' : limit}`);
dmlHandler = dmlHandler ?? new this.deps.DmlHandler(account);
return await dmlHandler.select(schemaName, tableName, object, limit);
}
};

return {
...prev,
...funcForTable
};
}, {});
return result;
} catch (error) {
console.warn('Caught error when generating context.db methods. Building no functions. You can still use other context object methods.\n', error);
}

return {}; // Default to empty object if error
}

async setStatus (functionName: string, blockHeight: number, status: string): Promise<any> {
Expand Down

0 comments on commit 67908d3

Please sign in to comment.