Skip to content

Commit

Permalink
refactor: Remove unused code in Indexer
Browse files Browse the repository at this point in the history
  • Loading branch information
@morgsmccauley
morgsmccauley committed Aug 2, 2023
1 parent 013f4dd commit aeaeaef
Show file tree
Hide file tree
Showing 3 changed files with 18 additions and 179 deletions.
4 changes: 2 additions & 2 deletions runner/src/indexer/__snapshots__/indexer.test.ts.snap
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`Indexer unit tests Indexer.buildImperativeContextForFunction() can fetch from the near social api 1`] = `
exports[`Indexer unit tests Indexer.buildContext() can fetch from the near social api 1`] = `
[
[
"https://api.near.social/index",
Expand Down Expand Up @@ -300,7 +300,7 @@ exports[`Indexer unit tests Indexer.runFunctions() supplies the required role to
[
"mock-hasura-endpoint/v1/graphql",
{
"body": "{"query":"mutation { set(functionName: \\"buildnear.testnet/test\\", key: \\"height\\", data: \\"82699904\\")}","variables":{}}",
"body": "{"query":"mutation { set(functionName: \\"buildnear.testnet/test\\", key: \\"height\\", data: \\"82699904\\")}"}",
"headers": {
"Content-Type": "application/json",
"X-Hasura-Admin-Secret": "mock-hasura-secret",
Expand Down
97 changes: 12 additions & 85 deletions runner/src/indexer/indexer.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,79 +60,6 @@ describe('Indexer unit tests', () => {
expect(mockFetch.mock.calls).toMatchSnapshot();
});

test('Indexer.writeMutations() should POST a graphQL mutation from a mutation string', async () => {
const mockFetch = jest.fn(() => ({
status: 200,
json: async () => ({
errors: null,
}),
}));
const indexer = new Indexer('mainnet', { fetch: mockFetch as unknown as typeof fetch });

const functionName = 'buildnear.testnet/test';
const mutations = { mutations: [`mutation { _0: set(functionName: "${functionName}", key: "foo2", data: "indexer test") }`], variables: {}, keysValues: {} };
await indexer.writeMutations(functionName, 'test', mutations, 1, 'morgs_near');

expect(mockFetch).toHaveBeenCalledTimes(1);
expect(mockFetch).toHaveBeenCalledWith(
`${HASURA_ENDPOINT}/v1/graphql`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Hasura-Use-Backend-Only-Permissions': 'true',
'X-Hasura-Role': 'morgs_near',
'X-Hasura-Admin-Secret': HASURA_ADMIN_SECRET
},
body: JSON.stringify({
query: `mutation { _0: set(functionName: "${functionName}", key: "foo2", data: "indexer test") }`,
variables: {}
}),
}
);
});

test('Indexer.writeMutations() should batch multiple mutations in to a single request', async () => {
const mockFetch = jest.fn(() => ({
status: 200,
json: async () => ({
errors: null,
}),
}));
const indexer = new Indexer('mainnet', { fetch: mockFetch as unknown as typeof fetch });

const functionName = 'buildnear.testnet/test';
const mutations = {
mutations: [
`mutation _0 { set(functionName: "${functionName}", key: "foo1", data: "indexer test") }`,
`mutation _1 { set(functionName: "${functionName}", key: "foo2", data: "indexer test") }`
],
variables: {},
keysValues: {}
};
await indexer.writeMutations(functionName, 'test', mutations, 1, 'morgs_near');

expect(mockFetch).toHaveBeenCalledTimes(1);
expect(mockFetch).toHaveBeenCalledWith(
`${HASURA_ENDPOINT}/v1/graphql`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
'X-Hasura-Use-Backend-Only-Permissions': 'true',
'X-Hasura-Role': 'morgs_near',
'X-Hasura-Admin-Secret': HASURA_ADMIN_SECRET
},
body: JSON.stringify({
query:
`mutation _0 { set(functionName: "buildnear.testnet/test", key: "foo1", data: "indexer test") }
mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "indexer test") }`,
variables: {}
}),
}
);
});

test('Indexer.fetchBlock() should fetch a block from the S3', async () => {
const author = 'dokiacapital.poolv1.near';
const mockS3 = {
Expand Down Expand Up @@ -242,7 +169,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
`);
});

test('Indexer.buildImperativeContextForFunction() allows execution of arbitrary GraphQL operations', async () => {
test('Indexer.buildContext() allows execution of arbitrary GraphQL operations', async () => {
const mockFetch = jest.fn()
.mockResolvedValueOnce({
status: 200,
Expand All @@ -264,7 +191,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
});
const indexer = new Indexer('mainnet', { fetch: mockFetch as unknown as typeof fetch });

const context = indexer.buildImperativeContextForFunction('test', 'morgs.near/test', 1, 'morgs_near');
const context = indexer.buildContext('test', 'morgs.near/test', 1, 'morgs_near');

const query = `
query {
Expand Down Expand Up @@ -312,11 +239,11 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
]);
});

test('Indexer.buildImperativeContextForFunction() can fetch from the near social api', async () => {
test('Indexer.buildContext() can fetch from the near social api', async () => {
const mockFetch = jest.fn();
const indexer = new Indexer('mainnet', { fetch: mockFetch as unknown as typeof fetch });

const context = indexer.buildImperativeContextForFunction('test', 'morgs.near/test', 1, 'role');
const context = indexer.buildContext('test', 'morgs.near/test', 1, 'role');

await context.fetchFromSocialApi('/index', {
method: 'POST',
Expand All @@ -336,7 +263,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
expect(mockFetch.mock.calls).toMatchSnapshot();
});

test('Indexer.buildImperativeContextForFunction() throws when a GraphQL response contains errors', async () => {
test('Indexer.buildContext() throws when a GraphQL response contains errors', async () => {
const mockFetch = jest.fn()
.mockResolvedValue({
json: async () => ({
Expand All @@ -345,12 +272,12 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
});
const indexer = new Indexer('mainnet', { fetch: mockFetch as unknown as typeof fetch });

const context = indexer.buildImperativeContextForFunction('test', 'morgs.near/test', 1, 'role');
const context = indexer.buildContext('test', 'morgs.near/test', 1, 'role');

await expect(async () => await context.graphql('query { hello }')).rejects.toThrow('boom');
});

test('Indexer.buildImperativeContextForFunction() handles GraphQL variables', async () => {
test('Indexer.buildContext() handles GraphQL variables', async () => {
const mockFetch = jest.fn()
.mockResolvedValue({
status: 200,
Expand All @@ -360,7 +287,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
});
const indexer = new Indexer('mainnet', { fetch: mockFetch as unknown as typeof fetch });

const context = indexer.buildImperativeContextForFunction('test', 'morgs.near/test', 1, 'morgs_near');
const context = indexer.buildContext('test', 'morgs.near/test', 1, 'morgs_near');

const query = 'query($name: String) { hello(name: $name) }';
const variables = { name: 'morgan' };
Expand Down Expand Up @@ -495,7 +422,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
`
};

await indexer.runFunctions(blockHeight, functions, false, { imperative: true });
await indexer.runFunctions(blockHeight, functions, false);

expect(mockFetch.mock.calls).toMatchSnapshot();
});
Expand Down Expand Up @@ -552,7 +479,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
`
};

await expect(indexer.runFunctions(blockHeight, functions, false, { imperative: true })).rejects.toThrow(new Error('boom'));
await expect(indexer.runFunctions(blockHeight, functions, false)).rejects.toThrow(new Error('boom'));
expect(mockFetch.mock.calls).toMatchSnapshot();
});

Expand Down Expand Up @@ -772,7 +699,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
});
const indexer = new Indexer('mainnet', { fetch: mockFetch as unknown as typeof fetch });
// @ts-expect-error legacy test
const context = indexer.buildImperativeContextForFunction('test', 'morgs.near/test', 1, null);
const context = indexer.buildContext('test', 'morgs.near/test', 1, null);

const mutation = `
mutation {
Expand Down Expand Up @@ -807,7 +734,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in
});
const role = 'morgs_near';
const indexer = new Indexer('mainnet', { fetch: mockFetch as unknown as typeof fetch });
const context = indexer.buildImperativeContextForFunction('test', 'morgs.near/test', 1, role);
const context = indexer.buildContext('test', 'morgs.near/test', 1, role);

const mutation = `
mutation {
Expand Down
96 changes: 4 additions & 92 deletions runner/src/indexer/indexer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,7 @@ interface Dependencies {
provisioner: Provisioner
};

interface MutationsReturnValue {
mutations: string[]
keysValues: Record<string, any>
variables: Record<string, any>
}

interface FunctionalContext {
graphql: (mutation: string, variables: Record<string, any>) => void
set: (key: string, value: any) => void
log: (...log: any[]) => Promise<void>
}

interface ImperativeContext {
interface Context {
graphql: (operation: string, variables?: Record<string, any>) => Promise<any>
set: (key: string, value: any) => Promise<any>
log: (...log: any[]) => Promise<void>
Expand Down Expand Up @@ -61,7 +49,7 @@ export default class Indexer {
blockHeight: number,
functions: Record<string, IndexerFunction>,
isHistorical: boolean,
options: { imperative?: boolean, provision?: boolean } = { imperative: false, provision: false }
options: { provision?: boolean } = { provision: false }
): Promise<string[]> {
const blockWithHelpers = Block.fromStreamerMessage(await this.fetchStreamerMessage(blockHeight));

Expand Down Expand Up @@ -102,15 +90,11 @@ export default class Indexer {
await this.setStatus(functionName, blockHeight, 'RUNNING');

const vm = new VM({ timeout: 3000, allowAsync: true });
const mutationsReturnValue: MutationsReturnValue = { mutations: [], variables: {}, keysValues: {} };
const context = options.imperative
? this.buildImperativeContextForFunction(functionName, functionNameWithoutAccount, blockHeight, hasuraRoleName)
: this.buildFunctionalContextForFunction(mutationsReturnValue, functionName, blockHeight);
const context = this.buildContext(functionName, functionNameWithoutAccount, blockHeight, hasuraRoleName);

vm.freeze(blockWithHelpers, 'block');
vm.freeze(context, 'context');
vm.freeze(context, 'console'); // provide console.log via context.log
vm.freeze(mutationsReturnValue, 'mutationsReturnValue'); // this still allows context.set to modify it

const modifiedFunction = this.transformIndexerFunction(indexerFunction.code);
try {
Expand All @@ -125,14 +109,6 @@ export default class Indexer {
throw e;
}

if (!options.imperative) {
console.log(`Function ${functionName} returned`, mutationsReturnValue); // debug output
const writtenMutations = await this.writeMutations(functionName, functionNameWithoutAccount, mutationsReturnValue, blockHeight, hasuraRoleName); // await can be dropped once it's all tested so writes can happen in parallel
if ((writtenMutations != null) && writtenMutations.length > 0) {
allMutations.push(...writtenMutations);
}
}

simultaneousPromises.push(this.writeFunctionState(functionName, blockHeight, isHistorical));
} catch (e) {
console.error(`${functionName}: Failed to run function`, e);
Expand All @@ -145,54 +121,6 @@ export default class Indexer {
return allMutations;
}

buildKeyValueMutations (
hasuraRoleName: string,
functionNameWithoutAccount: string,
keysValues: Record<string, string>
): string {
if (Object.keys(keysValues).length === 0) return '';

return `mutation writeKeyValues($function_name: String!, ${Object.keys(keysValues).map((_key, index) => `$key_name${index}: String!, $value${index}: String!`).join(', ')}) {
${Object.keys(keysValues).map((_key, index) => `_${index}: insert_${hasuraRoleName}_${functionNameWithoutAccount}_indexer_storage_one(object: {function_name: $function_name, key_name: $key_name${index}, value: $value${index}} on_conflict: {constraint: indexer_storage_pkey, update_columns: value}) {key_name}`).join('\n')}
}`;
}

buildKeyValueVariables (
functionName: string,
keysValues: Record<string, any>
): Record<string, any> {
if (Object.keys(keysValues).length === 0) return {};

return Object.keys(keysValues).reduce((acc: Record<string, any>, key, index) => {
acc[`key_name${index.toString()}`] = key;
acc[`value${index.toString()}`] = keysValues[key] ? JSON.stringify(keysValues[key]) : null;
return acc;
}, { function_name: functionName });
}

async writeMutations (
functionName: string,
functionNameWithoutAccount: string,
mutationsReturnValue: MutationsReturnValue,
blockHeight: number,
hasuraRoleName: string
): Promise<string[] | undefined> {
if (mutationsReturnValue?.mutations.length === 0 && Object.keys(mutationsReturnValue?.keysValues).length === 0) return undefined;
try {
const keyValuesMutations = this.buildKeyValueMutations(hasuraRoleName, functionNameWithoutAccount, mutationsReturnValue.keysValues);
const allMutations = mutationsReturnValue.mutations.join('\n') + keyValuesMutations;
const variablesPlusKeyValues = { ...mutationsReturnValue.variables, ...this.buildKeyValueVariables(functionName, mutationsReturnValue.keysValues) };

console.log('Writing mutations for function: ' + functionName, allMutations, variablesPlusKeyValues); // debug output
await this.runGraphQLQuery(allMutations, variablesPlusKeyValues, functionName, blockHeight, hasuraRoleName);

return keyValuesMutations.length > 0 ? mutationsReturnValue.mutations.concat(keyValuesMutations) : mutationsReturnValue.mutations;
} catch (e) {
console.error(`${functionName}: Failed to write mutations for function`, e);
return undefined;
}
}

// pad with 0s to 12 digits
normalizeBlockHeight (blockHeight: number): string {
return blockHeight.toString().padStart(12, '0');
Expand Down Expand Up @@ -255,23 +183,7 @@ export default class Indexer {
].reduce((acc, val) => val(acc), indexerFunction);
}

buildFunctionalContextForFunction (mutationsReturnValue: MutationsReturnValue, functionName: string, blockHeight: number): FunctionalContext {
return {
graphql: (mutation, variables) => {
console.log({ mutation });
mutationsReturnValue.mutations.push(mutation);
mutationsReturnValue.variables = Object.assign(mutationsReturnValue.variables, variables);
},
set: (key, value) => {
mutationsReturnValue.keysValues[key] = value;
},
log: async (...log) => {
return await this.writeLog(functionName, blockHeight, ...log);
},
};
}

buildImperativeContextForFunction (functionName: string, functionNameWithoutAccount: string, blockHeight: number, hasuraRoleName: string): ImperativeContext {
buildContext (functionName: string, functionNameWithoutAccount: string, blockHeight: number, hasuraRoleName: string): Context {
return {
graphql: async (operation, variables) => {
console.log(`${functionName}: Running context graphql`, operation);
Expand Down

0 comments on commit aeaeaef

Please sign in to comment.