Merge branch 'master' into upgrade-snapshot.js-to-0.8.0-beta.0

.env.example

@@ -1,6 +1,7 @@
 PORT=3000
 NETWORK=testnet
-DATABASE_URL=mysql://...
+HUB_DATABASE_URL=mysql://...
+SEQ_DATABASE_URL=mysql://
 SEQUENCER_URL=https://testnet.seq.snapshot.org
 KEYCARD_URL=https://keycard.snapshot.org
 KEYCARD_SECRET=

README.md

@@ -32,7 +32,8 @@ yarn
 
 2. Copy [`.env.example`](https://github.com/snapshot-labs/snapshot-hub/blob/master/.env.example), rename it to `.env` and set a value for these config vars:
 
-- `DATABASE_URL`: The database connection string. You will need to run your own MySQL database or use a Cloud service like [JawsDB](https://jawsdb.com).
+- `HUB_DATABASE_URL`: The database connection string. You will need to run your own MySQL database or use a Cloud service like [JawsDB](https://jawsdb.com).
+- `SEQ_DATABASE_URL`:  We now use `messages` from a different database, it can be same as `HUB_DATABASE_URL` in your local
 - `RELAYER_PK`: This is the private key of the hub. The hub counter-sign every accepted message with this key.
 
 3. Create the database schema

docker-compose.yml

@@ -8,8 +8,9 @@ services:
     env_file:
       - .env
     environment:
-      DATABASE_URL: mysql://admin:pwd@snapshot-mysql:3306/snapshot
-
+      HUB_DATABASE_URL: mysql://admin:pwd@snapshot-mysql:3306/snapshot
+      SEQ_DATABASE_URL: mysql://admin:pwd@snapshot-mysql:3306/snapshot
+
     depends_on:
       - snapshot-mysql
     ports:

jest.config.ts

@@ -16,6 +16,8 @@ export default {
   preset: 'ts-jest',
   testEnvironment: 'jest-environment-node-single-context',
   setupFiles: ['dotenv/config'],
+  globalSetup: '<rootDir>/test/setup.ts',
+  globalTeardown: '<rootDir>/test/teardown.ts',
   moduleFileExtensions: ['js', 'ts'],
   testPathIgnorePatterns: ['dist/'],
   verbose: true

package.json

@@ -11,6 +11,7 @@
     "build": "tsc && copyfiles src/graphql/*.gql dist/",
     "start": "node dist/src/index.js",
     "start:test": "dotenv -e test/.env.test yarn dev",
+    "test:setup": "dotenv -e test/.env.test yarn ts-node ./test/setupDb.ts",
     "test": "PORT=3030 start-server-and-test 'yarn start:test' 3030 'dotenv -e test/.env.test jest --runInBand'",
     "test:unit": "dotenv -e test/.env.test jest test/unit/",
     "test:e2e": "PORT=3030 start-server-and-test 'yarn start:test' 3030 'dotenv -e test/.env.test jest --runInBand --collectCoverage=false test/e2e/'"
@@ -20,6 +21,7 @@
   },
   "prettier": "@snapshot-labs/prettier-config",
   "dependencies": {
+    "@graphql-tools/schema": "^10.0.0",
     "@snapshot-labs/keycard": "^0.4.0",
     "@snapshot-labs/snapshot-metrics": "^1.3.1",
     "@snapshot-labs/snapshot-sentry": "^1.5.1",
@@ -35,7 +37,6 @@
     "graphql-depth-limit": "^1.1.0",
     "graphql-fields": "^2.0.3",
     "graphql-query-count-limit": "^1.0.0",
-    "graphql-tools": "^9.0.0",
     "lodash": "^4.17.21",
     "mysql": "^2.18.1",
     "rate-limit-redis": "^3.0.2",

src/api.ts

@@ -27,6 +27,11 @@ router.get('/', (req, res) => {
 
 router.get('/spaces/:key', (req, res) => {
   const { key } = req.params;
+
+  if (!spaces[key]) {
+    return sendError(res, 'space not found', 404);
+  }
+
   return res.json(spaces[key]);
 });
 

src/graphql/helpers.ts

@@ -2,7 +2,7 @@ import graphqlFields from 'graphql-fields';
 import { jsonParse } from '../helpers/utils';
 import { spacesMetadata } from '../helpers/spaces';
 import db from '../helpers/mysql';
-import { flaggedLinks, flaggedProposals } from '../helpers/moderation';
+import { flaggedLinks } from '../helpers/moderation';
 
 const network = process.env.NETWORK || 'testnet';
 
@@ -45,7 +45,7 @@ export function checkLimits(args: any = {}, type) {
   return true;
 }
 
-export function formatSpace(id, settings) {
+export function formatSpace({ id, settings, verified, flagged }) {
   const spaceMetadata = spacesMetadata[id] || {};
   const space = { ...jsonParse(settings, {}), ...spaceMetadata.counts };
 
@@ -86,8 +86,8 @@ export function formatSpace(id, settings) {
   space.validation = space.validation || { name: 'any', params: {} };
   space.treasuries = space.treasuries || [];
 
-  space.verified = spaceMetadata?.verified ?? null;
-  space.flagged = spaceMetadata?.flagged ?? null;
+  space.verified = verified ?? null;
+  space.flagged = flagged ?? null;
   space.rank = spaceMetadata?.rank ?? null;
 
   // always return parent and children in child node format
@@ -174,7 +174,7 @@ export async function fetchSpaces(args) {
   params.push(skip, first);
 
   const spaces = await db.queryAsync(query, params);
-  return spaces.map(space => Object.assign(space, formatSpace(space.id, space.settings)));
+  return spaces.map(space => Object.assign(space, formatSpace(space)));
 }
 
 function checkRelatedSpacesNesting(requestedFields): void {
@@ -258,12 +258,7 @@ export function formatUser(user) {
 
 function isFlaggedProposal(proposal) {
   const flaggedLinksRegex = new RegExp(flaggedLinks.join('|'), 'i');
-  return (
-    // TODO: remove this check once flagged proposals are migrated to hub db via script
-    flaggedProposals?.includes(proposal.id) ||
-    flaggedLinksRegex.test(proposal.body) ||
-    proposal.flagged
-  );
+  return flaggedLinksRegex.test(proposal.body) || proposal.flagged;
 }
 
 export function formatProposal(proposal) {
@@ -281,7 +276,12 @@ export function formatProposal(proposal) {
   if (ts > proposal.start) proposalState = 'active';
   if (ts > proposal.end) proposalState = 'closed';
   proposal.state = proposalState;
-  proposal.space = formatSpace(proposal.space, proposal.settings);
+  proposal.space = formatSpace({
+    id: proposal.space,
+    settings: proposal.settings,
+    verified: proposal.spaceVerified,
+    flagged: proposal.spaceFlagged
+  });
   const networkStr = network === 'testnet' ? 'demo.' : '';
   proposal.link = `https://${networkStr}snapshot.org/#/${proposal.space.id}/proposal/${proposal.id}`;
   proposal.strategies = proposal.strategies.map(strategy => ({
@@ -298,16 +298,31 @@ export function formatVote(vote) {
   vote.choice = jsonParse(vote.choice);
   vote.metadata = jsonParse(vote.metadata, {});
   vote.vp_by_strategy = jsonParse(vote.vp_by_strategy, []);
-  vote.space = formatSpace(vote.space, vote.settings);
+  vote.space = formatSpace({
+    id: vote.space,
+    settings: vote.settings,
+    verified: vote.spaceVerified,
+    flagged: vote.spaceFlagged
+  });
   return vote;
 }
 
 export function formatFollow(follow) {
-  follow.space = formatSpace(follow.space, follow.settings);
+  follow.space = formatSpace({
+    id: follow.space,
+    settings: follow.settings,
+    verified: follow.spaceVerified,
+    flagged: follow.spaceFlagged
+  });
   return follow;
 }
 
 export function formatSubscription(subscription) {
-  subscription.space = formatSpace(subscription.space, subscription.settings);
+  subscription.space = formatSpace({
+    id: subscription.space,
+    settings: subscription.settings,
+    verified: subscription.spaceVerified,
+    flagged: subscription.spaceFlagged
+  });
   return subscription;
 }

src/graphql/index.ts

@@ -1,7 +1,7 @@
 import path from 'path';
 import fs from 'fs';
 import { graphqlHTTP } from 'express-graphql';
-import { makeExecutableSchema } from 'graphql-tools';
+import { makeExecutableSchema } from '@graphql-tools/schema';
 import queryCountLimit from 'graphql-query-count-limit';
 import depthLimit from 'graphql-depth-limit';
 import Query from './operations';

src/graphql/operations/follows.ts

@@ -27,7 +27,7 @@ export default async function (parent, args) {
   if (!['ASC', 'DESC'].includes(orderDirection)) orderDirection = 'DESC';
 
   const query = `
-    SELECT f.*, spaces.settings FROM follows f
+    SELECT f.*, spaces.settings, spaces.flagged as spaceFlagged, spaces.verified as spaceVerified FROM follows f
     INNER JOIN spaces ON spaces.id = f.space
     WHERE spaces.settings IS NOT NULL ${queryStr}
     ORDER BY ${orderBy} ${orderDirection} LIMIT ?, ?

src/graphql/operations/messages.ts

@@ -1,4 +1,4 @@
-import db from '../../helpers/mysql';
+import { sequencerDB } from '../../helpers/mysql';
 import { buildWhereQuery, checkLimits } from '../helpers';
 import log from '../../helpers/log';
 import { capture } from '@snapshot-labs/snapshot-sentry';
@@ -34,7 +34,7 @@ export default async function (parent, args) {
   `;
   params.push(skip, first);
   try {
-    return await db.queryAsync(query, params);
+    return await sequencerDB.queryAsync(query, params);
   } catch (e: any) {
     log.error(`[graphql] messages, ${JSON.stringify(e)}`);
     capture(e, { args });

src/graphql/operations/proposal.ts

@@ -5,7 +5,7 @@ import { capture } from '@snapshot-labs/snapshot-sentry';
 
 export default async function (parent, { id }) {
   const query = `
-    SELECT p.*, spaces.settings FROM proposals p
+    SELECT p.*, spaces.settings, spaces.flagged as spaceFlagged, spaces.verified as spaceVerified FROM proposals p
     INNER JOIN spaces ON spaces.id = p.space
     WHERE p.id = ? AND spaces.settings IS NOT NULL
     LIMIT 1

src/graphql/operations/proposals.ts

@@ -1,7 +1,6 @@
 import db from '../../helpers/mysql';
 import { buildWhereQuery, formatProposal, checkLimits } from '../helpers';
 import log from '../../helpers/log';
-import { flaggedProposals, verifiedSpaces } from '../../helpers/moderation';
 import { capture } from '@snapshot-labs/snapshot-sentry';
 
 export default async function (parent, args) {
@@ -61,21 +60,16 @@ export default async function (parent, args) {
     params.push(`%"name": "${where.validation}"%`);
   }
 
-  if (where.space_verified && verifiedSpaces.length > 0) {
-    searchSql += ' AND spaces.id in (?)';
-    params.push(verifiedSpaces);
+  if (where.space_verified) {
+    searchSql += ' AND spaces.verified = 1';
   }
 
-  // TODO: remove part `p.id IN (?)` when flagged proposals are moved from laser DB to snapshot-sequencer DB
-  if (where.flagged === true && flaggedProposals.length > 0) {
-    searchSql += ' AND (p.id IN (?) OR p.flagged = 1)';
-    params.push(flaggedProposals);
+  if (where.flagged === true) {
+    searchSql += ' AND p.flagged = 1';
   }
 
-  // TODO: remove part `p.id NOT IN (?)` when flagged proposals are moved from laser DB to snapshot-sequencer DB
-  if (where.flagged === false && flaggedProposals.length > 0) {
-    searchSql += ' AND (p.id NOT IN (?) AND p.flagged = 0)';
-    params.push(flaggedProposals);
+  if (where.flagged === false) {
+    searchSql += ' AND p.flagged = 0';
   }
 
   let orderBy = args.orderBy || 'created';
@@ -86,7 +80,7 @@ export default async function (parent, args) {
   if (!['ASC', 'DESC'].includes(orderDirection)) orderDirection = 'DESC';
 
   const query = `
-    SELECT p.*, spaces.settings FROM proposals p
+    SELECT p.*, spaces.settings, spaces.flagged as spaceFlagged, spaces.verified as spaceVerified FROM proposals p
     INNER JOIN spaces ON spaces.id = p.space
     WHERE spaces.settings IS NOT NULL ${queryStr} ${searchSql}
     ORDER BY ${orderBy} ${orderDirection}, p.id ASC LIMIT ?, ?

src/graphql/operations/ranking.ts

@@ -46,7 +46,7 @@ export default async function (_parent, args, context, info) {
       ORDER BY FIELD(s.id, ?) ASC
     `;
     let spaces = await db.queryAsync(query, [filteredSpaces, filteredSpaces]);
-    spaces = spaces.map(space => Object.assign(space, formatSpace(space.id, space.settings)));
+    spaces = spaces.map(space => Object.assign(space, formatSpace(space)));
 
     const items = await handleRelatedSpaces(info, spaces);
 

src/graphql/operations/subscriptions.ts

@@ -29,7 +29,7 @@ export default async function (parent, args) {
   let subscriptions: any[] = [];
 
   const query = `
-    SELECT s.*, spaces.settings FROM subscriptions s
+    SELECT s.*, spaces.settings, spaces.flagged as spaceFlagged, spaces.verified as spaceVerified FROM subscriptions s
     INNER JOIN spaces ON spaces.id = s.space
     WHERE spaces.settings IS NOT NULL ${queryStr}
     ORDER BY ${orderBy} ${orderDirection} LIMIT ?, ?

src/graphql/operations/vote.ts

@@ -7,7 +7,7 @@ import { capture } from '@snapshot-labs/snapshot-sentry';
 export default async function (parent, { id }, context, info) {
   const requestedFields = info ? graphqlFields(info) : {};
   const query = `
-    SELECT v.*, spaces.settings FROM votes v
+    SELECT v.*, spaces.settings, spaces.flagged as spaceFlagged, spaces.verified as spaceVerified FROM votes v
     INNER JOIN spaces ON spaces.id = v.space
     WHERE v.id = ? AND spaces.settings IS NOT NULL
     LIMIT 1
@@ -18,7 +18,7 @@ export default async function (parent, { id }, context, info) {
     if (requestedFields.proposal && result?.proposal) {
       const proposalId = result.proposal;
       const query = `
-        SELECT p.*, spaces.settings FROM proposals p
+        SELECT p.*, spaces.settings, spaces.flagged as spaceFlagged, spaces.verified as spaceVerified FROM proposals p
         INNER JOIN spaces ON spaces.id = p.space
         WHERE spaces.settings IS NOT NULL AND p.id = ?
       `;

src/graphql/operations/votes.ts

@@ -55,15 +55,13 @@ async function query(parent, args, context?, info?) {
   if (requestedFields.space && votes.length > 0) {
     const spaceIds = votes.map(vote => vote.space.id).filter((v, i, a) => a.indexOf(v) === i);
     const query = `
-      SELECT id, settings FROM spaces
+      SELECT * FROM spaces
       WHERE id IN (?) AND settings IS NOT NULL AND deleted = 0
     `;
     try {
       let spaces = await db.queryAsync(query, [spaceIds]);
 
-      spaces = Object.fromEntries(
-        spaces.map(space => [space.id, formatSpace(space.id, space.settings)])
-      );
+      spaces = Object.fromEntries(spaces.map(space => [space.id, formatSpace(space)]));
       votes = votes.map(vote => {
         if (spaces[vote.space.id]) return { ...vote, space: spaces[vote.space.id] };
         return vote;
@@ -78,7 +76,7 @@ async function query(parent, args, context?, info?) {
   if (requestedFields.proposal && votes.length > 0) {
     const proposalIds = votes.map(vote => vote.proposal);
     const query = `
-      SELECT p.*, spaces.settings FROM proposals p
+      SELECT p.*, spaces.settings, spaces.flagged as spaceFlagged, spaces.verified as spaceVerified FROM proposals p
       INNER JOIN spaces ON spaces.id = p.space
       WHERE spaces.settings IS NOT NULL AND p.id IN (?)
     `;

src/helpers/metrics.ts

@@ -1,9 +1,11 @@
 import init, { client } from '@snapshot-labs/snapshot-metrics';
 import { capture } from '@snapshot-labs/snapshot-sentry';
-import { Express } from 'express';
+import { Express, type Request, type Response } from 'express';
+import { parse } from 'graphql';
 import { spacesMetadata } from './spaces';
 import { strategies } from './strategies';
 import db from './mysql';
+import operations from '../graphql/operations/';
 
 const whitelistedPath = [
   /^\/$/,
@@ -30,6 +32,8 @@ function instrumentRateLimitedRequests(req, res, next) {
 }
 
 export default function initMetrics(app: Express) {
+  const GRAPHQL_TYPES = Object.keys(operations);
+
   init(app, {
     normalizedPath: [
       ['^/api/scores/.+', '/api/scores/#id'],
@@ -41,6 +45,39 @@ export default function initMetrics(app: Express) {
   });
 
   app.use(instrumentRateLimitedRequests);
+
+  app.use((req: Request, res: Response, next) => {
+    if (req.originalUrl.startsWith('/graphql')) {
+      const end = graphqlRequest.startTimer();
+
+      res.on('finish', () => {
+        try {
+          const query = (req.body || req.query)?.query;
+          const operationName = (req.body || req.query)?.operationName;
+
+          if (query && operationName) {
+            const definition = parse(query).definitions.find(
+              // @ts-ignore
+              def => def.name.value === operationName
+            );
+
+            // @ts-ignore
+            const types = definition.selectionSet.selections.map(sel => sel.name.value);
+
+            for (const type of types) {
+              if (GRAPHQL_TYPES.includes(type)) {
+                end({ type });
+              }
+            }
+          }
+        } catch (e: any) {
+          capture(e);
+        }
+      });
+    }
+
+    next();
+  });
 }
 
 new client.Gauge({
@@ -137,3 +174,10 @@ export const requestDeduplicatorSize = new client.Gauge({
   name: 'request_deduplicator_size',
   help: 'Total number of items in the deduplicator queue'
 });
+
+export const graphqlRequest = new client.Histogram({
+  name: 'graphql_requests_duration_seconds',
+  help: 'Duration in seconds of graphql requests',
+  labelNames: ['type'],
+  buckets: [0.5, 1, 2, 5, 10, 15]
+});