diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..49fd9e4a7 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @near/queryapi-core diff --git a/frontend/README.md b/frontend/README.md index 965a1228c..ea5bb0f32 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,38 +1,51 @@ -This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app). +## What is this repo? + +Frontend for Near QueryAPI that allows users to create, manage, and explore indexers stored on-chain. You can visit the app [here](https://near.org/dataplatform.near/widget/QueryApi.App) + + +BOS widgets are stored in the `widgets/` folder while the main NextJS application lives in the root. ## Getting Started -First, run the development server: +First, download the bos-loader cli by following this guide [here](https://docs.near.org/bos/dev/bos-loader). + +From the root of QueryAPI Frontend repo, run the following command ```bash -npm run dev -# or -yarn dev -# or -pnpm dev +yarn serve:widgets ``` +> Near.org or any other BOS gateway queries the blockchain state to pull the latest widgets code and renders it. If we would like to test our BOS widgets, we need to override the path at which the gateway (near.org) queries for the widget code. We do this using the Bos-loader tool (the underlying CLI tool used in the `yarn serve:widgets` command) which allows us to serve out widgets locally (http://127.0.0.1:3030 by default). At this point, we have served our widgets locally but have not yet told the BOS gateway (near.org) where to load our local widgets from. + -Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. +**Then, Head to `near.org/flags` and enter `http://127.0.0.1:3030`** -You can start editing the page by modifying `pages/index.tsx`. The page auto-updates as you edit the file. +> In order to tell our BOS gateway (near.org), where to load the local widgets from, we head to `near.org/flags` and enter the local path we got from running the previous command. If you have not changed any configurations then the default should be `http://127.0.0.1:3030` -[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`. +**Finally**, run the following to serve the local NextJS frontend +```bash +yarn dev +``` -The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages. -This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font. +**Now, head to the path where the widgets are served on the BOS.** -## Learn More +- Prod Environment: `https://near.org/dataplatform.near/widget/QueryApi.App` +- Dev Environment: `https://near.org/dev-queryapi.dataplatform.near/widget/QueryApi.dev-App` + +--- +### Notes +> **Make sure to change your widgets code (while testing only) to point to where your local nextJS app is being served.** + +```QueryApi.App.jsx +---const EXTERNAL_APP_URL = "https://queryapi.io"; ++++const EXTERNAL_APP_URL = "http://localhost:3000"; +``` -To learn more about Next.js, take a look at the following resources: -- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. -- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. +> **You may need to change the accountId argument to the bos-loader CLI command in `package.json` to load from `dataplatform.near` or `dev-queryapi.dataplatform.near`. This depends on what environment you are testing for.** -You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome! +`bos-loader dev-queryapi.dataplatform.near --path widgets/src` +`bos-loader dataplatform.near --path widgets/src` -## Deploy on Vercel -The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. -Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details. diff --git a/frontend/package.json b/frontend/package.json index a7c3bc221..7a73280e4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -4,14 +4,14 @@ "private": true, "scripts": { "dev": "next dev", - "social::dev": "cd near.social && yarn run dev", + "serve:widgets": "bos-loader dev-queryapi.dataplatform.near --path widgets/src", "build": "next build", "start": "next start", "lint": "next lint" }, "dependencies": { - "@graphiql/plugin-explorer": "^0.1.20", - "@graphiql/plugin-code-exporter": "^0.1.2", + "@graphiql/plugin-explorer": "0.3.0", + "@graphiql/plugin-code-exporter": "0.3.0", "@monaco-editor/react": "^4.1.3", "@near-lake/primitives": "0.1.0", "@next/font": "13.1.6", diff --git a/frontend/src/components/Editor/Editor.js b/frontend/src/components/Editor/Editor.js index 18c133c4d..4d217e6b6 100644 --- a/frontend/src/components/Editor/Editor.js +++ b/frontend/src/components/Editor/Editor.js @@ -34,6 +34,7 @@ const Editor = ({ debugMode, isCreateNewIndexer, indexerNameField, + setAccountId, } = useContext(IndexerDetailsContext); const DEBUG_LIST_STORAGE_KEY = `QueryAPI:debugList:${indexerDetails.accountId}#${indexerDetails.indexerName}` @@ -90,28 +91,6 @@ const Editor = ({ localStorage.setItem(DEBUG_LIST_STORAGE_KEY, heights); }, [heights]); - // useEffect(() => { - // if (selectedOption == "latestBlockHeight") { - // setBlockHeightError(null); - // return; - // } - // - // if (height - blockHeight > BLOCKHEIGHT_LIMIT) { - // setBlockHeightError( - // `Warning: Please enter a valid start block height. At the moment we only support historical indexing of the last ${BLOCKHEIGHT_LIMIT} blocks or ${BLOCKHEIGHT_LIMIT / 3600 - // } hrs. Choose a start block height between ${height - BLOCKHEIGHT_LIMIT - // } - ${height}.` - // ); - // } else if (blockHeight > height) { - // setBlockHeightError( - // `Warning: Start Block Hieght can not be in the future. Please choose a value between ${height - BLOCKHEIGHT_LIMIT - // } - ${height}.` - // ); - // } else { - // setBlockHeightError(null); - // } - // }, [blockHeight, height, selectedOption]); - const checkSQLSchemaFormatting = () => { try { let formatted_sql = formatSQL(schema); @@ -127,10 +106,21 @@ const Editor = ({ } }; + + const forkIndexer = async(indexerName) => { + let code = indexingCode; + setAccountId(currentUserAccountId) + let prevAccountId = indexerDetails.accountId.replaceAll(".", "_"); + let newAccountId = currentUserAccountId.replaceAll(".", "_"); + let prevIndexerName = indexerDetails.indexerName.replaceAll("-", "_").trim().toLowerCase(); + let newIndexerName = indexerName.replaceAll("-", "_").trim().toLowerCase(); + code = code.replaceAll(prevAccountId, newAccountId); + code = code.replaceAll(prevIndexerName, newIndexerName); + setIndexingCode(formatIndexingCode(code)) + } + const registerFunction = async (indexerName, indexerConfig) => { let formatted_schema = checkSQLSchemaFormatting(); - let isForking = indexerDetails.accountId !== currentUserAccountId; - let innerCode = indexingCode.match(/getBlock\s*\([^)]*\)\s*{([\s\S]*)}/)[1]; indexerName = indexerName.replaceAll(" ", "_"); if (formatted_schema == undefined) { @@ -140,14 +130,6 @@ const Editor = ({ ); return; } - - if (isForking) { - let prevAccountName = indexerDetails.accountId.replace(".", "_"); - let newAccountName = currentUserAccountId.replace(".", "_"); - - innerCode = innerCode.replaceAll(prevAccountName, newAccountName); - } - setError(() => undefined); request("register-function", { @@ -170,8 +152,8 @@ const Editor = ({ const handleReload = async () => { if (isCreateNewIndexer) { setShowResetCodeModel(false); - setIndexingCode(defaultCode); - setSchema(defaultSchema); + setIndexingCode((formatIndexingCode(indexerDetails.code))); + setSchema(formatSQL(indexerDetails.schema)) return; } @@ -331,7 +313,7 @@ const Editor = ({ blockHeightError={blockHeightError} />
{accountId} - {!isCreateNewIndexer && ( + {indexerName && ( {indexerName} @@ -169,21 +169,33 @@ const EditorButtons = ({ - {currentUserAccountId && ( + {(!isUserIndexer && !isCreateNewIndexer) ? ( {getActionButtonText()}} + overlay={Fork Indexer} + > + + + ) : ( + Publish} > )} - @@ -208,6 +220,5 @@ const EditorButtons = ({ ); }; - export default EditorButtons; diff --git a/frontend/src/components/Form/IndexerConfigOptionsInputGroup.jsx b/frontend/src/components/Form/IndexerConfigOptionsInputGroup.jsx index ca84bd625..c728d0e9a 100644 --- a/frontend/src/components/Form/IndexerConfigOptionsInputGroup.jsx +++ b/frontend/src/components/Form/IndexerConfigOptionsInputGroup.jsx @@ -51,11 +51,11 @@ const IndexerConfigOptions = ({ updateConfig }) => { Indexer Name setIndexerNameField(e.target.value)} + onChange={(e) => setIndexerNameField(e.target.value.toLowerCase().trim())} /> diff --git a/frontend/src/components/Modals/ForkIndexerModal.jsx b/frontend/src/components/Modals/ForkIndexerModal.jsx index 7e427e238..71b7cb37b 100644 --- a/frontend/src/components/Modals/ForkIndexerModal.jsx +++ b/frontend/src/components/Modals/ForkIndexerModal.jsx @@ -1,46 +1,41 @@ import React, { useContext, useState } from "react"; -import { Button, Modal, Alert } from "react-bootstrap"; +import { Button, Modal, Alert, InputGroup, Form } from "react-bootstrap"; import IndexerConfigOptions from "../Form/IndexerConfigOptionsInputGroup"; -import { IndexerDetailsContext } from '../../contexts/IndexerDetailsContext'; +import { IndexerDetailsContext } from "../../contexts/IndexerDetailsContext"; import { validateContractId } from "../../utils/validators"; -export const ForkIndexerModal = ({ - registerFunction, -}) => { +export const ForkIndexerModal = ({ registerFunction, forkIndexer }) => { const { indexerDetails, showForkIndexerModal, setShowForkIndexerModal, + setIsCreateNewIndexer, + setIndexerName, + setIndexerConfig, + isCreateNewIndexer, } = useContext(IndexerDetailsContext); - const [indexerConfig, setIndexerConfig] = useState({ filter: "social.near", startBlockHeight: null }) - const [indexerName, setIndexerName] = useState("") - const [error, setError] = useState(null) + const [indexerName, setIndexerNameField] = useState(""); + const [error, setError] = useState(null); - const updateConfig = (indexerName, filter, startBlockHeight, option) => { - let finalStartBlockHeight = option === "latestBlockHeight" ? null : startBlockHeight; - setIndexerConfig({ filter, startBlockHeight: finalStartBlockHeight }) - setIndexerName(indexerName) - } - const register = async () => { + const fork = async () => { if (!indexerName) { - setError("Please provide an Indexer Name") - return + setError("Please provide an Indexer Name"); + return; } if (indexerName === indexerDetails.indexerName) { - setError("Please provide a different Indexer Name than the orginal Indexer") - return + setError( + "Please provide a different Indexer Name than the orginal Indexer" + ); + return; } - - if (!validateContractId(indexerConfig.filter)) { - setError("Please provide a valid contract name") - return - } - setError(null) - registerFunction(indexerName, indexerConfig) - setShowForkIndexerModal(false) - } + setError(null); + setIndexerName(indexerName); + setIsCreateNewIndexer(true); + forkIndexer(indexerName); + setShowForkIndexerModal(false); + }; return ( Enter Indexer Details - + + Indexer Name + setIndexerNameField(e.target.value.trim().toLowerCase())} + /> + {error && ( {error} @@ -60,11 +64,14 @@ export const ForkIndexerModal = ({ )} - - diff --git a/frontend/src/components/Playground/graphiql.jsx b/frontend/src/components/Playground/graphiql.jsx index 1e1917b01..e0aa80f91 100644 --- a/frontend/src/components/Playground/graphiql.jsx +++ b/frontend/src/components/Playground/graphiql.jsx @@ -1,9 +1,9 @@ -import React, { useContext, useState } from "react"; +import React, { useContext, useState, useMemo } from "react"; import GraphiQL from "graphiql"; import { sessionStorage } from "near-social-bridge"; import { IndexerDetailsContext } from '../../contexts/IndexerDetailsContext'; -import { useExporterPlugin } from '@graphiql/plugin-code-exporter'; -import { useExplorerPlugin } from '@graphiql/plugin-explorer'; +import { codeExporterPlugin } from '@graphiql/plugin-code-exporter'; +import { explorerPlugin } from '@graphiql/plugin-explorer'; import "graphiql/graphiql.min.css"; import '@graphiql/plugin-code-exporter/dist/style.css'; import '@graphiql/plugin-explorer/dist/style.css'; @@ -90,36 +90,28 @@ const renderData = (a) => { const renderedData = state.data.map(renderData); return ( + <> {renderedData} + );`; } } }; + +const explorer = explorerPlugin(); + export const GraphqlPlayground = () => { const { indexerDetails } = useContext(IndexerDetailsContext); - const snippets = [bosQuerySnippet(indexerDetails.accountId)]; - const [query, setQuery] = useState(""); - - const explorerPlugin = useExplorerPlugin({ - query, - onEdit: setQuery, - }); - const exporterPlugin = useExporterPlugin({ - query, - snippets, - codeMirrorTheme: 'graphiql', - }); + const snippets = useMemo(()=>[bosQuerySnippet(indexerDetails.accountId)], [indexerDetails.accountId]); + const exporter = useMemo(()=> codeExporterPlugin({snippets}), [snippets]) return (
graphQLFetcher(params, indexerDetails.accountId)} - defaultQuery="" storage={sessionStorage} - plugins={[explorerPlugin, exporterPlugin]} + plugins={[exporter, explorer]} />
); diff --git a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Comment.jsx b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Comment.jsx index 42a673856..96429ca0d 100644 --- a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Comment.jsx +++ b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Comment.jsx @@ -1,6 +1,6 @@ const GRAPHQL_ENDPOINT = - "https://queryapi-hasura-graphql-24ktefolwq-ew.a.run.app"; -const APP_OWNER = "dev-queryapi.dataplatform.near"; + props.GRAPHQL_ENDPOINT || "https://near-queryapi.api.pagoda.co"; +const APP_OWNER = props.APP_OWNER || "dataplatform.near"; const accountId = props.accountId; const blockHeight = props.blockHeight === "now" ? "now" : parseInt(props.blockHeight); @@ -26,7 +26,7 @@ const commentUrl = `https://alpha.near.org/#/${APP_OWNER}/widget/QueryApi.Exampl if (!state.content && accountId && blockHeight !== "now") { const commentQuery = ` query CommentQuery { - roshaan_near_feed_indexer_comments( + dataplatform_near_social_feed_comments( where: {_and: {account_id: {_eq: "${accountId}"}, block_height: {_eq: ${blockHeight}}}} ) { content @@ -44,7 +44,7 @@ query CommentQuery { `${GRAPHQL_ENDPOINT}/v1/graphql`, { method: "POST", - headers: { "x-hasura-role": "roshaan_near" }, + headers: { "x-hasura-role": "dataplatform_near" }, body: JSON.stringify({ query: operationsDoc, variables: variables, @@ -57,7 +57,7 @@ query CommentQuery { fetchGraphQL(commentQuery, "CommentQuery", {}).then((result) => { if (result.status === 200) { if (result.body.data) { - const comments = result.body.data.roshaan_near_feed_indexer_comments; + const comments = result.body.data.dataplatform_near_social_feed_comments; if (comments.length > 0) { const comment = comments[0]; let content = JSON.parse(comment.content); diff --git a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.LikeButton.jsx b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.LikeButton.jsx index a0c79896d..1eddbda39 100644 --- a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.LikeButton.jsx +++ b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.LikeButton.jsx @@ -1,7 +1,6 @@ - const GRAPHQL_ENDPOINT = - "https://queryapi-hasura-graphql-24ktefolwq-ew.a.run.app"; -const APP_OWNER = "dev-queryapi.dataplatform.near"; + props.GRAPHQL_ENDPOINT || "https://near-queryapi.api.pagoda.co"; +const APP_OWNER = props.APP_OWNER || "dataplatform.near"; const item = props.item; const likes = JSON.parse(props.likes?.length ? props.likes : "[]") ?? []; diff --git a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Post.jsx b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Post.jsx index c3817a435..4e7673a31 100644 --- a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Post.jsx +++ b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Post.jsx @@ -1,6 +1,6 @@ const GRAPHQL_ENDPOINT = - "https://queryapi-hasura-graphql-24ktefolwq-ew.a.run.app"; -const APP_OWNER = "dev-queryapi.dataplatform.near"; + props.GRAPHQL_ENDPOINT || "https://near-queryapi.api.pagoda.co"; +const APP_OWNER = props.APP_OWNER || "dataplatform.near"; const accountId = props.accountId; const blockHeight = props.blockHeight === "now" ? "now" : parseInt(props.blockHeight); @@ -23,10 +23,9 @@ const item = { // Load post if not contents and comments are not passed in if (!state.content || !state.comments || !state.likes) { - console.log("making call again"); const postsQuery = ` query IndexerQuery { - roshaan_near_feed_indexer_posts( + dataplatform_near_social_feed_posts( order_by: {block_height: desc} where: {_and: {block_height: {_eq: ${blockHeight}}, account_id: {_eq: "${accountId}"}}} ) { @@ -51,7 +50,7 @@ query IndexerQuery { `${GRAPHQL_ENDPOINT}/v1/graphql`, { method: "POST", - headers: { "x-hasura-role": "roshaan_near" }, + headers: { "x-hasura-role": "dataplatform_near" }, body: JSON.stringify({ query: operationsDoc, variables: variables, @@ -64,7 +63,7 @@ query IndexerQuery { fetchGraphQL(postsQuery, "IndexerQuery", {}).then((result) => { if (result.status === 200) { if (result.body.data) { - const posts = result.body.data.roshaan_near_feed_indexer_posts; + const posts = result.body.data.dataplatform_near_social_feed_posts; if (posts.length > 0) { const post = posts[0]; let content = JSON.parse(post.content); diff --git a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.PostPage.jsx b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.PostPage.jsx index d772ee4c1..27afc38bb 100644 --- a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.PostPage.jsx +++ b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.PostPage.jsx @@ -1,5 +1,5 @@ const GRAPHQL_ENDPOINT = - props.GRAPHQL_ENDPOINT || "https://queryapi-hasura-graphql-24ktefolwq-ew.a.run.app"; + props.GRAPHQL_ENDPOINT || "https://near-queryapi.api.pagoda.co"; const APP_OWNER = props.APP_OWNER || "dataplatform.near"; const accountId = props.accountId; const commentBlockHeight = parseInt(props.commentBlockHeight); @@ -13,7 +13,7 @@ State.init({ }); const parentPostByComment = `query ParentPostByComment { - roshaan_near_feed_indexer_comments( + dataplatform_near_social_feed_comments( where: {_and: {account_id: {_eq: "${accountId}"}, block_height: {_eq: ${commentBlockHeight}}}} ) { post { @@ -49,7 +49,7 @@ function fetchGraphQL(operationsDoc, operationName, variables) { `${GRAPHQL_ENDPOINT}/v1/graphql`, { method: "POST", - headers: { "x-hasura-role": "roshaan_near" }, + headers: { "x-hasura-role": "dataplatform_near" }, body: JSON.stringify({ query: operationsDoc, variables: variables, @@ -64,7 +64,7 @@ if (commentBlockHeight) { (result) => { if (result.status === 200) { if (result.body.data) { - const posts = result.body.data.roshaan_near_feed_indexer_comments; + const posts = result.body.data.dataplatform_near_social_feed_comments; if (posts.length > 0) { const post = posts[0].post; let content = JSON.parse(post.content); diff --git a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Posts.jsx b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Posts.jsx new file mode 100644 index 000000000..9c59d7a71 --- /dev/null +++ b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.Posts.jsx @@ -0,0 +1,304 @@ +const APP_OWNER = props.APP_OWNER || "dataplatform.near"; +const GRAPHQL_ENDPOINT = + props.GRAPHQL_ENDPOINT || "https://near-queryapi.api.pagoda.co"; +const sortOption = props.postsOrderOption || "blockHeight"; // following, blockHeight +const LIMIT = 25; +let accountsFollowing = props.accountsFollowing + +if (context.accountId && !accountsFollowing) { + const graph = Social.keys(`${context.accountId}/graph/follow/*`, "final"); + if (graph !== null) { + accountsFollowing = Object.keys(graph[context.accountId].graph.follow || {}); + } +} + +State.init({ + selectedTab: Storage.privateGet("selectedTab") || "all", + posts: [], + postsCountLeft: 0, + initLoadPosts: false, + initLoadPostsAll: false +}); + +function fetchGraphQL(operationsDoc, operationName, variables) { + return asyncFetch( + `${GRAPHQL_ENDPOINT}/v1/graphql`, + { + method: "POST", + headers: { "x-hasura-role": "dataplatform_near" }, + body: JSON.stringify({ + query: operationsDoc, + variables: variables, + operationName: operationName, + }), + } + ); +} + +const createQuery = (sortOption, type) => { +let querySortOption = ""; +switch (sortOption) { + case "recentComments": + querySortOption = `{ last_comment_timestamp: desc_nulls_last },`; + break; + // More options... + default: + querySortOption = ""; +} + +let queryFilter = ""; +switch (type) { + case "following": + let queryAccountsString = accountsFollowing.map(account => `"${account}"`).join(", "); + queryFilter = `account_id: { _in: [${queryAccountsString}]}`; + break; + // More options... + default: + queryFilter = ""; +} + +const indexerQueries = ` +query GetPostsQuery($offset: Int, $limit: Int) { + dataplatform_near_social_feed_posts(order_by: [${querySortOption} { block_height: desc }], offset: $offset, limit: $limit) { + account_id + block_height + block_timestamp + content + receipt_id + accounts_liked + last_comment_timestamp + comments(order_by: {block_height: asc}) { + account_id + block_height + block_timestamp + content + } + } + dataplatform_near_social_feed_posts_aggregate(order_by: [${querySortOption} { block_height: desc }], offset: $offset){ + aggregate { + count + } + } +} +query GetFollowingPosts($offset: Int, $limit: Int) { + dataplatform_near_social_feed_posts(where: {${queryFilter}}, order_by: [${querySortOption} { block_height: desc }], offset: $offset, limit: $limit) { + account_id + block_height + block_timestamp + content + receipt_id + accounts_liked + last_comment_timestamp + comments(order_by: {block_height: asc}) { + account_id + block_height + block_timestamp + content + } + } + dataplatform_near_social_feed_posts_aggregate(where: {${queryFilter}}, order_by: [${querySortOption} { block_height: desc }], offset: $offset) { + aggregate { + count + } + } +} +`; +return indexerQueries +} + +const loadMorePosts = () => { + const queryName = state.selectedTab == "following" && accountsFollowing ? "GetFollowingPosts" : "GetPostsQuery" + const type = state.selectedTab == "following" && accountsFollowing ? "following" : "all" + + if(state.selectedTab == "following" && accountsSelected && accountsSelected.length == 0) { + console.log("user has no followers") + return + } + fetchGraphQL(createQuery(sortOption, type), queryName, { + offset: state.posts.length, + limit: LIMIT + }).then((result) => { + if (result.status === 200 && result.body) { + if(result.body.errors) { + console.log('error:', result.body.errors) + return + } + let data = result.body.data; + if (data) { + const newPosts = data.dataplatform_near_social_feed_posts; + const postsCountLeft = + data.dataplatform_near_social_feed_posts_aggregate.aggregate.count; + if (newPosts.length > 0) { + State.update({ + posts: [...state.posts, ...newPosts], + postsCountLeft, + }); + } + } + } + }); +}; + +const previousSelectedTab = Storage.privateGet("selectedTab"); +if (previousSelectedTab && previousSelectedTab !== state.selectedTab) { + State.update({ + selectedTab: previousSelectedTab, + }); +} + +function selectTab(selectedTab) { + Storage.privateSet("selectedTab", selectedTab); + State.update({ + posts: [], + postsCountLeft: 0, + selectedTab + }); + loadMorePosts() +} + +const H2 = styled.h2` + font-size: 19px; + line-height: 22px; + color: #11181C; + margin: 0 0 24px; + padding: 0 24px; + + @media (max-width: 1200px) { + display: none; + } +`; + +const Content = styled.div` + @media (max-width: 1200px) { + > div:first-child { + border-top: none; + } + } +`; + +const ComposeWrapper = styled.div` + border-top: 1px solid #ECEEF0; +`; + +const FilterWrapper = styled.div` + border-top: 1px solid #ECEEF0; + padding: 24px 24px 0; + + @media (max-width: 1200px) { + padding: 12px; + } +`; + +const PillSelect = styled.div` + display: inline-flex; + align-items: center; + + @media (max-width: 600px) { + width: 100%; + + button { + flex: 1; + } + } +`; + +const PillSelectButton = styled.button` + display: block; + position: relative; + border: 1px solid #E6E8EB; + border-right: none; + padding: 3px 24px; + border-radius: 0; + font-size: 12px; + line-height: 18px; + color: ${(p) => (p.selected ? "#fff" : "#687076")}; + background: ${(p) => (p.selected ? "#006ADC !important" : "#FBFCFD")}; + font-weight: 600; + transition: all 200ms; + + &:hover { + background: #ECEDEE; + text-decoration: none; + } + + &:focus { + outline: none; + border-color: #006ADC !important; + box-shadow: 0 0 0 1px #006ADC; + z-index: 5; + } + + &:first-child { + border-radius: 6px 0 0 6px; + } + &:last-child { + border-radius: 0 6px 6px 0; + border-right: 1px solid #E6E8EB; + } +`; + +const FeedWrapper = styled.div` + .post { + padding-left: 24px; + padding-right: 24px; + + @media (max-width: 1200px) { + padding-left: 12px; + padding-right: 12px; + } + } +`; + +const hasMore = state.postsCountLeft != state.posts.length + +if(!state.initLoadPostsAll) { + loadMorePosts() + State.update({initLoadPostsAll: true}) +} + +if(state.initLoadPostsAll == true && !state.initLoadPosts && accountsFollowing) { + if (accountsFollowing.length > 0 && state.selectedTab == "following") { + selectTab("following") + } + State.update({initLoadPosts: true}) +} + +return ( + <> +

Posts

+ + + {context.accountId && ( + <> + + + + + + + selectTab("all")} + selected={state.selectedTab === "all"} + > + All + + + selectTab("following")} + selected={state.selectedTab === "following"} + > + Following + + + + + )} + + + + + + +); diff --git a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.jsx b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.jsx index 0d83123c8..f8855e248 100644 --- a/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.jsx +++ b/frontend/widgets/examples/feed/src/QueryApi.Examples.Feed.jsx @@ -1,14 +1,9 @@ - const GRAPHQL_ENDPOINT = - props.GRAPHQL_ENDPOINT || "https://queryapi-hasura-graphql-24ktefolwq-ew.a.run.app"; + props.GRAPHQL_ENDPOINT || "https://near-queryapi.api.pagoda.co"; const APP_OWNER = props.APP_OWNER || "dataplatform.near"; -const LIMIT = 10; -const option = props.postsOrderOption ?? "blockHeight"; - -State.init({ - posts: [], - postsCount: 0, -}); +const loadMorePosts = props.loadMorePosts; +const hasMore = props.hasMore || false; +const posts = props.posts || []; const Subheading = styled.h2` display: block; @@ -24,55 +19,6 @@ const Subheading = styled.h2` outline: none; `; -let querySortFilter = ""; -switch (option) { - case "recentComments": - querySortFilter = `{ last_comment_timestamp: desc_nulls_last },`; - break; - // More options... - default: - querySortFilter = ""; -} - -const indexerQueries = ` - query GetPostsQuery($offset: Int) { - roshaan_near_feed_indexer_posts(order_by: [${querySortFilter} { block_height: desc }], offset: $offset, limit: ${LIMIT}) { - account_id - block_height - block_timestamp - content - receipt_id - accounts_liked - last_comment_timestamp - comments(order_by: {block_height: asc}) { - account_id - block_height - block_timestamp - content - } - } - roshaan_near_feed_indexer_posts_aggregate { - aggregate { - count - } - } -} -`; - -function fetchGraphQL(operationsDoc, operationName, variables) { - return asyncFetch( - `${GRAPHQL_ENDPOINT}/v1/graphql`, - { - method: "POST", - headers: { "x-hasura-role": "roshaan_near" }, - body: JSON.stringify({ - query: operationsDoc, - variables: variables, - operationName: operationName, - }), - } - ); -} const Post = styled.div` border-bottom: 1px solid #ECEEF0; @@ -105,34 +51,27 @@ const renderItem = (item, i) => { ); }; -const loadMorePosts = () => { - fetchGraphQL(indexerQueries, "GetPostsQuery", { - offset: state.posts.length, - }).then((result) => { - if (result.status === 200) { - let data = result.body.data; - if (data) { - const newPosts = data.roshaan_near_feed_indexer_posts; - console.log(newPosts); - const postsCount = - data.roshaan_near_feed_indexer_posts_aggregate.aggregate.count; - if (newPosts.length > 0) { - State.update({ - posts: [...state.posts, ...newPosts], - postsCount: postsCount, - }); - } - } - } - }); -}; +const renderedItems = posts.map(renderItem); + +const Loader = () => { +return( +
+
) +} + +if (!posts) return() -const renderedItems = state.posts.map(renderItem); return ( (p.primary ? "1px solid #ECEEF0" : "none")}; + border-right: ${(p) => (p.primary ? "1px solid #ECEEF0" : "none")}; + + > div { + padding-bottom: 24px; + margin-bottom: 24px; + border-bottom: 1px solid #ECEEF0; + + &:last-child { + padding-bottom: 0; + margin-bottom: 0; + border-bottom: none; + } + } + + @media (max-width: 1200px) { + padding-top: 0px; + border-left: none; + border-right: none; + display: ${(p) => (p.active ? "block" : "none")}; + margin: ${(p) => (p.negativeMargin ? "0 -12px" : "0")}; + } +`; + +const Tabs = styled.div` + display: none; + height: 48px; + background: #F8F9FA; + border-bottom: 1px solid #ECEEF0; + margin-bottom: ${(p) => (p.noMargin ? "0" : p.halfMargin ? "24px" : "24px")}; + overflow: auto; + scroll-behavior: smooth; + + @media (max-width: 1200px) { + display: flex; + margin-left: -12px; + margin-right: -12px; + + > * { + flex: 1; + } + } +`; + +const TabsButton = styled.a` + display: inline-flex; + align-items: center; + justify-content: center; + height: 100%; + font-weight: 600; + font-size: 12px; + padding: 0 12px; + position: relative; + color: ${(p) => (p.selected ? "#11181C" : "#687076")}; + background: none; + border: none; + outline: none; + text-align: center; + text-decoration: none !important; + + &:hover { + color: #11181C; + } + + &::after { + content: ''; + display: ${(p) => (p.selected ? "block" : "none")}; + position: absolute; + bottom: 0; + left: 0; + right: 0; + height: 3px; + background: #59E692; + } +`; + +return ( + + + + Posts + + + + Components + + + + Explore + + + +
+
+ + +
+
+ +
+
+ +
+
+
+); + diff --git a/frontend/widgets/examples/feed/src/QueryApi.Feed.jsx b/frontend/widgets/examples/feed/src/QueryApi.Feed.jsx index 42028338e..e800bf49e 100644 --- a/frontend/widgets/examples/feed/src/QueryApi.Feed.jsx +++ b/frontend/widgets/examples/feed/src/QueryApi.Feed.jsx @@ -1,13 +1,75 @@ -const GRAPHQL_ENDPOINT = - "https://queryapi-hasura-graphql-24ktefolwq-ew.a.run.app"; +const GRAPHQL_ENDPOINT = "https://near-queryapi.api.pagoda.co"; const APP_OWNER = "dataplatform.near"; +let lastPostSocialApi = Social.index("post", "main", { + limit: 1, + order: "desc", +}); + +State.init({ + shouldFallback: props.shouldFallback ?? false, +}); + +function fetchGraphQL(operationsDoc, operationName, variables) { + return asyncFetch(`${GRAPHQL_ENDPOINT}/v1/graphql`, { + method: "POST", + headers: { "x-hasura-role": "dataplatform_near" }, + body: JSON.stringify({ + query: operationsDoc, + variables: variables, + operationName: operationName, + }), + }); +} + +const lastPostQuery = ` +query IndexerQuery { + dataplatform_near_social_feed_posts( limit: 1, order_by: { block_height: desc }) { + block_height + } +} +`; + +fetchGraphQL(lastPostQuery, "IndexerQuery", {}) + .then((feedIndexerResponse) => { + if (feedIndexerResponse && feedIndexerResponse.body.data.dataplatform_near_social_feed_posts.length > 0) { + const nearSocialBlockHeight = lastPostSocialApi[0].blockHeight; + const feedIndexerBlockHeight = + feedIndexerResponse.body.data.dataplatform_near_social_feed_posts[0] + .block_height; + + const lag = nearSocialBlockHeight - feedIndexerBlockHeight; + + let shouldFallback = lag > 2 || !feedIndexerBlockHeight; + + // console.log(`Social API block height: ${nearSocialBlockHeight}`); + // console.log(`Feed block height: ${feedIndexerBlockHeight}`); + // console.log(`Lag: ${lag}`); + // console.log(`Fallback to old widget? ${shouldFallback}`); + + State.update({ shouldFallback }); + } else { + console.log("Falling back to old widget."); + State.update({ shouldFallback: true }); + } + }) + .catch((error) => { + console.log("Error while fetching GraphQL(falling back to old widget): ", error); + State.update({ shouldFallback: true }); + }); + return ( - + <> + {state.shouldFallback == true ? ( + + ) : ( + + )} + ); diff --git a/frontend/widgets/examples/feed/src/QueryApi.dev.Feed.jsx b/frontend/widgets/examples/feed/src/QueryApi.dev.Feed.jsx index e6d76ebf4..f3e95cd02 100644 --- a/frontend/widgets/examples/feed/src/QueryApi.dev.Feed.jsx +++ b/frontend/widgets/examples/feed/src/QueryApi.dev.Feed.jsx @@ -1,14 +1,76 @@ const GRAPHQL_ENDPOINT = - "https://queryapi-hasura-graphql-vcqilefdcq-ew.a.run.app"; + "https://near-queryapi.dev.api.pagoda.co"; const APP_OWNER = "dev-queryapi.dataplatform.near"; +let lastPostSocialApi = Social.index("post", "main", { + limit: 1, + order: "desc", +}); + +State.init({ + shouldFallback: props.shouldFallback ?? false, +}); + +function fetchGraphQL(operationsDoc, operationName, variables) { + return asyncFetch(`${GRAPHQL_ENDPOINT}/v1/graphql`, { + method: "POST", + headers: { "x-hasura-role": "dataplatform_near" }, + body: JSON.stringify({ + query: operationsDoc, + variables: variables, + operationName: operationName, + }), + }); +} + +const lastPostQuery = ` +query IndexerQuery { + dataplatform_near_social_feed_posts( limit: 1, order_by: { block_height: desc }) { + block_height + } +} +`; + +fetchGraphQL(lastPostQuery, "IndexerQuery", {}) + .then((feedIndexerResponse) => { + if (feedIndexerResponse && feedIndexerResponse.body.data.dataplatform_near_social_feed_posts.length > 0) { + const nearSocialBlockHeight = lastPostSocialApi[0].blockHeight; + const feedIndexerBlockHeight = + feedIndexerResponse.body.data.dataplatform_near_social_feed_posts[0] + .block_height; + + const lag = nearSocialBlockHeight - feedIndexerBlockHeight; + + let shouldFallback = lag > 2 || !feedIndexerBlockHeight; + + // console.log(`Social API block height: ${nearSocialBlockHeight}`); + // console.log(`Feed block height: ${feedIndexerBlockHeight}`); + // console.log(`Lag: ${lag}`); + // console.log(`Fallback to old widget? ${shouldFallback}`); + + State.update({ shouldFallback }); + } else { + console.log("Falling back to old widget."); + State.update({ shouldFallback: true }); + } + }) + .catch((error) => { + console.log("Error while fetching GraphQL(falling back to old widget): ", error); + State.update({ shouldFallback: true }); + }); return ( - + <> + {state.shouldFallback == true ? ( + + ) : ( + + )} + ); diff --git a/frontend/widgets/src/NearQueryApi.metadata.json b/frontend/widgets/src/NearQueryApi.metadata.json index d574ec564..1e901891d 100644 --- a/frontend/widgets/src/NearQueryApi.metadata.json +++ b/frontend/widgets/src/NearQueryApi.metadata.json @@ -6,6 +6,7 @@ "name": "Near Query API waitlist", "tags": { "indexers": "", - "data-platform": "" + "data-platform": "", + "waitlist": "" } } diff --git a/frontend/widgets/src/QueryApi.App.metadata.json b/frontend/widgets/src/QueryApi.App.metadata.json new file mode 100644 index 000000000..0017edc7c --- /dev/null +++ b/frontend/widgets/src/QueryApi.App.metadata.json @@ -0,0 +1,12 @@ +{ + "description": "Main entrypoint to Near QueryAPI's production widget which allows you to seamlessly create, manage, and discover new indexers", + "image": { + "ipfs_cid": "bafkreihx3wowmjrv3taztqxwgubt6mijaqwzvo6573wi6lv4omxfh3ogdm" + }, + "name": "Near QueryAPI", + "tags": { + "app": "", + "indexers": "", + "data-platform": "" + } +} diff --git a/frontend/widgets/src/QueryApi.Dashboard.metadata.json b/frontend/widgets/src/QueryApi.Dashboard.metadata.json index 3a6291fd5..0c1f9ef89 100644 --- a/frontend/widgets/src/QueryApi.Dashboard.metadata.json +++ b/frontend/widgets/src/QueryApi.Dashboard.metadata.json @@ -1,11 +1,8 @@ { "description": "Main dashboard for Near QueryAPI which allows you to seamlessly create, manage, and discover indexers", "image": { - "ipfs_cid": "bafkreihx3wowmjrv3taztqxwgubt6mijaqwzvo6573wi6lv4omxfh3ogdm" }, - "name": "Near Query API", + "name": "Near QueryAPI Dashboard", "tags": { - "indexers": "", - "data-platform": "" } } diff --git a/frontend/widgets/src/QueryApi.Editor.metadata.json b/frontend/widgets/src/QueryApi.Editor.metadata.json index 4d0315bf9..abf5b7496 100644 --- a/frontend/widgets/src/QueryApi.Editor.metadata.json +++ b/frontend/widgets/src/QueryApi.Editor.metadata.json @@ -1,10 +1,11 @@ { - "description": "Helper widget for QueryApi.Dashboard to help write indexer", + "description": "Helper widget for QueryApi.Dashboard. Loads QueryAPI's React App which allows you to edit indexers inside the browser", "image": { }, "name": "Editor", "tags": { "indexers": "", - "data-platform": "" + "data-platform": "", + "react": "" } } diff --git a/frontend/widgets/src/QueryApi.dev-App.metadata.json b/frontend/widgets/src/QueryApi.dev-App.metadata.json new file mode 100644 index 000000000..bfbdf4a96 --- /dev/null +++ b/frontend/widgets/src/QueryApi.dev-App.metadata.json @@ -0,0 +1,9 @@ +{ + "description": "Entrypoint to Near QueryAPI's development widget which allows you to seamlessly create, manage, and discover new indexers", + "image": { + }, + "name": "QueryAPI Development", + "tags": { + "development": "" + } +} diff --git a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap index cd1464589..c5dccfbb8 100644 --- a/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap +++ b/indexer-js-queue-handler/__snapshots__/hasura-client.test.js.snap @@ -38,6 +38,40 @@ exports[`HasuraClient adds the specified permissions for the specified roles/tab }, "type": "pg_create_insert_permission", }, + { + "args": { + "permission": { + "check": {}, + "columns": "*", + "computed_fields": [], + "filter": {}, + }, + "role": "role", + "source": "default", + "table": { + "name": "height", + "schema": "schema", + }, + }, + "type": "pg_create_update_permission", + }, + { + "args": { + "permission": { + "check": {}, + "columns": "*", + "computed_fields": [], + "filter": {}, + }, + "role": "role", + "source": "default", + "table": { + "name": "height", + "schema": "schema", + }, + }, + "type": "pg_create_delete_permission", + }, { "args": { "permission": { @@ -73,6 +107,40 @@ exports[`HasuraClient adds the specified permissions for the specified roles/tab }, "type": "pg_create_insert_permission", }, + { + "args": { + "permission": { + "check": {}, + "columns": "*", + "computed_fields": [], + "filter": {}, + }, + "role": "role", + "source": "default", + "table": { + "name": "width", + "schema": "schema", + }, + }, + "type": "pg_create_update_permission", + }, + { + "args": { + "permission": { + "check": {}, + "columns": "*", + "computed_fields": [], + "filter": {}, + }, + "role": "role", + "source": "default", + "table": { + "name": "width", + "schema": "schema", + }, + }, + "type": "pg_create_delete_permission", + }, ], "type": "bulk", } diff --git a/indexer-js-queue-handler/__snapshots__/indexer.test.js.snap b/indexer-js-queue-handler/__snapshots__/indexer.test.js.snap index 1f11c303a..c92566108 100644 --- a/indexer-js-queue-handler/__snapshots__/indexer.test.js.snap +++ b/indexer-js-queue-handler/__snapshots__/indexer.test.js.snap @@ -1,5 +1,20 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Indexer unit tests Indexer.buildImperativeContextForFunction() can fetch from the near social api 1`] = ` +[ + [ + "https://api.near.social/index", + { + "body": "{"action":"post","key":"main","options":{"limit":1,"order":"desc"}}", + "headers": { + "Content-Type": "application/json", + }, + "method": "POST", + }, + ], +] +`; + exports[`Indexer unit tests Indexer.runFunctions() allows imperative execution of GraphQL operations 1`] = ` [ [ diff --git a/indexer-js-queue-handler/__snapshots__/metrics.test.js.snap b/indexer-js-queue-handler/__snapshots__/metrics.test.js.snap index 6886b82e0..ab32474fa 100644 --- a/indexer-js-queue-handler/__snapshots__/metrics.test.js.snap +++ b/indexer-js-queue-handler/__snapshots__/metrics.test.js.snap @@ -18,6 +18,10 @@ exports[`Metrics writes the block height for an indexer function 1`] = ` "Name": "STAGE", "Value": "dev", }, + { + "Name": "EXECUTION_TYPE", + "Value": "real-time", + }, ], "MetricName": "INDEXER_FUNCTION_LATEST_BLOCK_HEIGHT", "Unit": "None", diff --git a/indexer-js-queue-handler/hasura-client.js b/indexer-js-queue-handler/hasura-client.js index 4723928ff..318dc913d 100644 --- a/indexer-js-queue-handler/hasura-client.js +++ b/indexer-js-queue-handler/hasura-client.js @@ -230,7 +230,7 @@ export default class HasuraClient { check: {}, computed_fields: [], filter: {}, - ...(permission == 'select' && { allow_aggregations: true }) + ...(permission === "select" && { allow_aggregations: true }) }, source: 'default' }, diff --git a/indexer-js-queue-handler/hasura-client.test.js b/indexer-js-queue-handler/hasura-client.test.js index c64e5a173..150f846b9 100644 --- a/indexer-js-queue-handler/hasura-client.test.js +++ b/indexer-js-queue-handler/hasura-client.test.js @@ -114,7 +114,7 @@ describe('HasuraClient', () => { }); const client = new HasuraClient({ fetch }) - await client.addPermissionsToTables('schema', ['height', 'width'], 'role', ['select', 'insert']); + await client.addPermissionsToTables('schema', ['height', 'width'], 'role', ['select', 'insert', 'update', 'delete']); expect(fetch.mock.calls[0][1].headers['X-Hasura-Admin-Secret']).toBe(HASURA_ADMIN_SECRET) expect(JSON.parse(fetch.mock.calls[0][1].body)).toMatchSnapshot(); diff --git a/indexer-js-queue-handler/indexer.integration.test.js b/indexer-js-queue-handler/indexer.integration.test.js index 5a8c3fa14..1117863fa 100644 --- a/indexer-js-queue-handler/indexer.integration.test.js +++ b/indexer-js-queue-handler/indexer.integration.test.js @@ -20,11 +20,16 @@ const mockAwsXray = { }), }; +const mockMetrics = { + putBlockHeight: () => {}, +}; + + /** These tests require the following Environment Variables to be set: HASURA_ENDPOINT, HASURA_ADMIN_SECRET */ describe('Indexer integration tests', () => { test('Indexer.runFunctions() should execute an imperative style test function against a given block using key-value storage', async () => { - const indexer = new Indexer('mainnet', { fetch: fetch, awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { fetch: fetch, awsXray: mockAwsXray, metrics: mockMetrics }); const functions = {}; functions['buildnear.testnet/itest1'] = {provisioned: false, code: 'context.set("BlockHeight", block.header().height);', schema: 'create table indexer_storage (function_name text, key_name text, value text, primary key (function_name, key_name));'}; const block_height = 85376002; @@ -40,7 +45,7 @@ describe('Indexer integration tests', () => { }, 30000); test('Indexer.runFunctions() should execute a test function against a given block using key-value storage', async () => { - const indexer = new Indexer('mainnet', { awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { awsXray: mockAwsXray, metrics: mockMetrics }); const functions = {}; functions['buildnear.testnet/itest1'] = {code: 'context.set("BlockHeight", block.header().height);'}; const block_height = 85376546; @@ -51,7 +56,7 @@ describe('Indexer integration tests', () => { }, 30000); test('Indexer.runFunctions() should execute a test function against a given block using a full mutation to write to key-value storage', async () => { - const indexer = new Indexer('mainnet', { awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { awsXray: mockAwsXray, metrics: mockMetrics }); const functions = {}; functions['buildnear.testnet/itest1'] = {code: 'context.graphql(`mutation { insert_buildnear_testnet_itest1_indexer_storage_one(object: {function_name: "buildnear.testnet/itest3", key_name: "BlockHeight", value: "${block.header().height}"} on_conflict: {constraint: indexer_storage_pkey, update_columns: value}) {key_name}}`);'}; const block_height = 85376546; @@ -65,7 +70,7 @@ describe('Indexer integration tests', () => { * due to known Hasura issues with unique indexes vs unique constraints */ test('Indexer.runFunctions() should execute a near social function against a given block', async () => { - const indexer = new Indexer('mainnet', { awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { awsXray: mockAwsXray, metrics: mockMetrics }); const functions = {}; functions['buildnear.testnet/test'] = {code: @@ -128,7 +133,7 @@ describe('Indexer integration tests', () => { * due to known Hasura issues with unique indexes vs unique constraints */ // needs update to have schema test.skip('Indexer.runFunctions() should execute an imperative style near social function against a given block', async () => { - const indexer = new Indexer('mainnet', { awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { awsXray: mockAwsXray, metrics: mockMetrics }); const functions = {}; functions['buildnear.testnet/itest5'] = {code:` @@ -176,14 +181,14 @@ describe('Indexer integration tests', () => { }); test("writeLog() should write a log to the database", async () => { - const indexer = new Indexer('mainnet', { awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { awsXray: mockAwsXray, metrics: mockMetrics }); const id = await indexer.writeLog("buildnear.testnet/itest", 85376002, "test message"); expect(id).toBeDefined(); expect(id.length).toBe(36); }); test("writeFunctionState should write a function state to the database", async () => { - const indexer = new Indexer('mainnet', { awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { awsXray: mockAwsXray, metrics: mockMetrics }); const result = await indexer.writeFunctionState("buildnear.testnet/itest8", 85376002); expect(result).toBeDefined(); expect(result.insert_indexer_state.returning[0].current_block_height).toBe(85376002); @@ -191,7 +196,7 @@ describe('Indexer integration tests', () => { // Errors are now exposed to the lambda hander. This test will be relevant again if this changes. test.skip ("function that throws an error should catch the error", async () => { - const indexer = new Indexer('mainnet', { awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { awsXray: mockAwsXray, metrics: mockMetrics }); const functions = {}; functions['buildnear.testnet/test'] = {code:` @@ -205,7 +210,7 @@ describe('Indexer integration tests', () => { // Errors are now exposed to the lambda hander. This test will be relevant again if this changes. test.skip("rejected graphql promise is awaited and caught", async () => { - const indexer = new Indexer('mainnet', { awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { awsXray: mockAwsXray, metrics: mockMetrics }); const functions = {}; functions['buildnear.testnet/itest3'] = {code: @@ -219,7 +224,7 @@ describe('Indexer integration tests', () => { // Unreturned promise rejection seems to be uncatchable even with process.on('unhandledRejection' // However, the next function is run (in this test but not on Lambda). test.skip("function that rejects a promise should catch the error", async () => { - const indexer = new Indexer('mainnet', { awsXray: mockAwsXray }); + const indexer = new Indexer('mainnet', { awsXray: mockAwsXray, metrics: mockMetrics }); const functions = {}; functions['buildnear.testnet/fails'] = {code:` diff --git a/indexer-js-queue-handler/indexer.js b/indexer-js-queue-handler/indexer.js index 63e69a535..7ecb37464 100644 --- a/indexer-js-queue-handler/indexer.js +++ b/indexer-js-queue-handler/indexer.js @@ -46,7 +46,7 @@ export default class Indexer { functionSubsegment.addAnnotation('indexer_function', function_name); simultaneousPromises.push(this.writeLog(function_name, block_height, runningMessage)); - simultaneousPromises.push(this.deps.metrics.putBlockHeight(indexerFunction.account_id, indexerFunction.function_name, block_height)); + simultaneousPromises.push(this.deps.metrics.putBlockHeight(indexerFunction.account_id, indexerFunction.function_name, is_historical, block_height)); const hasuraRoleName = function_name.split('/')[0].replace(/[.-]/g, '_'); const functionNameWithoutAccount = function_name.split('/')[1].replace(/[.-]/g, '_'); @@ -74,7 +74,7 @@ export default class Indexer { const vm = new VM({timeout: 3000, allowAsync: true}); const mutationsReturnValue = {mutations: [], variables: {}, keysValues: {}}; const context = options.imperative - ? this.buildImperativeContextForFunction(function_name, functionNameWithoutAccount, block_height, hasuraRoleName) + ? this.buildImperativeContextForFunction(function_name, functionNameWithoutAccount, block_height, hasuraRoleName, is_historical) : this.buildFunctionalContextForFunction(mutationsReturnValue, function_name, block_height); vm.freeze(blockWithHelpers, 'block'); @@ -218,14 +218,13 @@ export default class Indexer { set: (key, value) => { mutationsReturnValue.keysValues[key] = value; }, - log: async (log) => { // starting with imperative logging for both imperative and functional contexts - return await this.writeLog(functionName, block_height, log); - } - + log: async (...log) => { // starting with imperative logging for both imperative and functional contexts + return await this.writeLog(functionName, block_height, ...log); + }, }; } - buildImperativeContextForFunction(functionName, functionNameWithoutAccount, block_height, hasuraRoleName) { + buildImperativeContextForFunction(functionName, functionNameWithoutAccount, block_height, hasuraRoleName, is_historical) { return { graphql: async (operation, variables) => { try { @@ -252,8 +251,21 @@ export default class Indexer { throw e; // allow catch outside of vm.run to receive the error } }, - log: async (log) => { - return await this.writeLog(functionName, block_height, log); + log: async (...log) => { + return await this.writeLog(functionName, block_height, ...log); + }, + putMetric: (name, value) => { + const [accountId, fnName] = functionName.split('/'); + return this.deps.metrics.putCustomMetric( + accountId, + fnName, + is_historical, + `CUSTOM_${name}`, + value + ); + }, + fetchFromSocialApi: async (path, options) => { + return this.deps.fetch(`https://api.near.social${path}`, options); } }; } diff --git a/indexer-js-queue-handler/indexer.test.js b/indexer-js-queue-handler/indexer.test.js index ce2f39b3b..daea27a65 100644 --- a/indexer-js-queue-handler/indexer.test.js +++ b/indexer-js-queue-handler/indexer.test.js @@ -318,6 +318,30 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in ]); }); + test('Indexer.buildImperativeContextForFunction() can fetch from the near social api', async () => { + const mockFetch = jest.fn(); + const indexer = new Indexer('mainnet', { fetch: mockFetch, awsXray: mockAwsXray, metrics: mockMetrics }); + + const context = indexer.buildImperativeContextForFunction(); + + await context.fetchFromSocialApi('/index', { + method: 'POST', + headers: { + ['Content-Type']: 'application/json', + }, + body: JSON.stringify({ + action: 'post', + key: 'main', + options: { + limit: 1, + order: 'desc' + } + }) + }); + + expect(mockFetch.mock.calls).toMatchSnapshot(); + }); + test('Indexer.buildImperativeContextForFunction() throws when a GraphQL response contains errors', async () => { const mockFetch = jest.fn() .mockResolvedValue({ @@ -480,14 +504,14 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in test('Indexer.runFunctions() console.logs', async () => { const logs = [] - const context = {log: (m) => { - logs.push(m) + const context = {log: (...m) => { + logs.push(...m) }}; const vm = new VM(); vm.freeze(context, 'context'); vm.freeze(context, 'console'); - await vm.run('console.log("hello"); context.log("world")'); - expect(logs).toEqual(['hello','world']); + await vm.run('console.log("hello", "brave new"); context.log("world")'); + expect(logs).toEqual(['hello','brave new','world']); }); test("Errors thrown in VM can be caught outside the VM", async () => { @@ -821,7 +845,7 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in }; await indexer.runFunctions(block_height, functions, false); - expect(metrics.putBlockHeight).toHaveBeenCalledWith('buildnear.testnet', 'test', block_height); + expect(metrics.putBlockHeight).toHaveBeenCalledWith('buildnear.testnet', 'test', false, block_height); }); test('does not attach the hasura admin secret header when no role specified', async () => { @@ -895,6 +919,49 @@ mutation _1 { set(functionName: "buildnear.testnet/test", key: "foo2", data: "in ]); }); + test('allows writing of custom metrics', async () => { + const mockFetch = jest.fn(() => ({ + status: 200, + json: async () => ({ + errors: null, + }), + })); + const block_height = 456; + const mockS3 = { + getObject: jest.fn(() => ({ + promise: () => Promise.resolve({ + Body: { + toString: () => JSON.stringify({ + chunks: [], + header: { + height: block_height + } + }) + } + }) + })), + }; + const metrics = { + putBlockHeight: () => {}, + putCustomMetric: jest.fn(), + }; + const indexer = new Indexer('mainnet', { fetch: mockFetch, s3: mockS3, awsXray: mockAwsXray, metrics }); + + const functions = {}; + functions['buildnear.testnet/test'] = {code:` + context.putMetric('TEST_METRIC', 1) + `}; + await indexer.runFunctions(block_height, functions, true, { imperative: true }); + + expect(metrics.putCustomMetric).toHaveBeenCalledWith( + 'buildnear.testnet', + 'test', + true, + 'CUSTOM_TEST_METRIC', + 1 + ); + }); + // The unhandled promise causes problems with test reporting. // Note unhandled promise rejections fail to proceed to the next function on AWS Lambda test.skip('Indexer.runFunctions() continues despite promise rejection, unable to log rejection', async () => { diff --git a/indexer-js-queue-handler/latest-post-metrics-writer.js b/indexer-js-queue-handler/latest-post-metrics-writer.js deleted file mode 100644 index 95f03c8f9..000000000 --- a/indexer-js-queue-handler/latest-post-metrics-writer.js +++ /dev/null @@ -1,33 +0,0 @@ -import fetch from "node-fetch"; -import AWS from "aws-sdk"; - -import Metrics from "./metrics.js"; - -export const handler = async () => { - const metrics = new Metrics("QueryAPI"); - - const response = await fetch("https://api.near.social/index", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - action: "post", - key: "main", - options: { - limit: 1, - order: "desc", - }, - }), - }); - - const body = await response.text(); - - if (response.status !== 200) { - throw new Error(body); - } - - const [{ blockHeight }] = JSON.parse(body); - - await metrics.putBlockHeight("social.near", "posts", blockHeight); -}; diff --git a/indexer-js-queue-handler/metrics.js b/indexer-js-queue-handler/metrics.js index 52909fe30..cd7a653ac 100644 --- a/indexer-js-queue-handler/metrics.js +++ b/indexer-js-queue-handler/metrics.js @@ -7,12 +7,16 @@ export default class Metrics { this.namespace = namespace; } - putBlockHeight(accountId, functionName, height) { + putBlockHeight(accountId, functionName, isHistorical, height) { + return this.putCustomMetric(accountId, functionName, isHistorical, "INDEXER_FUNCTION_LATEST_BLOCK_HEIGHT", height); + } + + putCustomMetric(accountId, functionName, isHistorical, metricName, value) { return this.cloudwatch .putMetricData({ MetricData: [ { - MetricName: "INDEXER_FUNCTION_LATEST_BLOCK_HEIGHT", + MetricName: metricName, Dimensions: [ { Name: "ACCOUNT_ID", @@ -26,9 +30,13 @@ export default class Metrics { Name: "STAGE", Value: process.env.STAGE, }, + { + Name: "EXECUTION_TYPE", + Value: isHistorical ? "historical" : "real-time", + }, ], Unit: "None", - Value: height, + Value: value, }, ], Namespace: this.namespace, diff --git a/indexer-js-queue-handler/metrics.test.js b/indexer-js-queue-handler/metrics.test.js index aa58b8875..fc9b74aaf 100644 --- a/indexer-js-queue-handler/metrics.test.js +++ b/indexer-js-queue-handler/metrics.test.js @@ -22,7 +22,7 @@ describe('Metrics', () => { }; const metrics = new Metrics('test', cloudwatch); - await metrics.putBlockHeight('morgs.near', 'test', 2); + await metrics.putBlockHeight('morgs.near', 'test', false, 2); expect(cloudwatch.putMetricData).toBeCalledTimes(1); expect(cloudwatch.putMetricData.mock.calls[0]).toMatchSnapshot() diff --git a/indexer-js-queue-handler/serverless.yml b/indexer-js-queue-handler/serverless.yml index a44657e3e..368400cc7 100644 --- a/indexer-js-queue-handler/serverless.yml +++ b/indexer-js-queue-handler/serverless.yml @@ -41,8 +41,8 @@ constructs: timeout: 15 # 1.5 minutes as lift multiplies this value by 6 (https://github.com/getlift/lift/blob/master/docs/queue.md#retry-delay) functions: - latestPostMetricsWriter: - handler: latest-post-metrics-writer.handler + socialLagMetricsWriter: + handler: social-lag-metrics-writer.handler events: - schedule: rate(1 minute) diff --git a/indexer-js-queue-handler/social-lag-metrics-writer.js b/indexer-js-queue-handler/social-lag-metrics-writer.js new file mode 100644 index 000000000..ec9d28f92 --- /dev/null +++ b/indexer-js-queue-handler/social-lag-metrics-writer.js @@ -0,0 +1,63 @@ +import fetch from "node-fetch"; +import AWS from "aws-sdk"; + +import Metrics from "./metrics.js"; + +const fetchJson = async (url, requestBody, requestHeaders) => { + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...requestHeaders, + }, + body: JSON.stringify(requestBody), + }); + + const responseBody = await response.json(); + + if (response.status !== 200 || responseBody.errors) { + throw new Error(JSON.stringify(responseBody)); + } + + return responseBody; +}; + +export const handler = async () => { + const metrics = new Metrics("QueryAPI"); + + const [nearSocialResponse, feedIndexerResponse] = await Promise.all([ + fetchJson(`https://api.near.social/index`, { + action: "post", + key: "main", + options: { + limit: 1, + order: "desc", + }, + }), + fetchJson( + `${process.env.HASURA_ENDPOINT}/v1/graphql`, + { + query: `{ + dataplatform_near_social_feed_posts( + limit: 1, + order_by: { block_height: desc } + ) { + block_height + } + }`, + }, + { + ["X-Hasura-Role"]: "dataplatform_near", + } + ), + ]); + + const nearSocialBlockHeight = nearSocialResponse[0].blockHeight; + const feedIndexerBlockHeight = + feedIndexerResponse.data.dataplatform_near_social_feed_posts[0] + .block_height; + + const lag = nearSocialBlockHeight - feedIndexerBlockHeight; + + await metrics.putCustomMetric("dataplatform.near", "social_feed", false, 'SOCIAL_LAG', lag); +}; diff --git a/indexer/queryapi_coordinator/src/historical_block_processing.rs b/indexer/queryapi_coordinator/src/historical_block_processing.rs index c8c4fbeaf..9800d25d5 100644 --- a/indexer/queryapi_coordinator/src/historical_block_processing.rs +++ b/indexer/queryapi_coordinator/src/historical_block_processing.rs @@ -29,6 +29,7 @@ pub fn spawn_historical_message_thread( tokio::spawn(process_historical_messages( block_height, new_indexer_function_copy, + Opts::parse(), )) }) } @@ -36,6 +37,7 @@ pub fn spawn_historical_message_thread( pub(crate) async fn process_historical_messages( block_height: BlockHeight, indexer_function: IndexerFunction, + opts: Opts, ) -> i64 { let start_block = indexer_function.start_block_height.unwrap(); let block_difference: i64 = (block_height - start_block) as i64; @@ -58,8 +60,6 @@ pub(crate) async fn process_historical_messages( indexer_function.function_name ); - let opts = Opts::parse(); - let chain_id = opts.chain_id().clone(); let aws_region = opts.aws_queue_region.clone(); let queue_client = queue::queue_client(aws_region, opts.queue_credentials()); @@ -117,16 +117,17 @@ pub(crate) async fn process_historical_messages( blocks_from_index.append(&mut blocks_between_indexed_and_current_block); + let first_block_in_data = *blocks_from_index.first().unwrap_or(&start_block); for current_block in blocks_from_index { send_execution_message( block_height, - start_block, + first_block_in_data, chain_id.clone(), &queue_client, queue_url.clone(), &mut indexer_function, current_block, - None, //alert_queue_message.payload.clone(), // future: populate with data from the Match + None, ) .await; } @@ -180,7 +181,7 @@ pub(crate) async fn filter_matching_blocks_from_index_files( let index_files_content = match &indexer_rule.matching_rule { MatchingRule::ActionAny { affected_account_id, - status, + .. } => { if affected_account_id.contains('*') || affected_account_id.contains(',') { needs_dedupe_and_sort = true; @@ -194,11 +195,7 @@ pub(crate) async fn filter_matching_blocks_from_index_files( ) .await } - MatchingRule::ActionFunctionCall { - affected_account_id, - status, - function, - } => { + MatchingRule::ActionFunctionCall { .. } => { tracing::error!( target: crate::INDEXER, "ActionFunctionCall matching rule not supported for historical processing" @@ -392,7 +389,7 @@ fn normalize_block_height(block_height: BlockHeight) -> String { async fn send_execution_message( block_height: BlockHeight, - start_block: u64, + first_block: BlockHeight, chain_id: ChainId, queue_client: &Client, queue_url: String, @@ -401,7 +398,7 @@ async fn send_execution_message( payload: Option, ) { // only request provisioning on the first block - if current_block != start_block { + if current_block != first_block { indexer_function.provisioned = true; } diff --git a/indexer/queryapi_coordinator/src/historical_block_processing_integration_tests.rs b/indexer/queryapi_coordinator/src/historical_block_processing_integration_tests.rs index 8a862b2eb..3457974e7 100644 --- a/indexer/queryapi_coordinator/src/historical_block_processing_integration_tests.rs +++ b/indexer/queryapi_coordinator/src/historical_block_processing_integration_tests.rs @@ -1,140 +1,167 @@ -use crate::historical_block_processing::filter_matching_blocks_from_index_files; -use crate::indexer_types::IndexerFunction; -use crate::opts::{Opts, Parser}; -use crate::{historical_block_processing, opts}; -use aws_types::SdkConfig; -use chrono::{DateTime, NaiveDate, Utc}; -use indexer_rule_type::indexer_rule::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; -use near_lake_framework::near_indexer_primitives::types::BlockHeight; -use std::ops::Range; +#[cfg(test)] +mod tests { + use crate::historical_block_processing::filter_matching_blocks_from_index_files; + use crate::indexer_types::IndexerFunction; + use crate::opts::{ChainId, Opts, StartOptions}; + use crate::{historical_block_processing, opts}; + use aws_types::SdkConfig; + use chrono::{DateTime, NaiveDate, Utc}; + use indexer_rule_type::indexer_rule::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; + use near_lake_framework::near_indexer_primitives::types::BlockHeight; + use std::env; + use std::ops::Range; -/// Parses env vars from .env, Run with -/// cargo test historical_block_processing_integration_tests::test_indexing_metadata_file -- mainnet from-latest; -#[tokio::test] -async fn test_indexing_metadata_file() { - let opts = Opts::parse(); - let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); + impl Opts { + pub fn test_opts_with_aws() -> Self { + dotenv::dotenv().ok(); + let lake_aws_access_key = env::var("LAKE_AWS_ACCESS_KEY").unwrap(); + let lake_aws_secret_access_key = env::var("LAKE_AWS_SECRET_ACCESS_KEY").unwrap(); + Opts { + redis_connection_string: "".to_string(), + lake_aws_access_key, + lake_aws_secret_access_key, + queue_aws_access_key: "".to_string(), + queue_aws_secret_access_key: "".to_string(), + aws_queue_region: "".to_string(), + queue_url: "".to_string(), + start_from_block_queue_url: "".to_string(), + registry_contract_id: "".to_string(), + port: 0, + chain_id: ChainId::Mainnet(StartOptions::FromLatest), + } + } + } - let last_indexed_block = - historical_block_processing::last_indexed_block_from_metadata(aws_config) - .await - .unwrap(); - let a: Range = 90000000..9000000000; // valid for the next 300 years - assert!(a.contains(&last_indexed_block)); -} + /// Parses env vars from .env, Run with + /// cargo test historical_block_processing_integration_tests::test_indexing_metadata_file -- mainnet from-latest; + #[tokio::test] + async fn test_indexing_metadata_file() { + let opts = Opts::test_opts_with_aws(); + let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); -/// Parses env vars from .env, Run with -/// cargo test historical_block_processing_integration_tests::test_process_historical_messages -- mainnet from-latest; -#[tokio::test] -async fn test_process_historical_messages() { - opts::init_tracing(); + let last_indexed_block = + historical_block_processing::last_indexed_block_from_metadata(aws_config) + .await + .unwrap(); + let a: Range = 90000000..9000000000; // valid for the next 300 years + assert!(a.contains(&last_indexed_block)); + } - let contract = "queryapi.dataplatform.near"; - let matching_rule = MatchingRule::ActionAny { - affected_account_id: contract.to_string(), - status: Status::Any, - }; - let filter_rule = IndexerRule { - indexer_rule_kind: IndexerRuleKind::Action, - matching_rule, - id: None, - name: None, - }; - let indexer_function = IndexerFunction { - account_id: "buildnear.testnet".to_string().parse().unwrap(), - function_name: "index_stuff".to_string(), - code: "".to_string(), - start_block_height: Some(85376002), - schema: None, - provisioned: false, - indexer_rule: filter_rule, - }; + /// Parses env vars from .env, Run with + /// cargo test historical_block_processing_integration_tests::test_process_historical_messages -- mainnet from-latest; + #[tokio::test] + async fn test_process_historical_messages() { + opts::init_tracing(); - let opts = Opts::parse(); - let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); - let fake_block_height = - historical_block_processing::last_indexed_block_from_metadata(aws_config) - .await - .unwrap(); - historical_block_processing::process_historical_messages( - fake_block_height + 1, - indexer_function, - ) - .await; -} + let contract = "queryapi.dataplatform.near"; + let matching_rule = MatchingRule::ActionAny { + affected_account_id: contract.to_string(), + status: Status::Any, + }; + let filter_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule, + id: None, + name: None, + }; + let indexer_function = IndexerFunction { + account_id: "buildnear.testnet".to_string().parse().unwrap(), + function_name: "index_stuff".to_string(), + code: "".to_string(), + start_block_height: Some(85376002), + schema: None, + provisioned: false, + indexer_rule: filter_rule, + }; -/// Parses env vars from .env, Run with -/// cargo test historical_block_processing_integration_tests::test_filter_matching_wildcard_blocks_from_index_files -- mainnet from-latest; -#[tokio::test] -async fn test_filter_matching_wildcard_blocks_from_index_files() { - let contract = "*.keypom.near"; - let matching_rule = MatchingRule::ActionAny { - affected_account_id: contract.to_string(), - status: Status::Any, - }; - let filter_rule = IndexerRule { - indexer_rule_kind: IndexerRuleKind::Action, - matching_rule, - id: None, - name: None, - }; + let opts = Opts::test_opts_with_aws(); + let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); + let fake_block_height = + historical_block_processing::last_indexed_block_from_metadata(aws_config) + .await + .unwrap(); + historical_block_processing::process_historical_messages( + fake_block_height + 1, + indexer_function, + opts, + ) + .await; + } - let opts = Opts::parse(); - let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); + /// Parses env vars from .env, Run with + /// cargo test historical_block_processing_integration_tests::test_filter_matching_wildcard_blocks_from_index_files -- mainnet from-latest; + #[tokio::test] + async fn test_filter_matching_wildcard_blocks_from_index_files() { + let contract = "*.keypom.near"; + let matching_rule = MatchingRule::ActionAny { + affected_account_id: contract.to_string(), + status: Status::Any, + }; + let filter_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule, + id: None, + name: None, + }; - let start_block_height = 75472603; - let naivedatetime_utc = NaiveDate::from_ymd_opt(2022, 10, 03) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap(); - let datetime_utc = DateTime::::from_utc(naivedatetime_utc, Utc); - let blocks = filter_matching_blocks_from_index_files( - start_block_height, - &filter_rule, - aws_config, - datetime_utc, - ) - .await; + let opts = Opts::test_opts_with_aws(); + let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); - // // remove any blocks from after when the test was written -- not working, due to new contracts? - // let fixed_blocks: Vec = blocks.into_iter().filter(|&b| b <= 95175853u64).collect(); // 95175853u64 95242647u64 - assert!(blocks.len() > 21830); // 22913 raw, deduped to 21830 -} + let start_block_height = 77016214; + let naivedatetime_utc = NaiveDate::from_ymd_opt(2022, 10, 03) + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap(); + let datetime_utc = DateTime::::from_utc(naivedatetime_utc, Utc); + let blocks = filter_matching_blocks_from_index_files( + start_block_height, + &filter_rule, + aws_config, + datetime_utc, + ) + .await; -/// Parses env vars from .env, Run with -/// cargo test historical_block_processing_integration_tests::test_filter_matching_blocks_from_index_files -- mainnet from-latest; -#[tokio::test] -async fn test_filter_matching_blocks_from_index_files() { - let contract = "*.agency.near"; - let matching_rule = MatchingRule::ActionAny { - affected_account_id: contract.to_string(), - status: Status::Any, - }; - let filter_rule = IndexerRule { - indexer_rule_kind: IndexerRuleKind::Action, - matching_rule, - id: None, - name: None, - }; + // // remove any blocks from after when the test was written -- not working, due to new contracts? + // let fixed_blocks: Vec = blocks.into_iter().filter(|&b| b <= 95175853u64).collect(); // 95175853u64 95242647u64 + assert!(blocks.len() >= 71899); + } - let opts = Opts::parse(); - let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); + /// Parses env vars from .env, Run with + /// cargo test historical_block_processing_integration_tests::test_filter_matching_blocks_from_index_files -- mainnet from-latest; + #[tokio::test] + async fn test_filter_matching_blocks_from_index_files() { + let contract = "*.agency.near"; + let matching_rule = MatchingRule::ActionAny { + affected_account_id: contract.to_string(), + status: Status::Any, + }; + let filter_rule = IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + matching_rule, + id: None, + name: None, + }; - let start_block_height = 45894620; - let naivedatetime_utc = NaiveDate::from_ymd_opt(2021, 08, 01) - .unwrap() - .and_hms_opt(0, 0, 0) - .unwrap(); - let datetime_utc = DateTime::::from_utc(naivedatetime_utc, Utc); - let blocks = filter_matching_blocks_from_index_files( - start_block_height, - &filter_rule, - aws_config, - datetime_utc, - ) - .await; + let opts = Opts::test_opts_with_aws(); + let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); + + let start_block_height = 45894620; + let naivedatetime_utc = NaiveDate::from_ymd_opt(2021, 08, 01) + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap(); + let datetime_utc = DateTime::::from_utc(naivedatetime_utc, Utc); + let blocks = filter_matching_blocks_from_index_files( + start_block_height, + &filter_rule, + aws_config, + datetime_utc, + ) + .await; - // remove any blocks from after when the test was written - let fixed_blocks: Vec = blocks.into_iter().filter(|&b| b <= 95175853u64).collect(); - assert_eq!(fixed_blocks.len(), 6); // hackathon.agency.near = 45894627,45898423, hacker.agency.near = 45897358, hack.agency.near = 45894872,45895120,45896237 + // remove any blocks from after when the test was written + let fixed_blocks: Vec = + blocks.into_iter().filter(|&b| b <= 95175853u64).collect(); + assert_eq!(fixed_blocks.len(), 6); // hackathon.agency.near = 45894627,45898423, hacker.agency.near = 45897358, hack.agency.near = 45894872,45895120,45896237 + } } diff --git a/indexer/queryapi_coordinator/src/main.rs b/indexer/queryapi_coordinator/src/main.rs index d663f7c8c..e447f3316 100644 --- a/indexer/queryapi_coordinator/src/main.rs +++ b/indexer/queryapi_coordinator/src/main.rs @@ -46,7 +46,6 @@ pub(crate) struct QueryApiContext<'a> { pub registry_contract_id: &'a str, pub balance_cache: &'a BalanceCache, pub redis_connection_manager: &'a ConnectionManager, - pub json_rpc_client: &'a JsonRpcClient, } #[tokio::main] @@ -102,7 +101,6 @@ async fn main() -> anyhow::Result<()> { let context = QueryApiContext { redis_connection_manager: &redis_connection_manager, queue_url: &queue_url, - json_rpc_client: &json_rpc_client, balance_cache: &balances_cache, registry_contract_id: ®istry_contract_id, streamer_message, @@ -245,7 +243,7 @@ async fn handle_streamer_message( fn set_provisioned_flag( indexer_registry_locked: &mut MutexGuard, - indexer_function: &&IndexerFunction, + indexer_function: &IndexerFunction, ) { match indexer_registry_locked.get_mut(&indexer_function.account_id) { Some(account_functions) => { @@ -297,3 +295,49 @@ async fn reduce_rule_matches_for_indexer_function<'x>( #[cfg(test)] mod historical_block_processing_integration_tests; + +use indexer_rule_type::indexer_rule::{IndexerRule, IndexerRuleKind, MatchingRule, Status}; +use std::collections::HashMap; + +#[tokio::test] +async fn set_provisioning_finds_functions_in_registry() { + let mut indexer_registry = IndexerRegistry::new(); + let indexer_function = IndexerFunction { + account_id: "test_near".to_string().parse().unwrap(), + function_name: "test_indexer".to_string(), + code: "".to_string(), + start_block_height: None, + schema: None, + provisioned: false, + indexer_rule: IndexerRule { + indexer_rule_kind: IndexerRuleKind::Action, + id: None, + name: None, + matching_rule: MatchingRule::ActionAny { + affected_account_id: "social.near".to_string(), + status: Status::Success, + }, + }, + }; + + let mut functions: HashMap = HashMap::new(); + functions.insert( + indexer_function.function_name.clone(), + indexer_function.clone(), + ); + indexer_registry.insert(indexer_function.account_id.clone(), functions); + + let indexer_registry: SharedIndexerRegistry = std::sync::Arc::new(Mutex::new(indexer_registry)); + let mut indexer_registry_locked = indexer_registry.lock().await; + + set_provisioned_flag(&mut indexer_registry_locked, &&indexer_function); + + let account_functions = indexer_registry_locked + .get(&indexer_function.account_id) + .unwrap(); + let indexer_function = account_functions + .get(&indexer_function.function_name) + .unwrap(); + + assert!(indexer_function.provisioned); +} diff --git a/indexer/queryapi_coordinator/src/s3.rs b/indexer/queryapi_coordinator/src/s3.rs index 775958af8..f5e63e753 100644 --- a/indexer/queryapi_coordinator/src/s3.rs +++ b/indexer/queryapi_coordinator/src/s3.rs @@ -253,7 +253,6 @@ mod tests { #[tokio::test] async fn list_with_single_contract() { let opts = Opts::parse(); - let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); let list = find_index_files_by_pattern( &opts.lake_aws_sdk_config(), @@ -269,7 +268,6 @@ mod tests { #[tokio::test] async fn list_with_csv_contracts() { let opts = Opts::parse(); - let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); let list = find_index_files_by_pattern( &opts.lake_aws_sdk_config(), @@ -285,7 +283,6 @@ mod tests { #[tokio::test] async fn list_with_wildcard_contracts() { let opts = Opts::parse(); - let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); let list = find_index_files_by_pattern( &opts.lake_aws_sdk_config(), @@ -301,7 +298,6 @@ mod tests { #[tokio::test] async fn list_with_csv_and_wildcard_contracts() { let opts = Opts::parse(); - let aws_config: &SdkConfig = &opts.lake_aws_sdk_config(); let list = find_index_files_by_pattern( &opts.lake_aws_sdk_config(),