feat: explorers page indexer to find most popular indexers (desc) (#926)

display indexers by numQueries Desc. **High level Approach:** onLoad set metadata from databricks to local state. immediately after QueryAPI indexer grabs all results and any matching metadata there is from the above state and sets all data new state that has all indexer data. pagination loads 50 and loads locally from all indexer data leading to an instant interaction and loading
near · Jul 30, 2024 · 221ef09 · 221ef09
1 parent a8f91a9
commit 221ef09
Show file tree

Hide file tree

Showing 2 changed files with 86 additions and 76 deletions.
diff --git a/frontend/widgets/src/QueryApi.IndexerCard.jsx b/frontend/widgets/src/QueryApi.IndexerCard.jsx
@@ -1,15 +1,12 @@
-const { accountId, indexerName, indexerMetadata } = props;
-const sanitizedAccountID = accountId.replace(/\./g, '_');
-const key = `${sanitizedAccountID}/${indexerName}`;
+const { accountId, indexerName, numQueries } = props;
 
 const indexer = {
   accountId,
   indexerName,
-  ...(indexerMetadata.has(key) && indexerMetadata.get(key))
+  numQueries,
 };
 
 const editUrl = `https://dev.near.org/${REPL_ACCOUNT_ID}/widget/QueryApi.App?selectedIndexerPath=${accountId}/${indexerName}`;
-
 const playgroundLink = `https://cloud.hasura.io/public/graphiql?endpoint=${REPL_GRAPHQL_ENDPOINT}/v1/graphql&header=x-hasura-role%3A${accountId.replace(/\./g, '_')}`;
 const formatNumberWithCommas = (number) => number.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
 

diff --git a/frontend/widgets/src/QueryApi.IndexerExplorer.jsx b/frontend/widgets/src/QueryApi.IndexerExplorer.jsx
@@ -3,11 +3,9 @@ const myAccountId = context.accountId;
 const PAGE_SIZE = 50;
 const TABLE_NAME = "dataplatform_near_queryapi_indexer_indexers";
 const GET_ALL_ACTIVE_INDEXERS = `
-query getAllActiveIndexers($limit: Int!, $offset: Int!) {
+query getAllActiveIndexers {
   ${TABLE_NAME}(
     where: {is_removed: {_eq: false}}
-    limit: $limit
-    offset: $offset
   ) {
     author_account_id
     indexer_name
@@ -19,8 +17,7 @@ query getAllActiveIndexers($limit: Int!, $offset: Int!) {
       count
     }
   }
-}
-`;
+}`;
 
 const GET_MY_ACTIVE_INDEXERS = `
 query getMyActiveIndexers($authorAccountId: String!) {
@@ -37,16 +34,17 @@ query getMyActiveIndexers($authorAccountId: String!) {
 `;
 
 const [selectedTab, setSelectedTab] = useState(props.tab && props.tab !== "all" ? props.tab : "all");
-const [indexerMetadata, setIndexerMetaData] = useState(new Map());
+const [indexerMetaData, setIndexerMetaData] = useState(new Map());
+const [hasMetadataRendered, setHasMetadataRendered] = useState(false);
 
 const [indexers, setIndexers] = useState([]);
 const [total, setTotal] = useState(0);
-const [myIndexers, setMyIndexers] = useState([]);
+const [currentPageIndexer, setCurrentPageIndexer] = useState([]);
 const [page, setPage] = useState(0);
 
+const [myIndexers, setMyIndexers] = useState([]);
+
 const [loading, setLoading] = useState(false);
-const [isLoadingMore, setIsLoadingMore] = useState(false);
-const [isFetching, setIsFetching] = useState(false);
 const [error, setError] = useState(null);
 
 const fetchGraphQL = (operationsDoc, operationName, variables) => {
@@ -64,87 +62,101 @@ const fetchGraphQL = (operationsDoc, operationName, variables) => {
   });
 }
 
-const fetchMyIndexerData = () => {
+const fetchIndexerData = () => {
   setLoading(true);
-
-  fetchGraphQL(GET_MY_ACTIVE_INDEXERS, 'getMyActiveIndexers', {
-    authorAccountId: myAccountId,
-  })
-    .then((result) => {
-      if (result.status === 200) {
-        const data = result?.body?.data?.[TABLE_NAME];
-        if (Array.isArray(data)) {
-          const newIndexers = data.map(({ author_account_id, indexer_name }) => ({
+  fetchGraphQL(GET_ALL_ACTIVE_INDEXERS, 'getAllActiveIndexers').then((result) => {
+    if (result.status === 200) {
+      const data = result?.body?.data?.[TABLE_NAME];
+      const totalCount = result?.body?.data?.[`${TABLE_NAME}_aggregate`]?.aggregate?.count;
+      if (Array.isArray(data)) {
+        //inject metadata if exist
+        const newIndexers = data.map(({ author_account_id, indexer_name }) => {
+          const sanitizedAccountID = author_account_id.replace(/\./g, '_');
+          const key = `${sanitizedAccountID}/${indexer_name}`;
+          return ({
             accountId: author_account_id,
             indexerName: indexer_name,
-          }));
-          setMyIndexers(newIndexers);
-        } else {
-          throw new Error('Data is not an array:', data);
-        }
+            ...(indexerMetaData.has(key) && indexerMetaData.get(key))
+          })
+        });
+        // sort by numQueries
+        const sortedIndexers = newIndexers.sort((a, b) => (b.numQueries ?? 0) - (a.numQueries ?? 0));
+        setTotal(totalCount);
+        setIndexers(sortedIndexers);
+        setCurrentPageIndexer(sortedIndexers.slice(0, PAGE_SIZE));
       } else {
-        throw new Error('Failed to fetch data:', result);
+        throw new Error('Data is not an array:', data);
       }
-      setLoading(false);
-    })
+    } else {
+      throw new Error('Failed to fetch data:', result);
+    }
+    setLoading(false);
+  })
     .catch((error) => {
       setError('An error occurred while retrieving indexer data. Attempting to fetch from NEAR RPC...', error);
       backupNearRPCRequest();
     });
 }
 
-const fetchIndexerData = (page, append) => {
-  if (isFetching) return;
-  setIsFetching(true);
-  append ? setIsLoadingMore(true) : setLoading(true);
+const fetchMyIndexerData = () => {
+  setLoading(true);
 
-  fetchGraphQL(GET_ALL_ACTIVE_INDEXERS, 'getAllActiveIndexers', {
-    limit: PAGE_SIZE,
-    offset: page * PAGE_SIZE,
+  fetchGraphQL(GET_MY_ACTIVE_INDEXERS, 'getMyActiveIndexers', {
+    authorAccountId: myAccountId,
   })
     .then((result) => {
       if (result.status === 200) {
         const data = result?.body?.data?.[TABLE_NAME];
-        const totalCount = result?.body?.data?.[`${TABLE_NAME}_aggregate`]?.aggregate?.count;
         if (Array.isArray(data)) {
-          const newIndexers = data.map(({ author_account_id, indexer_name }) => ({
-            accountId: author_account_id,
-            indexerName: indexer_name,
-          }));
-          setTotal(totalCount);
-          setIndexers((prevIndexers) => append ? [...prevIndexers, ...newIndexers] : newIndexers);
+          //inject metadata if exist
+          const newIndexers = data.map(({ author_account_id, indexer_name }) => {
+            const sanitizedAccountID = author_account_id.replace(/\./g, '_');
+            const key = `${sanitizedAccountID}/${indexer_name}`;
+            return ({
+              accountId: author_account_id,
+              indexerName: indexer_name,
+              ...(indexerMetaData.has(key) && indexerMetaData.get(key))
+            })
+          });
+          // sort by numQueries
+          const sortedIndexers = newIndexers.sort((a, b) => (b.numQueries ?? 0) - (a.numQueries ?? 0));
+          setMyIndexers(sortedIndexers);
         } else {
           throw new Error('Data is not an array:', data);
         }
       } else {
         throw new Error('Failed to fetch data:', result);
       }
-      append ? setIsLoadingMore(false) : setLoading(false);
-      setIsFetching(false);
+      setLoading(false);
     })
     .catch((error) => {
       setError('An error occurred while retrieving indexer data. Attempting to fetch from NEAR RPC...', error);
-      append ? setIsLoadingMore(false) : setLoading(false);
-      setIsFetching(false);
       backupNearRPCRequest();
     });
-};
+}
 
-const storeIndexerMetaData = () => {
+const fetchIndexerMetadata = () => {
   const url = `${REPL_QUERY_API_USAGE_URL}`;
+  const map = new Map();
 
   asyncFetch(url)
     .then(response => {
       if (!response.ok) {
         setError('There was an error fetching the data');
         return;
       }
-      const { data } = JSON.parse(response.body);
-      const map = new Map();
 
+      const { data } = JSON.parse(response.body);
       data.forEach(entry => {
         const { indexer_account_id, indexers } = entry;
-        indexers.forEach(({ indexer_name, last_deployment_date, num_deployements, num_queries, original_deployment_date }) => {
+
+        indexers.forEach(({
+          indexer_name,
+          last_deployment_date,
+          num_deployements,
+          num_queries,
+          original_deployment_date
+        }) => {
           const indexer = {
             accountId: indexer_account_id,
             indexerName: indexer_name,
@@ -155,30 +167,34 @@ const storeIndexerMetaData = () => {
           };
           map.set(`${indexer_account_id}/${indexer_name}`, indexer);
         });
+        setIndexerMetaData(map);
+        setHasMetadataRendered(true);
       });
-      setIndexerMetaData(map);
     })
 }
 
-useEffect(() => {
-  storeIndexerMetaData();
-}, []);
 
 useEffect(() => {
-  fetchIndexerData(page, page > 0);
-}, [page]);
+  fetchIndexerMetadata();
+}, []);
 
 useEffect(() => {
-  if (selectedTab === "my-indexers") {
-    if (myIndexers.length <= 0) fetchMyIndexerData();
-  }
-  if (selectedTab === "all") {
-    if (indexers.length <= 0) fetchIndexerData(page, page > 0);
+  if (hasMetadataRendered) {
+    if (selectedTab === "my-indexers") {
+      if (myIndexers.length <= 0) fetchMyIndexerData();
+    }
+    if (selectedTab === "all") {
+      if (indexers.length <= 0) fetchIndexerData();
+    }
   }
-}, [selectedTab]);
+}, [selectedTab, hasMetadataRendered]);
 
 const handleLoadMore = () => {
-  if (!isLoadingMore) setPage((prevPage) => prevPage + 1);
+  const start = page * PAGE_SIZE;
+  const end = start + PAGE_SIZE;
+  const newIndexers = indexers.slice(start, end);
+  setCurrentPageIndexer([...currentPageIndexer, ...newIndexers]);
+  setPage(page + 1);
 };
 
 const backupNearRPCRequest = () => {
@@ -301,7 +317,6 @@ const Item = styled.div`
   }
 `;
 
-
 const Button = styled.button`
   display: block;
   width: 100%;
@@ -407,7 +422,6 @@ const TextLink = styled.a`
   }
 `;
 
-
 const NavBarContainer = styled.div`
   display: flex;
   align-items: center;
@@ -595,19 +609,18 @@ return (
         ) : (
           <>
             <Items>
-              {indexers.map((indexer, i) => (
+              {currentPageIndexer.map((indexer, i) => (
                 <Item key={i}>
                   <Widget
                     src={`${REPL_ACCOUNT_ID}/widget/QueryApi.IndexerCard`}
-                    props={{ ...indexer, indexerMetadata }}
+                    props={{ ...indexer }}
                   />
                 </Item>
               ))}
             </Items>
-            {isLoadingMore && <LoadingSpinner />}
             <LoadMoreContainer>
-              <Button onClick={handleLoadMore} disabled={isLoadingMore || isFetching || error !== null || total === 0 || (total === indexers.length)}>
-                Load More
+              <Button onClick={handleLoadMore} disabled={indexers.length <= currentPageIndexer.length}>
+                {indexers.length <= currentPageIndexer.length ? "No more indexers" : "Load More"}
               </Button>
             </LoadMoreContainer>
           </>
@@ -640,7 +653,7 @@ return (
               <Item key={i}>
                 <Widget
                   src={`${REPL_ACCOUNT_ID}/widget/QueryApi.IndexerCard`}
-                  props={{ ...indexer, indexerMetadata }}
+                  props={{ ...indexer }}
                 />
               </Item>
             ))}