feat: cursor-based pagination on blocks endpoint (#2060)

* feat: cursor-based pagination for `/v2/blocks` endpoint

* feat: support `offset` with cursor-based pagination for `/v2/blocks`

* feat: support negative offsets for random-page access in either direction

* chore: limit on `/v2/blocks` offset (10 pages max)

* fix: offset working without cursor

* chore: create CursorOffsetParam

* chore: create PaginatedCursorResponse

* chore: use index_block_hash for cursor value

* chore: return 404 when cursor is re-orged

src/api/routes/v2/blocks.ts

@@ -6,11 +6,12 @@ import { parseDbTx } from '../../../api/controllers/db-controller';
 import { FastifyPluginAsync } from 'fastify';
 import { Type, TypeBoxTypeProvider } from '@fastify/type-provider-typebox';
 import { Server } from 'node:http';
-import { LimitParam, OffsetParam } from '../../schemas/params';
-import { ResourceType } from '../../pagination';
+import { CursorOffsetParam, LimitParam, OffsetParam } from '../../schemas/params';
+import { getPagingQueryLimit, pagingQueryLimits, ResourceType } from '../../pagination';
 import { PaginatedResponse } from '../../schemas/util';
 import { NakamotoBlock, NakamotoBlockSchema } from '../../schemas/entities/block';
 import { TransactionSchema } from '../../schemas/entities/transactions';
+import { BlockListV2ResponseSchema } from '../../schemas/responses/responses';
 
 export const BlockRoutesV2: FastifyPluginAsync<
   Record<never, never>,
@@ -28,21 +29,29 @@ export const BlockRoutesV2: FastifyPluginAsync<
         tags: ['Blocks'],
         querystring: Type.Object({
           limit: LimitParam(ResourceType.Block),
-          offset: OffsetParam(),
+          offset: CursorOffsetParam({ resource: ResourceType.Block }),
+          cursor: Type.Optional(Type.String({ description: 'Cursor for pagination' })),
         }),
         response: {
-          200: PaginatedResponse(NakamotoBlockSchema),
+          200: BlockListV2ResponseSchema,
         },
       },
     },
     async (req, reply) => {
       const query = req.query;
-      const { limit, offset, results, total } = await fastify.db.v2.getBlocks(query);
-      const blocks: NakamotoBlock[] = results.map(r => parseDbNakamotoBlock(r));
+      const limit = getPagingQueryLimit(ResourceType.Block, req.query.limit);
+      const blockQuery = await fastify.db.v2.getBlocks({ ...query, limit });
+      if (query.cursor && !blockQuery.current_cursor) {
+        throw new NotFoundError('Cursor not found');
+      }
+      const blocks: NakamotoBlock[] = blockQuery.results.map(r => parseDbNakamotoBlock(r));
       await reply.send({
-        limit,
-        offset,
-        total,
+        limit: blockQuery.limit,
+        offset: blockQuery.offset,
+        total: blockQuery.total,
+        next_cursor: blockQuery.next_cursor,
+        prev_cursor: blockQuery.prev_cursor,
+        cursor: blockQuery.current_cursor,
         results: blocks,
       });
     }

src/api/schemas/params.ts

@@ -28,6 +28,23 @@ export const LimitParam = (
     })
   );
 
+export const CursorOffsetParam = (args: {
+  resource: ResourceType;
+  title?: string;
+  description?: string;
+  limitOverride?: number;
+  maxPages?: number;
+}) =>
+  Type.Optional(
+    Type.Integer({
+      default: 0,
+      maximum: pagingQueryLimits[args.resource].maxLimit * (args.maxPages ?? 10),
+      minimum: -pagingQueryLimits[args.resource].maxLimit * (args.maxPages ?? 10),
+      title: args.title ?? 'Offset',
+      description: args.description ?? 'Result offset',
+    })
+  );
+
 export const UnanchoredParamSchema = Type.Optional(
   Type.Boolean({
     default: false,

src/api/schemas/responses/responses.ts

@@ -1,5 +1,5 @@
 import { Static, Type } from '@sinclair/typebox';
-import { OptionalNullable, PaginatedResponse } from '../util';
+import { Nullable, OptionalNullable, PaginatedCursorResponse, PaginatedResponse } from '../util';
 import { MempoolStatsSchema } from '../entities/mempool-transactions';
 import { MempoolTransactionSchema, TransactionSchema } from '../entities/transactions';
 import { MicroblockSchema } from '../entities/microblock';
@@ -12,6 +12,7 @@ import {
   BurnchainRewardSchema,
   BurnchainRewardSlotHolderSchema,
 } from '../entities/burnchain-rewards';
+import { NakamotoBlockSchema } from '../entities/block';
 
 export const ErrorResponseSchema = Type.Object(
   {
@@ -178,3 +179,6 @@ export const RunFaucetResponseSchema = Type.Object(
   }
 );
 export type RunFaucetResponse = Static<typeof RunFaucetResponseSchema>;
+
+export const BlockListV2ResponseSchema = PaginatedCursorResponse(NakamotoBlockSchema);
+export type BlockListV2Response = Static<typeof BlockListV2ResponseSchema>;

src/api/schemas/util.ts

@@ -12,3 +12,17 @@ export const PaginatedResponse = <T extends TSchema>(type: T, options?: ObjectOp
     },
     options
   );
+
+export const PaginatedCursorResponse = <T extends TSchema>(type: T, options?: ObjectOptions) =>
+  Type.Object(
+    {
+      limit: Type.Integer({ examples: [20] }),
+      offset: Type.Integer({ examples: [0] }),
+      total: Type.Integer({ examples: [1] }),
+      next_cursor: Nullable(Type.String({ description: 'Next page cursor' })),
+      prev_cursor: Nullable(Type.String({ description: 'Previous page cursor' })),
+      cursor: Nullable(Type.String({ description: 'Current page cursor' })),
+      results: Type.Array(type),
+    },
+    options
+  );

src/datastore/common.ts

@@ -1143,6 +1143,16 @@ export type DbPaginatedResult<T> = {
   results: T[];
 };
 
+export type DbCursorPaginatedResult<T> = {
+  limit: number;
+  offset: number;
+  next_cursor: string | null;
+  prev_cursor: string | null;
+  current_cursor: string | null;
+  total: number;
+  results: T[];
+};
+
 export interface BlocksWithMetadata {
   results: {
     block: DbBlock;

src/datastore/pg-store-v2.ts

@@ -36,6 +36,7 @@ import {
   PoxCycleQueryResult,
   DbPoxCycleSigner,
   DbPoxCycleSignerStacker,
+  DbCursorPaginatedResult,
 } from './common';
 import {
   BLOCK_COLUMNS,
@@ -59,37 +60,97 @@ async function assertTxIdExists(sql: PgSqlClient, tx_id: string) {
 }
 
 export class PgStoreV2 extends BasePgStoreModule {
-  async getBlocks(args: BlockPaginationQueryParams): Promise<DbPaginatedResult<DbBlock>> {
+  async getBlocks(args: {
+    limit: number;
+    offset?: number;
+    cursor?: string;
+  }): Promise<DbCursorPaginatedResult<DbBlock>> {
     return await this.sqlTransaction(async sql => {
-      const limit = args.limit ?? BlockLimitParamSchema.default;
+      const limit = args.limit;
       const offset = args.offset ?? 0;
-      const blocksQuery = await sql<(BlockQueryResult & { total: number })[]>`
-        WITH block_count AS (
-          SELECT block_count AS count FROM chain_tip
+      const cursor = args.cursor ?? null;
+
+      const blocksQuery = await sql<
+        (BlockQueryResult & { total: number; next_block_hash: string; prev_block_hash: string })[]
+      >`
+      WITH cursor_block AS (
+        WITH ordered_blocks AS (
+          SELECT *, LEAD(block_height, ${offset}) OVER (ORDER BY block_height DESC) offset_block_height
+          FROM blocks
+          WHERE canonical = true
+          ORDER BY block_height DESC
         )
-        SELECT
-          ${sql(BLOCK_COLUMNS)},
-          (SELECT count FROM block_count)::int AS total
+        SELECT offset_block_height as block_height
+        FROM ordered_blocks 
+        WHERE index_block_hash = ${cursor ?? sql`(SELECT index_block_hash FROM chain_tip LIMIT 1)`}
+        LIMIT 1
+      ),
+      selected_blocks AS (
+        SELECT ${sql(BLOCK_COLUMNS)}
         FROM blocks
         WHERE canonical = true
+        AND block_height <= (SELECT block_height FROM cursor_block)
         ORDER BY block_height DESC
         LIMIT ${limit}
-        OFFSET ${offset}
+      ),
+      prev_page AS (
+        SELECT index_block_hash as prev_block_hash
+        FROM blocks
+        WHERE canonical = true
+        AND block_height < (
+          SELECT block_height
+          FROM selected_blocks
+          ORDER BY block_height DESC
+          LIMIT 1
+        )
+        ORDER BY block_height DESC
+        OFFSET ${limit - 1}
+        LIMIT 1
+      ),
+      next_page AS (
+        SELECT index_block_hash as next_block_hash
+        FROM blocks
+        WHERE canonical = true
+        AND block_height > (
+          SELECT block_height
+          FROM selected_blocks
+          ORDER BY block_height DESC
+          LIMIT 1
+        )
+        ORDER BY block_height ASC
+        OFFSET ${limit - 1}
+        LIMIT 1
+      )
+      SELECT
+        (SELECT block_count FROM chain_tip)::int AS total,
+        sb.*,
+        nb.next_block_hash,
+        pb.prev_block_hash
+      FROM selected_blocks sb
+      LEFT JOIN next_page nb ON true
+      LEFT JOIN prev_page pb ON true
+      ORDER BY sb.block_height DESC
       `;
-      if (blocksQuery.count === 0)
-        return {
-          limit,
-          offset,
-          results: [],
-          total: 0,
-        };
+
+      // Parse blocks
       const blocks = blocksQuery.map(b => parseBlockQueryResult(b));
-      return {
+      const total = blocksQuery[0]?.total ?? 0;
+
+      // Determine cursors
+      const nextCursor = blocksQuery[0]?.next_block_hash ?? null;
+      const prevCursor = blocksQuery[0]?.prev_block_hash ?? null;
+      const currentCursor = blocksQuery[0]?.index_block_hash ?? null;
+
+      const result: DbCursorPaginatedResult<DbBlock> = {
         limit,
-        offset,
+        offset: offset,
         results: blocks,
-        total: blocksQuery[0].total,
+        total: total,
+        next_cursor: nextCursor,
+        prev_cursor: prevCursor,
+        current_cursor: currentCursor,
       };
+      return result;
     });
   }