diff --git a/src/api/routes/v2/blocks.ts b/src/api/routes/v2/blocks.ts index ed7f138e8..bf647e1b5 100644 --- a/src/api/routes/v2/blocks.ts +++ b/src/api/routes/v2/blocks.ts @@ -6,11 +6,12 @@ import { parseDbTx } from '../../../api/controllers/db-controller'; import { FastifyPluginAsync } from 'fastify'; import { Type, TypeBoxTypeProvider } from '@fastify/type-provider-typebox'; import { Server } from 'node:http'; -import { LimitParam, OffsetParam } from '../../schemas/params'; -import { ResourceType } from '../../pagination'; +import { CursorOffsetParam, LimitParam, OffsetParam } from '../../schemas/params'; +import { getPagingQueryLimit, pagingQueryLimits, ResourceType } from '../../pagination'; import { PaginatedResponse } from '../../schemas/util'; import { NakamotoBlock, NakamotoBlockSchema } from '../../schemas/entities/block'; import { TransactionSchema } from '../../schemas/entities/transactions'; +import { BlockListV2ResponseSchema } from '../../schemas/responses/responses'; export const BlockRoutesV2: FastifyPluginAsync< Record, @@ -28,21 +29,29 @@ export const BlockRoutesV2: FastifyPluginAsync< tags: ['Blocks'], querystring: Type.Object({ limit: LimitParam(ResourceType.Block), - offset: OffsetParam(), + offset: CursorOffsetParam({ resource: ResourceType.Block }), + cursor: Type.Optional(Type.String({ description: 'Cursor for pagination' })), }), response: { - 200: PaginatedResponse(NakamotoBlockSchema), + 200: BlockListV2ResponseSchema, }, }, }, async (req, reply) => { const query = req.query; - const { limit, offset, results, total } = await fastify.db.v2.getBlocks(query); - const blocks: NakamotoBlock[] = results.map(r => parseDbNakamotoBlock(r)); + const limit = getPagingQueryLimit(ResourceType.Block, req.query.limit); + const blockQuery = await fastify.db.v2.getBlocks({ ...query, limit }); + if (query.cursor && !blockQuery.current_cursor) { + throw new NotFoundError('Cursor not found'); + } + const blocks: NakamotoBlock[] = blockQuery.results.map(r => parseDbNakamotoBlock(r)); await reply.send({ - limit, - offset, - total, + limit: blockQuery.limit, + offset: blockQuery.offset, + total: blockQuery.total, + next_cursor: blockQuery.next_cursor, + prev_cursor: blockQuery.prev_cursor, + cursor: blockQuery.current_cursor, results: blocks, }); } diff --git a/src/api/schemas/params.ts b/src/api/schemas/params.ts index 0b0c230e0..de4892bf3 100644 --- a/src/api/schemas/params.ts +++ b/src/api/schemas/params.ts @@ -28,6 +28,23 @@ export const LimitParam = ( }) ); +export const CursorOffsetParam = (args: { + resource: ResourceType; + title?: string; + description?: string; + limitOverride?: number; + maxPages?: number; +}) => + Type.Optional( + Type.Integer({ + default: 0, + maximum: pagingQueryLimits[args.resource].maxLimit * (args.maxPages ?? 10), + minimum: -pagingQueryLimits[args.resource].maxLimit * (args.maxPages ?? 10), + title: args.title ?? 'Offset', + description: args.description ?? 'Result offset', + }) + ); + export const UnanchoredParamSchema = Type.Optional( Type.Boolean({ default: false, diff --git a/src/api/schemas/responses/responses.ts b/src/api/schemas/responses/responses.ts index a926dc10a..7b15eacba 100644 --- a/src/api/schemas/responses/responses.ts +++ b/src/api/schemas/responses/responses.ts @@ -1,5 +1,5 @@ import { Static, Type } from '@sinclair/typebox'; -import { OptionalNullable, PaginatedResponse } from '../util'; +import { Nullable, OptionalNullable, PaginatedCursorResponse, PaginatedResponse } from '../util'; import { MempoolStatsSchema } from '../entities/mempool-transactions'; import { MempoolTransactionSchema, TransactionSchema } from '../entities/transactions'; import { MicroblockSchema } from '../entities/microblock'; @@ -12,6 +12,7 @@ import { BurnchainRewardSchema, BurnchainRewardSlotHolderSchema, } from '../entities/burnchain-rewards'; +import { NakamotoBlockSchema } from '../entities/block'; export const ErrorResponseSchema = Type.Object( { @@ -178,3 +179,6 @@ export const RunFaucetResponseSchema = Type.Object( } ); export type RunFaucetResponse = Static; + +export const BlockListV2ResponseSchema = PaginatedCursorResponse(NakamotoBlockSchema); +export type BlockListV2Response = Static; diff --git a/src/api/schemas/util.ts b/src/api/schemas/util.ts index ea60a75d3..5905e21f6 100644 --- a/src/api/schemas/util.ts +++ b/src/api/schemas/util.ts @@ -12,3 +12,17 @@ export const PaginatedResponse = (type: T, options?: ObjectOp }, options ); + +export const PaginatedCursorResponse = (type: T, options?: ObjectOptions) => + Type.Object( + { + limit: Type.Integer({ examples: [20] }), + offset: Type.Integer({ examples: [0] }), + total: Type.Integer({ examples: [1] }), + next_cursor: Nullable(Type.String({ description: 'Next page cursor' })), + prev_cursor: Nullable(Type.String({ description: 'Previous page cursor' })), + cursor: Nullable(Type.String({ description: 'Current page cursor' })), + results: Type.Array(type), + }, + options + ); diff --git a/src/datastore/common.ts b/src/datastore/common.ts index 3b3b78ee6..e0f3cdfdd 100644 --- a/src/datastore/common.ts +++ b/src/datastore/common.ts @@ -1143,6 +1143,16 @@ export type DbPaginatedResult = { results: T[]; }; +export type DbCursorPaginatedResult = { + limit: number; + offset: number; + next_cursor: string | null; + prev_cursor: string | null; + current_cursor: string | null; + total: number; + results: T[]; +}; + export interface BlocksWithMetadata { results: { block: DbBlock; diff --git a/src/datastore/pg-store-v2.ts b/src/datastore/pg-store-v2.ts index a962fbe4f..de7a5efaf 100644 --- a/src/datastore/pg-store-v2.ts +++ b/src/datastore/pg-store-v2.ts @@ -36,6 +36,7 @@ import { PoxCycleQueryResult, DbPoxCycleSigner, DbPoxCycleSignerStacker, + DbCursorPaginatedResult, } from './common'; import { BLOCK_COLUMNS, @@ -59,37 +60,97 @@ async function assertTxIdExists(sql: PgSqlClient, tx_id: string) { } export class PgStoreV2 extends BasePgStoreModule { - async getBlocks(args: BlockPaginationQueryParams): Promise> { + async getBlocks(args: { + limit: number; + offset?: number; + cursor?: string; + }): Promise> { return await this.sqlTransaction(async sql => { - const limit = args.limit ?? BlockLimitParamSchema.default; + const limit = args.limit; const offset = args.offset ?? 0; - const blocksQuery = await sql<(BlockQueryResult & { total: number })[]>` - WITH block_count AS ( - SELECT block_count AS count FROM chain_tip + const cursor = args.cursor ?? null; + + const blocksQuery = await sql< + (BlockQueryResult & { total: number; next_block_hash: string; prev_block_hash: string })[] + >` + WITH cursor_block AS ( + WITH ordered_blocks AS ( + SELECT *, LEAD(block_height, ${offset}) OVER (ORDER BY block_height DESC) offset_block_height + FROM blocks + WHERE canonical = true + ORDER BY block_height DESC ) - SELECT - ${sql(BLOCK_COLUMNS)}, - (SELECT count FROM block_count)::int AS total + SELECT offset_block_height as block_height + FROM ordered_blocks + WHERE index_block_hash = ${cursor ?? sql`(SELECT index_block_hash FROM chain_tip LIMIT 1)`} + LIMIT 1 + ), + selected_blocks AS ( + SELECT ${sql(BLOCK_COLUMNS)} FROM blocks WHERE canonical = true + AND block_height <= (SELECT block_height FROM cursor_block) ORDER BY block_height DESC LIMIT ${limit} - OFFSET ${offset} + ), + prev_page AS ( + SELECT index_block_hash as prev_block_hash + FROM blocks + WHERE canonical = true + AND block_height < ( + SELECT block_height + FROM selected_blocks + ORDER BY block_height DESC + LIMIT 1 + ) + ORDER BY block_height DESC + OFFSET ${limit - 1} + LIMIT 1 + ), + next_page AS ( + SELECT index_block_hash as next_block_hash + FROM blocks + WHERE canonical = true + AND block_height > ( + SELECT block_height + FROM selected_blocks + ORDER BY block_height DESC + LIMIT 1 + ) + ORDER BY block_height ASC + OFFSET ${limit - 1} + LIMIT 1 + ) + SELECT + (SELECT block_count FROM chain_tip)::int AS total, + sb.*, + nb.next_block_hash, + pb.prev_block_hash + FROM selected_blocks sb + LEFT JOIN next_page nb ON true + LEFT JOIN prev_page pb ON true + ORDER BY sb.block_height DESC `; - if (blocksQuery.count === 0) - return { - limit, - offset, - results: [], - total: 0, - }; + + // Parse blocks const blocks = blocksQuery.map(b => parseBlockQueryResult(b)); - return { + const total = blocksQuery[0]?.total ?? 0; + + // Determine cursors + const nextCursor = blocksQuery[0]?.next_block_hash ?? null; + const prevCursor = blocksQuery[0]?.prev_block_hash ?? null; + const currentCursor = blocksQuery[0]?.index_block_hash ?? null; + + const result: DbCursorPaginatedResult = { limit, - offset, + offset: offset, results: blocks, - total: blocksQuery[0].total, + total: total, + next_cursor: nextCursor, + prev_cursor: prevCursor, + current_cursor: currentCursor, }; + return result; }); } diff --git a/src/tests/block-tests.ts b/src/tests/block-tests.ts index b5a1b2335..be0774443 100644 --- a/src/tests/block-tests.ts +++ b/src/tests/block-tests.ts @@ -14,6 +14,7 @@ import { TestBlockBuilder, TestMicroblockStreamBuilder } from '../test-utils/tes import { PgWriteStore } from '../datastore/pg-write-store'; import { PgSqlClient, bufferToHex } from '@hirosystems/api-toolkit'; import { migrate } from '../test-utils/test-helpers'; +import { BlockListV2Response } from 'src/api/schemas/responses/responses'; describe('block tests', () => { let db: PgWriteStore; @@ -754,6 +755,209 @@ describe('block tests', () => { expect(fetch.status).not.toBe(200); }); + test('blocks v2 cursor pagination', async () => { + for (let i = 1; i <= 14; i++) { + const block = new TestBlockBuilder({ + block_height: i, + block_hash: `0x11${i.toString().padStart(62, '0')}`, + index_block_hash: `0x${i.toString().padStart(64, '0')}`, + parent_index_block_hash: `0x${(i - 1).toString().padStart(64, '0')}`, + parent_block_hash: `0x${(i - 1).toString().padStart(64, '0')}`, + burn_block_height: 700000, + burn_block_hash: '0x00000000000000000001e2ee7f0c6bd5361b5e7afd76156ca7d6f524ee5ca3d8', + }) + .addTx({ tx_id: `0x${i.toString().padStart(64, '0')}` }) + .build(); + await db.update(block); + } + + let body: BlockListV2Response; + + // Fetch latest page + ({ body } = await supertest(api.server).get(`/extended/v2/blocks?limit=3`)); + expect(body).toEqual( + expect.objectContaining({ + limit: 3, + offset: 0, + total: 14, + cursor: '0x0000000000000000000000000000000000000000000000000000000000000014', + next_cursor: null, + prev_cursor: '0x0000000000000000000000000000000000000000000000000000000000000011', + results: [ + expect.objectContaining({ height: 14 }), + expect.objectContaining({ height: 13 }), + expect.objectContaining({ height: 12 }), + ], + }) + ); + const latestPageCursor = body.cursor; + const latestBlock = body.results[0]; + + // Can fetch same page using cursor + ({ body } = await supertest(api.server).get( + `/extended/v2/blocks?limit=3&cursor=${body.cursor}` + )); + expect(body).toEqual( + expect.objectContaining({ + limit: 3, + offset: 0, + total: 14, + cursor: '0x0000000000000000000000000000000000000000000000000000000000000014', + next_cursor: null, + prev_cursor: '0x0000000000000000000000000000000000000000000000000000000000000011', + results: [ + expect.objectContaining({ height: 14 }), + expect.objectContaining({ height: 13 }), + expect.objectContaining({ height: 12 }), + ], + }) + ); + + // Fetch previous page + ({ body } = await supertest(api.server).get( + `/extended/v2/blocks?limit=3&cursor=${body.prev_cursor}` + )); + expect(body).toEqual( + expect.objectContaining({ + limit: 3, + offset: 0, + total: 14, + cursor: '0x0000000000000000000000000000000000000000000000000000000000000011', + next_cursor: '0x0000000000000000000000000000000000000000000000000000000000000014', + prev_cursor: '0x0000000000000000000000000000000000000000000000000000000000000008', + results: [ + expect.objectContaining({ height: 11 }), + expect.objectContaining({ height: 10 }), + expect.objectContaining({ height: 9 }), + ], + }) + ); + + // Oldest page has no prev_cursor + ({ body } = await supertest(api.server).get( + `/extended/v2/blocks?limit=3&cursor=0x0000000000000000000000000000000000000000000000000000000000000002` + )); + expect(body).toEqual( + expect.objectContaining({ + limit: 3, + offset: 0, + total: 14, + cursor: '0x0000000000000000000000000000000000000000000000000000000000000002', + next_cursor: '0x0000000000000000000000000000000000000000000000000000000000000005', + prev_cursor: null, + results: [expect.objectContaining({ height: 2 }), expect.objectContaining({ height: 1 })], + }) + ); + + // Offset + cursor works + ({ body } = await supertest(api.server).get( + `/extended/v2/blocks?limit=3&cursor=0x0000000000000000000000000000000000000000000000000000000000000011&offset=2` + )); + expect(body).toEqual( + expect.objectContaining({ + limit: 3, + offset: 2, + total: 14, + cursor: '0x0000000000000000000000000000000000000000000000000000000000000009', + next_cursor: '0x0000000000000000000000000000000000000000000000000000000000000012', + prev_cursor: '0x0000000000000000000000000000000000000000000000000000000000000006', + results: [ + expect.objectContaining({ height: 9 }), + expect.objectContaining({ height: 8 }), + expect.objectContaining({ height: 7 }), + ], + }) + ); + + // Negative offset + cursor + ({ body } = await supertest(api.server).get( + `/extended/v2/blocks?limit=3&cursor=0x0000000000000000000000000000000000000000000000000000000000000008&offset=-2` + )); + expect(body).toEqual( + expect.objectContaining({ + limit: 3, + offset: -2, + total: 14, + cursor: '0x0000000000000000000000000000000000000000000000000000000000000010', + next_cursor: '0x0000000000000000000000000000000000000000000000000000000000000013', + prev_cursor: '0x0000000000000000000000000000000000000000000000000000000000000007', + results: [ + expect.objectContaining({ height: 10 }), + expect.objectContaining({ height: 9 }), + expect.objectContaining({ height: 8 }), + ], + }) + ); + + // Offset (no cursor) works, has original behavior + ({ body } = await supertest(api.server).get(`/extended/v2/blocks?limit=3&offset=5`)); + expect(body).toEqual( + expect.objectContaining({ + limit: 3, + offset: 5, + total: 14, + cursor: '0x0000000000000000000000000000000000000000000000000000000000000009', + next_cursor: '0x0000000000000000000000000000000000000000000000000000000000000012', + prev_cursor: '0x0000000000000000000000000000000000000000000000000000000000000006', + results: [ + expect.objectContaining({ height: 9 }), + expect.objectContaining({ height: 8 }), + expect.objectContaining({ height: 7 }), + ], + }) + ); + + // Re-org the the cursor for the latest block, should get a 404 on use + const blockB1 = new TestBlockBuilder({ + block_height: latestBlock.height, + block_hash: `0x22${latestBlock.height.toString().padStart(62, '0')}`, + index_block_hash: `0xbb${latestBlock.height.toString().padStart(62, '0')}`, + parent_index_block_hash: `0x${(latestBlock.height - 1).toString().padStart(64, '0')}`, + parent_block_hash: `0x${(latestBlock.height - 1).toString().padStart(64, '0')}`, + burn_block_height: 700000, + burn_block_hash: '0x00000000000000000001e2ee7f0c6bd5361b5e7afd76156ca7d6f524ee5ca3d8', + }) + .addTx({ tx_id: `0x${latestBlock.height.toString().padStart(64, '0')}` }) + .build(); + await db.update(blockB1); + const blockB2 = new TestBlockBuilder({ + block_height: latestBlock.height + 1, + block_hash: `0x22${(latestBlock.height + 1).toString().padStart(62, '0')}`, + index_block_hash: `0xbb${(latestBlock.height + 1).toString().padStart(62, '0')}`, + parent_index_block_hash: `0xbb${latestBlock.height.toString().padStart(62, '0')}`, + parent_block_hash: `0x${latestBlock.height.toString().padStart(64, '0')}`, + burn_block_height: 700000, + burn_block_hash: '0x00000000000000000001e2ee7f0c6bd5361b5e7afd76156ca7d6f524ee5ca3d8', + }) + .addTx({ tx_id: `0x${(latestBlock.height + 1).toString().padStart(64, '0')}` }) + .build(); + await db.update(blockB2); + + // Should get a 404 when using cursor for re-orged block + const req = await supertest(api.server).get( + `/extended/v2/blocks?limit=3&cursor=${latestPageCursor}` + ); + expect(req.statusCode).toBe(404); + + // Latest page should have the re-org blocks + ({ body } = await supertest(api.server).get(`/extended/v2/blocks?limit=3`)); + expect(body).toEqual( + expect.objectContaining({ + limit: 3, + offset: 0, + total: 15, + cursor: '0xbb00000000000000000000000000000000000000000000000000000000000015', + next_cursor: null, + prev_cursor: '0x0000000000000000000000000000000000000000000000000000000000000012', + results: [ + expect.objectContaining({ height: 15 }), + expect.objectContaining({ height: 14 }), + expect.objectContaining({ height: 13 }), + ], + }) + ); + }); + test('blocks v2 retrieved by hash or height', async () => { for (let i = 1; i < 6; i++) { const block = new TestBlockBuilder({