diff --git a/cli/src/commands/pull/index.ts b/cli/src/commands/pull/index.ts index aea162d63..b5819e924 100644 --- a/cli/src/commands/pull/index.ts +++ b/cli/src/commands/pull/index.ts @@ -51,20 +51,22 @@ export default class Pull extends BaseCommand { const details = await getBranchDetailsWithPgRoll(xata, { workspace, region, database, branch }); - let logs: Schemas.MigrationHistoryItem[] | Schemas.Commit[] = []; + let logs: (Schemas.MigrationHistoryItem | Schemas.Commit)[] = []; + let cursor = undefined; if (isBranchPgRollEnabled(details)) { - const { migrations } = await xata.api.migrations.getMigrationHistory({ - pathParams: { workspace, region, dbBranchName: `${database}:${branch}` } - }); - logs = migrations; + do { + const { migrations, cursor: newCursor } = await xata.api.migrations.getMigrationHistory({ + pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, + queryParams: { cursor, limit: 200 } + }); + + logs = logs.concat(migrations); + cursor = newCursor; + } while (cursor !== undefined); } else { const data = await xata.api.migrations.getBranchSchemaHistory({ pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, - body: { - // TODO: Fix pagination in the API to start from last known migration and not from the beginning - // Also paginate until we get all migrations - page: { size: 200 } - } + body: { page: { size: 200 } } }); logs = data.logs; } diff --git a/cli/src/commands/pull/pull.test.ts b/cli/src/commands/pull/pull.test.ts index 609815eff..290aee7e7 100644 --- a/cli/src/commands/pull/pull.test.ts +++ b/cli/src/commands/pull/pull.test.ts @@ -230,7 +230,7 @@ const pgrollFetchSingle = (url: string, request: any) => { schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] } }) }; - } else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') { + } else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') { return { ok: true, json: async () => ({ @@ -251,7 +251,7 @@ const pgrollFetchMultiple = (url: string, request: any) => { schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] } }) }; - } else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') { + } else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') { return { ok: true, json: async () => ({ diff --git a/cli/src/commands/push/index.ts b/cli/src/commands/push/index.ts index 247c48458..a3db0a089 100644 --- a/cli/src/commands/push/index.ts +++ b/cli/src/commands/push/index.ts @@ -48,20 +48,22 @@ export default class Push extends BaseCommand { const details = await getBranchDetailsWithPgRoll(xata, { workspace, region, database, branch }); - let logs: Schemas.MigrationHistoryItem[] | Schemas.Commit[] = []; + let logs: (Schemas.MigrationHistoryItem | Schemas.Commit)[] = []; + let cursor = undefined; if (isBranchPgRollEnabled(details)) { - const { migrations } = await xata.api.migrations.getMigrationHistory({ - pathParams: { workspace, region, dbBranchName: `${database}:${branch}` } - }); - logs = migrations; + do { + const { migrations, cursor: newCursor } = await xata.api.migrations.getMigrationHistory({ + pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, + queryParams: { cursor, limit: 200 } + }); + + logs = logs.concat(migrations); + cursor = newCursor; + } while (cursor !== undefined); } else { const data = await xata.api.migrations.getBranchSchemaHistory({ pathParams: { workspace, region, dbBranchName: `${database}:${branch}` }, - body: { - // TODO: Fix pagination in the API to start from last known migration and not from the beginning - // Also paginate until we get all migrations - page: { size: 200 } - } + body: { page: { size: 200 } } }); logs = data.logs; } diff --git a/cli/src/commands/push/push.test.ts b/cli/src/commands/push/push.test.ts index 0cfca5376..2a8f1f227 100644 --- a/cli/src/commands/push/push.test.ts +++ b/cli/src/commands/push/push.test.ts @@ -244,7 +244,7 @@ const pgrollFetchSingle = (url: string, request: any, type: 'inferred' | 'pgroll schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] } }) }; - } else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') { + } else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') { return { ok: true, json: async () => (type === 'inferred' ? { migrations: [pgrollMigration3] } : { migrations: [pgrollMigration1] }) @@ -263,7 +263,7 @@ const pgrollFetchEmpty = (url: string, request: any) => { schema: { tables: [{ name: 'table1', columns: [{ name: 'a', type: 'string' }] }] } }) }; - } else if (url === `${baseUrl}/migrations/history` && request.method === 'GET') { + } else if (url === `${baseUrl}/migrations/history?limit=200` && request.method === 'GET') { return { ok: true, json: async () => ({ diff --git a/cli/src/migrations/files.ts b/cli/src/migrations/files.ts index e6149bb47..fccfd6d4d 100644 --- a/cli/src/migrations/files.ts +++ b/cli/src/migrations/files.ts @@ -97,7 +97,7 @@ export async function removeLocalMigrations() { } } -export function commitToMigrationFile(logs: Schemas.Commit[] | Schemas.MigrationHistoryItem[]): LocalMigrationFile[] { +export function commitToMigrationFile(logs: (Schemas.Commit | Schemas.MigrationHistoryItem)[]): LocalMigrationFile[] { // Schema history comes in reverse order, so we need to reverse it return logs.reverse().map((log) => isMigrationPgRollFormat(log) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c0239110e..31cc52e3d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -583,7 +583,7 @@ packages: /@aws-crypto/util@3.0.0: resolution: {integrity: sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==} dependencies: - '@aws-sdk/types': 3.567.0 + '@aws-sdk/types': 3.577.0 '@aws-sdk/util-utf8-browser': 3.259.0 tslib: 1.14.1 dev: true @@ -9494,7 +9494,7 @@ packages: eslint-plugin-import: 2.29.1(@typescript-eslint/parser@7.11.0)(eslint-import-resolver-typescript@3.6.1)(eslint@9.3.0) fast-glob: 3.3.1 get-tsconfig: 4.7.2 - is-core-module: 2.13.0 + is-core-module: 2.13.1 is-glob: 4.0.3 transitivePeerDependencies: - '@typescript-eslint/parser' @@ -10948,12 +10948,6 @@ packages: ci-info: 2.0.0 dev: true - /is-core-module@2.13.0: - resolution: {integrity: sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ==} - dependencies: - has: 1.0.3 - dev: true - /is-core-module@2.13.1: resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==} dependencies: