diff --git a/genkit-tools/common/src/manager/manager.ts b/genkit-tools/common/src/manager/manager.ts index 9bcfb3c9f..b5fb7d6eb 100644 --- a/genkit-tools/common/src/manager/manager.ts +++ b/genkit-tools/common/src/manager/manager.ts @@ -27,7 +27,7 @@ import { import * as apis from '../types/apis'; import { TraceData } from '../types/trace'; import { logger } from '../utils/logger'; -import { checkServerHealth, findRuntimesDir } from '../utils/utils'; +import { checkServerHealth, findRuntimesDir, retriable } from '../utils/utils'; import { GenkitToolsError, RuntimeEvent, @@ -308,8 +308,15 @@ export class RuntimeManager { */ private async handleNewRuntime(filePath: string) { try { - const content = await fs.readFile(filePath, 'utf-8'); - const runtimeInfo = JSON.parse(content) as RuntimeInfo; + const { content, runtimeInfo } = await retriable( + async () => { + const content = await fs.readFile(filePath, 'utf-8'); + const runtimeInfo = JSON.parse(content) as RuntimeInfo; + return { content, runtimeInfo }; + }, + { maxRetries: 10, delayMs: 500 } + ); + if (isValidRuntimeInfo(runtimeInfo)) { const fileName = path.basename(filePath); if (await checkServerHealth(runtimeInfo.reflectionServerUrl)) { diff --git a/genkit-tools/common/src/utils/utils.ts b/genkit-tools/common/src/utils/utils.ts index 795715207..7f6a07882 100644 --- a/genkit-tools/common/src/utils/utils.ts +++ b/genkit-tools/common/src/utils/utils.ts @@ -148,3 +148,26 @@ export async function waitUntilUnresponsive( } return false; } + +export async function retriable( + fn: () => Promise, + opts: { maxRetries?: number; delayMs?: number } +): Promise { + const maxRetries = opts.maxRetries ?? 3; + const delayMs = opts.delayMs ?? 0; + + let attempt = 0; + while (true) { + try { + return await fn(); + } catch (e) { + if (attempt >= maxRetries - 1) { + throw e; + } + if (delayMs > 0) { + await new Promise((r) => setTimeout(r, delayMs)); + } + } + attempt++; + } +}