From f493e914a4d24728901afb92dd07e478dc894a35 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 14:54:44 +0000 Subject: [PATCH 01/11] =?UTF-8?q?Refactor=20file=20edit=20logic=20to=20imp?= =?UTF-8?q?rove=20modularity=20and=20reusability=20=F0=9F=93=A6=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/genaisrc/genaiscript.d.ts | 4 +- eval/extrism/genaisrc/genaiscript.d.ts | 4 +- genaisrc/genaiscript.d.ts | 4 +- packages/auto/genaiscript.d.ts | 4 +- packages/core/src/fileedits.ts | 258 ++++++++++++++++++ packages/core/src/genaisrc/genaiscript.d.ts | 4 +- packages/core/src/promptrunner.ts | 174 +----------- packages/core/src/runpromptcontext.ts | 19 ++ packages/core/src/types/prompt_template.d.ts | 4 +- .../sample/genaisrc/blog/genaiscript.d.ts | 4 +- packages/sample/genaisrc/genaiscript.d.ts | 4 +- .../sample/genaisrc/node/genaiscript.d.ts | 4 +- .../sample/genaisrc/python/genaiscript.d.ts | 4 +- .../sample/genaisrc/style/genaiscript.d.ts | 4 +- packages/sample/src/aici/genaiscript.d.ts | 4 +- packages/sample/src/errors/genaiscript.d.ts | 4 +- packages/sample/src/genaiscript.d.ts | 4 +- packages/sample/src/makecode/genaiscript.d.ts | 4 +- packages/sample/src/tla/genaiscript.d.ts | 4 +- packages/sample/src/vision/genaiscript.d.ts | 4 +- packages/vscode/genaisrc/genaiscript.d.ts | 4 +- slides/genaisrc/genaiscript.d.ts | 4 +- 22 files changed, 325 insertions(+), 202 deletions(-) create mode 100644 packages/core/src/fileedits.ts diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/eval/extrism/genaisrc/genaiscript.d.ts b/eval/extrism/genaisrc/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/eval/extrism/genaisrc/genaiscript.d.ts +++ b/eval/extrism/genaisrc/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/auto/genaiscript.d.ts b/packages/auto/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/auto/genaiscript.d.ts +++ b/packages/auto/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/core/src/fileedits.ts b/packages/core/src/fileedits.ts new file mode 100644 index 0000000000..75a3db0294 --- /dev/null +++ b/packages/core/src/fileedits.ts @@ -0,0 +1,258 @@ +import { applyChangeLog, parseChangeLogs } from "./changelog" +import { applyLLMDiff, applyLLMPatch, parseLLMDiffs } from "./diff" +import { errorMessage } from "./error" +import { unquote } from "./fence" +import { fileExists, readText } from "./fs" +import { isGlobMatch } from "./glob" +import { runtimeHost } from "./host" +import { JSON5parse } from "./json5" +import { stringToPos } from "./parser" +import { validateJSONWithSchema } from "./schema" +import { MarkdownTrace, TraceOptions } from "./trace" +import { logError, logVerbose, relativePath } from "./util" +import { YAMLParse } from "./yaml" + +export async function computeFileEdits( + res: RunPromptResult, + options: TraceOptions & { + fileOutputs: FileOutput[] + schemas?: Record + fileMerges?: FileMergeHandler[] + outputProcessors?: PromptOutputProcessorHandler[] + } +): Promise<{ + edits: Edits[] + fileEdits: Record + changelogs: string[] +}> { + const { trace, fileOutputs, fileMerges, outputProcessors, schemas } = + options || {} + const { fences, frames, genVars } = res + let text = res.text + let annotations = res.annotations?.slice(0) + const fileEdits: Record = {} + const changelogs: string[] = [] + const edits: Edits[] = [] + const projFolder = runtimeHost.projectFolder() + + // Helper function to get or create file edit object + const getFileEdit = async (fn: string) => { + fn = relativePath(projFolder, fn) + let fileEdit = fileEdits[fn] + if (!fileEdit) { + let before: string = null + let after: string = undefined + if (await fileExists(fn)) before = await readText(fn) + else if (await fileExists(fn)) after = await readText(fn) + fileEdit = fileEdits[fn] = { before, after } + } + return fileEdit + } + + for (const fence of fences.filter( + ({ validation }) => validation?.valid !== false + )) { + const { label: name, content: val, language } = fence + const pm = /^((file|diff):?)\s+/i.exec(name) + if (pm) { + const kw = pm[1].toLowerCase() + const n = unquote(name.slice(pm[0].length).trim()) + const fn = /^[^\/]/.test(n) + ? runtimeHost.resolvePath(projFolder, n) + : n + const fileEdit = await getFileEdit(fn) + if (kw === "file") { + if (fileMerges.length) { + try { + for (const fileMerge of fileMerges) + fileEdit.after = + (await fileMerge( + fn, + "", // todo + fileEdit.after ?? fileEdit.before, + val + )) ?? val + } catch (e) { + logVerbose(e) + trace.error(`error custom merging diff in ${fn}`, e) + } + } else fileEdit.after = val + } else if (kw === "diff") { + const chunks = parseLLMDiffs(val) + try { + fileEdit.after = applyLLMPatch( + fileEdit.after || fileEdit.before, + chunks + ) + } catch (e) { + logVerbose(e) + trace.error(`error applying patch to ${fn}`, e) + try { + fileEdit.after = applyLLMDiff( + fileEdit.after || fileEdit.before, + chunks + ) + } catch (e) { + logVerbose(e) + trace.error(`error merging diff in ${fn}`, e) + } + } + } + } else if (/^changelog$/i.test(name) || /^changelog/i.test(language)) { + changelogs.push(val) + const cls = parseChangeLogs(val) + for (const changelog of cls) { + const { filename } = changelog + const fn = /^[^\/]/.test(filename) // TODO + ? runtimeHost.resolvePath(projFolder, filename) + : filename + const fileEdit = await getFileEdit(fn) + fileEdit.after = applyChangeLog( + fileEdit.after || fileEdit.before || "", + changelog + ) + } + } + } + + // Apply user-defined output processors + if (outputProcessors?.length) { + try { + trace.startDetails("🖨️ output processors") + for (const outputProcessor of outputProcessors) { + const { + text: newText, + files, + annotations: oannotations, + } = (await outputProcessor({ + text, + fileEdits, + fences, + frames, + genVars, + annotations, + schemas, + })) || {} + + if (newText !== undefined) { + text = newText + trace.detailsFenced(`📝 text`, text) + } + + if (files) + for (const [n, content] of Object.entries(files)) { + const fn = runtimeHost.path.isAbsolute(n) + ? n + : runtimeHost.resolvePath(projFolder, n) + trace.detailsFenced(`📁 file ${fn}`, content) + const fileEdit = await getFileEdit(fn) + fileEdit.after = content + fileEdit.validation = { valid: true } + } + if (oannotations) annotations = oannotations.slice(0) + } + } catch (e) { + logError(e) + trace.error(`output processor failed`, e) + } finally { + trace.endDetails() + } + } + + // Validate and apply file outputs + validateFileOutputs(fileOutputs, trace, fileEdits, schemas) + + // Convert file edits into structured edits + Object.entries(fileEdits) + .filter(([, { before, after }]) => before !== after) // ignore unchanged files + .forEach(([fn, { before, after, validation }]) => { + if (before) { + edits.push({ + label: `Update ${fn}`, + filename: fn, + type: "replace", + range: [[0, 0], stringToPos(after)], + text: after, + validated: validation?.valid, + }) + } else { + edits.push({ + label: `Create ${fn}`, + filename: fn, + type: "createfile", + text: after, + overwrite: true, + validated: validation?.valid, + }) + } + }) + + return { fileEdits, changelogs, edits } +} + +// Validate file outputs against specified schemas and patterns +/** + * Validates file outputs based on provided patterns and schemas. + * @param fileOutputs List of file outputs to validate. + * @param trace The markdown trace for logging. + * @param fileEdits Record of file updates. + * @param schemas The JSON schemas for validation. + */ +function validateFileOutputs( + fileOutputs: FileOutput[], + trace: MarkdownTrace, + fileEdits: Record, + schemas: Record +) { + if (fileOutputs?.length && Object.keys(fileEdits || {}).length) { + trace.startDetails("🗂 file outputs") + for (const fileEditName of Object.keys(fileEdits)) { + const fe = fileEdits[fileEditName] + for (const fileOutput of fileOutputs) { + const { pattern, options } = fileOutput + if (isGlobMatch(fileEditName, pattern)) { + try { + trace.startDetails(`📁 ${fileEditName}`) + trace.itemValue(`pattern`, pattern) + const { schema: schemaId } = options || {} + if (/\.(json|yaml)$/i.test(fileEditName)) { + const { after } = fileEdits[fileEditName] + const data = /\.json$/i.test(fileEditName) + ? JSON5parse(after) + : YAMLParse(after) + trace.detailsFenced("📝 data", data) + if (schemaId) { + const schema = schemas[schemaId] + if (!schema) + fe.validation = { + valid: false, + error: `schema ${schemaId} not found`, + } + else + fe.validation = validateJSONWithSchema( + data, + schema, + { + trace, + } + ) + } + } else { + fe.validation = { valid: true } + } + } catch (e) { + trace.error(errorMessage(e)) + fe.validation = { + valid: false, + error: errorMessage(e), + } + } finally { + trace.endDetails() + } + break + } + } + } + trace.endDetails() + } +} diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts index 696333c568..fb48fa8cb3 100644 --- a/packages/core/src/promptrunner.ts +++ b/packages/core/src/promptrunner.ts @@ -24,6 +24,7 @@ import { YAMLParse } from "./yaml" import { expandTemplate } from "./expander" import { resolveLanguageModel } from "./lm" import { Stats } from "fs" +import { computeFileEdits } from "./fileedits" // Asynchronously resolve expansion variables needed for a template /** @@ -188,24 +189,6 @@ export async function runTemplate( frames: [], } } - const fileEdits: Record = {} - const changelogs: string[] = [] - const edits: Edits[] = [] - const projFolder = runtimeHost.projectFolder() - - // Helper function to get or create file edit object - const getFileEdit = async (fn: string) => { - fn = relativePath(projFolder, fn) - let fileEdit = fileEdits[fn] - if (!fileEdit) { - let before: string = null - let after: string = undefined - if (await fileExists(fn)) before = await readText(fn) - else if (await fileExists(fn)) after = await readText(fn) - fileEdit = fileEdits[fn] = { before, after } - } - return fileEdit - } // Resolve model connection information const connection = await resolveModelConnectionInfo( @@ -259,153 +242,16 @@ export async function runTemplate( } = output let { text, annotations } = output - // Handle fenced code regions within the output - if (json === undefined) { - for (const fence of fences.filter( - ({ validation }) => validation?.valid !== false - )) { - const { label: name, content: val, language } = fence - const pm = /^((file|diff):?)\s+/i.exec(name) - if (pm) { - const kw = pm[1].toLowerCase() - const n = unquote(name.slice(pm[0].length).trim()) - const fn = /^[^\/]/.test(n) - ? runtimeHost.resolvePath(projFolder, n) - : n - const fileEdit = await getFileEdit(fn) - if (kw === "file") { - if (fileMerges.length) { - try { - for (const fileMerge of fileMerges) - fileEdit.after = - (await fileMerge( - fn, - label, - fileEdit.after ?? fileEdit.before, - val - )) ?? val - } catch (e) { - logVerbose(e) - trace.error( - `error custom merging diff in ${fn}`, - e - ) - } - } else fileEdit.after = val - } else if (kw === "diff") { - const chunks = parseLLMDiffs(val) - try { - fileEdit.after = applyLLMPatch( - fileEdit.after || fileEdit.before, - chunks - ) - } catch (e) { - logVerbose(e) - trace.error(`error applying patch to ${fn}`, e) - try { - fileEdit.after = applyLLMDiff( - fileEdit.after || fileEdit.before, - chunks - ) - } catch (e) { - logVerbose(e) - trace.error(`error merging diff in ${fn}`, e) - } - } - } - } else if ( - /^changelog$/i.test(name) || - /^changelog/i.test(language) - ) { - changelogs.push(val) - const cls = parseChangeLogs(val) - for (const changelog of cls) { - const { filename } = changelog - const fn = /^[^\/]/.test(filename) // TODO - ? runtimeHost.resolvePath(projFolder, filename) - : filename - const fileEdit = await getFileEdit(fn) - fileEdit.after = applyChangeLog( - fileEdit.after || fileEdit.before || "", - changelog - ) - } - } + const { fileEdits, changelogs, edits } = await computeFileEdits( + output, + { + trace, + fileOutputs, + schemas, + fileMerges, + outputProcessors, } - } - - // Apply user-defined output processors - if (outputProcessors?.length) { - try { - trace.startDetails("🖨️ output processors") - for (const outputProcessor of outputProcessors) { - const { - text: newText, - files, - annotations: oannotations, - } = (await outputProcessor({ - text, - fileEdits, - fences, - frames, - genVars, - annotations, - schemas, - })) || {} - - if (newText !== undefined) { - text = newText - trace.detailsFenced(`📝 text`, text) - } - - if (files) - for (const [n, content] of Object.entries(files)) { - const fn = runtimeHost.path.isAbsolute(n) - ? n - : runtimeHost.resolvePath(projFolder, n) - trace.detailsFenced(`📁 file ${fn}`, content) - const fileEdit = await getFileEdit(fn) - fileEdit.after = content - fileEdit.validation = { valid: true } - } - if (oannotations) annotations = oannotations.slice(0) - } - } catch (e) { - logError(e) - trace.error(`output processor failed`, e) - } finally { - trace.endDetails() - } - } - - // Validate and apply file outputs - validateFileOutputs(fileOutputs, trace, fileEdits, schemas) - - // Convert file edits into structured edits - Object.entries(fileEdits) - .filter(([, { before, after }]) => before !== after) // ignore unchanged files - .forEach(([fn, { before, after, validation }]) => { - if (before) { - edits.push({ - label: `Update ${fn}`, - filename: fn, - type: "replace", - range: [[0, 0], stringToPos(after)], - text: after, - validated: validation?.valid, - }) - } else { - edits.push({ - label: `Create ${fn}`, - filename: fn, - type: "createfile", - text: after, - overwrite: true, - validated: validation?.valid, - }) - } - }) - + ) // Reporting and tracing output if (fences?.length) trace.details("📩 code regions", renderFencedVariables(fences)) diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts index cb99e166dc..ba48aa25a7 100644 --- a/packages/core/src/runpromptcontext.ts +++ b/packages/core/src/runpromptcontext.ts @@ -56,6 +56,7 @@ import { concurrentLimit } from "./concurrency" import { Project } from "./ast" import { dedent } from "./indent" import { runtimeHost } from "./host" +import { computeFileEdits } from "./fileedits" export function createChatTurnGenerationContext( options: GenerationOptions, @@ -505,6 +506,9 @@ export function createChatGenerationContext( let tools: ToolCallback[] = undefined let schemas: Record = undefined let chatParticipants: ChatParticipant[] = undefined + const fileMerges: FileMergeHandler[] = [] + const outputProcessors: PromptOutputProcessorHandler[] = [] + const fileOutputs: FileOutput[] = [] // expand template const { provider } = parseModelIdentifier(genOptions.model) @@ -519,6 +523,9 @@ export function createChatGenerationContext( functions: fns, messages: msgs, chatParticipants: cps, + fileMerges: fms, + outputProcessors: ops, + fileOutputs: fos, } = await renderPromptNode(genOptions.model, node, { flexTokens: genOptions.flexTokens, trace: runTrace, @@ -528,6 +535,9 @@ export function createChatGenerationContext( tools = fns chatParticipants = cps messages.push(...msgs) + fileMerges.push(...fms) + outputProcessors.push(...ops) + fileOutputs.push(...fos) if (errors?.length) { logError(errors.map((err) => errorMessage(err)).join("\n")) @@ -630,6 +640,15 @@ export function createChatGenerationContext( ) ) tracePromptResult(runTrace, resp) + + const { edits } = await computeFileEdits(resp, { + trace, + schemas, + fileOutputs, + fileMerges, + outputProcessors, + }) + return resp } catch (e) { runTrace.error(e) diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 34678dfba4..c3611e1eaa 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -2116,6 +2116,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2890,8 +2892,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/blog/genaiscript.d.ts b/packages/sample/genaisrc/blog/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/genaisrc/blog/genaiscript.d.ts +++ b/packages/sample/genaisrc/blog/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/genaiscript.d.ts b/packages/sample/src/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/src/genaiscript.d.ts +++ b/packages/sample/src/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/packages/vscode/genaisrc/genaiscript.d.ts b/packages/vscode/genaisrc/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/packages/vscode/genaisrc/genaiscript.d.ts +++ b/packages/vscode/genaisrc/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index faed3976ba..a3795140d1 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -2197,6 +2197,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { strings: TemplateStringsArray, ...args: any[] ): RunPromptResultPromiseWithOptions + defFileMerge(fn: FileMergeHandler): void + defOutputProcessor(fn: PromptOutputProcessorHandler): void } interface GenerationOutput { @@ -2971,8 +2973,6 @@ interface ContainerHost extends ShellHost { interface PromptContext extends ChatGenerationContext { script(options: PromptArgs): void system(options: PromptSystemArgs): void - defFileMerge(fn: FileMergeHandler): void - defOutputProcessor(fn: PromptOutputProcessorHandler): void env: ExpansionVariables path: Path parsers: Parsers From 76ec6fd9f724448f018a3a8a9c22e2adcd2bfd42 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 15:06:07 +0000 Subject: [PATCH 02/11] =?UTF-8?q?Refactor=20file=20edit=20handling=20by=20?= =?UTF-8?q?updating=20imports=20and=20enhancing=20functionality=20?= =?UTF-8?q?=E2=9C=8F=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/genaisrc/genaiscript.d.ts | 5 ++ eval/extrism/genaisrc/genaiscript.d.ts | 5 ++ genaisrc/genaiscript.d.ts | 5 ++ packages/auto/genaiscript.d.ts | 5 ++ packages/cli/src/run.ts | 6 +-- packages/core/src/edits.ts | 37 -------------- packages/core/src/fileedits.ts | 51 ++++++++++++++++++- packages/core/src/genaisrc/genaiscript.d.ts | 5 ++ packages/core/src/promptrunner.ts | 7 --- packages/core/src/runpromptcontext.ts | 15 +++--- packages/core/src/types/prompt_template.d.ts | 5 ++ .../sample/genaisrc/blog/genaiscript.d.ts | 5 ++ packages/sample/genaisrc/genaiscript.d.ts | 5 ++ .../sample/genaisrc/node/genaiscript.d.ts | 5 ++ .../sample/genaisrc/python/genaiscript.d.ts | 5 ++ .../sample/genaisrc/style/genaiscript.d.ts | 5 ++ packages/sample/src/aici/genaiscript.d.ts | 5 ++ packages/sample/src/errors/genaiscript.d.ts | 5 ++ packages/sample/src/genaiscript.d.ts | 5 ++ packages/sample/src/makecode/genaiscript.d.ts | 5 ++ packages/sample/src/tla/genaiscript.d.ts | 5 ++ packages/sample/src/vision/genaiscript.d.ts | 5 ++ packages/vscode/genaisrc/genaiscript.d.ts | 5 ++ slides/genaisrc/genaiscript.d.ts | 5 ++ 24 files changed, 156 insertions(+), 55 deletions(-) delete mode 100644 packages/core/src/edits.ts diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/eval/extrism/genaisrc/genaiscript.d.ts b/eval/extrism/genaisrc/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/eval/extrism/genaisrc/genaiscript.d.ts +++ b/eval/extrism/genaisrc/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/auto/genaiscript.d.ts b/packages/auto/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/auto/genaiscript.d.ts +++ b/packages/auto/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/cli/src/run.ts b/packages/cli/src/run.ts index fe654acfc6..3b9ac677e3 100644 --- a/packages/cli/src/run.ts +++ b/packages/cli/src/run.ts @@ -59,7 +59,7 @@ import { } from "../../core/src/util" import { YAMLStringify } from "../../core/src/yaml" import { PromptScriptRunOptions } from "../../core/src/server/messages" -import { writeFileEdits } from "../../core/src/edits" +import { writeFileEdits } from "../../core/src/fileedits" import { azureDevOpsCreateIssueComment, AzureDevOpsEnv, @@ -355,8 +355,8 @@ export async function runScript( if (isJSONLFilename(outData)) await appendJSONL(outData, result.frames) else await writeText(outData, JSON.stringify(result.frames, null, 2)) - if (result.status === "success" && result.fileEdits) - await writeFileEdits(result, applyEdits) + if (result.status === "success" && result.fileEdits && applyEdits) + await writeFileEdits(result.fileEdits, { trace }) const promptjson = result.messages?.length ? JSON.stringify(result.messages, null, 2) diff --git a/packages/core/src/edits.ts b/packages/core/src/edits.ts deleted file mode 100644 index 52173d140f..0000000000 --- a/packages/core/src/edits.ts +++ /dev/null @@ -1,37 +0,0 @@ -// This module provides functionality to write file edits to disk, -// supporting conditional application of edits based on validation results. - -import { GenerationResult } from "./generation" // Import type for generation results -import { writeText } from "./fs" // Import function to write text to files -import { logVerbose } from "./util" // Import function for verbose logging - -/** - * Asynchronously writes file edits to disk. - * - * @param res - The result of a generation process containing file edits. - * @param applyEdits - A flag indicating whether edits should be applied even if validation fails. - */ -export async function writeFileEdits( - res: GenerationResult, // Contains the edits to be applied to files - applyEdits: boolean // Determines whether to apply edits unconditionally -) { - // Iterate over each file edit entry - for (const fileEdit of Object.entries(res.fileEdits)) { - // Destructure the filename, before content, after content, and validation from the entry - const [fn, { before, after, validation }] = fileEdit - - // Skip writing if the edit is invalid and applyEdits is false - if (!validation?.valid && !applyEdits) continue - - // Check if there's a change between before and after content - if (after !== before) { - // Log whether the file is being updated or created - logVerbose( - `${before !== undefined ? `updating` : `creating`} ${fn}` - ) - - // Write the new content to the file - await writeText(fn, after ?? before) // Write 'after' content if available, otherwise 'before' - } - } -} diff --git a/packages/core/src/fileedits.ts b/packages/core/src/fileedits.ts index 75a3db0294..1033182b1a 100644 --- a/packages/core/src/fileedits.ts +++ b/packages/core/src/fileedits.ts @@ -1,5 +1,6 @@ import { applyChangeLog, parseChangeLogs } from "./changelog" -import { applyLLMDiff, applyLLMPatch, parseLLMDiffs } from "./diff" +import { CSVToMarkdown } from "./csv" +import { applyLLMDiff, applyLLMPatch, createDiff, parseLLMDiffs } from "./diff" import { errorMessage } from "./error" import { unquote } from "./fence" import { fileExists, readText } from "./fs" @@ -11,6 +12,7 @@ import { validateJSONWithSchema } from "./schema" import { MarkdownTrace, TraceOptions } from "./trace" import { logError, logVerbose, relativePath } from "./util" import { YAMLParse } from "./yaml" +import { writeText } from "./fs" export async function computeFileEdits( res: RunPromptResult, @@ -187,6 +189,14 @@ export async function computeFileEdits( } }) + if (edits.length) + trace.details( + "✏️ edits", + CSVToMarkdown(edits, { + headers: ["type", "filename", "message", "validated"], + }) + ) + return { fileEdits, changelogs, edits } } @@ -256,3 +266,42 @@ function validateFileOutputs( trace.endDetails() } } + +/** + * Asynchronously writes file edits to disk. + * + * @param res - The result of a generation process containing file edits. + * @param applyEdits - A flag indicating whether edits should be applied even if validation fails. + */ +export async function writeFileEdits( + fileEdits: Record, // Contains the edits to be applied to files + options?: TraceOptions +) { + const { trace } = options || {} + // Iterate over each file edit entry + for (const fileEdit of Object.entries(fileEdits || {})) { + // Destructure the filename, before content, after content, and validation from the entry + const [fn, { before, after, validation }] = fileEdit + + // Skip writing if the edit is invalid and applyEdits is false + if (!validation?.valid) continue + + // Check if there's a change between before and after content + if (after !== before) { + // Log whether the file is being updated or created + logVerbose( + `${before !== undefined ? `updating` : `creating`} ${fn}` + ) + trace.detailsFenced( + `updating ${fn}`, + createDiff( + { filename: fn, content: before }, + { filename: fn, content: after } + ), + "diff" + ) + // Write the new content to the file + await writeText(fn, after ?? before) // Write 'after' content if available, otherwise 'before' + } + } +} diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts index fb48fa8cb3..5b405578cc 100644 --- a/packages/core/src/promptrunner.ts +++ b/packages/core/src/promptrunner.ts @@ -255,13 +255,6 @@ export async function runTemplate( // Reporting and tracing output if (fences?.length) trace.details("📩 code regions", renderFencedVariables(fences)) - if (edits.length) - trace.details( - "✏️ edits", - CSVToMarkdown(edits, { - headers: ["type", "filename", "message", "validated"], - }) - ) if (annotations?.length) trace.details( "⚠️ annotations", diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts index ba48aa25a7..42dc100236 100644 --- a/packages/core/src/runpromptcontext.ts +++ b/packages/core/src/runpromptcontext.ts @@ -56,7 +56,7 @@ import { concurrentLimit } from "./concurrency" import { Project } from "./ast" import { dedent } from "./indent" import { runtimeHost } from "./host" -import { computeFileEdits } from "./fileedits" +import { computeFileEdits, writeFileEdits } from "./fileedits" export function createChatTurnGenerationContext( options: GenerationOptions, @@ -478,7 +478,7 @@ export function createChatGenerationContext( generator: string | PromptGenerator, runOptions?: PromptGeneratorOptions ): Promise => { - const { label } = runOptions || {} + const { label, applyEdits } = runOptions || {} const runTrace = trace.startTraceDetails(`🎁 run prompt ${label || ""}`) try { infoCb?.({ text: `run prompt ${label || ""}` }) @@ -574,13 +574,13 @@ export function createChatGenerationContext( if (sysr.schemas) Object.assign(schemas, sysr.schemas) if (sysr.functions) tools.push(...sysr.functions) if (sysr.fileMerges?.length) - throw new NotSupportedError("fileMerges") + fileMerges.push(...sysr.fileMerges) if (sysr.outputProcessors?.length) - throw new NotSupportedError("outputProcessors") + outputProcessors.push(...sysr.outputProcessors) if (sysr.chatParticipants) chatParticipants.push(...sysr.chatParticipants) if (sysr.fileOutputs?.length) - throw new NotSupportedError("fileOutputs") + fileOutputs.push(...sysr.fileOutputs) if (sysr.logs?.length) runTrace.details("📝 console.log", sysr.logs) if (sysr.text) { @@ -641,14 +641,15 @@ export function createChatGenerationContext( ) tracePromptResult(runTrace, resp) - const { edits } = await computeFileEdits(resp, { + const { fileEdits } = await computeFileEdits(resp, { trace, schemas, fileOutputs, fileMerges, outputProcessors, }) - + if (fileEdits?.length && applyEdits) + await writeFileEdits(fileEdits, { trace }) return resp } catch (e) { runTrace.error(e) diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index c3611e1eaa..02b724336b 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -1964,6 +1964,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/genaisrc/blog/genaiscript.d.ts b/packages/sample/genaisrc/blog/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/genaisrc/blog/genaiscript.d.ts +++ b/packages/sample/genaisrc/blog/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/src/genaiscript.d.ts b/packages/sample/src/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/src/genaiscript.d.ts +++ b/packages/sample/src/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/packages/vscode/genaisrc/genaiscript.d.ts b/packages/vscode/genaisrc/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/packages/vscode/genaisrc/genaiscript.d.ts +++ b/packages/vscode/genaisrc/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index a3795140d1..ffb6c4db5b 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -2045,6 +2045,11 @@ interface PromptGeneratorOptions extends ModelOptions, PromptSystemOptions { * Label for trace */ label?: string + + /** + * Write file edits to the file system + */ + applyEdits?: boolean } interface FileOutputOptions { From 0a926676a6967ddf875d70663d922307b46f7a61 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 15:55:49 +0000 Subject: [PATCH 03/11] =?UTF-8?q?Enhance=20file=20edit=20handling=20and=20?= =?UTF-8?q?update=20model=20configurations=20=F0=9F=93=82=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/genaisrc/genaiscript.d.ts | 3 + eval/extrism/genaisrc/extrism-tool.genai.mts | 122 +++++++++++------- eval/extrism/genaisrc/genaiscript.d.ts | 3 + genaisrc/genaiscript.d.ts | 3 + packages/auto/genaiscript.d.ts | 3 + packages/core/src/chat.ts | 45 ++++++- packages/core/src/fileedits.ts | 12 +- packages/core/src/genaisrc/genaiscript.d.ts | 3 + packages/core/src/promptcontext.ts | 9 -- packages/core/src/promptrunner.ts | 18 +-- packages/core/src/runpromptcontext.ts | 27 ++-- packages/core/src/types/prompt_template.d.ts | 3 + .../sample/genaisrc/blog/genaiscript.d.ts | 3 + packages/sample/genaisrc/genaiscript.d.ts | 3 + .../sample/genaisrc/node/genaiscript.d.ts | 3 + .../sample/genaisrc/python/genaiscript.d.ts | 3 + .../sample/genaisrc/style/genaiscript.d.ts | 3 + .../sample/genaisrc/style/runprompt.genai.js | 27 +++- packages/sample/src/aici/genaiscript.d.ts | 3 + packages/sample/src/errors/genaiscript.d.ts | 3 + packages/sample/src/genaiscript.d.ts | 3 + packages/sample/src/makecode/genaiscript.d.ts | 3 + packages/sample/src/tla/genaiscript.d.ts | 3 + packages/sample/src/vision/genaiscript.d.ts | 3 + packages/vscode/genaisrc/genaiscript.d.ts | 3 + slides/genaisrc/genaiscript.d.ts | 3 + 26 files changed, 224 insertions(+), 93 deletions(-) diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/eval/extrism/genaisrc/extrism-tool.genai.mts b/eval/extrism/genaisrc/extrism-tool.genai.mts index 5b05f81dff..3aaa001beb 100644 --- a/eval/extrism/genaisrc/extrism-tool.genai.mts +++ b/eval/extrism/genaisrc/extrism-tool.genai.mts @@ -1,11 +1,9 @@ script({ model: "openai:gpt-4o", title: "Runs a extrism sample", - system: ["system", "system.explanations", "system.python", "system.files"], }) const practiceDir = "python/exercises/practice" -const { sample = "anagram" } = env.vars // create container const container = await host.container({ @@ -23,22 +21,32 @@ await container.exec("pip install -r requirements.txt --user", { await container.exec("pip install pytest --user", { cwd: "python" }) await container.disconnect() -// generate sample -const cwd = path.join(practiceDir, sample) -const { files: samplefiles } = JSON.parse( - await container.readText(path.join(cwd, ".meta/config.json")) -) -const { solution } = samplefiles -const filename = path.join(cwd, solution[0]) -let instructions = "" -for (const introname of ["introduction", "instructions", "instructions.app"]) { - const intro = await container.readText( - path.join(cwd, `.docs/${introname}.md`) +const samples = (await container.exec("ls -1", { cwd: practiceDir })).stdout + .trim() + .split(/\n/g) +for (const sample of samples) { + const cwd = path.join(practiceDir, sample) + console.log(cwd) + const { files: samplefiles } = JSON.parse( + await container.readText(path.join(cwd, ".meta/config.json")) ) - if (intro) instructions += intro + "\n\n" -} + const { solution } = samplefiles + const filename = path.join(cwd, solution[0]) + let instructions = "" + for (const introname of [ + "introduction", + "instructions", + "instructions.app", + ]) { + const intro = await container.readText( + path.join(cwd, `.docs/${introname}.md`) + ) + if (intro) instructions += intro + "\n\n" + } -$` + await runPrompt( + (ctx) => { + ctx.$` ## Task 1: @@ -62,39 +70,53 @@ If the tests fail, update the generated code in Task 1 and re-run the tests in T If the tests passed, stop. ` -def("INSTRUCTIONS", instructions, { language: "markdown" }) -def("TEMPLATE", { filename }, { language: "python" }) - -let generatedCode = "" -let testPassed = true -defTool( - "test_code", - "Run unit tests against generated solution", - { - code: { - type: "string", - description: "Generated Python code to solve the problem", + ctx.def("INSTRUCTIONS", instructions, { language: "markdown" }) + ctx.def("TEMPLATE", { filename }, { language: "python" }) + + let generatedCode = "" + let testPassed = true + ctx.defTool( + "test_code", + "Run unit tests against generated solution", + { + code: { + type: "string", + description: + "Generated Python code to solve the problem", + }, + }, + async ({ code }) => { + generatedCode = code + console.log(code) + await container.writeText(filename, code) + const res = await container.exec( + `python3.11 -m pytest -o markers=task ${sample}_test.py`, + { cwd } + ) + if (res.exitCode) { + console.log(res.stdout || "") + console.log(res.stderr || "") + } + testPassed = true + return res + } + ) + + ctx.defOutputProcessor(async (res) => { + if (!generatedCode) throw new Error("Generated code is missing") + if (!testPassed) throw new Error("Unit tests failed") + return { text: res.text + "\n\n" } + }) }, - }, - async ({ code }) => { - generatedCode = code - console.log(code) - await container.writeText(filename, code) - const res = await container.exec( - `python3.11 -m pytest -o markers=task ${sample}_test.py`, - { cwd } - ) - if (res.exitCode) { - console.log(res.stdout || "") - console.log(res.stderr || "") + { + label: sample, + applyEdits: true, + system: [ + "system", + "system.explanations", + "system.python", + "system.files", + ], } - testPassed = true - return res - } -) - -defOutputProcessor(async (res) => { - if (!generatedCode) throw new Error("Generated code is missing") - if (!testPassed) throw new Error("Unit tests failed") - return { text: res.text + "\n\n" } -}) + ) +} diff --git a/eval/extrism/genaisrc/genaiscript.d.ts b/eval/extrism/genaisrc/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/eval/extrism/genaisrc/genaiscript.d.ts +++ b/eval/extrism/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/auto/genaiscript.d.ts b/packages/auto/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/auto/genaiscript.d.ts +++ b/packages/auto/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts index ef3a51611c..953fd9ed7d 100644 --- a/packages/core/src/chat.ts +++ b/packages/core/src/chat.ts @@ -47,6 +47,7 @@ import { fenceMD, prettifyMarkdown } from "./markdown" import { YAMLStringify } from "./yaml" import { resolveTokenEncoder } from "./encoders" import { estimateTokens, truncateTextToTokens } from "./tokens" +import { computeFileEdits } from "./fileedits" export function toChatCompletionUserMessage( expanded: string, @@ -401,16 +402,19 @@ function assistantText( return text } -function structurifyChatSession( +async function structurifyChatSession( messages: ChatCompletionMessageParam[], schemas: Record, genVars: Record, + fileOutputs: FileOutput[], + outputProcessors: PromptOutputProcessorHandler[], + fileMerges: FileMergeHandler[], options: GenerationOptions, others?: { resp?: ChatCompletionResponse err?: any } -): RunPromptResult { +): Promise { const { trace, responseType, responseSchema } = options const { resp, err } = others || {} const text = assistantText(messages, responseType) @@ -458,7 +462,7 @@ function structurifyChatSession( if (fences?.length) frames.push(...validateFencesWithSchema(fences, schemas, { trace })) - return { + const res = { text, annotations, finishReason, @@ -469,6 +473,14 @@ function structurifyChatSession( genVars, schemas, } + await computeFileEdits(res, { + trace, + schemas, + fileOutputs, + fileMerges, + outputProcessors, + }) + return res } async function processChatMessage( @@ -479,6 +491,9 @@ async function processChatMessage( chatParticipants: ChatParticipant[], schemas: Record, genVars: Record, + fileOutputs: FileOutput[], + outputProcessors: PromptOutputProcessorHandler[], + fileMerges: FileMergeHandler[], options: GenerationOptions ): Promise { const { @@ -554,9 +569,18 @@ async function processChatMessage( if (needsNewTurn) return undefined } - return structurifyChatSession(messages, schemas, genVars, options, { - resp, - }) + return structurifyChatSession( + messages, + schemas, + genVars, + fileOutputs, + outputProcessors, + fileMerges, + options, + { + resp, + } + ) } export function mergeGenerationOptions( @@ -585,6 +609,9 @@ export async function executeChatSession( messages: ChatCompletionMessageParam[], toolDefinitions: ToolCallback[], schemas: Record, + fileOutputs: FileOutput[], + outputProcessors: PromptOutputProcessorHandler[], + fileMerges: FileMergeHandler[], completer: ChatCompletionHandler, chatParticipants: ChatParticipant[], genOptions: GenerationOptions @@ -683,6 +710,9 @@ export async function executeChatSession( chatParticipants, schemas, genVars, + fileOutputs, + outputProcessors, + fileMerges, genOptions ) if (output) return output @@ -691,6 +721,9 @@ export async function executeChatSession( messages, schemas, genVars, + fileOutputs, + outputProcessors, + fileMerges, genOptions, { resp, err } ) diff --git a/packages/core/src/fileedits.ts b/packages/core/src/fileedits.ts index 1033182b1a..876f5dfa32 100644 --- a/packages/core/src/fileedits.ts +++ b/packages/core/src/fileedits.ts @@ -22,11 +22,7 @@ export async function computeFileEdits( fileMerges?: FileMergeHandler[] outputProcessors?: PromptOutputProcessorHandler[] } -): Promise<{ - edits: Edits[] - fileEdits: Record - changelogs: string[] -}> { +): Promise { const { trace, fileOutputs, fileMerges, outputProcessors, schemas } = options || {} const { fences, frames, genVars } = res @@ -197,7 +193,11 @@ export async function computeFileEdits( }) ) - return { fileEdits, changelogs, edits } + res.text = text + res.fileEdits = fileEdits + res.changelogs = changelogs + res.annotations = annotations + res.edits = edits } // Validate file outputs against specified schemas and patterns diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/core/src/promptcontext.ts b/packages/core/src/promptcontext.ts index 800b7e5c70..e8cdbc515b 100644 --- a/packages/core/src/promptcontext.ts +++ b/packages/core/src/promptcontext.ts @@ -193,11 +193,6 @@ export async function createPromptContext( }, } - // Default output processor for the prompt - const defOutputProcessor = (fn: PromptOutputProcessorHandler) => { - if (fn) appendPromptChild(createOutputProcessor(fn)) - } - // Define the host for executing commands, browsing, and other operations const promptHost: PromptHost = Object.freeze({ exec: async ( @@ -263,10 +258,6 @@ export async function createPromptContext( parsers, retrieval, host: promptHost, - defOutputProcessor, - defFileMerge: (fn) => { - appendPromptChild(createFileMerge(fn)) - }, } env.generator = ctx ctx.env = Object.freeze(env) diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts index 5b405578cc..9fe895b8f6 100644 --- a/packages/core/src/promptrunner.ts +++ b/packages/core/src/promptrunner.ts @@ -23,8 +23,6 @@ import { validateJSONWithSchema } from "./schema" import { YAMLParse } from "./yaml" import { expandTemplate } from "./expander" import { resolveLanguageModel } from "./lm" -import { Stats } from "fs" -import { computeFileEdits } from "./fileedits" // Asynchronously resolve expansion variables needed for a template /** @@ -225,6 +223,9 @@ export async function runTemplate( messages, functions, schemas, + fileOutputs, + outputProcessors, + fileMerges, completer, chatParticipants, genOptions @@ -239,19 +240,12 @@ export async function runTemplate( error, finishReason, usages, + fileEdits, + changelogs, + edits, } = output let { text, annotations } = output - const { fileEdits, changelogs, edits } = await computeFileEdits( - output, - { - trace, - fileOutputs, - schemas, - fileMerges, - outputProcessors, - } - ) // Reporting and tracing output if (fences?.length) trace.details("📩 code regions", renderFencedVariables(fences)) diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts index 42dc100236..54826388ca 100644 --- a/packages/core/src/runpromptcontext.ts +++ b/packages/core/src/runpromptcontext.ts @@ -14,6 +14,8 @@ import { createStringTemplateNode, createTextNode, renderPromptNode, + createOutputProcessor, + createFileMerge, } from "./promptdom" import { MarkdownTrace } from "./trace" import { GenerationOptions } from "./generation" @@ -56,7 +58,7 @@ import { concurrentLimit } from "./concurrency" import { Project } from "./ast" import { dedent } from "./indent" import { runtimeHost } from "./host" -import { computeFileEdits, writeFileEdits } from "./fileedits" +import { writeFileEdits } from "./fileedits" export function createChatTurnGenerationContext( options: GenerationOptions, @@ -243,6 +245,11 @@ export function createChatGenerationContext( const turnCtx = createChatTurnGenerationContext(options, trace) const node = turnCtx.node + // Default output processor for the prompt + const defOutputProcessor = (fn: PromptOutputProcessorHandler) => { + if (fn) appendChild(node, createOutputProcessor(fn)) + } + const defTool: ( name: | string @@ -634,20 +641,16 @@ export function createChatGenerationContext( messages, tools, schemas, + fileOutputs, + outputProcessors, + fileMerges, completer, chatParticipants, genOptions ) ) tracePromptResult(runTrace, resp) - - const { fileEdits } = await computeFileEdits(resp, { - trace, - schemas, - fileOutputs, - fileMerges, - outputProcessors, - }) + const { fileEdits } = resp if (fileEdits?.length && applyEdits) await writeFileEdits(fileEdits, { trace }) return resp @@ -663,6 +666,10 @@ export function createChatGenerationContext( } } + const defFileMerge = (fn: FileMergeHandler) => { + appendChild(node, createFileMerge(fn)) + } + const ctx = { ...turnCtx, defAgent, @@ -671,6 +678,8 @@ export function createChatGenerationContext( defImages, defChatParticipant, defFileOutput, + defOutputProcessor, + defFileMerge, prompt, runPrompt, } diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 02b724336b..6dcf80d156 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -922,6 +922,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/genaisrc/blog/genaiscript.d.ts b/packages/sample/genaisrc/blog/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/genaisrc/blog/genaiscript.d.ts +++ b/packages/sample/genaisrc/blog/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/genaisrc/style/runprompt.genai.js b/packages/sample/genaisrc/style/runprompt.genai.js index 1d8346b902..fd18f02b8f 100644 --- a/packages/sample/genaisrc/style/runprompt.genai.js +++ b/packages/sample/genaisrc/style/runprompt.genai.js @@ -1,5 +1,5 @@ script({ - model: "openai:gpt-3.5-turbo", + model: "openai:gpt-4o-mini", tests: {}, }) @@ -39,3 +39,28 @@ fence(resPoem.text) $`Is this JSON? Respond yes or no.` fence(resJSON.text) + +const { text, fileEdits } = await runPrompt( + (_) => { + _.$`Create a file named "test.txt" with the following content: "hello world"` + _.defOutputProcessor((output) => { + console.log(`processing output: ${output.text}`) + return { text: output.text + "\n" } + }) + _.defFileMerge((filename, label, before, after) => { + console.log({ filename, label, before, after }) + if (path.basename(filename) !== "test.txt") + throw new Error("wrong file name") + return after + "\n" + }) + }, + { applyEdits: true, label: "outputs", system: ["system", "system.files"] } +) +console.log(text) +console.log(YAML.stringify(fileEdits)) +if (!fileEdits?.["test.txt"]?.after?.includes("hello world")) + throw new Error("File not created") +if (!text.includes("")) + throw new Error("output processor did not run") +if (!fileEdits?.["test.txt"]?.after?.includes("")) + throw new Error("file merge did not run") diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/src/genaiscript.d.ts b/packages/sample/src/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/src/genaiscript.d.ts +++ b/packages/sample/src/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/packages/vscode/genaisrc/genaiscript.d.ts b/packages/vscode/genaisrc/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/packages/vscode/genaisrc/genaiscript.d.ts +++ b/packages/vscode/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index ffb6c4db5b..391b02d15d 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -1003,6 +1003,9 @@ interface RunPromptResult { | "cancel" | "fail" usages?: ChatCompletionUsages + fileEdits?: Record + edits?: Edits[] + changelogs?: ChangeLog[] } /** From 5f1fa462c063cd3c30b792f9902d1ee45cd33614 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 16:05:27 +0000 Subject: [PATCH 04/11] =?UTF-8?q?Add=20YAML=20serialization=20and=20enhanc?= =?UTF-8?q?e=20logging=20=F0=9F=9A=80=F0=9F=93=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{extrism-tool.genai.mts => extrism.genai.mts} | 7 ++++++- packages/core/src/importprompt.ts | 11 +++++------ 2 files changed, 11 insertions(+), 7 deletions(-) rename eval/extrism/genaisrc/{extrism-tool.genai.mts => extrism.genai.mts} (96%) diff --git a/eval/extrism/genaisrc/extrism-tool.genai.mts b/eval/extrism/genaisrc/extrism.genai.mts similarity index 96% rename from eval/extrism/genaisrc/extrism-tool.genai.mts rename to eval/extrism/genaisrc/extrism.genai.mts index 3aaa001beb..85d0d45301 100644 --- a/eval/extrism/genaisrc/extrism-tool.genai.mts +++ b/eval/extrism/genaisrc/extrism.genai.mts @@ -44,7 +44,7 @@ for (const sample of samples) { if (intro) instructions += intro + "\n\n" } - await runPrompt( + const res = await runPrompt( (ctx) => { ctx.$` @@ -119,4 +119,9 @@ If the tests passed, stop. ], } ) + + await workspace.writeText( + `eval/extrism/results/${sample}.yaml`, + YAML.stringify(res) + ) } diff --git a/packages/core/src/importprompt.ts b/packages/core/src/importprompt.ts index a89eade576..6cbc3b19d3 100644 --- a/packages/core/src/importprompt.ts +++ b/packages/core/src/importprompt.ts @@ -1,6 +1,5 @@ -import { assert } from "console" import { host } from "./host" -import { logError } from "./util" +import { logError, logVerbose } from "./util" import { TraceOptions } from "./trace" import { pathToFileURL } from "url" import { resolveGlobal } from "./globals" @@ -33,10 +32,10 @@ export async function importPrompt( try { // override global context for (const field of Object.keys(ctx0)) { - assert( - field === "console" || leakables.includes(field) || !glb[field], - `overriding global field ${field}` - ) + //logVerbose( + // field === "console" || leakables.includes(field) || !glb[field], + // `overriding global field ${field}` + //) oldGlb[field] = glb[field] glb[field] = (ctx0 as any)[field] } From db6de9b90c473c86db7f68c8605887ddf4a3a3f0 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 16:16:27 +0000 Subject: [PATCH 05/11] =?UTF-8?q?Add=20test=20improvements=20and=20caching?= =?UTF-8?q?=20enhancements=20=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/genaisrc/genaiscript.d.ts | 3 ++- eval/extrism/genaisrc/extrism.genai.mts | 13 ++++++++++--- eval/extrism/genaisrc/genaiscript.d.ts | 3 ++- genaisrc/genaiscript.d.ts | 3 ++- packages/auto/genaiscript.d.ts | 3 ++- packages/core/src/genaisrc/genaiscript.d.ts | 3 ++- packages/core/src/types/prompt_template.d.ts | 3 ++- packages/sample/genaisrc/blog/genaiscript.d.ts | 3 ++- packages/sample/genaisrc/genaiscript.d.ts | 3 ++- packages/sample/genaisrc/node/genaiscript.d.ts | 3 ++- packages/sample/genaisrc/python/genaiscript.d.ts | 3 ++- packages/sample/genaisrc/style/genaiscript.d.ts | 3 ++- packages/sample/src/aici/genaiscript.d.ts | 3 ++- packages/sample/src/errors/genaiscript.d.ts | 3 ++- packages/sample/src/genaiscript.d.ts | 3 ++- packages/sample/src/makecode/genaiscript.d.ts | 3 ++- packages/sample/src/tla/genaiscript.d.ts | 3 ++- packages/sample/src/vision/genaiscript.d.ts | 3 ++- packages/vscode/genaisrc/genaiscript.d.ts | 3 ++- slides/genaisrc/genaiscript.d.ts | 3 ++- 20 files changed, 48 insertions(+), 22 deletions(-) diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/eval/extrism/genaisrc/extrism.genai.mts b/eval/extrism/genaisrc/extrism.genai.mts index 85d0d45301..8d32edcb9a 100644 --- a/eval/extrism/genaisrc/extrism.genai.mts +++ b/eval/extrism/genaisrc/extrism.genai.mts @@ -44,6 +44,8 @@ for (const sample of samples) { if (intro) instructions += intro + "\n\n" } + let generatedCode = "" + let testPassed = true const res = await runPrompt( (ctx) => { ctx.$` @@ -73,8 +75,6 @@ If the tests passed, stop. ctx.def("INSTRUCTIONS", instructions, { language: "markdown" }) ctx.def("TEMPLATE", { filename }, { language: "python" }) - let generatedCode = "" - let testPassed = true ctx.defTool( "test_code", "Run unit tests against generated solution", @@ -99,7 +99,8 @@ If the tests passed, stop. } testPassed = true return res - } + }, + { maxTokens: 20000 } ) ctx.defOutputProcessor(async (res) => { @@ -111,6 +112,7 @@ If the tests passed, stop. { label: sample, applyEdits: true, + cache: "extrism", system: [ "system", "system.explanations", @@ -124,4 +126,9 @@ If the tests passed, stop. `eval/extrism/results/${sample}.yaml`, YAML.stringify(res) ) + if (generatedCode) + await workspace.writeText( + `eval/extrism/results/${testPassed ? "success" : "failed"}/${solution[0]}`, + generatedCode + ) } diff --git a/eval/extrism/genaisrc/genaiscript.d.ts b/eval/extrism/genaisrc/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/eval/extrism/genaisrc/genaiscript.d.ts +++ b/eval/extrism/genaisrc/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/auto/genaiscript.d.ts b/packages/auto/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/auto/genaiscript.d.ts +++ b/packages/auto/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index 6dcf80d156..f3c3beb21a 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -2099,7 +2099,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/genaisrc/blog/genaiscript.d.ts b/packages/sample/genaisrc/blog/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/genaisrc/blog/genaiscript.d.ts +++ b/packages/sample/genaisrc/blog/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/src/genaiscript.d.ts b/packages/sample/src/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/src/genaiscript.d.ts +++ b/packages/sample/src/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/packages/vscode/genaisrc/genaiscript.d.ts b/packages/vscode/genaisrc/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/packages/vscode/genaisrc/genaiscript.d.ts +++ b/packages/vscode/genaisrc/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 391b02d15d..f1e2965211 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -2180,7 +2180,8 @@ interface ChatGenerationContext extends ChatTurnGenerationContext { name: string, description: string, parameters: PromptParametersSchema | JSONSchema, - fn: ChatFunctionHandler + fn: ChatFunctionHandler, + options?: DefToolOptions ): void defAgent( name: string, From 45747ae2e91b664de74136a1a9e3c6837abbeb49 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 16:29:00 +0000 Subject: [PATCH 06/11] =?UTF-8?q?Update=20test=20execution=20and=20result?= =?UTF-8?q?=20paths=20in=20`extrism.genai.mts`=20=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eval/extrism/genaisrc/extrism.genai.mts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/eval/extrism/genaisrc/extrism.genai.mts b/eval/extrism/genaisrc/extrism.genai.mts index 8d32edcb9a..9e3a91b2cf 100644 --- a/eval/extrism/genaisrc/extrism.genai.mts +++ b/eval/extrism/genaisrc/extrism.genai.mts @@ -30,7 +30,7 @@ for (const sample of samples) { const { files: samplefiles } = JSON.parse( await container.readText(path.join(cwd, ".meta/config.json")) ) - const { solution } = samplefiles + const { solution, test } = samplefiles const filename = path.join(cwd, solution[0]) let instructions = "" for (const introname of [ @@ -45,7 +45,7 @@ for (const sample of samples) { } let generatedCode = "" - let testPassed = true + let testPassed = false const res = await runPrompt( (ctx) => { ctx.$` @@ -90,14 +90,13 @@ If the tests passed, stop. console.log(code) await container.writeText(filename, code) const res = await container.exec( - `python3.11 -m pytest -o markers=task ${sample}_test.py`, + `python3.11 -m pytest -o markers=task ${test[0]}`, { cwd } ) if (res.exitCode) { console.log(res.stdout || "") console.log(res.stderr || "") - } - testPassed = true + } else testPassed = true return res }, { maxTokens: 20000 } @@ -123,7 +122,7 @@ If the tests passed, stop. ) await workspace.writeText( - `eval/extrism/results/${sample}.yaml`, + `eval/extrism/results/res/${sample}.yaml`, YAML.stringify(res) ) if (generatedCode) From 13ba3511966b75d5f92c35c845973320a80af43b Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 16:30:19 +0000 Subject: [PATCH 07/11] =?UTF-8?q?Add=20comments=20to=20explain=20generated?= =?UTF-8?q?=20Python=20code=20=E2=9C=8F=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eval/extrism/genaisrc/extrism.genai.mts | 1 + 1 file changed, 1 insertion(+) diff --git a/eval/extrism/genaisrc/extrism.genai.mts b/eval/extrism/genaisrc/extrism.genai.mts index 9e3a91b2cf..18c60b56fd 100644 --- a/eval/extrism/genaisrc/extrism.genai.mts +++ b/eval/extrism/genaisrc/extrism.genai.mts @@ -59,6 +59,7 @@ Analyze INSTRUCTIONS and generate Python code that the requirements. You mush NOT change the function signature. Implement the functions. - do NOT generate unit tests. - you can only use built-in python libraries. pip packages are not allowed. +- add comments to explain the generated code ## Task 2: From a4a8229f5a8323ff57278223fe42b975fae2ab8b Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 16:39:54 +0000 Subject: [PATCH 08/11] =?UTF-8?q?Add=20timestamped=20results=20directory?= =?UTF-8?q?=20and=20optimize=20sample=20processing=20=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eval/extrism/genaisrc/extrism.genai.mts | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/eval/extrism/genaisrc/extrism.genai.mts b/eval/extrism/genaisrc/extrism.genai.mts index 18c60b56fd..0a1fffcffc 100644 --- a/eval/extrism/genaisrc/extrism.genai.mts +++ b/eval/extrism/genaisrc/extrism.genai.mts @@ -3,6 +3,7 @@ script({ title: "Runs a extrism sample", }) +const results = `eval/extrism/results/${new Date().toISOString().replace(/:/g, "-")}` const practiceDir = "python/exercises/practice" // create container @@ -21,10 +22,11 @@ await container.exec("pip install -r requirements.txt --user", { await container.exec("pip install pytest --user", { cwd: "python" }) await container.disconnect() +const q = host.promiseQueue(5) const samples = (await container.exec("ls -1", { cwd: practiceDir })).stdout .trim() .split(/\n/g) -for (const sample of samples) { +await q.mapAll(samples, async (sample) => { const cwd = path.join(practiceDir, sample) console.log(cwd) const { files: samplefiles } = JSON.parse( @@ -59,7 +61,6 @@ Analyze INSTRUCTIONS and generate Python code that the requirements. You mush NOT change the function signature. Implement the functions. - do NOT generate unit tests. - you can only use built-in python libraries. pip packages are not allowed. -- add comments to explain the generated code ## Task 2: @@ -102,12 +103,6 @@ If the tests passed, stop. }, { maxTokens: 20000 } ) - - ctx.defOutputProcessor(async (res) => { - if (!generatedCode) throw new Error("Generated code is missing") - if (!testPassed) throw new Error("Unit tests failed") - return { text: res.text + "\n\n" } - }) }, { label: sample, @@ -123,12 +118,12 @@ If the tests passed, stop. ) await workspace.writeText( - `eval/extrism/results/res/${sample}.yaml`, + `${results}/res/${sample}.yaml`, YAML.stringify(res) ) if (generatedCode) await workspace.writeText( - `eval/extrism/results/${testPassed ? "success" : "failed"}/${solution[0]}`, + `${results}/${testPassed ? "success" : "failed"}/${solution[0]}`, generatedCode ) -} +}) From 04e66eced812c62117a11a22f91d1e97fe8008ac Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 18:33:04 +0000 Subject: [PATCH 09/11] =?UTF-8?q?Add=20placeholder=20text=20to=20poem=20ge?= =?UTF-8?q?neration=20file=20=F0=9F=93=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/sample/genaisrc/poem.genai.mts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/sample/genaisrc/poem.genai.mts b/packages/sample/genaisrc/poem.genai.mts index df55f31d3a..d2b6675d6e 100644 --- a/packages/sample/genaisrc/poem.genai.mts +++ b/packages/sample/genaisrc/poem.genai.mts @@ -1 +1,2 @@ $`Write a short poem.` +xsd sadf asdf dsaf \ No newline at end of file From 1a2c89ece288298e3509ce5f81b2386556f90a0b Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 19:09:24 +0000 Subject: [PATCH 10/11] =?UTF-8?q?Enhance=20Git=20system=20integration=20an?= =?UTF-8?q?d=20update=20failure=20investigation=20steps=20=F0=9F=9B=A0?= =?UTF-8?q?=EF=B8=8F=F0=9F=94=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/content/docs/reference/scripts/system.mdx | 5 ++++- .../core/src/genaisrc/system.agent_git.genai.mjs | 5 ++++- packages/sample/genaisrc/github-agent.genai.mts | 15 +++++++++------ packages/sample/genaisrc/poem.genai.mts | 1 - 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx index b020953551..4437ef699a 100644 --- a/docs/src/content/docs/reference/scripts/system.mdx +++ b/docs/src/content/docs/reference/scripts/system.mdx @@ -206,7 +206,10 @@ defAgent( - The current repository is the same as github repository. - Prefer using diff to compare files rather than listing files. Listing files is only useful when you need to read the content of the files. `, - { model, system: ["system.git_info", "system.github_info"], tools: ["git"] } + { + model, + system: ["system.git_info", "system.github_info", "system.git"], + } ) ````` diff --git a/packages/core/src/genaisrc/system.agent_git.genai.mjs b/packages/core/src/genaisrc/system.agent_git.genai.mjs index 4304e4703c..35aa448c6d 100644 --- a/packages/core/src/genaisrc/system.agent_git.genai.mjs +++ b/packages/core/src/genaisrc/system.agent_git.genai.mjs @@ -12,5 +12,8 @@ defAgent( - The current repository is the same as github repository. - Prefer using diff to compare files rather than listing files. Listing files is only useful when you need to read the content of the files. `, - { model, system: ["system.git_info", "system.github_info"], tools: ["git"] } + { + model, + system: ["system.git_info", "system.github_info", "system.git"], + } ) diff --git a/packages/sample/genaisrc/github-agent.genai.mts b/packages/sample/genaisrc/github-agent.genai.mts index edb130283b..adb99f1010 100644 --- a/packages/sample/genaisrc/github-agent.genai.mts +++ b/packages/sample/genaisrc/github-agent.genai.mts @@ -15,15 +15,18 @@ script({ }) const { - workflow = "latest failed", + workflow = "build.yml", failure_run_id = "latest", - branch = await git.defaultBranch(), + branch = await git.branch(), } = env.vars -$`Investigate the status of the ${workflow} workflow and identify the root cause of the failure of run ${failure_run_id} in branch ${branch}. - -- Correlate the failure with the relevant commits, pull requests or issues. -- Compare the source code between the failed run and the last successful run before that run. +$` +0. Find the worflow ${workflow} in the repository +1. Find the latest failed run of ${workflow} +2. Find the last successful run before the failed run +3. Compare the run job logs between the failed run and the last successful run +4. Compare the source code between the failed run commit (head_sha) and the last successful run commit (head_sha) +5. Analyze all the above information and identify the root cause of the failure In your report, include html links to the relevant runs, commits, pull requests or issues. ` diff --git a/packages/sample/genaisrc/poem.genai.mts b/packages/sample/genaisrc/poem.genai.mts index d2b6675d6e..df55f31d3a 100644 --- a/packages/sample/genaisrc/poem.genai.mts +++ b/packages/sample/genaisrc/poem.genai.mts @@ -1,2 +1 @@ $`Write a short poem.` -xsd sadf asdf dsaf \ No newline at end of file From aa53d5d467fef855ae3893e443dc8c0482adca74 Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 8 Oct 2024 19:13:35 +0000 Subject: [PATCH 11/11] =?UTF-8?q?Fix=20typo=20in=20tool=20name=20from=20md?= =?UTF-8?q?=5Fread=5Ffrontmatterm=20to=20md=5Fread=5Ffrontmatter=20?= =?UTF-8?q?=F0=9F=94=A7=E2=9C=8F=EF=B8=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/core/src/genaisrc/system.agent_docs.genai.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/src/genaisrc/system.agent_docs.genai.mjs b/packages/core/src/genaisrc/system.agent_docs.genai.mjs index 513a3898ea..4288050780 100644 --- a/packages/core/src/genaisrc/system.agent_docs.genai.mjs +++ b/packages/core/src/genaisrc/system.agent_docs.genai.mjs @@ -30,7 +30,7 @@ defAgent( system: ["system.explanations", "system.github_info"], tools: [ "md_find_files", - "md_read_frontmatterm", + "md_read_frontmatter", "fs_find_files", "fs_read_file", ],