Refactor: Remove file validation and add sample scripts

packages/cli/src/run.ts

@@ -367,8 +367,7 @@
         if (isJSONLFilename(outData)) await appendJSONL(outData, result.frames)
         else await writeText(outData, JSON.stringify(result.frames, null, 2))
 
-    if (result.status === "success" && result.fileEdits && applyEdits)
-        await writeFileEdits(result.fileEdits, { trace })
+    await writeFileEdits(result.fileEdits, { applyEdits, trace })
 
     const promptjson = result.messages?.length
         ? JSON.stringify(result.messages, null, 2)

packages/core/src/fileedits.ts

@@ -282,18 +282,23 @@
  */
 export async function writeFileEdits(
     fileEdits: Record<string, FileUpdate>, // Contains the edits to be applied to files
-    options?: TraceOptions
+    options?: { applyEdits?: boolean } & TraceOptions
 ) {
-    const { trace } = options || {}
+    const { applyEdits, trace } = options || {}
     // Iterate over each file edit entry
     for (const fileEdit of Object.entries(fileEdits || {})) {
         // Destructure the filename, before content, after content, and validation from the entry
         const [fn, { before, after, validation }] = fileEdit
 
+        if (!applyEdits && !validation?.valid) {
+            // path not validated
+            continue
+        }
+
         // Skip writing if the edit is invalid and applyEdits is false
-        if (validation?.valid === false) {
+        if (validation?.error) {
             trace.detailsFenced(
-                `skipping ${fn}, invalid`,
+                `skipping ${fn}, invalid schema`,
                 validation.error,
                 "text"
             )

packages/core/src/promptrunner.ts

@@ -292,70 +292,3 @@ export async function runTemplate(
         await runtimeHost.removeBrowsers()
     }
 }
-
-// Validate file outputs against specified schemas and patterns
-/**
- * Validates file outputs based on provided patterns and schemas.
- * @param fileOutputs List of file outputs to validate.
- * @param trace The markdown trace for logging.
- * @param fileEdits Record of file updates.
- * @param schemas The JSON schemas for validation.
- */
-function validateFileOutputs(
-    fileOutputs: FileOutput[],
-    trace: MarkdownTrace,
-    fileEdits: Record<string, FileUpdate>,
-    schemas: Record<string, JSONSchema>
-) {
-    if (fileOutputs?.length && Object.keys(fileEdits || {}).length) {
-        trace.startDetails("🗂 file outputs")
-        for (const fileEditName of Object.keys(fileEdits)) {
-            const fe = fileEdits[fileEditName]
-            for (const fileOutput of fileOutputs) {
-                const { pattern, options } = fileOutput
-                if (isGlobMatch(fileEditName, pattern)) {
-                    try {
-                        trace.startDetails(`📁 ${fileEditName}`)
-                        trace.itemValue(`pattern`, pattern)
-                        const { schema: schemaId } = options || {}
-                        if (/\.(json|yaml)$/i.test(fileEditName)) {
-                            const { after } = fileEdits[fileEditName]
-                            const data = /\.json$/i.test(fileEditName)
-                                ? JSON.parse(after)
-                                : YAMLParse(after)
-                            trace.detailsFenced("📝 data", data)
-                            if (schemaId) {
-                                const schema = schemas[schemaId]
-                                if (!schema)
-                                    fe.validation = {
-                                        valid: false,
-                                        error: `schema ${schemaId} not found`,
-                                    }
-                                else
-                                    fe.validation = validateJSONWithSchema(
-                                        data,
-                                        schema,
-                                        {
-                                            trace,
-                                        }
-                                    )
-                            }
-                        } else {
-                            fe.validation = { valid: true }
-                        }
-                    } catch (e) {
-                        trace.error(errorMessage(e))
-                        fe.validation = {
-                            valid: false,
-                            error: errorMessage(e),
-                        }
-                    } finally {
-                        trace.endDetails()
-                    }
-                    break
-                }
-            }
-        }
-        trace.endDetails()
-    }
-}

packages/core/src/runpromptcontext.ts

@@ -691,9 +691,7 @@ export function createChatGenerationContext(
                 )
             )
             tracePromptResult(runTrace, resp)
-            const { fileEdits } = resp
-            if (fileEdits?.length && applyEdits)
-                await writeFileEdits(fileEdits, { trace })
+            await writeFileEdits(resp.fileEdits, { applyEdits, trace })
             return resp
         } catch (e) {
             runTrace.error(e)

packages/core/src/schema.ts

@@ -168,7 +168,7 @@ export function validateJSONWithSchema(
     object: any,
     schema: JSONSchema,
     options?: { trace: MarkdownTrace }
-): JSONSchemaValidation {
+): FileEditValidation {
     const { trace } = options || {}
     if (!schema)
         return {

packages/core/src/types/prompt_template.d.ts

@@ -932,7 +932,7 @@ interface JSONSchemaArray {
 
 type JSONSchema = JSONSchemaObject | JSONSchemaArray
 
-interface JSONSchemaValidation {
+interface FileEditValidation {
     schema?: JSONSchema
     valid: boolean
     error?: string
@@ -941,7 +941,7 @@ interface JSONSchemaValidation {
 interface DataFrame {
     schema?: string
     data: unknown
-    validation?: JSONSchemaValidation
+    validation?: FileEditValidation
 }
 
 interface RunPromptResult {
@@ -1023,7 +1023,7 @@ interface Fenced {
     content: string
     args?: { schema?: string } & Record<string, string>
 
-    validation?: JSONSchemaValidation
+    validation?: FileEditValidation
 }
 
 interface XMLParseOptions {
@@ -1257,7 +1257,7 @@ interface Parsers {
      * @param schema JSON schema instance
      * @param content object to validate
      */
-    validateJSON(schema: JSONSchema, content: any): JSONSchemaValidation
+    validateJSON(schema: JSONSchema, content: any): FileEditValidation
 
     /**
      * Renders a mustache template
@@ -2107,7 +2107,7 @@ interface ChatTurnGenerationContext {
 interface FileUpdate {
     before: string
     after: string
-    validation?: JSONSchemaValidation
+    validation?: FileEditValidation
 }
 
 interface RunPromptResultPromiseWithOptions extends Promise<RunPromptResult> {

packages/sample/src/edits/.gitattributes

@@ -0,0 +1 @@
+genaiscript.d.ts -diff merge=ours linguist-generated

packages/sample/src/edits/.gitignore

@@ -0,0 +1,4 @@
+# auto-generated
+genaiscript.d.ts
+tsconfig.json
+jsconfig.json

packages/sample/src/edits/edits_changelog.genai.mts

@@ -0,0 +1,9 @@
+import { editTest } from "./fileedittest.mts"
+script({
+    model: "large",
+    title: "system.diff test",
+    files: "src/edits/fib.ts",
+    system: ["system", "system.changelog"],
+})
+
+editTest()

packages/sample/src/edits/edits_diff.genai.mts

@@ -0,0 +1,9 @@
+import { editTest } from "./fileedittest.mts"
+script({
+    model: "large",
+    title: "system.diff test",
+    files: "src/edits/fib.ts",
+    system: ["system", "system.diff"],
+})
+
+editTest()

packages/sample/src/edits/edits_files.genai.mts

@@ -0,0 +1,37 @@
+script({
+    model: "large",
+    title: "system.files test",
+    files: "src/edits/fib.*",
+    system: ["system", "system.files"],
+    tests: [
+        {
+            files: "src/edits/fib.ts",
+        },
+        {
+            files: "src/edits/fib.cpp",
+        },
+        {
+            files: "src/edits/fib.cs",
+        },
+        {
+            files: "src/edits/fib.go",
+        },
+        {
+            files: "src/edits/fib.java",
+        },
+        {
+            files: "src/edits/fib.js",
+        },
+        {
+            files: "src/edits/fib.kt",
+        },
+        {
+            files: "src/edits/fib.py",
+        },
+        {
+            files: "src/edits/fib.swift",
+        },
+    ],
+})
+import { editTest } from "./fileedittest.mts"
+editTest()

packages/sample/src/edits/editsgen.genai.mjs

@@ -0,0 +1,10 @@
+script({
+    files: "src/edits/fib.ts",
+})
+$`Generation 1 variations of the SNIPPET in various programming languages and save them in files.
+
+- there should be lines with comments
+- there should be a function with a TODO comment and a BODY comment
+`
+def("FILE", env.files)
+defFileOutput("src/edits/*")

packages/sample/src/edits/fib.cpp

@@ -0,0 +1,7 @@
+#include <iostream>
+
+int fibonacci(int n) {
+    // TODO: implement fibonacci algorithm
+    return 0; // BODY
+} // this one needs to go
+// another comment

packages/sample/src/edits/fib.cs

@@ -0,0 +1,7 @@
+public class Fibonacci {
+    public static int fibonacci(int n) {
+        // TODO: implement fibonacci algorithm
+        return 0; // BODY
+    } // this one needs to go
+    // another comment
+}

packages/sample/src/edits/fib.go

@@ -0,0 +1,7 @@
+package main
+
+func fibonacci(n int) int {
+    // TODO: implement fibonacci algorithm
+    return 0 // BODY
+} // this one needs to go
+// another comment

packages/sample/src/edits/fib.java

@@ -0,0 +1,7 @@
+public class Fibonacci {
+    public static int fibonacci(int n) {
+        // TODO: implement fibonacci algorithm
+        return 0; // BODY
+    } // this one needs to go
+    // another comment
+}

packages/sample/src/edits/fib.js

@@ -0,0 +1,5 @@
+function fibonacci(n) {
+    // TODO: implement fibonacci algorithm
+    return 0; // BODY
+} // this one needs to go
+// another comment

packages/sample/src/edits/fib.kt

@@ -0,0 +1,5 @@
+fun fibonacci(n: Int): Int {
+    // TODO: implement fibonacci algorithm
+    return 0 // BODY
+} // this one needs to go
+// another comment

packages/sample/src/edits/fib.php

@@ -0,0 +1,7 @@
+<?php
+function fibonacci($n) {
+    // TODO: implement fibonacci algorithm
+    return 0; // BODY
+} // this one needs to go
+// another comment
+?>

packages/sample/src/edits/fib.py

@@ -0,0 +1,5 @@
+def fibonacci(n):
+    # TODO: implement fibonacci algorithm
+    return 0  # BODY
+# this one needs to go
+# another comment

packages/sample/src/edits/fib.rb

@@ -0,0 +1,5 @@
+def fibonacci(n)
+    # TODO: implement fibonacci algorithm
+    return 0 # BODY
+end # this one needs to go
+# another comment

packages/sample/src/edits/fib.swift

@@ -0,0 +1,5 @@
+func fibonacci(_ n: Int) -> Int {
+    // TODO: implement fibonacci algorithm
+    return 0 // BODY
+} // this one needs to go
+// another comment

packages/sample/src/edits/fib.ts

@@ -0,0 +1,5 @@
+function fibonacci(n: number): number {
+    // TODO: implement fibonacci algorithm
+    return 0 // BODY
+} // this one needs to go
+// another comment

packages/sample/src/edits/fileedittest.mts

@@ -0,0 +1,17 @@
+export function editTest() {
+    def("FILE", env.files)
+
+    $`- Implement the functions with TODO.
+- Remove all comments.
+`
+
+    defOutputProcessor((output) => {
+        const { fileEdits } = output
+        //console.log(YAML.stringify(fileEdits))
+        const res = fileEdits[Object.keys(fileEdits)[0]].after
+        if (/^\s*\/\/.*$/.test(res))
+            throw new Error("some comments were not removed")
+        if (res.includes("// BODY"))
+            throw new Error("the // BODY comment was not removed")
+    })
+}