AI + CI test synchronization tool (#755)

* Add AI effort on code generation and test improvements

---------

Co-authored-by: Liangying.Wei <[email protected]>

.github/workflows/integration-tests-python.yml

@@ -40,4 +40,27 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r requirements.txt
         pytest
-
+  post-test:
+    runs-on: ubuntu-latest
+    needs: test
+    if: failure()
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Download test output
+        uses: actions/download-artifact@v3
+        with:
+          name: test-output
+          path: ./test-output
+      - name: Read test output
+        id: read-test-output
+        run: echo "TEST_OUTPUT=$(cat ./test-output/test-output.txt)" >> $GITHUB_ENV
+      - name: Fix error with deep prompt
+        run: |
+          node eng/copilot/fixErrorWithDeepPrompt.js
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          API_BASE: ${{ secrets.OPENAI_API_BASE }}
+          PR_ID: ${{ github.event.pull_request.number }}
+          TEST_OUTPUT: ${{ env.TEST_OUTPUT }}

.github/workflows/tranalate-test.yml

@@ -0,0 +1,36 @@
+name: Translate tests with deep prompt  
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  check-and-translate:
+    runs-on: ubuntu-latest
+    if: github.event.review.state == 'approved'
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Check if all checks passed
+        id: checks
+        uses: LouisBrunner/[email protected]
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          timeout: 300
+
+      - name: Run translation only if all checks are successful
+        if: steps.checks.outputs.conclusion == 'success'
+        run: |
+          echo "All checks passed. Starting translation."
+          cd eng/translation
+          yarn install
+          node translate.js
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          API_BASE: ${{ secrets.OPENAI_API_BASE }}
+          PR_ID: ${{ github.event.pull_request.number }}

eng/copilot/constants.js

@@ -0,0 +1,8 @@
+export const githubToken = process.env.GITHUB_TOKEN;
+export const apiKey = process.env.API_KEY;
+export const apiBase = process.env.API_BASE;
+export const basePrId = process.env.BASE_PR_ID;
+export const prId = process.env.PR_ID;
+export const errorMessage = process.env.TEST_OUTPUT;
+export const targetRepoOwner = "Azure";
+export const targetRepo = "azure-webpubsub";

eng/copilot/deepPromptFunctions.js

@@ -0,0 +1,55 @@
+import { githubToken, apiBase, apiKey } from "./constants.js";
+
+export async function getSessionAccess() {
+    try {
+        return fetch("https://data-ai.microsoft.com/deepprompt/api/v1/exchange", {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+                token: githubToken,
+                provider: "github",
+            }),
+        }).then(res => res.json());
+    } catch (error) {
+        console.error("Failed to exchange github token");
+        throw error;
+    }
+}
+
+export async function fetchDeepPromptWithQuery(query, sessionId, accessToken){
+    try {
+        const dpResponse = await fetch("https://data-ai.microsoft.com/deepprompt/api/v1/query", {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                "DeepPrompt-Session-ID": sessionId,
+                "Authorization": `Bearer ${accessToken}`,
+                "DeepPrompt-OpenAI-API-Base": apiBase,
+                "DeepPrompt-OpenAI-API-Key": apiKey,
+                "DeepPrompt-OpenAI-Deployment": "gpt-4o",
+            },
+            body: JSON.stringify({
+                query: query
+            }),
+        }).then((response) => response.json());
+        return dpResponse.response_text;
+
+    } catch (err) {
+        console.error("Failed to fetch deep prompt rest api:", err.message);
+        throw error;
+    }
+}
+
+export function parseResponseToJson(response) {
+    let trimmed = response.trim();
+    if (trimmed.startsWith("```json\n") && trimmed.endsWith("```")) {
+        trimmed = trimmed.substring(7, trimmed.length - 3);
+    }
+    trimmed = trimmed.trim();
+    try {
+        return JSON.parse(trimmed);
+    } catch (error) {
+        console.error("Failed to parse the deep prompt response to JSON:", error);
+        throw error;
+    }
+}

eng/copilot/fixErrorWithDeepPrompt.js

@@ -0,0 +1,87 @@
+import { Octokit } from "@octokit/rest";
+import { getSessionAccess, fetchDeepPromptWithQuery, parseResponseToJson } from './deepPromptFunctions.js'
+import { getChangedFiles, createChangeBranch, createBlob, createCommit, updateBranch, createPR, getLatestCommitShaOnPr, createTree } from './octokitFunctions.js'
+import { githubToken, prId, errorMessage } from "./constants.js";
+
+function getErrorMessage(ciError) {
+    const lines = ciError.split('\n');
+    let errors = [];
+    let currentError = null;
+
+    lines.forEach(line => {
+        const errorCollectingPrefix = 'ERROR collecting ';
+        const errorSummaryPrefix = 'short test summary info ';
+        if (line.includes(errorSummaryPrefix)) {
+            if (currentError) {
+                errors.push(currentError);
+            }
+        }
+        else if (line.includes(errorCollectingPrefix)) {
+            if (currentError) {
+                errors.push(currentError);
+            }
+            const filename = line.substring(line.indexOf(errorCollectingPrefix) + 16, line.indexOf(' ________________'));
+            currentError = { filename: filename.trim(), errorMessage: '' };
+        } else if (currentError) {
+            currentError.errorMessage += line + '\n';
+        }
+    });
+
+    return errors;
+}
+
+async function fixErrorWithDP(file, errorMessage, sessionId, accessToken) {
+    const query = `
+                Below is a file change patch from git hub pull request, it failed to pass the ci test.
+                Base on the error message, rewrite the code of this file to fix the error.
+                Return the respinses in stringfied json format with fileName and fileContent.
+                file name: ###${file.filename}###
+                file dispatch: ###${file.patch}###
+                error message: ###${errorMessage}###`;
+    try {
+        while (true) {
+            const dpResponse = await fetchDeepPromptWithQuery(query, sessionId, accessToken);
+            if (dpResponse && dpResponse.includes("fileName") && dpResponse.includes("fileContent")) {
+                return parseResponseToJson(dpResponse);
+            }
+        }
+
+    } catch (err) {
+        console.error("Failed to fetch deep prompt rest api:", err.message);
+    }
+}
+
+async function fix() {
+    try {
+        const errors = getErrorMessage(errorMessage);
+        const accessSession = await getSessionAccess();
+        const files = await getChangedFiles("Azure", "azure-webpubsub", prId);
+        let fixedFiles = [];
+        const errorFixPromises = errors.map(async (error) => {
+            const filesWithError = files.filter(file => file.filename.includes(error.filename));
+            const fileFixPromises = filesWithError.map(async (file) => {
+                console.log(`start fixing error in ${file.filename} ...`);
+                const fixedFile = await fixErrorWithDP(file, error.errorMessage, accessSession.session_id, accessSession.access_token);
+                console.log(`${file.filename} error fix complete`);
+                return fixedFile;
+            });
+            const fixedFilesForError = await Promise.all(fileFixPromises);
+            fixedFiles.push(...fixedFilesForError);
+        });
+        await Promise.all(errorFixPromises);
+        // const sha = await getLatestCommitShaOnPr(targetRepoOwner, targetRepo, prId);
+        // const blobs = await createBlob(targetRepoOwner, targetRepo, fixedFiles);
+        // const treeSha = await createTree(targetRepoOwner, targetRepo, blobs, sha);
+        // const commitSha = await createCommit(targetRepoOwner, targetRepo, treeSha, sha);
+        // await updateBranch(targetRepoOwner, targetRepo, commitSha);
+        // console.log(`fix attempt completed and pushed to pr ${prId}`);
+
+    } catch (error) {
+        console.error('Error occurred during fix:', error.message);
+    }
+}
+
+fix();
+
+
+