telemetry(amazonq): add telemetry for code generated and code accepted (

#6034)

## Problem
- we need to define a way to measure code generated and code accepted
from `/dev` feature


## Solution
- we compute the char and line difference between the two files
- all generated codes are immediately reported in telemetry
- within the same session, de-duplication is done to avoid reporting the
exact same changes twice across different code gen iterations
- currently, only accepted changes with new lines added will be reported

## Special Note
`charsAdded` and `charRemoved` may be slightly different from the count
you manually get from the IDE.
The reason is because new line character `\n` is not counted here in the
computeDiff method.

---

<!--- REMINDER: Ensure that your PR meets the guidelines in
CONTRIBUTING.md -->

License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.

packages/amazonq/test/unit/amazonqFeatureDev/session/session.test.ts

@@ -109,6 +109,7 @@ describe('session', () => {
         it('only insert non rejected files', async () => {
             const fsSpyWriteFile = sinon.spy(fs, 'writeFile')
             const session = await createCodeGenState()
+            sinon.stub(session, 'sendLinesOfCodeAcceptedTelemetry').resolves()
             await sessionWriteFile(session, uri, encodedContent)
             await session.insertChanges()
 

packages/core/package.json

@@ -435,7 +435,6 @@
         "c8": "^9.0.0",
         "circular-dependency-plugin": "^5.2.2",
         "css-loader": "^6.10.0",
-        "diff": "^5.1.0",
         "esbuild-loader": "2.20.0",
         "file-loader": "^6.2.0",
         "jsdom": "^23.0.1",
@@ -488,6 +487,7 @@
         "bytes": "^3.1.2",
         "cross-fetch": "^4.0.0",
         "cross-spawn": "^7.0.3",
+        "diff": "^5.1.0",
         "fast-json-patch": "^3.1.1",
         "glob": "^10.3.10",
         "got": "^11.8.5",

packages/core/src/amazonq/commons/diff.ts

@@ -6,6 +6,7 @@
 import * as vscode from 'vscode'
 import { featureDevScheme } from '../../amazonqFeatureDev/constants'
 import { fs } from '../../shared'
+import { diffLines } from 'diff'
 
 export async function openDiff(leftPath: string, rightPath: string, tabId: string) {
     const { left, right } = await getFileDiffUris(leftPath, rightPath, tabId)
@@ -29,6 +30,34 @@ export async function getFileDiffUris(leftPath: string, rightPath: string, tabId
     return { left, right }
 }
 
+export async function computeDiff(leftPath: string, rightPath: string, tabId: string) {
+    const { left, right } = await getFileDiffUris(leftPath, rightPath, tabId)
+    const leftFile = await vscode.workspace.openTextDocument(left)
+    const rightFile = await vscode.workspace.openTextDocument(right)
+
+    const changes = diffLines(leftFile.getText(), rightFile.getText(), {
+        ignoreWhitespace: true,
+    })
+
+    let charsAdded = 0
+    let charsRemoved = 0
+    let linesAdded = 0
+    let linesRemoved = 0
+    changes.forEach((change) => {
+        const lines = change.value.split('\n')
+        const charCount = lines.reduce((sum, line) => sum + line.length, 0)
+        const lineCount = change.count ?? lines.length - 1 // ignoring end-of-file empty line
+        if (change.added) {
+            charsAdded += charCount
+            linesAdded += lineCount
+        } else if (change.removed) {
+            charsRemoved += charCount
+            linesRemoved += lineCount
+        }
+    })
+    return { changes, charsAdded, linesAdded, charsRemoved, linesRemoved }
+}
+
 export function createAmazonQUri(path: string, tabId: string) {
     // TODO change the featureDevScheme to a more general amazon q scheme
     return vscode.Uri.from({ scheme: featureDevScheme, path, query: `tabID=${tabId}` })

packages/core/src/amazonq/index.ts

@@ -27,7 +27,14 @@ export { amazonQHelpUrl } from '../shared/constants'
 export { listCodeWhispererCommandsWalkthrough } from '../codewhisperer/ui/statusBarMenu'
 export { focusAmazonQPanel, focusAmazonQPanelKeybinding } from '../codewhispererChat/commands/registerCommands'
 export { TryChatCodeLensProvider, tryChatCodeLensCommand } from '../codewhispererChat/editor/codelens'
-export { createAmazonQUri, openDiff, openDeletedDiff, getOriginalFileUri, getFileDiffUris } from './commons/diff'
+export {
+    createAmazonQUri,
+    openDiff,
+    openDeletedDiff,
+    getOriginalFileUri,
+    getFileDiffUris,
+    computeDiff,
+} from './commons/diff'
 export { CodeReference } from '../codewhispererChat/view/connector/connector'
 export { AuthMessageDataMap, AuthFollowUpType } from './auth/model'
 export { extractAuthFollowUp } from './util/authUtils'

packages/core/src/amazonqFeatureDev/client/codewhispererruntime-2022-11-11.json

@@ -873,6 +873,44 @@
             "max": 100,
             "min": 0
         },
+        "FeatureDevCodeAcceptanceEvent": {
+            "type": "structure",
+            "required": ["conversationId", "linesOfCodeAccepted", "charactersOfCodeAccepted"],
+            "members": {
+                "conversationId": { "shape": "ConversationId" },
+                "linesOfCodeAccepted": { "shape": "FeatureDevCodeAcceptanceEventLinesOfCodeAcceptedInteger" },
+                "charactersOfCodeAccepted": { "shape": "FeatureDevCodeAcceptanceEventCharactersOfCodeAcceptedInteger" },
+                "programmingLanguage": { "shape": "ProgrammingLanguage" }
+            }
+        },
+        "FeatureDevCodeAcceptanceEventCharactersOfCodeAcceptedInteger": {
+            "type": "integer",
+            "min": 0
+        },
+        "FeatureDevCodeAcceptanceEventLinesOfCodeAcceptedInteger": {
+            "type": "integer",
+            "min": 0
+        },
+        "FeatureDevCodeGenerationEvent": {
+            "type": "structure",
+            "required": ["conversationId", "linesOfCodeGenerated", "charactersOfCodeGenerated"],
+            "members": {
+                "conversationId": { "shape": "ConversationId" },
+                "linesOfCodeGenerated": { "shape": "FeatureDevCodeGenerationEventLinesOfCodeGeneratedInteger" },
+                "charactersOfCodeGenerated": {
+                    "shape": "FeatureDevCodeGenerationEventCharactersOfCodeGeneratedInteger"
+                },
+                "programmingLanguage": { "shape": "ProgrammingLanguage" }
+            }
+        },
+        "FeatureDevCodeGenerationEventCharactersOfCodeGeneratedInteger": {
+            "type": "integer",
+            "min": 0
+        },
+        "FeatureDevCodeGenerationEventLinesOfCodeGeneratedInteger": {
+            "type": "integer",
+            "min": 0
+        },
         "FeatureDevEvent": {
             "type": "structure",
             "required": ["conversationId"],
@@ -1741,6 +1779,8 @@
                 "chatUserModificationEvent": { "shape": "ChatUserModificationEvent" },
                 "terminalUserInteractionEvent": { "shape": "TerminalUserInteractionEvent" },
                 "featureDevEvent": { "shape": "FeatureDevEvent" },
+                "featureDevCodeGenerationEvent": { "shape": "FeatureDevCodeGenerationEvent" },
+                "featureDevCodeAcceptanceEvent": { "shape": "FeatureDevCodeAcceptanceEvent" },
                 "inlineChatEvent": { "shape": "InlineChatEvent" }
             },
             "union": true

packages/core/src/amazonqFeatureDev/client/featureDev.ts

@@ -25,6 +25,7 @@ import { createCodeWhispererChatStreamingClient } from '../../shared/clients/cod
 import { getClientId, getOptOutPreference, getOperatingSystem } from '../../shared/telemetry/util'
 import { extensionVersion } from '../../shared/vscode/env'
 import apiConfig = require('./codewhispererruntime-2022-11-11.json')
+import { FeatureDevCodeAcceptanceEvent, FeatureDevCodeGenerationEvent, TelemetryEvent } from './featuredevproxyclient'
 
 // Re-enable once BE is able to handle retries.
 const writeAPIRetryOptions = {
@@ -260,13 +261,34 @@ export class FeatureDevClient {
      * @param conversationId
      */
     public async sendFeatureDevTelemetryEvent(conversationId: string) {
+        await this.sendFeatureDevEvent('featureDevEvent', {
+            conversationId,
+        })
+    }
+
+    public async sendFeatureDevCodeGenerationEvent(event: FeatureDevCodeGenerationEvent) {
+        getLogger().debug(
+            `featureDevCodeGenerationEvent: conversationId: ${event.conversationId} charactersOfCodeGenerated: ${event.charactersOfCodeGenerated} linesOfCodeGenerated: ${event.linesOfCodeGenerated}`
+        )
+        await this.sendFeatureDevEvent('featureDevCodeGenerationEvent', event)
+    }
+
+    public async sendFeatureDevCodeAcceptanceEvent(event: FeatureDevCodeAcceptanceEvent) {
+        getLogger().debug(
+            `featureDevCodeAcceptanceEvent: conversationId: ${event.conversationId} charactersOfCodeAccepted: ${event.charactersOfCodeAccepted} linesOfCodeAccepted: ${event.linesOfCodeAccepted}`
+        )
+        await this.sendFeatureDevEvent('featureDevCodeAcceptanceEvent', event)
+    }
+
+    public async sendFeatureDevEvent<T extends keyof TelemetryEvent>(
+        eventName: T,
+        event: NonNullable<TelemetryEvent[T]>
+    ) {
         try {
             const client = await this.getClient()
             const params: FeatureDevProxyClient.SendTelemetryEventRequest = {
                 telemetryEvent: {
-                    featureDevEvent: {
-                        conversationId,
-                    },
+                    [eventName]: event,
                 },
                 optOutPreference: getOptOutPreference(),
                 userContext: {
@@ -279,11 +301,11 @@ export class FeatureDevClient {
             }
             const response = await client.sendTelemetryEvent(params).promise()
             getLogger().debug(
-                `${featureName}: successfully sent featureDevEvent: ConversationId: ${conversationId} RequestId: ${response.$response.requestId}`
+                `${featureName}: successfully sent ${eventName} telemetryEvent:${'conversationId' in event ? ' ConversationId: ' + event.conversationId : ''} RequestId: ${response.$response.requestId}`
             )
         } catch (e) {
             getLogger().error(
-                `${featureName}: failed to send feature dev telemetry: ${(e as Error).name}: ${
+                `${featureName}: failed to send ${eventName} telemetry: ${(e as Error).name}: ${
                     (e as Error).message
                 } RequestId: ${(e as any).requestId}`
             )

packages/core/src/amazonqFeatureDev/controllers/chat/controller.ts

@@ -481,6 +481,7 @@ export class FeatureDevController {
                     messageId,
                 })
                 await session.updateChatAnswer(tabID, i18n('AWS.amazonq.featureDev.pillText.acceptAllChanges'))
+                await session.sendLinesOfCodeGeneratedTelemetry()
             }
             this.messenger.sendUpdatePlaceholder(tabID, i18n('AWS.amazonq.featureDev.pillText.selectOption'))
         } finally {

packages/core/src/amazonqFeatureDev/session/session.ts

@@ -32,6 +32,7 @@ import { CodeReference } from '../../amazonq/webview/ui/connector'
 import { UpdateAnswerMessage } from '../views/connector/connector'
 import { MynahIcons } from '@aws/mynah-ui'
 import { i18n } from '../../shared/i18n-helper'
+import { computeDiff } from '../../amazonq/commons/diff'
 export class Session {
     private _state?: SessionState | Omit<SessionState, 'uploadId'>
     private task: string = ''
@@ -44,6 +45,7 @@ export class Session {
     private _codeResultMessageId: string | undefined = undefined
     private _acceptCodeMessageId: string | undefined = undefined
     private _acceptCodeTelemetrySent = false
+    private _reportedCodeChanges: Set<string>
 
     // Used to keep track of whether or not the current session is currently authenticating/needs authenticating
     public isAuthenticating: boolean
@@ -62,6 +64,7 @@ export class Session {
 
         this._telemetry = new TelemetryHelper()
         this.isAuthenticating = false
+        this._reportedCodeChanges = new Set()
     }
 
     /**
@@ -209,6 +212,7 @@ export class Session {
     }
 
     public async insertNewFiles(newFilePaths: NewFileInfo[]) {
+        await this.sendLinesOfCodeAcceptedTelemetry(newFilePaths)
         for (const filePath of newFilePaths) {
             const absolutePath = path.join(filePath.workspaceFolder.uri.fsPath, filePath.relativePath)
 
@@ -273,6 +277,50 @@ export class Session {
         return i18n('AWS.amazonq.featureDev.pillText.acceptAllChanges')
     }
 
+    public async computeFilePathDiff(filePath: NewFileInfo) {
+        const leftPath = `${filePath.workspaceFolder.uri.fsPath}/${filePath.relativePath}`
+        const rightPath = filePath.virtualMemoryUri.path
+        const diff = await computeDiff(leftPath, rightPath, this.tabID)
+        return { leftPath, rightPath, ...diff }
+    }
+
+    public async sendLinesOfCodeGeneratedTelemetry() {
+        let charactersOfCodeGenerated = 0
+        let linesOfCodeGenerated = 0
+        // deleteFiles are currently not counted because the number of lines added is always 0
+        const filePaths = this.state.filePaths ?? []
+        for (const filePath of filePaths) {
+            const { leftPath, changes, charsAdded, linesAdded } = await this.computeFilePathDiff(filePath)
+            const codeChangeKey = `${leftPath}#@${JSON.stringify(changes)}`
+            if (this._reportedCodeChanges.has(codeChangeKey)) {
+                continue
+            }
+            charactersOfCodeGenerated += charsAdded
+            linesOfCodeGenerated += linesAdded
+            this._reportedCodeChanges.add(codeChangeKey)
+        }
+        await this.proxyClient.sendFeatureDevCodeGenerationEvent({
+            conversationId: this.conversationId,
+            charactersOfCodeGenerated,
+            linesOfCodeGenerated,
+        })
+    }
+
+    public async sendLinesOfCodeAcceptedTelemetry(filePaths: NewFileInfo[]) {
+        let charactersOfCodeAccepted = 0
+        let linesOfCodeAccepted = 0
+        for (const filePath of filePaths) {
+            const { charsAdded, linesAdded } = await this.computeFilePathDiff(filePath)
+            charactersOfCodeAccepted += charsAdded
+            linesOfCodeAccepted += linesAdded
+        }
+        await this.proxyClient.sendFeatureDevCodeAcceptanceEvent({
+            conversationId: this.conversationId,
+            charactersOfCodeAccepted,
+            linesOfCodeAccepted,
+        })
+    }
+
     get state() {
         if (!this._state) {
             throw new Error("State should be initialized before it's read")