Add lost_communication_with_server_error_total (#445)

README.md

@@ -120,6 +120,9 @@ This action sends the following metrics if enabled.
   - Time from a job is started to completed
 - `github.actions.job.queued_duration_second`
   - Time from a job is started until the first step is started
+- `github.actions.job.lost_communication_with_server_error_total`
+  - Count of "lost communication with the server" errors of self-hosted runners.
+    See the issue [#444](https://github.com/int128/datadog-actions-metrics/issues/444) for details
 
 It has the following tags:
 

src/queries/completedCheckSuite.ts

@@ -1,5 +1,5 @@
 import { CompletedCheckSuiteQuery, CompletedCheckSuiteQueryVariables } from '../generated/graphql'
-import { CheckRun, CheckStep } from '../generated/graphql-types'
+import { CheckAnnotation, CheckRun, CheckStep } from '../generated/graphql-types'
 import { Octokit } from '../types'
 
 const query = /* GraphQL */ `
@@ -15,6 +15,11 @@ const query = /* GraphQL */ `
             completedAt
             conclusion
             status
+            annotations(first: 10) {
+              nodes {
+                message
+              }
+            }
             steps(first: 50) {
               nodes {
                 number
@@ -64,6 +69,9 @@ export type CompletedCheckSuite = {
 
 type CompletedCheckRun = Pick<CheckRun, 'databaseId' | 'name' | 'status'> &
   NonNullablePick<CheckRun, 'startedAt' | 'completedAt' | 'conclusion'> & {
+    annotations: {
+      nodes: Pick<CheckAnnotation, 'message'>[]
+    }
     steps: {
       nodes: CompletedStep[]
     }
@@ -97,6 +105,14 @@ const extractCheckRuns = (r: CompletedCheckSuiteQuery): CompletedCheckSuite['nod
       continue
     }
 
+    const annotations = []
+    for (const annotation of checkRun.annotations?.nodes ?? []) {
+      if (annotation == null) {
+        continue
+      }
+      annotations.push(annotation)
+    }
+
     const steps: CompletedStep[] = []
     for (const step of checkRun.steps?.nodes ?? []) {
       if (step == null) {
@@ -123,6 +139,7 @@ const extractCheckRuns = (r: CompletedCheckSuiteQuery): CompletedCheckSuite['nod
       startedAt,
       completedAt,
       conclusion,
+      annotations: { nodes: annotations },
       steps: { nodes: steps },
     })
   }

src/workflowRun/metrics.ts

@@ -141,6 +141,16 @@ export const computeJobMetrics = (
       points: [[completedAt, duration]],
     })
 
+    if (checkRun.annotations.nodes.some((a) => isLostCommunicationWithServerError(a.message))) {
+      series.push({
+        host: 'github.com',
+        tags,
+        metric: 'github.actions.job.lost_communication_with_server_error_total',
+        type: 'count',
+        points: [[completedAt, 1]],
+      })
+    }
+
     if (checkRun.steps.nodes.length > 0) {
       const firstStepStartedAt = Math.min(...checkRun.steps.nodes.map((s) => unixTime(s.startedAt)))
       const queued = firstStepStartedAt - startedAt
@@ -156,6 +166,9 @@ export const computeJobMetrics = (
   return series
 }
 
+export const isLostCommunicationWithServerError = (message: string): boolean =>
+  /^The self-hosted runner: .+? lost communication with the server./.test(message)
+
 export const computeStepMetrics = (
   e: WorkflowRunCompletedEvent,
   checkSuite: CompletedCheckSuite,

tests/workflowRun/fixtures/completedCheckSuite.ts

@@ -26,6 +26,7 @@ export const exampleCompletedCheckSuite: CompletedCheckSuiteQuery & CompletedChe
           name: 'build',
           status: CheckStatusState.Completed,
           conclusion: CheckConclusionState.Success,
+          annotations: { nodes: [] },
           steps: {
             nodes: [
               {

tests/workflowRun/metrics.test.ts

@@ -1,5 +1,10 @@
 import { WorkflowDefinition } from '../../src/workflowRun/parse'
-import { computeJobMetrics, computeStepMetrics, computeWorkflowRunMetrics } from '../../src/workflowRun/metrics'
+import {
+  computeJobMetrics,
+  computeStepMetrics,
+  computeWorkflowRunMetrics,
+  isLostCommunicationWithServerError,
+} from '../../src/workflowRun/metrics'
 import { exampleCompletedCheckSuite } from './fixtures/completedCheckSuite'
 import { exampleJobMetrics, exampleStepMetrics, exampleWorkflowRunMetrics } from './fixtures/metrics'
 import { exampleWorkflowRunEvent } from './fixtures/workflowRunEvent'
@@ -26,3 +31,16 @@ test('computeStepMetrics', () => {
   const series = computeStepMetrics(exampleWorkflowRunEvent, exampleCompletedCheckSuite, exampleWorkflowDefinition)
   expect(series).toStrictEqual(exampleStepMetrics)
 })
+
+describe('isLostCommunicationWithServerError', () => {
+  test('matched', () => {
+    expect(
+      isLostCommunicationWithServerError(
+        `The self-hosted runner: POD-NAME lost communication with the server. Verify the machine is running and has a healthy network connection. Anything in your workflow that terminates the runner process, starves it for CPU/Memory, or blocks its network access can cause this error.`
+      )
+    ).toBeTruthy()
+  })
+  test('not related error', () => {
+    expect(isLostCommunicationWithServerError(`Process exit 1`)).toBeFalsy()
+  })
+})