Lay groundwork for benchmark tests and reporting

.github/workflows/pull_request.yml

@@ -94,6 +94,80 @@ jobs:
           name: test-logs
           path: /tmp/receptor-testing
 
+      - name: Archive receptor binary
+        uses: actions/upload-artifact@v2
+        with:
+          name: receptor
+          path: /usr/local/bin/receptor
+  benchmark:
+    name: Performance regression check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "1.20"
+
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - name: build and install receptor
+        run: |
+          make build-all
+          sudo cp ./receptor /usr/local/bin/receptor
+
+      - name: Download kind binary
+        run: curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.11.1/kind-linux-amd64 && chmod +x ./kind
+
+      - name: Create k8s cluster
+        run: ./kind create cluster
+
+      - name: Interact with the cluster
+        run: kubectl get nodes
+
+      # Run benchmark with `go test -bench` and stores the output to a file
+      - name: Run benchmark
+        run: make benchmark | tee output.txt
+
+      - name: get k8s logs
+        if: ${{ failure() }}
+        run: .github/workflows/artifact-k8s-logs.sh
+
+      - name: remove sockets before archiving logs
+        if: ${{ failure() }}
+        run: find /tmp/receptor-testing -name controlsock -delete
+
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'go'
+          output-file-path: output.txt
+          external-data-json-path: ./cache/benchmark-data.json
+          fail-on-alert: true
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          comment-on-alert: true
+          summary-always: true
+
+      - name: Artifact receptor data
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: test-logs
+          path: /tmp/receptor-testing
+
       - name: Archive receptor binary
         uses: actions/upload-artifact@v2
         with:

Makefile

@@ -98,6 +98,11 @@ testloop: receptor
 	  make test; do \
 	  i=$$((i+1)); done
 
+benchmark:
+	PATH="${PWD}:${PATH}" \
+	go test -bench=. -run=^# ./...
+
+
 kubectl:
 	curl -LO "https://storage.googleapis.com/kubernetes-release/release/$$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl"
 	chmod a+x kubectl

tests/functional/mesh/work_test.go

@@ -47,6 +47,37 @@ func TestWorkSubmitWithTLSClient(t *testing.T) {
 	}
 }
 
+func BenchmarkWorkSubmitWithTLSClient(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		for _, plugin := range workPlugins {
+			plugin := plugin
+
+			b.Run(string(plugin), func(b *testing.B) {
+				controllers, m, expectedResults := benchWorkSetup(plugin, b)
+
+				defer m.WaitForShutdown()
+				defer m.Destroy()
+
+				command := `{"command":"work","subcommand":"submit","worktype":"echosleepshort","tlsclient":"client","node":"node2","params":"", "ttl":"10h"}`
+				unitID, err := controllers["node1"].WorkSubmitJSON(command)
+				if err != nil {
+					b.Fatal(err, m.DataDir)
+				}
+				ctx, _ := context.WithTimeout(context.Background(), 60*time.Second)
+				err = controllers["node1"].AssertWorkSucceeded(ctx, unitID)
+				if err != nil {
+					b.Fatal(err, m.DataDir)
+				}
+
+				err = controllers["node1"].AssertWorkResults(unitID, expectedResults)
+				if err != nil {
+					b.Fatal(err, m.GetDataDir())
+				}
+			})
+		}
+	}
+}
+
 // Tests that submitting work with wrong cert CN immediately fails the job
 // also tests that releasing a job that has not been started on remote
 // will not attempt to connect to remote.
@@ -490,7 +521,9 @@ func TestRuntimeParams(t *testing.T) {
 }
 
 func TestKubeRuntimeParams(t *testing.T) {
-	checkSkipKube(t)
+	if checkSkipKube() {
+		t.Skip("Kubernetes tests are set to skip, unset SKIP_KUBE to run them")
+	}
 
 	m := NewLibMesh()
 	node1 := m.NewLibNode("node1")
@@ -610,7 +643,9 @@ func TestRuntimeParamsNotAllowed(t *testing.T) {
 }
 
 func TestKubeContainerFailure(t *testing.T) {
-	checkSkipKube(t)
+	if checkSkipKube() {
+		t.Skip("Kubernetes tests are set to skip, unset SKIP_KUBE to run them")
+	}
 
 	m := NewLibMesh()
 	node1 := m.NewLibNode("node1")

tests/functional/mesh/work_utils.go

@@ -11,20 +11,18 @@ import (
 	"github.com/ansible/receptor/tests/utils"
 )
 
-func workSetup(workPluginName workPlugin, t *testing.T) (map[string]*ReceptorControl, *LibMesh, []byte) {
-	checkSkipKube(t)
-
+func setupCommon(workPluginName workPlugin, name string) (map[string]*ReceptorControl, *LibMesh, error) {
 	m := workTestMesh(workPluginName)
 
-	err := m.Start(t.Name())
+	err := m.Start(name)
 	if err != nil {
-		t.Fatal(err)
+		return nil, nil, err
 	}
 
 	ctx, _ := context.WithTimeout(context.Background(), 120*time.Second)
 	err = m.WaitForReady(ctx)
 	if err != nil {
-		t.Fatal(err, m.DataDir)
+		return nil, nil, err
 	}
 
 	nodes := m.GetNodes()
@@ -33,11 +31,37 @@ func workSetup(workPluginName workPlugin, t *testing.T) (map[string]*ReceptorCon
 		controller := NewReceptorControl()
 		err = controller.Connect(nodes[k].GetControlSocket())
 		if err != nil {
-			t.Fatal(err, m.DataDir)
+			return nil, nil, err
 		}
 		controllers[k] = controller
 	}
 
+	return controllers, m, nil
+}
+
+func workSetup(workPluginName workPlugin, t *testing.T) (map[string]*ReceptorControl, *LibMesh, []byte) {
+	if checkSkipKube() {
+		t.Skip("Kubernetes tests are set to skip, unset SKIP_KUBE to run them")
+	}
+
+	controllers, m, err := setupCommon(workPluginName, t.Name())
+	if err != nil {
+		t.Fatal(err, m.DataDir)
+	}
+
+	return controllers, m, []byte("1\n2\n3\n4\n5\n")
+}
+
+func benchWorkSetup(workPluginName workPlugin, b *testing.B) (map[string]*ReceptorControl, *LibMesh, []byte) {
+	if checkSkipKube() {
+		b.Skip("Kubernetes tests are set to skip, unset SKIP_KUBE to run them")
+	}
+
+	controllers, m, err := setupCommon(workPluginName, b.Name())
+	if err != nil {
+		b.Fatal(err, m.DataDir)
+	}
+
 	return controllers, m, []byte("1\n2\n3\n4\n5\n")
 }
 
@@ -73,8 +97,10 @@ func assertStdoutFizeSize(ctx context.Context, dataDir, nodeID, unitID string, w
 	return nil
 }
 
-func checkSkipKube(t *testing.T) {
+func checkSkipKube() bool {
 	if skip := os.Getenv("SKIP_KUBE"); skip == "1" {
-		t.Skip("Kubernetes tests are set to skip, unset SKIP_KUBE to run them")
+		return true
 	}
+
+	return false
 }