Add : Support e2e( under 'concurrency' mode) outputting the summary o…

…f 'verify' in YAML format (#117)

commands/verify/verify.go

@@ -108,6 +108,7 @@ func concurrentlyVerifySingleCase(
 	verifyInfo *verifyInfo,
 ) (res *output.CaseResult) {
 	res = &output.CaseResult{}
+	res.Name = caseName(v)
 	defer func() {
 		if res.Err != nil && verifyInfo.failFast {
 			cancel()
@@ -164,6 +165,7 @@ func verifyCasesConcurrently(verify *config.Verify, verifyInfo *verifyInfo) erro
 			select {
 			case <-ctx.Done():
 				res[i].Skip = true
+				res[i].Name = caseName(&verify.Cases[i])
 				return
 			default:
 				// It's safe to do this, since each goroutine only modifies a single, different, designated slice element.
@@ -173,9 +175,13 @@ func verifyCasesConcurrently(verify *config.Verify, verifyInfo *verifyInfo) erro
 	}
 	wg.Wait()
 
-	_, errNum, _ := printer.PrintResult(res)
-	if errNum > 0 {
-		return fmt.Errorf("failed to verify %d case(s)", errNum)
+	if output.SummaryOnly {
+		output.PrintResult(res)
+	} else {
+		_, errNum, _ := printer.PrintResult(res)
+		if errNum > 0 {
+			return fmt.Errorf("failed to verify %d case(s)", errNum)
+		}
 	}
 
 	return nil
@@ -188,6 +194,7 @@ func verifyCasesSerially(verify *config.Verify, verifyInfo *verifyInfo) (err err
 	for i := range res {
 		res[i] = &output.CaseResult{
 			Skip: true,
+			Name: caseName(&verify.Cases[i]),
 		}
 	}
 
@@ -205,7 +212,6 @@ func verifyCasesSerially(verify *config.Verify, verifyInfo *verifyInfo) (err err
 	for idx := range verify.Cases {
 		printer.Start()
 		v := &verify.Cases[idx]
-		res[idx].Name = caseName(v)
 
 		if v.GetExpected() == "" {
 			res[idx].Skip = false
@@ -294,7 +300,7 @@ func DoVerifyAccordingConfig() error {
 	concurrency := e2eConfig.Verify.Concurrency
 	if concurrency {
 		// enable batch output mode when concurrency is enabled
-		printer = output.NewPrinter(output.WithBatchOutput(true))
+		printer = output.NewPrinter(output.WithBatchOutput(true), output.WithSummaryOnly(output.SummaryOnly))
 		return verifyCasesConcurrently(&e2eConfig.Verify, &VerifyInfo)
 	}
 	printer = output.NewPrinter(output.WithBatchOutput(util.BatchMode), output.WithSummaryOnly(output.SummaryOnly))

test/e2e/concurrency/fail-fast/expected.yaml

@@ -0,0 +1,29 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+passed:
+{{range .passed}}
+- {{ if hasPrefix . "passed" }}{{.}}{{ end }}
+{{end}}
+failed:
+{{range .failed}}
+- {{ if hasPrefix . "failed" }}{{.}}{{ end }}
+{{end}}
+skipped:
+{{range .skipped}}
+- {{.}}
+{{end}}

test/e2e/concurrency/fail-fast/internal/expected.yaml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used to show how to write configuration files and can be used to test.
+
+args:
+    case: success
+headers:
+    Accept: application/json
+    Host: 127.0.0.1:8080
+    User-Agent: curl/7.81.0
+origin: 172.18.0.1
+url: http://127.0.0.1:8080/get?case=success

test/e2e/concurrency/fail-fast/internal/verify.yaml

@@ -0,0 +1,124 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used to show how to write configuration files and can be used to test.
+
+verify:
+  # verify with retry strategy
+  retry:
+    # max retry count
+    count: 1
+    # the interval between two attempts, e.g. 10s, 1m.
+    interval: 1s
+
+  # when a case fails, whether to stop verifying other cases. This property defaults to true.
+  fail-fast: true
+  # Whether to verify cases concurrently. This property defaults to false.
+  concurrency: true
+
+
+  cases:
+    - name: passed-case-1
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-2
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: failed-case-3
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-4
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-5
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: failed-case-6
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-7
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-8
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: failed-case-9
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-10
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+
+    - name: passed-case-11
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-12
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: failed-case-13
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-14
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-15
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: failed-case-16
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-17
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-18
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: failed-case-19
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-20
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+
+    - name: passed-case-21
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-22
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: failed-case-23
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-24
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-25
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: failed-case-26
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-27
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-28
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: failed-case-29
+      query: 'curl -s 127.0.0.1:8080/get?case=failure -H "accept: application/json"'
+      expected: ./expected.yaml
+    - name: passed-case-30
+      query: 'curl -s 127.0.0.1:8080/get?case=success -H "accept: application/json"'
+      expected: ./expected.yaml

test/e2e/concurrency/non-fail-fast/expected.yaml

@@ -0,0 +1,54 @@
+# Licensed to Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Apache Software Foundation (ASF) licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+passed:
+{{- contains .passed}}
+- passed-case-1
+- passed-case-2
+- passed-case-4
+- passed-case-5
+- passed-case-7
+- passed-case-8
+- passed-case-10
+- passed-case-11
+- passed-case-12
+- passed-case-14
+- passed-case-15
+- passed-case-17
+- passed-case-18
+- passed-case-20
+- passed-case-21
+- passed-case-22
+- passed-case-24
+- passed-case-25
+- passed-case-27
+- passed-case-28
+- passed-case-30
+{{- end}}
+failed:
+{{- contains .failed }}
+- failed-case-3
+- failed-case-6
+- failed-case-9
+- failed-case-13
+- failed-case-16
+- failed-case-19
+- failed-case-23
+- failed-case-26
+- failed-case-29
+{{- end }}
+skipped: []

test/e2e/concurrency/non-fail-fast/internal/expected.yaml

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used to show how to write configuration files and can be used to test.
+
+args:
+    case: success
+headers:
+    Accept: application/json
+    Host: 127.0.0.1:8080
+    User-Agent: curl/7.81.0
+origin: 172.18.0.1
+url: http://127.0.0.1:8080/get?case=success