[Dvaas]: Validating test stats.

dvaas/BUILD.bazel

@@ -20,6 +20,24 @@ package(
     licenses = ["notice"],
 )
 
+cc_library(
+    name = "validation_result",
+    srcs = ["validation_result.cc"],
+    hdrs = ["validation_result.h"],
+    deps = [
+        ":test_run_validation",
+        ":test_vector_cc_proto",
+        ":test_vector_stats",
+        "//gutil:status",
+        "@com_github_google_glog//:glog",
+        "@com_google_absl//absl/algorithm:container",
+        "@com_google_absl//absl/container:flat_hash_set",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/types:span",
+        "@com_google_protobuf//:protobuf",
+    ],
+)
+
 cc_library(
     name = "output_writer",
     hdrs = ["output_writer.h"],

dvaas/validation_result.cc

@@ -0,0 +1,91 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "dvaas/validation_result.h"
+
+#include <string>
+#include <vector>
+
+#include "absl/algorithm/container.h"
+#include "absl/container/flat_hash_set.h"
+#include "absl/status/status.h"
+#include "dvaas/test_run_validation.h"
+#include "dvaas/test_vector.pb.h"
+#include "dvaas/test_vector_stats.h"
+#include "glog/logging.h"
+#include "google/protobuf/descriptor.h"
+#include "gutil/status.h"
+
+namespace dvaas {
+
+ValidationResult::ValidationResult(
+    const PacketTestRuns& test_runs,
+    std::vector<const google::protobuf::FieldDescriptor*> ignored_fields,
+    const absl::flat_hash_set<std::string>& ignored_metadata) {
+  test_outcomes_.mutable_outcomes()->Reserve(test_runs.test_runs_size());
+  for (const auto& test_run : test_runs.test_runs()) {
+    PacketTestOutcome& outcome = *test_outcomes_.add_outcomes();
+    *outcome.mutable_test_run() = test_run;
+    *outcome.mutable_test_result() =
+        ValidateTestRun(test_run, ignored_metadata, ignored_fields);
+  }
+
+  test_vector_stats_ = ComputeTestVectorStats(test_outcomes_);
+}
+
+absl::Status ValidationResult::HasSuccessRateOfAtLeast(
+    double expected_success_rate) const {
+  // Avoid division by 0.
+  if (test_vector_stats_.num_vectors == 0) return absl::OkStatus();
+
+  double success_rate =
+      static_cast<double>(test_vector_stats_.num_vectors_passed) /
+      static_cast<double>(test_vector_stats_.num_vectors);
+  if (success_rate < expected_success_rate) {
+    auto it =
+        absl::c_find_if(test_outcomes_.outcomes(), [](const auto& outcome) {
+          return outcome.test_result().has_failure();
+        });
+    if (it == test_outcomes_.outcomes().end()) return absl::OkStatus();
+    return gutil::OutOfRangeErrorBuilder()
+           << ExplainTestVectorStats(test_vector_stats_)
+           << "\nShowing the first failure only. Refer to the test artifacts "
+              "for the full list of errors.\n"
+           << it->test_result().failure().description();
+  }
+  return absl::OkStatus();
+}
+
+const ValidationResult& ValidationResult::LogStatistics() const {
+  LOG(INFO) << ExplainTestVectorStats(test_vector_stats_);
+  return *this;
+}
+ValidationResult& ValidationResult::LogStatistics() {
+  LOG(INFO) << ExplainTestVectorStats(test_vector_stats_);
+  return *this;
+}
+
+std::vector<std::string> ValidationResult::GetAllFailures() const {
+  std::vector<std::string> failures;
+  failures.reserve(test_vector_stats_.num_vectors -
+                   test_vector_stats_.num_vectors_passed);
+  for (const auto& outcome : test_outcomes_.outcomes()) {
+    if (outcome.test_result().has_failure()) {
+      failures.push_back(outcome.test_result().failure().description());
+    }
+  }
+  return failures;
+}
+
+}  // namespace dvaas

dvaas/validation_result.h

@@ -0,0 +1,75 @@
+// Tools to analyze and consume the result of dataplane validation, as returned
+// to DVaaS users.
+
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PINS_DVAAS_VALIDATION_RESULT_H_
+#define PINS_DVAAS_VALIDATION_RESULT_H_
+
+#include <string>
+#include <vector>
+
+#include "absl/container/flat_hash_set.h"
+#include "absl/status/status.h"
+#include "absl/types/span.h"
+#include "dvaas/test_vector.pb.h"
+#include "dvaas/test_vector_stats.h"
+#include "google/protobuf/descriptor.h"
+
+namespace dvaas {
+
+// The result of dataplane validation, as returned to DVaaS users.
+class [[nodiscard]] ValidationResult {
+ public:
+  // Asserts that at least an `expected_success_rate` fraction of test vectors
+  // succeeded, returning either:
+  // * an `Ok` status if that is the case, or
+  // * an `OutOfRange` error with details to assist debugging otherwise.
+  //
+  // Example uses:
+  // ```
+  //   // Expect all tests to pass, e.g. when the switch is stable.
+  //   EXPECT_OK(validation_result.HasSuccessRateOfAtLeast(1.0));
+  //
+  //   // Expect at least 70% of tests to pass, e.g. during development.
+  //   EXPECT_OK(validation_result.HasSuccessRateOfAtLeast(0.75));
+  // ```
+  absl::Status HasSuccessRateOfAtLeast(double expected_success_rate) const;
+
+  // Logs various statistics about the number of test vectors and how many of
+  // them passed.
+  const ValidationResult& LogStatistics() const;
+  ValidationResult& LogStatistics();
+
+  // Returns a list of all test failures. Prefer using `HasSuccessRateOfAtLeast`
+  // as it includes additional information to ease debugging.
+  std::vector<std::string> GetAllFailures() const;
+
+  // Constructs a `ValidationResult` from the given `test_vectors`. Ignores
+  // `ignored_fields` and `ignored_metadata` during validation, see
+  // `test_run_validation.h` for details.
+  ValidationResult(
+      const PacketTestRuns& test_runs,
+      std::vector<const google::protobuf::FieldDescriptor*> ignored_fields,
+      const absl::flat_hash_set<std::string>& ignored_metadata);
+
+ private:
+  PacketTestOutcomes test_outcomes_;
+  TestVectorStats test_vector_stats_;
+};
+
+}  // namespace dvaas
+
+#endif  // PINS_DVAAS_VALIDATION_RESULT_H_