WIP: feat(cli): add moz-webgpu-cts adjust-exps subcmd.

Cargo.toml

@@ -24,8 +24,10 @@ miette = { version = "5.10.0", features = ["fancy"] }
 natord = "1.0.9"
 path-dsl = "0.6.1"
 regex = "1.9.5"
+serde = { version = "1.0.188", features = ["derive"] }
+serde_json = "1.0.107"
 thiserror = "1.0.49"
-wax = "0.6.0"
+wax = { version = "0.6.0", features = ["miette"], git = "https://github.com/ErichDonGubler/wax", branch = "static-miette-diags"}
 
 [dev-dependencies]
 insta = "1.33.0"

src/bin/moz-webgpu-cts/main.rs

@@ -1,22 +1,26 @@
 mod metadata;
+mod report;
 
-use self::metadata::{
-    AnalyzeableProps, Applicability, Expectation, Platform, SubtestOutcome, Test, TestOutcome,
+use self::{
+    metadata::{
+        AnalyzeableProps, Applicability, Expectation, Platform, SubtestOutcome, Test, TestOutcome,
+    },
+    report::ExecutionReport,
 };
 
 use std::{
     collections::{BTreeMap, BTreeSet},
     fmt::Display,
     fs,
-    io::{self, BufWriter},
+    io::{self, BufReader, BufWriter},
     path::{Path, PathBuf},
     process::ExitCode,
     sync::Arc,
 };
 
 use clap::Parser;
 use indexmap::{IndexMap, IndexSet};
-use miette::{miette, Diagnostic, NamedSource, Report, SourceSpan, WrapErr};
+use miette::{miette, Diagnostic, IntoDiagnostic, NamedSource, Report, SourceSpan, WrapErr};
 use path_dsl::path;
 
 use regex::Regex;
@@ -39,6 +43,12 @@ struct Cli {
 
 #[derive(Debug, Parser)]
 enum Subcommand {
+    #[clap(name = "adjust-exps")]
+    AdjustExpectations {
+        #[clap(long = "glob", value_name = "REPORT_GLOB")]
+        report_globs: Vec<String>,
+        report_paths: Vec<PathBuf>,
+    },
     #[clap(name = "fmt")]
     Format,
     Triage,
@@ -111,6 +121,181 @@ fn run(cli: Cli) -> ExitCode {
     }
 
     match subcommand {
+        Subcommand::AdjustExpectations {
+            report_globs,
+            report_paths,
+        } => {
+            let report_globs = {
+                let mut found_glob_parse_err = false;
+                // TODO: I think these can just be args. directly?
+                let globs = report_globs
+                    .into_iter()
+                    .filter_map(|glob| match Glob::diagnosed(&glob) {
+                        Ok((glob, _diagnostics)) => Some(glob.into_owned().partition()),
+                        Err(diagnostics) => {
+                            found_glob_parse_err = true;
+                            let error_reports = diagnostics
+                                .into_iter()
+                                .filter(|diag| {
+                                    // N.B.: There should be at least one of these!
+                                    diag.severity()
+                                        .map_or(true, |sev| sev == miette::Severity::Error)
+                                })
+                                .map(Report::new_boxed);
+                            for report in error_reports {
+                                eprintln!("{report:?}");
+                            }
+                            todo!("render glob parse diagnostics in hard error")
+                        }
+                    })
+                    .collect::<Vec<_>>();
+
+                if found_glob_parse_err {
+                    log::error!("failed to parse one or more WPT report globs; bailing");
+                    return ExitCode::FAILURE;
+                }
+
+                globs
+            };
+
+            if report_paths.is_empty() && report_globs.is_empty() {
+                log::error!("no report paths specified, bailing");
+                return ExitCode::FAILURE;
+            }
+
+            let exec_report_paths = {
+                let mut found_glob_walk_err = false;
+                let files = report_globs
+                    .iter()
+                    .flat_map(|(base_path, glob)| {
+                        glob.walk(base_path)
+                            .filter_map(|entry| match entry {
+                                // TODO: warn when not ending in `wptreport.json`
+                                Ok(entry) => Some(entry.into_path()),
+                                Err(e) => {
+                                    found_glob_walk_err = true;
+                                    let ctx_msg = if let Some(path) = e.path() {
+                                        format!(
+                                            "failed to enumerate files for glob `{}` at path {}",
+                                            glob,
+                                            path.display()
+                                        )
+                                    } else {
+                                        format!("failed to enumerate files for glob `{glob}`")
+                                    };
+                                    let e = Report::msg(e).wrap_err(ctx_msg);
+                                    eprintln!("{e:?}");
+                                    None
+                                }
+                            })
+                            .collect::<Vec<_>>() // OPT: Can we get rid of this somehow?
+                    })
+                    .chain(report_paths)
+                    .collect::<Vec<_>>();
+
+                if found_glob_walk_err {
+                    log::error!("failed to enumerate files with WPT report globs; bailing");
+                    return ExitCode::FAILURE;
+                }
+
+                files
+            };
+
+            if exec_report_paths.is_empty() {
+                log::error!("no WPT report files found, bailing");
+                return ExitCode::FAILURE;
+            }
+
+            log::trace!("working with the following WPT report files: {exec_report_paths:#?}");
+            log::info!("working with {} WPT report files", exec_report_paths.len());
+
+            let meta_files_by_path = {
+                let raw_meta_files_by_path = match read_metadata() {
+                    Ok(paths) => paths,
+                    Err(()) => return ExitCode::FAILURE,
+                };
+
+                log::info!("parsing metadata…");
+                let mut parse_err_found = false;
+
+                let files = raw_meta_files_by_path
+                    .into_iter()
+                    .filter_map(|(path, file_contents)| {
+                        match chumsky::Parser::parse(&metadata::File::parser(), &*file_contents)
+                            .into_result()
+                        {
+                            Err(errors) => {
+                                parse_err_found = true;
+                                render_metadata_parse_errors(&path, &file_contents, errors);
+                                None
+                            }
+                            Ok(file) => Some((path, file)),
+                        }
+                    })
+                    .collect::<BTreeMap<_, _>>();
+
+                if parse_err_found {
+                    log::error!(concat!(
+                        "found one or more failures while parsing metadata, ",
+                        "see above for more details"
+                    ));
+                    return ExitCode::FAILURE;
+                }
+
+                files
+            };
+
+            let exec_reports = {
+                let mut found_read_err = false;
+                let reports = exec_report_paths
+                    .into_iter()
+                    .filter_map(|path| {
+                        let res = fs::File::open(&path)
+                            .map(BufReader::new)
+                            .map_err(Report::msg)
+                            .wrap_err("failed to open file")
+                            .and_then(|reader| {
+                                serde_json::from_reader(reader)
+                                    .into_diagnostic()
+                                    .wrap_err("failed to parse JSON")
+                            })
+                            .wrap_err_with(|| {
+                                format!(
+                                    "failed to read WPT execution report from {}",
+                                    path.display()
+                                )
+                            });
+                        match res {
+                            Ok(parsed) => Some((path, parsed)),
+                            Err(e) => {
+                                found_read_err = true;
+                                log::error!("{e:?}");
+                                None
+                            }
+                        }
+                    })
+                    .collect::<Vec<(_, ExecutionReport)>>();
+
+                if found_read_err {
+                    log::error!("failed to read one or more");
+                    return ExitCode::FAILURE;
+                }
+
+                reports
+            };
+
+            // TODO: What if there are no reports?
+
+            log::info!("printing report…");
+            println!("{:#?}", exec_reports);
+
+            log::info!("comparing report(s) results to metadata…");
+
+            todo!(concat!(
+                "compare reports with expectations, ",
+                "write out new metadata",
+            ));
+        }
         Subcommand::Format => {
             let raw_test_files_by_path = match read_metadata() {
                 Ok(paths) => paths,
@@ -221,7 +406,7 @@ fn run(cli: Cli) -> ExitCode {
             #[derive(Clone, Debug, Default)]
             struct PerPlatformAnalysis {
                 tests_with_runner_errors: BTreeSet<Arc<SectionHeader>>,
-                tests_with_disabled: BTreeSet<Arc<SectionHeader>>,
+                tests_with_disabled_or_skip: BTreeSet<Arc<SectionHeader>>,
                 tests_with_crashes: BTreeSet<Arc<SectionHeader>>,
                 subtests_with_failures_by_test:
                     BTreeMap<Arc<SectionHeader>, IndexSet<Arc<SectionHeader>>>,
@@ -308,7 +493,9 @@ fn run(cli: Cli) -> ExitCode {
 
                 if is_disabled {
                     analysis.for_each_platform_mut(|analysis| {
-                        analysis.tests_with_disabled.insert(test_name.clone());
+                        analysis
+                            .tests_with_disabled_or_skip
+                            .insert(test_name.clone());
                     })
                 }
 
@@ -340,6 +527,11 @@ fn run(cli: Cli) -> ExitCode {
                             TestOutcome::Error => receiver(&mut |analysis| {
                                 analysis.tests_with_runner_errors.insert(test_name.clone());
                             }),
+                            TestOutcome::Skip => receiver(&mut |analysis| {
+                                analysis
+                                    .tests_with_disabled_or_skip
+                                    .insert(test_name.clone());
+                            }),
                         }
                     }
 
@@ -396,7 +588,7 @@ fn run(cli: Cli) -> ExitCode {
                     if is_disabled {
                         analysis
                             .windows
-                            .tests_with_disabled
+                            .tests_with_disabled_or_skip
                             .insert(test_name.clone());
                     }
 
@@ -482,7 +674,7 @@ fn run(cli: Cli) -> ExitCode {
             analysis.for_each_platform(|platform, analysis| {
                 let PerPlatformAnalysis {
                     tests_with_runner_errors,
-                    tests_with_disabled,
+                    tests_with_disabled_or_skip: tests_with_disabled,
                     tests_with_crashes,
                     subtests_with_failures_by_test,
                     subtests_with_timeouts_by_test,
@@ -586,7 +778,7 @@ fn read_gecko_files_at(
                     None => &"",
                 };
                 log::error!(
-                    "failed to enumerate `cts.https.html` files{}\n  caused by: {e}",
+                    "failed to enumerate {glob_pattern:?} files{}\n  caused by: {e}",
                     path_disp
                 );
                 found_read_err = true;