Add update-backlog subcommand with first promote-perma-passing preset

moz-webgpu-cts/src/main.rs

@@ -21,6 +21,7 @@ use std::{
     fs,
     hash::Hash,
     io::{self, BufReader, BufWriter},
+    iter,
     path::{Path, PathBuf},
     process::ExitCode,
     sync::{
@@ -104,6 +105,12 @@ enum Subcommand {
         #[clap(value_enum, long, default_value_t = Default::default())]
         on_zero_item: OnZeroItem,
     },
+    /// Identify and promote tests that are ready to come out of the `backlog` implementation
+    /// status.
+    UpdateBacklog {
+        /// The heuristic to use for identifying tests that can be promoted.
+        preset: UpdateBacklogPreset,
+    },
 }
 
 #[derive(Clone, Copy, Debug, ValueEnum)]
@@ -126,6 +133,12 @@ enum OnZeroItem {
     Hide,
 }
 
+#[derive(Clone, Copy, Debug, ValueEnum)]
+enum UpdateBacklogPreset {
+    /// Remove tests that expect only `PASS` outcomes on all platforms from `backlog`.
+    PromotePermaPassing,
+}
+
 fn main() -> ExitCode {
     env_logger::builder()
         .filter_level(log::LevelFilter::Info)
@@ -1354,6 +1367,103 @@ fn run(cli: Cli) -> ExitCode {
             println!("Full analysis: {analysis:#?}");
             ExitCode::SUCCESS
         }
+        Subcommand::UpdateBacklog { preset } => {
+            let mut files = {
+                let mut found_parse_err = false;
+                let extracted = read_and_parse_all_metadata(browser, &checkout)
+                    .filter_map(|res| match res {
+                        Ok(ok) => Some(ok),
+                        Err(AlreadyReportedToCommandline) => {
+                            found_parse_err = true;
+                            None
+                        }
+                    })
+                    .collect::<BTreeMap<_, _>>();
+                if found_parse_err {
+                    log::error!(concat!(
+                        "found one or more failures while parsing metadata, ",
+                        "see above for more details"
+                    ));
+                    return ExitCode::FAILURE;
+                }
+                extracted
+            };
+
+            #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+            enum Case {
+                #[default]
+                PermaPass,
+                Other,
+            }
+            let mut found_write_err = false;
+            for (file_path, file) in files.iter_mut() {
+                let File {
+                    properties: _,
+                    tests,
+                } = file;
+
+                for test in tests.values_mut() {
+                    let Test {
+                        properties,
+                        subtests,
+                    } = test;
+                    let mut cases = ExpandedPropertyValue::default();
+                    for ((platform, build_profile), expected) in properties
+                        .expected
+                        .as_ref()
+                        .unwrap_or(&Default::default())
+                        .iter()
+                    {
+                        let case = match expected.as_permanent() {
+                            Some(TestOutcome::Ok) => Case::PermaPass,
+                            _ => Case::Other,
+                        };
+                        cases[(platform, build_profile)] = case;
+                    }
+                    if !subtests.is_empty() {
+                        cases = ExpandedPropertyValue::from_query(|platform, build_profile| {
+                            let consistent_expected = subtests
+                                .iter()
+                                .map(|subtest| {
+                                    let (_name, Subtest { properties }) = subtest;
+                                    let expected =
+                                        properties.expected.as_ref().unwrap_or(&Default::default())
+                                            [(platform, build_profile)];
+                                    if let Some(SubtestOutcome::Pass) = expected.as_permanent() {
+                                        Case::PermaPass
+                                    } else {
+                                        Case::Other
+                                    }
+                                })
+                                .chain(iter::once(cases[(platform, build_profile)]))
+                                .all_equal_value()
+                                .ok();
+                            consistent_expected.unwrap_or(Case::Other)
+                        });
+                    }
+                    // TODO: Just compare this multiple times (here _and_ above), and compare
+                    // subtests afterwards.
+                    let value_across_all_platforms =
+                        || cases.into_iter().map(|(_, case)| case).all_equal_value();
+                    match preset {
+                        UpdateBacklogPreset::PromotePermaPassing => {
+                            if matches!(value_across_all_platforms(), Ok(Case::PermaPass)) {
+                                properties.implementation_status = None;
+                            }
+                        }
+                    }
+                }
+                match write_to_file(file_path, metadata::format_file(file)) {
+                    Ok(()) => (),
+                    Err(AlreadyReportedToCommandline) => found_write_err = true,
+                }
+            }
+            if found_write_err {
+                ExitCode::FAILURE
+            } else {
+                ExitCode::SUCCESS
+            }
+        }
     }
 }
 
@@ -1533,26 +1643,22 @@ fn search_for_repo_root() -> Result<PathBuf, AlreadyReportedToCommandline> {
                 dir
             })
     };
-    let gecko_source_root =
-        find_up("Mercurial", ".hg").or_else(|e| match find_up("Git", ".git") {
-            Ok(path) => {
-                log::debug!("{e:?}");
-                Ok(path)
-            }
-            Err(e2) => {
-                log::warn!("{e:?}");
-                log::warn!("{e2:?}");
-                log::error!("failed to automatically find a repository root");
-                Err(AlreadyReportedToCommandline)
-            }
-        })?;
+    let source_root = find_up("Mercurial", ".hg").or_else(|e| match find_up("Git", ".git") {
+        Ok(path) => {
+            log::debug!("{e:?}");
+            Ok(path)
+        }
+        Err(e2) => {
+            log::warn!("{e:?}");
+            log::warn!("{e2:?}");
+            log::error!("failed to automatically find a repository root");
+            Err(AlreadyReportedToCommandline)
+        }
+    })?;
 
-    log::info!(
-        "detected repository root at {}",
-        gecko_source_root.display()
-    );
+    log::info!("detected repository root at {}", source_root.display());
 
-    Ok(gecko_source_root)
+    Ok(source_root)
 }
 
 struct AlreadyReportedToCommandline;