From f406347a6e8af835f52bfb6868a64f01be2ee533 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 11 Nov 2024 15:00:34 -0600 Subject: [PATCH] Improve fuzzing of `*.wast` tests (#9587) * Improve fuzzing of `*.wast` tests Currently we have a fuzzer which is tasked with running `*.wast` tests with fuzz-generated configurations. This asserts that we at least satisfy all basic wasm semantics regardless of how various knobs in `Config` are turned (modulo limits to resources). The current fuzzing though is not comprehensive in that it doesn't include all the spec tests that we pass from all proposals. This runs the risk of we don't actually fuzz anything until the spec tests are merged upstream, which can take a significant amount of time. This commit refactors the `*.wast`-management infrastructure to share test discovery and feature calculation between `tests/wast.rs` and fuzzing. This new support crate centralizes limits and discovery for both to use. Additionally fuzzing is updated to no longer throw out test cases if configuration isn't applicable but instead clamp configuration to the minimum required values (e.g. features + resource limits). This means that we should now be fuzzing all spec tests that pass in all configurations. This new fuzzer discovered a few minor issues with the GC proposal implementation, for example, such as: * Some instructions were translated using trapping methods directly on `FunctionBuilder` rather than `FuncEnvironment` meaning they didn't properly handle `signals-based-traps` configuration. * Fuel handling for `return_call_ref` wasn't correct because it was accidentally omitted from the list of return-call instructions that need special treatment. * Add some manifest metadata --- Cargo.lock | 12 + Cargo.toml | 1 + crates/cranelift/src/func_environ.rs | 1 + crates/cranelift/src/gc/enabled.rs | 37 +- crates/cranelift/src/gc/enabled/drc.rs | 4 +- crates/cranelift/src/gc/enabled/null.rs | 19 +- crates/fuzzing/Cargo.toml | 6 +- crates/fuzzing/build.rs | 46 +- crates/fuzzing/src/generators/config.rs | 141 ++++-- crates/fuzzing/src/generators/module.rs | 17 +- crates/fuzzing/src/generators/wast_test.rs | 14 +- crates/fuzzing/src/oracles.rs | 24 +- crates/wast-util/Cargo.toml | 19 + crates/wast-util/src/lib.rs | 422 ++++++++++++++++++ tests/disas.rs | 4 +- .../memory64/more-than-4gb.wast | 1 + tests/support/mod.rs | 20 - tests/wast.rs | 411 +++-------------- 18 files changed, 707 insertions(+), 492 deletions(-) create mode 100644 crates/wast-util/Cargo.toml create mode 100644 crates/wast-util/src/lib.rs delete mode 100644 tests/support/mod.rs diff --git a/Cargo.lock b/Cargo.lock index b4feaf0af158..f3fa59b15917 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4017,6 +4017,7 @@ dependencies = [ "wasmtime-wasi-nn", "wasmtime-wasi-threads", "wasmtime-wast", + "wasmtime-wast-util", "wast 219.0.1", "wat", "windows-sys 0.59.0", @@ -4208,6 +4209,7 @@ dependencies = [ "wasmprinter", "wasmtime", "wasmtime-wast", + "wasmtime-wast-util", "wat", ] @@ -4376,6 +4378,16 @@ dependencies = [ "wast 219.0.1", ] +[[package]] +name = "wasmtime-wast-util" +version = "28.0.0" +dependencies = [ + "anyhow", + "serde", + "serde_derive", + "toml", +] + [[package]] name = "wasmtime-winch" version = "28.0.0" diff --git a/Cargo.toml b/Cargo.toml index 3b367932470d..cf7b2da6950e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,6 +120,7 @@ capstone = { workspace = true } object = { workspace = true, features = ['std'] } wasmtime-test-macros = { path = "crates/test-macros" } pulley-interpreter = { workspace = true, features = ["disas"] } +wasmtime-wast-util = { path = 'crates/wast-util' } [target.'cfg(windows)'.dev-dependencies] windows-sys = { workspace = true, features = ["Win32_System_Memory"] } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 9ce0d93a2017..939ef6fde9e6 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -382,6 +382,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { | Operator::CallIndirect { .. } | Operator::Call { .. } | Operator::ReturnCall { .. } + | Operator::ReturnCallRef { .. } | Operator::ReturnCallIndirect { .. } => { self.fuel_increment_var(builder); self.fuel_save_from_var(builder); diff --git a/crates/cranelift/src/gc/enabled.rs b/crates/cranelift/src/gc/enabled.rs index 8fc9a171c79a..5dea690293a9 100644 --- a/crates/cranelift/src/gc/enabled.rs +++ b/crates/cranelift/src/gc/enabled.rs @@ -1,7 +1,7 @@ use super::GcCompiler; use crate::func_environ::FuncEnvironment; use crate::gc::ArrayInit; -use crate::translate::{StructFieldsVec, TargetEnvironment}; +use crate::translate::{FuncEnvironment as _, StructFieldsVec, TargetEnvironment}; use crate::TRAP_INTERNAL_ASSERT; use cranelift_codegen::{ cursor::FuncCursor, @@ -304,7 +304,7 @@ pub fn translate_struct_get( // TODO: If we know we have a `(ref $my_struct)` here, instead of maybe a // `(ref null $my_struct)`, we could omit the `trapz`. But plumbing that // type info from `wasmparser` and through to here is a bit funky. - builder.ins().trapz(struct_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE); let field_index = usize::try_from(field_index).unwrap(); let interned_type_index = func_env.module.types[struct_type_index]; @@ -337,7 +337,7 @@ fn translate_struct_get_and_extend( extension: Extension, ) -> WasmResult { // TODO: See comment in `translate_struct_get` about the `trapz`. - builder.ins().trapz(struct_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE); let field_index = usize::try_from(field_index).unwrap(); let interned_type_index = func_env.module.types[struct_type_index]; @@ -410,7 +410,7 @@ pub fn translate_struct_set( new_val: ir::Value, ) -> WasmResult<()> { // TODO: See comment in `translate_struct_get` about the `trapz`. - builder.ins().trapz(struct_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE); let field_index = usize::try_from(field_index).unwrap(); let interned_type_index = func_env.module.types[struct_type_index]; @@ -640,15 +640,11 @@ pub fn translate_array_fill( let len = translate_array_len(func_env, builder, array_ref)?; // Check that the full range of elements we want to fill is within bounds. - let end_index = builder - .ins() - .uadd_overflow_trap(index, n, crate::TRAP_ARRAY_OUT_OF_BOUNDS); + let end_index = func_env.uadd_overflow_trap(builder, index, n, crate::TRAP_ARRAY_OUT_OF_BOUNDS); let out_of_bounds = builder .ins() .icmp(IntCC::UnsignedGreaterThan, end_index, len); - builder - .ins() - .trapnz(out_of_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); + func_env.trapnz(builder, out_of_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); // Get the address of the first element we want to fill. let interned_type_index = func_env.module.types[array_type_index]; @@ -695,7 +691,7 @@ pub fn translate_array_len( builder: &mut FunctionBuilder, array_ref: ir::Value, ) -> WasmResult { - builder.ins().trapz(array_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, array_ref, crate::TRAP_NULL_REFERENCE); let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset(); let len_field = func_env.prepare_gc_ref_access( @@ -794,9 +790,7 @@ fn array_elem_addr( let len = translate_array_len(func_env, builder, array_ref).unwrap(); let in_bounds = builder.ins().icmp(IntCC::UnsignedLessThan, index, len); - builder - .ins() - .trapz(in_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); + func_env.trapz(builder, in_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); // Compute the size (in bytes) of the whole array object. let ArraySizeInfo { @@ -1175,6 +1169,7 @@ fn uextend_i32_to_pointer_type( /// Traps if the size overflows. #[cfg_attr(not(any(feature = "gc-drc", feature = "gc-null")), allow(dead_code))] fn emit_array_size( + func_env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, array_layout: &GcArrayLayout, init: ArrayInit<'_>, @@ -1200,17 +1195,17 @@ fn emit_array_size( .ins() .imul_imm(len, i64::from(array_layout.elem_size)); let high_bits = builder.ins().ushr_imm(elems_size_64, 32); - builder - .ins() - .trapnz(high_bits, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.trapnz(builder, high_bits, crate::TRAP_ALLOCATION_TOO_LARGE); let elems_size = builder.ins().ireduce(ir::types::I32, elems_size_64); // And if adding the base size and elements size overflows, then the // allocation is too large. - let size = - builder - .ins() - .uadd_overflow_trap(base_size, elems_size, crate::TRAP_ALLOCATION_TOO_LARGE); + let size = func_env.uadd_overflow_trap( + builder, + base_size, + elems_size, + crate::TRAP_ALLOCATION_TOO_LARGE, + ); size } diff --git a/crates/cranelift/src/gc/enabled/drc.rs b/crates/cranelift/src/gc/enabled/drc.rs index 238c60287da6..e0b565530126 100644 --- a/crates/cranelift/src/gc/enabled/drc.rs +++ b/crates/cranelift/src/gc/enabled/drc.rs @@ -303,14 +303,14 @@ impl GcCompiler for DrcCompiler { let interned_type_index = func_env.module.types[array_type_index]; let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset(); - let array_layout = func_env.array_layout(interned_type_index); + let array_layout = func_env.array_layout(interned_type_index).clone(); let base_size = array_layout.base_size; let align = array_layout.align; let len_to_elems_delta = base_size.checked_sub(len_offset).unwrap(); // First, compute the array's total size from its base size, element // size, and length. - let size = emit_array_size(builder, array_layout, init); + let size = emit_array_size(func_env, builder, &array_layout, init); // Second, now that we have the array object's total size, call the // `gc_alloc_raw` builtin libcall to allocate the array. diff --git a/crates/cranelift/src/gc/enabled/null.rs b/crates/cranelift/src/gc/enabled/null.rs index ad042b36f418..a5367893a4ee 100644 --- a/crates/cranelift/src/gc/enabled/null.rs +++ b/crates/cranelift/src/gc/enabled/null.rs @@ -54,9 +54,7 @@ impl NullCompiler { .ins() .iconst(ir::types::I32, i64::from(VMGcKind::MASK)); let masked = builder.ins().band(size, mask); - builder - .ins() - .trapnz(masked, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.trapnz(builder, masked, crate::TRAP_ALLOCATION_TOO_LARGE); // Load the bump "pointer" (it is actually an index into the GC heap, // not a raw pointer). @@ -81,7 +79,8 @@ impl NullCompiler { // a power of two. let minus_one = builder.ins().iconst(ir::types::I32, -1); let align_minus_one = builder.ins().iadd(align, minus_one); - let next_plus_align_minus_one = builder.ins().uadd_overflow_trap( + let next_plus_align_minus_one = func_env.uadd_overflow_trap( + builder, next, align_minus_one, crate::TRAP_ALLOCATION_TOO_LARGE, @@ -93,9 +92,7 @@ impl NullCompiler { // Check whether the allocation fits in the heap space we have left. let end_of_object = - builder - .ins() - .uadd_overflow_trap(aligned, size, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.uadd_overflow_trap(builder, aligned, size, crate::TRAP_ALLOCATION_TOO_LARGE); let uext_end_of_object = uextend_i32_to_pointer_type(builder, pointer_type, end_of_object); let (base, bound) = func_env.get_gc_heap_base_bound(builder); let is_in_bounds = builder.ins().icmp( @@ -103,9 +100,7 @@ impl NullCompiler { uext_end_of_object, bound, ); - builder - .ins() - .trapz(is_in_bounds, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.trapz(builder, is_in_bounds, crate::TRAP_ALLOCATION_TOO_LARGE); // Write the header, update the bump "pointer", and return the newly // allocated object. @@ -162,14 +157,14 @@ impl GcCompiler for NullCompiler { let interned_type_index = func_env.module.types[array_type_index]; let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset(); - let array_layout = func_env.array_layout(interned_type_index); + let array_layout = func_env.array_layout(interned_type_index).clone(); let base_size = array_layout.base_size; let align = array_layout.align; let len_to_elems_delta = base_size.checked_sub(len_offset).unwrap(); // First, compute the array's total size from its base size, element // size, and length. - let size = emit_array_size(builder, array_layout, init); + let size = emit_array_size(func_env, builder, &array_layout, init); // Next, allocate the array. assert!(align.is_power_of_two()); diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index 91682ac86119..66d512a7c596 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -11,6 +11,9 @@ license = "Apache-2.0 WITH LLVM-exception" [lints] workspace = true +[build-dependencies] +wasmtime-wast-util = { path = '../wast-util' } + [dependencies] anyhow = { workspace = true } arbitrary = { workspace = true, features = ["derive"] } @@ -24,13 +27,14 @@ tempfile = "3.3.0" wasmparser = { workspace = true } wasmprinter = { workspace = true } wasmtime = { workspace = true, features = ['default', 'winch', 'gc', 'memory-protection-keys'] } -wasmtime-wast = { workspace = true } +wasmtime-wast = { workspace = true, features = ['component-model'] } wasm-encoder = { workspace = true } wasm-smith = { workspace = true } wasm-mutate = { workspace = true } wasm-spec-interpreter = { path = "./wasm-spec-interpreter", optional = true } wasmi = "0.39.1" futures = { workspace = true } +wasmtime-wast-util = { path = '../wast-util' } # We rely on precompiled v8 binaries, but rusty-v8 doesn't have a precompiled # binary for MinGW which is built on our CI. It does have one for Windows-msvc, diff --git a/crates/fuzzing/build.rs b/crates/fuzzing/build.rs index 316a15f21e5b..08b9bddfe3a0 100644 --- a/crates/fuzzing/build.rs +++ b/crates/fuzzing/build.rs @@ -4,39 +4,39 @@ use std::env; use std::path::PathBuf; +use wasmtime_wast_util::WastTest; fn main() { println!("cargo:rerun-if-changed=build.rs"); let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - let dirs = [ - "tests/spec_testsuite", - "tests/misc_testsuite", - "tests/misc_testsuite/multi-memory", - "tests/misc_testsuite/simd", - "tests/misc_testsuite/threads", - ]; let mut root = env::current_dir().unwrap(); root.pop(); // chop off 'fuzzing' root.pop(); // chop off 'crates' - let mut code = format!("static FILES: &[(&str, &str)] = &[\n"); - - let mut entries = Vec::new(); - for dir in dirs { - for entry in root.join(dir).read_dir().unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - if path.extension().and_then(|s| s.to_str()) == Some("wast") { - entries.push(path); - } - } - } - entries.sort(); - for path in entries { - let path = path.to_str().expect("path is not valid utf-8"); - code.push_str(&format!("({path:?}, include_str!({path:?})),\n")); + + let tests = wasmtime_wast_util::find_tests(&root).unwrap(); + + let mut code = format!("static FILES: &[fn() -> wasmtime_wast_util::WastTest] = &[\n"); + + for test in tests { + let WastTest { + path, + contents: _, + config, + } = test; + println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); + code.push_str(&format!( + "|| {{ + wasmtime_wast_util::WastTest {{ + path: {path:?}.into(), + contents: include_str!({path:?}).into(), + config: wasmtime_wast_util::{config:?}, + }} + }}," + )); } + code.push_str("];\n"); std::fs::write(out_dir.join("wasttests.rs"), code).unwrap(); } diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs index bfeb5718c434..18de6bfccfd9 100644 --- a/crates/fuzzing/src/generators/config.rs +++ b/crates/fuzzing/src/generators/config.rs @@ -10,6 +10,7 @@ use arbitrary::{Arbitrary, Unstructured}; use std::sync::Arc; use std::time::Duration; use wasmtime::{Engine, Module, MpkEnabled, Store}; +use wasmtime_wast_util::{limits, WastConfig, WastTest}; /// Configuration for `wasmtime::Config` and generated modules for a session of /// fuzzing. @@ -114,49 +115,111 @@ impl Config { self.module_config.generate(input, default_fuel) } - /// Tests whether this configuration is capable of running all wast tests. - pub fn is_wast_test_compliant(&self) -> bool { - let config = &self.module_config.config; - - // Check for wasm features that must be disabled to run spec tests - if config.memory64_enabled { - return false; + /// Updates this configuration to be able to run the `test` specified. + /// + /// This primarily updates `self.module_config` to ensure that it enables + /// all features and proposals necessary to execute the `test` specified. + /// This will additionally update limits in the pooling allocator to be able + /// to execute all tests. + pub fn make_wast_test_compliant(&mut self, test: &WastTest) -> WastConfig { + // Enable/disable some proposals that aren't configurable in wasm-smith + // but are configurable in Wasmtime. + self.module_config.extended_const_enabled = test.config.extended_const.unwrap_or(false); + self.module_config.function_references_enabled = test + .config + .function_references + .or(test.config.gc) + .unwrap_or(false); + self.module_config.component_model_more_flags = + test.config.component_model_more_flags.unwrap_or(false); + + // Enable/disable proposals that wasm-smith has knobs for which will be + // read when creating `wasmtime::Config`. + let config = &mut self.module_config.config; + config.bulk_memory_enabled = true; + config.multi_value_enabled = true; + config.simd_enabled = true; + config.wide_arithmetic_enabled = test.config.wide_arithmetic.unwrap_or(false); + config.memory64_enabled = test.config.memory64.unwrap_or(false); + config.tail_call_enabled = test.config.tail_call.unwrap_or(false); + config.custom_page_sizes_enabled = test.config.custom_page_sizes.unwrap_or(false); + config.threads_enabled = test.config.threads.unwrap_or(false); + config.gc_enabled = test.config.gc.unwrap_or(false); + config.reference_types_enabled = config.gc_enabled + || self.module_config.function_references_enabled + || test.config.reference_types.unwrap_or(false); + if test.config.multi_memory.unwrap_or(false) { + config.max_memories = limits::MEMORIES_PER_MODULE as usize; + } else { + config.max_memories = 1; } - // Check for wasm features that must be enabled to run spec tests - if !config.bulk_memory_enabled - || !config.reference_types_enabled - || !config.multi_value_enabled - || !config.simd_enabled - || !config.threads_enabled - || config.max_memories <= 1 - { - return false; + match &mut self.wasmtime.memory_config { + MemoryConfig::Normal(config) => { + if let Some(n) = &mut config.memory_reservation { + *n = (*n).max(limits::MEMORY_SIZE as u64); + } + } + MemoryConfig::CustomUnaligned => {} } - // Make sure the runtime limits allow for the instantiation of all spec - // tests. Note that the max memories must be precisely one since 0 won't - // instantiate spec tests and more than one is multi-memory which is - // disabled for spec tests. - if config.max_memories != 1 || config.max_tables < 5 { - return false; + // FIXME: it might be more ideal to avoid the need for this entirely + // and to just let the test fail. If a test fails due to a pooling + // allocator resource limit being met we could ideally detect that and + // let the fuzz test case pass. That would avoid the need to hardcode + // so much here and in theory wouldn't reduce the usefulness of fuzzers + // all that much. At this time though we can't easily test this configuration. + if let InstanceAllocationStrategy::Pooling(pooling) = &mut self.wasmtime.strategy { + // Clamp protection keys between 1 & 2 to reduce the number of + // slots and then multiply the total memories by the number of keys + // we have since a single store has access to only one key. + pooling.max_memory_protection_keys = pooling.max_memory_protection_keys.max(1).min(2); + pooling.total_memories = pooling + .total_memories + .max(limits::MEMORIES * (pooling.max_memory_protection_keys as u32)); + + // For other limits make sure they meet the minimum threshold + // required for our wast tests. + pooling.total_component_instances = pooling + .total_component_instances + .max(limits::COMPONENT_INSTANCES); + pooling.total_tables = pooling.total_tables.max(limits::TABLES); + pooling.max_tables_per_module = + pooling.max_tables_per_module.max(limits::TABLES_PER_MODULE); + pooling.max_memories_per_module = pooling + .max_memories_per_module + .max(limits::MEMORIES_PER_MODULE); + pooling.max_memories_per_component = pooling + .max_memories_per_component + .max(limits::MEMORIES_PER_MODULE); + pooling.total_core_instances = pooling.total_core_instances.max(limits::CORE_INSTANCES); + pooling.max_memory_size = pooling.max_memory_size.max(limits::MEMORY_SIZE); + pooling.table_elements = pooling.table_elements.max(limits::TABLE_ELEMENTS); + pooling.core_instance_size = pooling.core_instance_size.max(limits::CORE_INSTANCE_SIZE); + pooling.component_instance_size = pooling + .component_instance_size + .max(limits::CORE_INSTANCE_SIZE); } - if let InstanceAllocationStrategy::Pooling(pooling) = &self.wasmtime.strategy { - // Check to see if any item limit is less than the required - // threshold to execute the spec tests. - if pooling.total_memories < 1 - || pooling.total_tables < 5 - || pooling.table_elements < 1_000 - || pooling.max_memory_size < (900 << 16) - || pooling.total_core_instances < 500 - || pooling.core_instance_size < 64 * 1024 - { - return false; - } + // Return the test configuration that this fuzz configuration represents + // which is used afterwards to test if the `test` here is expected to + // fail or not. + WastConfig { + collector: match self.wasmtime.collector { + Collector::Null => wasmtime_wast_util::Collector::Null, + Collector::DeferredReferenceCounting => { + wasmtime_wast_util::Collector::DeferredReferenceCounting + } + }, + pooling: matches!( + self.wasmtime.strategy, + InstanceAllocationStrategy::Pooling(_) + ), + compiler: match self.wasmtime.compiler_strategy { + CompilerStrategy::Cranelift => wasmtime_wast_util::Compiler::Cranelift, + CompilerStrategy::Winch => wasmtime_wast_util::Compiler::Winch, + }, } - - true } /// Converts this to a `wasmtime::Config` object @@ -166,7 +229,7 @@ impl Config { let mut cfg = wasmtime::Config::new(); cfg.wasm_bulk_memory(true) - .wasm_reference_types(true) + .wasm_reference_types(self.module_config.config.reference_types_enabled) .wasm_multi_value(self.module_config.config.multi_value_enabled) .wasm_multi_memory(self.module_config.config.max_memories > 1) .wasm_simd(self.module_config.config.simd_enabled) @@ -174,10 +237,12 @@ impl Config { .wasm_tail_call(self.module_config.config.tail_call_enabled) .wasm_custom_page_sizes(self.module_config.config.custom_page_sizes_enabled) .wasm_threads(self.module_config.config.threads_enabled) - .wasm_function_references(self.module_config.config.gc_enabled) + .wasm_function_references(self.module_config.function_references_enabled) .wasm_gc(self.module_config.config.gc_enabled) .wasm_custom_page_sizes(self.module_config.config.custom_page_sizes_enabled) .wasm_wide_arithmetic(self.module_config.config.wide_arithmetic_enabled) + .wasm_extended_const(self.module_config.extended_const_enabled) + .wasm_component_model_more_flags(self.module_config.component_model_more_flags) .native_unwind_info(cfg!(target_os = "windows") || self.wasmtime.native_unwind_info) .cranelift_nan_canonicalization(self.wasmtime.canonicalize_nans) .cranelift_opt_level(self.wasmtime.opt_level.to_wasmtime()) diff --git a/crates/fuzzing/src/generators/module.rs b/crates/fuzzing/src/generators/module.rs index f3fcd4a51636..51ef41edae8b 100644 --- a/crates/fuzzing/src/generators/module.rs +++ b/crates/fuzzing/src/generators/module.rs @@ -10,6 +10,16 @@ use arbitrary::{Arbitrary, Unstructured}; pub struct ModuleConfig { #[allow(missing_docs)] pub config: wasm_smith::Config, + + // These knobs aren't exposed in `wasm-smith` at this time but are exposed + // in our `*.wast` testing so keep knobs here so they can be read during + // config-to-`wasmtime::Config` translation. + #[allow(missing_docs)] + pub extended_const_enabled: bool, + #[allow(missing_docs)] + pub function_references_enabled: bool, + #[allow(missing_docs)] + pub component_model_more_flags: bool, } impl<'a> Arbitrary<'a> for ModuleConfig { @@ -53,7 +63,12 @@ impl<'a> Arbitrary<'a> for ModuleConfig { // do that most of the time. config.disallow_traps = u.ratio(9, 10)?; - Ok(ModuleConfig { config }) + Ok(ModuleConfig { + extended_const_enabled: false, + component_model_more_flags: false, + function_references_enabled: config.gc_enabled, + config, + }) } } diff --git a/crates/fuzzing/src/generators/wast_test.rs b/crates/fuzzing/src/generators/wast_test.rs index f93a387861ec..63200b36fb9a 100644 --- a/crates/fuzzing/src/generators/wast_test.rs +++ b/crates/fuzzing/src/generators/wast_test.rs @@ -8,18 +8,16 @@ include!(concat!(env!("OUT_DIR"), "/wasttests.rs")); /// A wast test from this repository. #[derive(Debug)] pub struct WastTest { - /// The filename of the spec test - pub file: &'static str, - /// The `*.wast` contents of the spec test - pub contents: &'static str, + #[allow(missing_docs)] + pub test: wasmtime_wast_util::WastTest, } impl<'a> Arbitrary<'a> for WastTest { fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - // NB: this does get a uniform value in the provided range. - let i = u.int_in_range(0..=FILES.len() - 1)?; - let (file, contents) = FILES[i]; - Ok(WastTest { file, contents }) + log::debug!("{}", u.is_empty()); + Ok(WastTest { + test: u.choose(FILES)?(), + }) } fn size_hint(_depth: usize) -> (usize, Option) { diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index ebbe9bceb77f..168c267ef49f 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -90,7 +90,7 @@ impl StoreLimits { // `memcpy`. As more data is added over time growths get more and // more expensive meaning that fuel may not be effective at limiting // execution time. - remaining_growths: AtomicUsize::new(100), + remaining_growths: AtomicUsize::new(1000), oom: AtomicBool::new(false), })) } @@ -640,9 +640,21 @@ pub fn make_api_calls(api: generators::api::ApiCalls) { /// Executes the wast `test` with the `config` specified. /// /// Ensures that wast tests pass regardless of the `Config`. -pub fn wast_test(fuzz_config: generators::Config, test: generators::WastTest) { +pub fn wast_test(mut fuzz_config: generators::Config, test: generators::WastTest) { crate::init_fuzzing(); - if !fuzz_config.is_wast_test_compliant() { + let test = &test.test; + + // Discard tests that allocate a lot of memory as we don't want to OOM the + // fuzzer and we also limit memory growth which would cause the test to + // fail. + if test.config.hogs_memory.unwrap_or(false) { + return; + } + + // Transform `fuzz_config` to be valid for `test` and make sure that this + // test is supposed to pass. + let wast_config = fuzz_config.make_wast_test_compliant(test); + if test.should_fail(&wast_config) { return; } @@ -654,16 +666,16 @@ pub fn wast_test(fuzz_config: generators::Config, test: generators::WastTest) { } } - log::debug!("running {:?}", test.file); + log::debug!("running {:?}", test.path); let mut wast_context = WastContext::new(fuzz_config.to_store()); wast_context .register_spectest(&wasmtime_wast::SpectestConfig { - use_shared_memory: false, + use_shared_memory: true, suppress_prints: true, }) .unwrap(); wast_context - .run_buffer(test.file, test.contents.as_bytes()) + .run_buffer(test.path.to_str().unwrap(), test.contents.as_bytes()) .unwrap(); } diff --git a/crates/wast-util/Cargo.toml b/crates/wast-util/Cargo.toml new file mode 100644 index 000000000000..a69da0a6b724 --- /dev/null +++ b/crates/wast-util/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "wasmtime-wast-util" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +publish = false +description = "Utilities for `*.wast` files and tests in Wasmtime" +license = "Apache-2.0 WITH LLVM-exception" +repository = "https://github.com/bytecodealliance/wasmtime" + +[dependencies] +anyhow = { workspace = true } +serde = { workspace = true } +serde_derive = { workspace = true } +toml = { workspace = true } + +[lints] +workspace = true diff --git a/crates/wast-util/src/lib.rs b/crates/wast-util/src/lib.rs new file mode 100644 index 000000000000..c1177ec099ec --- /dev/null +++ b/crates/wast-util/src/lib.rs @@ -0,0 +1,422 @@ +use anyhow::{Context, Result}; +use serde::de::DeserializeOwned; +use serde_derive::Deserialize; +use std::fmt; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +/// Limits for running wast tests. +/// +/// This is useful for sharing between `tests/wast.rs` and fuzzing, for +/// example, and is used as the minimum threshold for configuration when +/// fuzzing. +/// +/// Note that it's ok to increase these numbers if a test comes along and needs +/// it, they're just here as empirically found minimum thresholds so far and +/// they're not too scientific. +pub mod limits { + pub const MEMORY_SIZE: usize = 805 << 16; + pub const MEMORIES: u32 = 450; + pub const TABLES: u32 = 200; + pub const MEMORIES_PER_MODULE: u32 = 9; + pub const TABLES_PER_MODULE: u32 = 5; + pub const COMPONENT_INSTANCES: u32 = 50; + pub const CORE_INSTANCES: u32 = 900; + pub const TABLE_ELEMENTS: usize = 1000; + pub const CORE_INSTANCE_SIZE: usize = 64 * 1024; +} + +/// Local all `*.wast` tests under `root` which should be the path to the root +/// of the wasmtime repository. +pub fn find_tests(root: &Path) -> Result> { + let mut tests = Vec::new(); + add_tests(&mut tests, &root.join("tests/spec_testsuite"), false)?; + add_tests(&mut tests, &root.join("tests/misc_testsuite"), true)?; + Ok(tests) +} + +fn add_tests(tests: &mut Vec, path: &Path, has_config: bool) -> Result<()> { + for entry in path.read_dir().context("failed to read directory")? { + let entry = entry.context("failed to read directory entry")?; + let path = entry.path(); + if entry + .file_type() + .context("failed to get file type")? + .is_dir() + { + add_tests(tests, &path, has_config).context("failed to read sub-directory")?; + continue; + } + + if path.extension().and_then(|s| s.to_str()) != Some("wast") { + continue; + } + + let contents = + fs::read_to_string(&path).with_context(|| format!("failed to read test: {path:?}"))?; + let config = if has_config { + parse_test_config(&contents) + .with_context(|| format!("failed to parse test configuration: {path:?}"))? + } else { + spec_test_config(&path) + }; + tests.push(WastTest { + path, + contents, + config, + }) + } + Ok(()) +} + +fn spec_test_config(test: &Path) -> TestConfig { + let mut ret = TestConfig::default(); + match spec_proposal_from_path(test) { + Some("multi-memory") => { + ret.multi_memory = Some(true); + ret.reference_types = Some(true); + } + Some("wide-arithmetic") => { + ret.wide_arithmetic = Some(true); + } + Some("threads") => { + ret.threads = Some(true); + ret.reference_types = Some(false); + } + Some("tail-call") => { + ret.tail_call = Some(true); + ret.reference_types = Some(true); + } + Some("relaxed-simd") => { + ret.relaxed_simd = Some(true); + } + Some("memory64") => { + ret.memory64 = Some(true); + ret.tail_call = Some(true); + ret.gc = Some(true); + ret.extended_const = Some(true); + ret.multi_memory = Some(true); + ret.relaxed_simd = Some(true); + } + Some("extended-const") => { + ret.extended_const = Some(true); + ret.reference_types = Some(true); + } + Some("custom-page-sizes") => { + ret.custom_page_sizes = Some(true); + ret.multi_memory = Some(true); + } + Some("exception-handling") => { + ret.reference_types = Some(true); + } + Some("gc") => { + ret.gc = Some(true); + ret.tail_call = Some(true); + } + Some("function-references") => { + ret.function_references = Some(true); + ret.tail_call = Some(true); + } + Some("annotations") => {} + Some(proposal) => panic!("unsuported proposal {proposal:?}"), + None => ret.reference_types = Some(true), + } + + ret +} + +/// Parse test configuration from the specified test, comments starting with +/// `;;!`. +pub fn parse_test_config(wat: &str) -> Result +where + T: DeserializeOwned, +{ + // The test config source is the leading lines of the WAT file that are + // prefixed with `;;!`. + let config_lines: Vec<_> = wat + .lines() + .take_while(|l| l.starts_with(";;!")) + .map(|l| &l[3..]) + .collect(); + let config_text = config_lines.join("\n"); + + toml::from_str(&config_text).context("failed to parse the test configuration") +} + +/// A `*.wast` test with its path, contents, and configuration. +#[derive(Clone)] +pub struct WastTest { + pub path: PathBuf, + pub contents: String, + pub config: TestConfig, +} + +impl fmt::Debug for WastTest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("WastTest") + .field("path", &self.path) + .field("contents", &"...") + .field("config", &self.config) + .finish() + } +} + +/// Per-test configuration which is written down in the test file itself for +/// `misc_testsuite/**/*.wast` or in `spec_test_config` above for spec tests. +#[derive(Debug, PartialEq, Default, Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct TestConfig { + pub memory64: Option, + pub custom_page_sizes: Option, + pub multi_memory: Option, + pub threads: Option, + pub gc: Option, + pub function_references: Option, + pub relaxed_simd: Option, + pub reference_types: Option, + pub tail_call: Option, + pub extended_const: Option, + pub wide_arithmetic: Option, + pub hogs_memory: Option, + pub nan_canonicalization: Option, + pub component_model_more_flags: Option, +} + +/// Configuration that spec tests can run under. +pub struct WastConfig { + /// Compiler chosen to run this test. + pub compiler: Compiler, + /// Whether or not the pooling allocator is enabled. + pub pooling: bool, + /// What garbage collector is being used. + pub collector: Collector, +} + +#[derive(PartialEq, Debug, Copy, Clone)] +pub enum Compiler { + Cranelift, + Winch, +} + +#[derive(PartialEq, Debug, Copy, Clone)] +pub enum Collector { + Auto, + Null, + DeferredReferenceCounting, +} + +impl WastTest { + /// Returns whether this test exercises the GC types and might want to use + /// multiple different garbage collectors. + pub fn test_uses_gc_types(&self) -> bool { + self.config + .gc + .or(self.config.function_references) + .unwrap_or(false) + } + + /// Returns the optional spec proposal that this test is associated with. + pub fn spec_proposal(&self) -> Option<&str> { + spec_proposal_from_path(&self.path) + } + + /// Returns whether this test should fail under the specified extra + /// configuration. + pub fn should_fail(&self, config: &WastConfig) -> bool { + // Winch only supports x86_64 at this time. + if config.compiler == Compiler::Winch && !cfg!(target_arch = "x86_64") { + return true; + } + + // Disable spec tests for proposals that Winch does not implement yet. + if config.compiler == Compiler::Winch { + // A few proposals that winch has no support for. + if self.config.gc == Some(true) + || self.config.threads == Some(true) + || self.config.tail_call == Some(true) + || self.config.function_references == Some(true) + || self.config.gc == Some(true) + || self.config.relaxed_simd == Some(true) + { + return true; + } + + let unsupported = [ + // externref/reference-types related + "component-model/modules.wast", + "extended-const/elem.wast", + "extended-const/global.wast", + "memory64/threads.wast", + "misc_testsuite/externref-id-function.wast", + "misc_testsuite/externref-segment.wast", + "misc_testsuite/externref-segments.wast", + "misc_testsuite/externref-table-dropped-segment-issue-8281.wast", + "misc_testsuite/linking-errors.wast", + "misc_testsuite/many_table_gets_lead_to_gc.wast", + "misc_testsuite/mutable_externref_globals.wast", + "misc_testsuite/no-mixup-stack-maps.wast", + "misc_testsuite/no-panic.wast", + "misc_testsuite/simple_ref_is_null.wast", + "misc_testsuite/table_grow_with_funcref.wast", + "spec_testsuite/br_table.wast", + "spec_testsuite/data-invalid.wast", + "spec_testsuite/elem.wast", + "spec_testsuite/global.wast", + "spec_testsuite/linking.wast", + "spec_testsuite/ref_func.wast", + "spec_testsuite/ref_is_null.wast", + "spec_testsuite/ref_null.wast", + "spec_testsuite/select.wast", + "spec_testsuite/table-sub.wast", + "spec_testsuite/table_fill.wast", + "spec_testsuite/table_get.wast", + "spec_testsuite/table_grow.wast", + "spec_testsuite/table_set.wast", + "spec_testsuite/table_size.wast", + "spec_testsuite/unreached-invalid.wast", + "spec_testsuite/call_indirect.wast", + // simd-related failures + "annotations/simd_lane.wast", + "memory64/simd.wast", + "misc_testsuite/int-to-float-splat.wast", + "misc_testsuite/issue6562.wast", + "misc_testsuite/simd/almost-extmul.wast", + "misc_testsuite/simd/canonicalize-nan.wast", + "misc_testsuite/simd/cvt-from-uint.wast", + "misc_testsuite/simd/issue4807.wast", + "misc_testsuite/simd/issue6725-no-egraph-panic.wast", + "misc_testsuite/simd/issue_3327_bnot_lowering.wast", + "misc_testsuite/simd/load_splat_out_of_bounds.wast", + "misc_testsuite/simd/replace-lane-preserve.wast", + "misc_testsuite/simd/spillslot-size-fuzzbug.wast", + "misc_testsuite/simd/unaligned-load.wast", + "multi-memory/simd_memory-multi.wast", + "spec_testsuite/simd_align.wast", + "spec_testsuite/simd_bit_shift.wast", + "spec_testsuite/simd_bitwise.wast", + "spec_testsuite/simd_boolean.wast", + "spec_testsuite/simd_const.wast", + "spec_testsuite/simd_conversions.wast", + "spec_testsuite/simd_f32x4.wast", + "spec_testsuite/simd_f32x4_arith.wast", + "spec_testsuite/simd_f32x4_cmp.wast", + "spec_testsuite/simd_f32x4_pmin_pmax.wast", + "spec_testsuite/simd_f32x4_rounding.wast", + "spec_testsuite/simd_f64x2.wast", + "spec_testsuite/simd_f64x2_arith.wast", + "spec_testsuite/simd_f64x2_cmp.wast", + "spec_testsuite/simd_f64x2_pmin_pmax.wast", + "spec_testsuite/simd_f64x2_rounding.wast", + "spec_testsuite/simd_i16x8_arith.wast", + "spec_testsuite/simd_i16x8_arith2.wast", + "spec_testsuite/simd_i16x8_cmp.wast", + "spec_testsuite/simd_i16x8_extadd_pairwise_i8x16.wast", + "spec_testsuite/simd_i16x8_extmul_i8x16.wast", + "spec_testsuite/simd_i16x8_q15mulr_sat_s.wast", + "spec_testsuite/simd_i16x8_sat_arith.wast", + "spec_testsuite/simd_i32x4_arith.wast", + "spec_testsuite/simd_i32x4_arith2.wast", + "spec_testsuite/simd_i32x4_cmp.wast", + "spec_testsuite/simd_i32x4_dot_i16x8.wast", + "spec_testsuite/simd_i32x4_extadd_pairwise_i16x8.wast", + "spec_testsuite/simd_i32x4_extmul_i16x8.wast", + "spec_testsuite/simd_i32x4_trunc_sat_f32x4.wast", + "spec_testsuite/simd_i32x4_trunc_sat_f64x2.wast", + "spec_testsuite/simd_i64x2_arith.wast", + "spec_testsuite/simd_i64x2_arith2.wast", + "spec_testsuite/simd_i64x2_cmp.wast", + "spec_testsuite/simd_i64x2_extmul_i32x4.wast", + "spec_testsuite/simd_i8x16_arith.wast", + "spec_testsuite/simd_i8x16_arith2.wast", + "spec_testsuite/simd_i8x16_cmp.wast", + "spec_testsuite/simd_i8x16_sat_arith.wast", + "spec_testsuite/simd_int_to_int_extend.wast", + "spec_testsuite/simd_lane.wast", + "spec_testsuite/simd_load.wast", + "spec_testsuite/simd_load16_lane.wast", + "spec_testsuite/simd_load32_lane.wast", + "spec_testsuite/simd_load64_lane.wast", + "spec_testsuite/simd_load8_lane.wast", + "spec_testsuite/simd_load_extend.wast", + "spec_testsuite/simd_load_splat.wast", + "spec_testsuite/simd_load_zero.wast", + "spec_testsuite/simd_splat.wast", + "spec_testsuite/simd_store16_lane.wast", + "spec_testsuite/simd_store32_lane.wast", + "spec_testsuite/simd_store64_lane.wast", + "spec_testsuite/simd_store8_lane.wast", + ]; + + if unsupported.iter().any(|part| self.path.ends_with(part)) { + return true; + } + } + + for part in self.path.iter() { + // Not implemented in Wasmtime yet + if part == "exception-handling" { + return !self.path.ends_with("binary.wast"); + } + + if part == "memory64" { + if [ + // wasmtime doesn't implement exceptions yet + "imports.wast", + "ref_null.wast", + "exports.wast", + "throw.wast", + "throw_ref.wast", + "try_table.wast", + "tag.wast", + "instance.wast", + ] + .iter() + .any(|i| self.path.ends_with(i)) + { + return true; + } + } + } + + // Some tests are known to fail with the pooling allocator + if config.pooling { + let unsupported = [ + // allocates too much memory for the pooling configuration here + "misc_testsuite/memory64/more-than-4gb.wast", + // shared memories + pooling allocator aren't supported yet + "misc_testsuite/memory-combos.wast", + "misc_testsuite/threads/LB.wast", + "misc_testsuite/threads/LB_atomic.wast", + "misc_testsuite/threads/MP.wast", + "misc_testsuite/threads/MP_atomic.wast", + "misc_testsuite/threads/MP_wait.wast", + "misc_testsuite/threads/SB.wast", + "misc_testsuite/threads/SB_atomic.wast", + "misc_testsuite/threads/atomics_notify.wast", + "misc_testsuite/threads/atomics_wait_address.wast", + "misc_testsuite/threads/wait_notify.wast", + "spec_testsuite/proposals/threads/atomic.wast", + "spec_testsuite/proposals/threads/exports.wast", + "spec_testsuite/proposals/threads/memory.wast", + ]; + + if unsupported.iter().any(|part| self.path.ends_with(part)) { + return true; + } + } + + false + } +} + +fn spec_proposal_from_path(path: &Path) -> Option<&str> { + let mut iter = path.iter(); + loop { + match iter.next()?.to_str()? { + "proposals" => break, + _ => {} + } + } + Some(iter.next()?.to_str()?) +} diff --git a/tests/disas.rs b/tests/disas.rs index 8dabf786fd4c..4494ac6e7188 100644 --- a/tests/disas.rs +++ b/tests/disas.rs @@ -55,8 +55,6 @@ use tempfile::TempDir; use wasmtime::{Engine, OptLevel, Strategy}; use wasmtime_cli_flags::CommonOptions; -mod support; - fn main() -> Result<()> { if cfg!(miri) { return Ok(()); @@ -153,7 +151,7 @@ impl Test { fn new(path: &Path) -> Result { let contents = std::fs::read_to_string(path).with_context(|| format!("failed to read {path:?}"))?; - let config: TestConfig = support::parse_test_config(&contents) + let config: TestConfig = wasmtime_wast_util::parse_test_config(&contents) .context("failed to parse test configuration as TOML")?; let mut flags = vec!["wasmtime"]; match &config.flags { diff --git a/tests/misc_testsuite/memory64/more-than-4gb.wast b/tests/misc_testsuite/memory64/more-than-4gb.wast index 3a23af02a600..8c7f4057ba2e 100644 --- a/tests/misc_testsuite/memory64/more-than-4gb.wast +++ b/tests/misc_testsuite/memory64/more-than-4gb.wast @@ -1,4 +1,5 @@ ;;! memory64 = true +;;! hogs_memory = true ;; try to create as few 4gb memories as we can to reduce the memory consumption ;; of this test, so create one up front here and use it below. diff --git a/tests/support/mod.rs b/tests/support/mod.rs deleted file mode 100644 index 10d071d4fab1..000000000000 --- a/tests/support/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -use anyhow::{Context, Result}; -use serde::de::DeserializeOwned; - -/// Parse test configuration from the specified test, comments starting with -/// `;;!`. -pub fn parse_test_config(wat: &str) -> Result -where - T: DeserializeOwned, -{ - // The test config source is the leading lines of the WAT file that are - // prefixed with `;;!`. - let config_lines: Vec<_> = wat - .lines() - .take_while(|l| l.starts_with(";;!")) - .map(|l| &l[3..]) - .collect(); - let config_text = config_lines.join("\n"); - - toml::from_str(&config_text).context("failed to parse the test configuration") -} diff --git a/tests/wast.rs b/tests/wast.rs index 3f56eea02627..e7ff317849a7 100644 --- a/tests/wast.rs +++ b/tests/wast.rs @@ -1,58 +1,29 @@ use anyhow::{bail, Context}; use libtest_mimic::{Arguments, FormatSetting, Trial}; -use serde_derive::Deserialize; -use std::path::Path; use std::sync::{Condvar, LazyLock, Mutex}; use wasmtime::{ - Collector, Config, Engine, InstanceAllocationStrategy, MpkEnabled, PoolingAllocationConfig, - Store, Strategy, + Config, Engine, InstanceAllocationStrategy, MpkEnabled, PoolingAllocationConfig, Store, }; use wasmtime_environ::Memory; use wasmtime_wast::{SpectestConfig, WastContext}; - -mod support; +use wasmtime_wast_util::{limits, Collector, Compiler, WastConfig, WastTest}; fn main() { env_logger::init(); - let mut trials = Vec::new(); - if !cfg!(miri) { - add_tests(&mut trials, "tests/spec_testsuite".as_ref()); - add_tests(&mut trials, "tests/misc_testsuite".as_ref()); - } - - // There's a lot of tests so print only a `.` to keep the output a - // bit more terse by default. - let mut args = Arguments::from_args(); - if args.format.is_none() { - args.format = Some(FormatSetting::Terse); - } - libtest_mimic::run(&args, trials).exit() -} - -fn add_tests(trials: &mut Vec, path: &Path) { - for entry in path.read_dir().unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - if entry.file_type().unwrap().is_dir() { - add_tests(trials, &path); - continue; - } - - if path.extension().and_then(|s| s.to_str()) != Some("wast") { - continue; - } + let tests = if cfg!(miri) { + Vec::new() + } else { + wasmtime_wast_util::find_tests(".".as_ref()).unwrap() + }; - let test_uses_gc_types = path.iter().any(|part| { - part.to_str().map_or(false, |s| { - s.contains("gc") - || s.contains("function-references") - || s.contains("reference-types") - || s.contains("exception-handling") - }) - }); + let mut trials = Vec::new(); - for strategy in [Strategy::Cranelift, Strategy::Winch] { + // For each test generate a combinatorial matrix of all configurations to + // run this test in. + for test in tests { + let test_uses_gc_types = test.test_uses_gc_types(); + for compiler in [Compiler::Cranelift, Compiler::Winch] { for pooling in [true, false] { let collectors: &[_] = if !pooling && test_uses_gc_types { &[Collector::DeferredReferenceCounting, Collector::Null] @@ -63,22 +34,22 @@ fn add_tests(trials: &mut Vec, path: &Path) { for collector in collectors.iter().copied() { let trial = Trial::test( format!( - "{strategy:?}/{}{}{}", + "{compiler:?}/{}{}{}", if pooling { "pooling/" } else { "" }, if collector != Collector::Auto { format!("{collector:?}/") } else { String::new() }, - path.to_str().unwrap() + test.path.to_str().unwrap() ), { - let path = path.clone(); + let test = test.clone(); move || { run_wast( - &path, + &test, WastConfig { - strategy, + compiler, pooling, collector, }, @@ -92,306 +63,21 @@ fn add_tests(trials: &mut Vec, path: &Path) { } } } -} - -fn should_fail(test: &Path, wast_config: &WastConfig, test_config: &TestConfig) -> bool { - // Winch only supports x86_64 at this time. - if wast_config.strategy == Strategy::Winch && !cfg!(target_arch = "x86_64") { - return true; - } - - // Disable spec tests for proposals that Winch does not implement yet. - if wast_config.strategy == Strategy::Winch { - // A few proposals that winch has no support for. - if test_config.gc == Some(true) - || test_config.threads == Some(true) - || test_config.tail_call == Some(true) - || test_config.function_references == Some(true) - || test_config.gc == Some(true) - || test_config.relaxed_simd == Some(true) - { - return true; - } - - let unsupported = [ - // externref/reference-types related - "component-model/modules.wast", - "extended-const/elem.wast", - "extended-const/global.wast", - "memory64/threads.wast", - "misc_testsuite/externref-id-function.wast", - "misc_testsuite/externref-segment.wast", - "misc_testsuite/externref-segments.wast", - "misc_testsuite/externref-table-dropped-segment-issue-8281.wast", - "misc_testsuite/linking-errors.wast", - "misc_testsuite/many_table_gets_lead_to_gc.wast", - "misc_testsuite/mutable_externref_globals.wast", - "misc_testsuite/no-mixup-stack-maps.wast", - "misc_testsuite/no-panic.wast", - "misc_testsuite/simple_ref_is_null.wast", - "misc_testsuite/table_grow_with_funcref.wast", - "spec_testsuite/br_table.wast", - "spec_testsuite/data-invalid.wast", - "spec_testsuite/elem.wast", - "spec_testsuite/global.wast", - "spec_testsuite/linking.wast", - "spec_testsuite/ref_func.wast", - "spec_testsuite/ref_is_null.wast", - "spec_testsuite/ref_null.wast", - "spec_testsuite/select.wast", - "spec_testsuite/table-sub.wast", - "spec_testsuite/table_fill.wast", - "spec_testsuite/table_get.wast", - "spec_testsuite/table_grow.wast", - "spec_testsuite/table_set.wast", - "spec_testsuite/table_size.wast", - "spec_testsuite/unreached-invalid.wast", - "spec_testsuite/call_indirect.wast", - // simd-related failures - "annotations/simd_lane.wast", - "memory64/simd.wast", - "misc_testsuite/int-to-float-splat.wast", - "misc_testsuite/issue6562.wast", - "misc_testsuite/simd/almost-extmul.wast", - "misc_testsuite/simd/canonicalize-nan.wast", - "misc_testsuite/simd/cvt-from-uint.wast", - "misc_testsuite/simd/issue4807.wast", - "misc_testsuite/simd/issue6725-no-egraph-panic.wast", - "misc_testsuite/simd/issue_3327_bnot_lowering.wast", - "misc_testsuite/simd/load_splat_out_of_bounds.wast", - "misc_testsuite/simd/replace-lane-preserve.wast", - "misc_testsuite/simd/spillslot-size-fuzzbug.wast", - "misc_testsuite/simd/unaligned-load.wast", - "multi-memory/simd_memory-multi.wast", - "spec_testsuite/simd_align.wast", - "spec_testsuite/simd_bit_shift.wast", - "spec_testsuite/simd_bitwise.wast", - "spec_testsuite/simd_boolean.wast", - "spec_testsuite/simd_const.wast", - "spec_testsuite/simd_conversions.wast", - "spec_testsuite/simd_f32x4.wast", - "spec_testsuite/simd_f32x4_arith.wast", - "spec_testsuite/simd_f32x4_cmp.wast", - "spec_testsuite/simd_f32x4_pmin_pmax.wast", - "spec_testsuite/simd_f32x4_rounding.wast", - "spec_testsuite/simd_f64x2.wast", - "spec_testsuite/simd_f64x2_arith.wast", - "spec_testsuite/simd_f64x2_cmp.wast", - "spec_testsuite/simd_f64x2_pmin_pmax.wast", - "spec_testsuite/simd_f64x2_rounding.wast", - "spec_testsuite/simd_i16x8_arith.wast", - "spec_testsuite/simd_i16x8_arith2.wast", - "spec_testsuite/simd_i16x8_cmp.wast", - "spec_testsuite/simd_i16x8_extadd_pairwise_i8x16.wast", - "spec_testsuite/simd_i16x8_extmul_i8x16.wast", - "spec_testsuite/simd_i16x8_q15mulr_sat_s.wast", - "spec_testsuite/simd_i16x8_sat_arith.wast", - "spec_testsuite/simd_i32x4_arith.wast", - "spec_testsuite/simd_i32x4_arith2.wast", - "spec_testsuite/simd_i32x4_cmp.wast", - "spec_testsuite/simd_i32x4_dot_i16x8.wast", - "spec_testsuite/simd_i32x4_extadd_pairwise_i16x8.wast", - "spec_testsuite/simd_i32x4_extmul_i16x8.wast", - "spec_testsuite/simd_i32x4_trunc_sat_f32x4.wast", - "spec_testsuite/simd_i32x4_trunc_sat_f64x2.wast", - "spec_testsuite/simd_i64x2_arith.wast", - "spec_testsuite/simd_i64x2_arith2.wast", - "spec_testsuite/simd_i64x2_cmp.wast", - "spec_testsuite/simd_i64x2_extmul_i32x4.wast", - "spec_testsuite/simd_i8x16_arith.wast", - "spec_testsuite/simd_i8x16_arith2.wast", - "spec_testsuite/simd_i8x16_cmp.wast", - "spec_testsuite/simd_i8x16_sat_arith.wast", - "spec_testsuite/simd_int_to_int_extend.wast", - "spec_testsuite/simd_lane.wast", - "spec_testsuite/simd_load.wast", - "spec_testsuite/simd_load16_lane.wast", - "spec_testsuite/simd_load32_lane.wast", - "spec_testsuite/simd_load64_lane.wast", - "spec_testsuite/simd_load8_lane.wast", - "spec_testsuite/simd_load_extend.wast", - "spec_testsuite/simd_load_splat.wast", - "spec_testsuite/simd_load_zero.wast", - "spec_testsuite/simd_splat.wast", - "spec_testsuite/simd_store16_lane.wast", - "spec_testsuite/simd_store32_lane.wast", - "spec_testsuite/simd_store64_lane.wast", - "spec_testsuite/simd_store8_lane.wast", - ]; - - if unsupported.iter().any(|part| test.ends_with(part)) { - return true; - } - } - - for part in test.iter() { - // Not implemented in Wasmtime yet - if part == "exception-handling" { - return !test.ends_with("binary.wast"); - } - - if part == "memory64" { - if [ - // wasmtime doesn't implement exceptions yet - "imports.wast", - "ref_null.wast", - "exports.wast", - "throw.wast", - "throw_ref.wast", - "try_table.wast", - "tag.wast", - "instance.wast", - ] - .iter() - .any(|i| test.ends_with(i)) - { - return true; - } - } - } - // Some tests are known to fail with the pooling allocator - if wast_config.pooling { - let unsupported = [ - // allocates too much memory for the pooling configuration here - "misc_testsuite/memory64/more-than-4gb.wast", - // shared memories + pooling allocator aren't supported yet - "misc_testsuite/memory-combos.wast", - "misc_testsuite/threads/LB.wast", - "misc_testsuite/threads/LB_atomic.wast", - "misc_testsuite/threads/MP.wast", - "misc_testsuite/threads/MP_atomic.wast", - "misc_testsuite/threads/MP_wait.wast", - "misc_testsuite/threads/SB.wast", - "misc_testsuite/threads/SB_atomic.wast", - "misc_testsuite/threads/atomics_notify.wast", - "misc_testsuite/threads/atomics_wait_address.wast", - "misc_testsuite/threads/wait_notify.wast", - "spec_testsuite/proposals/threads/atomic.wast", - "spec_testsuite/proposals/threads/exports.wast", - "spec_testsuite/proposals/threads/memory.wast", - ]; - - if unsupported.iter().any(|part| test.ends_with(part)) { - return true; - } - } - - false -} - -/// Configuration where the main function will generate a combinatorial -/// matrix of these top-level configurations to run the entire test suite with -/// that configuration. -struct WastConfig { - strategy: Strategy, - pooling: bool, - collector: Collector, -} - -/// Per-test configuration which is written down in the test file itself for -/// `misc_testsuite/**/*.wast` or in `spec_test_config` below for spec tests. -#[derive(Debug, PartialEq, Default, Deserialize)] -#[serde(deny_unknown_fields)] -struct TestConfig { - memory64: Option, - custom_page_sizes: Option, - multi_memory: Option, - threads: Option, - gc: Option, - function_references: Option, - relaxed_simd: Option, - reference_types: Option, - tail_call: Option, - extended_const: Option, - wide_arithmetic: Option, - hogs_memory: Option, - nan_canonicalization: Option, - component_model_more_flags: Option, -} - -fn spec_test_config(wast: &Path) -> TestConfig { - let mut ret = TestConfig::default(); - - match wast.strip_prefix("proposals") { - // This lists the features require to run the various spec tests suites - // in their `proposals` folder. - Ok(rest) => { - let proposal = rest.iter().next().unwrap().to_str().unwrap(); - match proposal { - "multi-memory" => { - ret.multi_memory = Some(true); - ret.reference_types = Some(true); - } - "wide-arithmetic" => { - ret.wide_arithmetic = Some(true); - } - "threads" => { - ret.threads = Some(true); - ret.reference_types = Some(false); - } - "tail-call" => { - ret.tail_call = Some(true); - ret.reference_types = Some(true); - } - "relaxed-simd" => { - ret.relaxed_simd = Some(true); - } - "memory64" => { - ret.memory64 = Some(true); - ret.tail_call = Some(true); - ret.gc = Some(true); - ret.extended_const = Some(true); - ret.multi_memory = Some(true); - ret.relaxed_simd = Some(true); - } - "extended-const" => { - ret.extended_const = Some(true); - ret.reference_types = Some(true); - } - "custom-page-sizes" => { - ret.custom_page_sizes = Some(true); - ret.multi_memory = Some(true); - } - "exception-handling" => { - ret.reference_types = Some(true); - } - "gc" => { - ret.gc = Some(true); - ret.tail_call = Some(true); - } - "function-references" => { - ret.function_references = Some(true); - ret.tail_call = Some(true); - } - "annotations" => {} - _ => panic!("unsuported proposal {proposal:?}"), - } - } - - // This lists the features required to run the top-level of spec tests - // outside of the `proposals` directory. - Err(_) => { - ret.reference_types = Some(true); - } + // There's a lot of tests so print only a `.` to keep the output a + // bit more terse by default. + let mut args = Arguments::from_args(); + if args.format.is_none() { + args.format = Some(FormatSetting::Terse); } - ret + libtest_mimic::run(&args, trials).exit() } // Each of the tests included from `wast_testsuite_tests` will call this // function which actually executes the `wast` test suite given the `strategy` // to compile it. -fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { - let wast_contents = std::fs::read_to_string(wast) - .with_context(|| format!("failed to read `{}`", wast.display()))?; - - // If this is a spec test then the configuration for it is loaded via - // `spec_test_config`, but otherwise it's required to be listed in the top - // of the file as we control the contents of the file. - let mut test_config = match wast.strip_prefix("tests/spec_testsuite") { - Ok(test) => spec_test_config(test), - Err(_) => support::parse_test_config(&wast_contents)?, - }; +fn run_wast(test: &WastTest, config: WastConfig) -> anyhow::Result<()> { + let mut test_config = test.config.clone(); // FIXME: this is a bit of a hack to get Winch working here for now. Winch // passes some tests on aarch64 so returning `true` from `should_fail` @@ -410,9 +96,14 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { test_config.reference_types = Some(true); } - let should_fail = should_fail(wast, &config, &test_config); - - let wast = Path::new(wast); + // Determine whether this test is expected to fail or pass. Regardless the + // test is executed and the result of the execution is asserted to match + // this expectation. Note that this means that the test can't, for example, + // panic or segfault as a result. + // + // Updates to whether a test should pass or fail should be done in the + // `crates/wast-util/src/lib.rs` file. + let should_fail = test.should_fail(&config); // Note that all of these proposals/features are currently default-off to // ensure that we annotate all tests accurately with what features they @@ -445,8 +136,8 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { .or(test_config.gc) .unwrap_or(false); - let is_cranelift = match config.strategy { - Strategy::Cranelift => true, + let is_cranelift = match config.compiler { + Compiler::Cranelift => true, _ => false, }; @@ -463,8 +154,15 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { .wasm_extended_const(extended_const) .wasm_wide_arithmetic(wide_arithmetic) .wasm_component_model_more_flags(component_model_more_flags) - .strategy(config.strategy) - .collector(config.collector) + .strategy(match config.compiler { + Compiler::Cranelift => wasmtime::Strategy::Cranelift, + Compiler::Winch => wasmtime::Strategy::Winch, + }) + .collector(match config.collector { + Collector::Auto => wasmtime::Collector::Auto, + Collector::Null => wasmtime::Collector::Null, + Collector::DeferredReferenceCounting => wasmtime::Collector::DeferredReferenceCounting, + }) .cranelift_nan_canonicalization(nan_canonicalization); if is_cranelift { @@ -521,24 +219,23 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { // When multiple memories are used and are configured in the pool then // force the usage of static memories without guards to reduce the VM // impact. - let max_memory_size = 805 << 16; + let max_memory_size = limits::MEMORY_SIZE; if multi_memory { cfg.memory_reservation(max_memory_size as u64); cfg.memory_reservation_for_growth(0); cfg.memory_guard_size(0); } - // The limits here are crafted such that the wast tests should pass. - // However, these limits may become insufficient in the future as the - // wast tests change. If a wast test fails because of a limit being - // "exceeded" or if memory/table fails to grow, the values here will - // need to be adjusted. let mut pool = PoolingAllocationConfig::default(); - pool.total_memories(450 * 2) + pool.total_memories(limits::MEMORIES * 2) .max_memory_protection_keys(2) .max_memory_size(max_memory_size) - .max_memories_per_module(if multi_memory { 9 } else { 1 }) - .max_tables_per_module(5); + .max_memories_per_module(if multi_memory { + limits::MEMORIES_PER_MODULE + } else { + 1 + }) + .max_tables_per_module(limits::TABLES_PER_MODULE); // When testing, we may choose to start with MPK force-enabled to ensure // we use that functionality. @@ -573,7 +270,7 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { suppress_prints: true, })?; wast_context - .run_buffer(wast.to_str().unwrap(), wast_contents.as_bytes()) + .run_buffer(test.path.to_str().unwrap(), test.contents.as_bytes()) .with_context(|| format!("failed to run spec test with {desc} engine")) });