diff --git a/Cargo.lock b/Cargo.lock index b4feaf0af158..f3fa59b15917 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4017,6 +4017,7 @@ dependencies = [ "wasmtime-wasi-nn", "wasmtime-wasi-threads", "wasmtime-wast", + "wasmtime-wast-util", "wast 219.0.1", "wat", "windows-sys 0.59.0", @@ -4208,6 +4209,7 @@ dependencies = [ "wasmprinter", "wasmtime", "wasmtime-wast", + "wasmtime-wast-util", "wat", ] @@ -4376,6 +4378,16 @@ dependencies = [ "wast 219.0.1", ] +[[package]] +name = "wasmtime-wast-util" +version = "28.0.0" +dependencies = [ + "anyhow", + "serde", + "serde_derive", + "toml", +] + [[package]] name = "wasmtime-winch" version = "28.0.0" diff --git a/Cargo.toml b/Cargo.toml index 3b367932470d..cf7b2da6950e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -120,6 +120,7 @@ capstone = { workspace = true } object = { workspace = true, features = ['std'] } wasmtime-test-macros = { path = "crates/test-macros" } pulley-interpreter = { workspace = true, features = ["disas"] } +wasmtime-wast-util = { path = 'crates/wast-util' } [target.'cfg(windows)'.dev-dependencies] windows-sys = { workspace = true, features = ["Win32_System_Memory"] } diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 9ce0d93a2017..939ef6fde9e6 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -382,6 +382,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { | Operator::CallIndirect { .. } | Operator::Call { .. } | Operator::ReturnCall { .. } + | Operator::ReturnCallRef { .. } | Operator::ReturnCallIndirect { .. } => { self.fuel_increment_var(builder); self.fuel_save_from_var(builder); diff --git a/crates/cranelift/src/gc/enabled.rs b/crates/cranelift/src/gc/enabled.rs index 8fc9a171c79a..5dea690293a9 100644 --- a/crates/cranelift/src/gc/enabled.rs +++ b/crates/cranelift/src/gc/enabled.rs @@ -1,7 +1,7 @@ use super::GcCompiler; use crate::func_environ::FuncEnvironment; use crate::gc::ArrayInit; -use crate::translate::{StructFieldsVec, TargetEnvironment}; +use crate::translate::{FuncEnvironment as _, StructFieldsVec, TargetEnvironment}; use crate::TRAP_INTERNAL_ASSERT; use cranelift_codegen::{ cursor::FuncCursor, @@ -304,7 +304,7 @@ pub fn translate_struct_get( // TODO: If we know we have a `(ref $my_struct)` here, instead of maybe a // `(ref null $my_struct)`, we could omit the `trapz`. But plumbing that // type info from `wasmparser` and through to here is a bit funky. - builder.ins().trapz(struct_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE); let field_index = usize::try_from(field_index).unwrap(); let interned_type_index = func_env.module.types[struct_type_index]; @@ -337,7 +337,7 @@ fn translate_struct_get_and_extend( extension: Extension, ) -> WasmResult { // TODO: See comment in `translate_struct_get` about the `trapz`. - builder.ins().trapz(struct_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE); let field_index = usize::try_from(field_index).unwrap(); let interned_type_index = func_env.module.types[struct_type_index]; @@ -410,7 +410,7 @@ pub fn translate_struct_set( new_val: ir::Value, ) -> WasmResult<()> { // TODO: See comment in `translate_struct_get` about the `trapz`. - builder.ins().trapz(struct_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE); let field_index = usize::try_from(field_index).unwrap(); let interned_type_index = func_env.module.types[struct_type_index]; @@ -640,15 +640,11 @@ pub fn translate_array_fill( let len = translate_array_len(func_env, builder, array_ref)?; // Check that the full range of elements we want to fill is within bounds. - let end_index = builder - .ins() - .uadd_overflow_trap(index, n, crate::TRAP_ARRAY_OUT_OF_BOUNDS); + let end_index = func_env.uadd_overflow_trap(builder, index, n, crate::TRAP_ARRAY_OUT_OF_BOUNDS); let out_of_bounds = builder .ins() .icmp(IntCC::UnsignedGreaterThan, end_index, len); - builder - .ins() - .trapnz(out_of_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); + func_env.trapnz(builder, out_of_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); // Get the address of the first element we want to fill. let interned_type_index = func_env.module.types[array_type_index]; @@ -695,7 +691,7 @@ pub fn translate_array_len( builder: &mut FunctionBuilder, array_ref: ir::Value, ) -> WasmResult { - builder.ins().trapz(array_ref, crate::TRAP_NULL_REFERENCE); + func_env.trapz(builder, array_ref, crate::TRAP_NULL_REFERENCE); let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset(); let len_field = func_env.prepare_gc_ref_access( @@ -794,9 +790,7 @@ fn array_elem_addr( let len = translate_array_len(func_env, builder, array_ref).unwrap(); let in_bounds = builder.ins().icmp(IntCC::UnsignedLessThan, index, len); - builder - .ins() - .trapz(in_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); + func_env.trapz(builder, in_bounds, crate::TRAP_ARRAY_OUT_OF_BOUNDS); // Compute the size (in bytes) of the whole array object. let ArraySizeInfo { @@ -1175,6 +1169,7 @@ fn uextend_i32_to_pointer_type( /// Traps if the size overflows. #[cfg_attr(not(any(feature = "gc-drc", feature = "gc-null")), allow(dead_code))] fn emit_array_size( + func_env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, array_layout: &GcArrayLayout, init: ArrayInit<'_>, @@ -1200,17 +1195,17 @@ fn emit_array_size( .ins() .imul_imm(len, i64::from(array_layout.elem_size)); let high_bits = builder.ins().ushr_imm(elems_size_64, 32); - builder - .ins() - .trapnz(high_bits, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.trapnz(builder, high_bits, crate::TRAP_ALLOCATION_TOO_LARGE); let elems_size = builder.ins().ireduce(ir::types::I32, elems_size_64); // And if adding the base size and elements size overflows, then the // allocation is too large. - let size = - builder - .ins() - .uadd_overflow_trap(base_size, elems_size, crate::TRAP_ALLOCATION_TOO_LARGE); + let size = func_env.uadd_overflow_trap( + builder, + base_size, + elems_size, + crate::TRAP_ALLOCATION_TOO_LARGE, + ); size } diff --git a/crates/cranelift/src/gc/enabled/drc.rs b/crates/cranelift/src/gc/enabled/drc.rs index 238c60287da6..e0b565530126 100644 --- a/crates/cranelift/src/gc/enabled/drc.rs +++ b/crates/cranelift/src/gc/enabled/drc.rs @@ -303,14 +303,14 @@ impl GcCompiler for DrcCompiler { let interned_type_index = func_env.module.types[array_type_index]; let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset(); - let array_layout = func_env.array_layout(interned_type_index); + let array_layout = func_env.array_layout(interned_type_index).clone(); let base_size = array_layout.base_size; let align = array_layout.align; let len_to_elems_delta = base_size.checked_sub(len_offset).unwrap(); // First, compute the array's total size from its base size, element // size, and length. - let size = emit_array_size(builder, array_layout, init); + let size = emit_array_size(func_env, builder, &array_layout, init); // Second, now that we have the array object's total size, call the // `gc_alloc_raw` builtin libcall to allocate the array. diff --git a/crates/cranelift/src/gc/enabled/null.rs b/crates/cranelift/src/gc/enabled/null.rs index ad042b36f418..a5367893a4ee 100644 --- a/crates/cranelift/src/gc/enabled/null.rs +++ b/crates/cranelift/src/gc/enabled/null.rs @@ -54,9 +54,7 @@ impl NullCompiler { .ins() .iconst(ir::types::I32, i64::from(VMGcKind::MASK)); let masked = builder.ins().band(size, mask); - builder - .ins() - .trapnz(masked, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.trapnz(builder, masked, crate::TRAP_ALLOCATION_TOO_LARGE); // Load the bump "pointer" (it is actually an index into the GC heap, // not a raw pointer). @@ -81,7 +79,8 @@ impl NullCompiler { // a power of two. let minus_one = builder.ins().iconst(ir::types::I32, -1); let align_minus_one = builder.ins().iadd(align, minus_one); - let next_plus_align_minus_one = builder.ins().uadd_overflow_trap( + let next_plus_align_minus_one = func_env.uadd_overflow_trap( + builder, next, align_minus_one, crate::TRAP_ALLOCATION_TOO_LARGE, @@ -93,9 +92,7 @@ impl NullCompiler { // Check whether the allocation fits in the heap space we have left. let end_of_object = - builder - .ins() - .uadd_overflow_trap(aligned, size, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.uadd_overflow_trap(builder, aligned, size, crate::TRAP_ALLOCATION_TOO_LARGE); let uext_end_of_object = uextend_i32_to_pointer_type(builder, pointer_type, end_of_object); let (base, bound) = func_env.get_gc_heap_base_bound(builder); let is_in_bounds = builder.ins().icmp( @@ -103,9 +100,7 @@ impl NullCompiler { uext_end_of_object, bound, ); - builder - .ins() - .trapz(is_in_bounds, crate::TRAP_ALLOCATION_TOO_LARGE); + func_env.trapz(builder, is_in_bounds, crate::TRAP_ALLOCATION_TOO_LARGE); // Write the header, update the bump "pointer", and return the newly // allocated object. @@ -162,14 +157,14 @@ impl GcCompiler for NullCompiler { let interned_type_index = func_env.module.types[array_type_index]; let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset(); - let array_layout = func_env.array_layout(interned_type_index); + let array_layout = func_env.array_layout(interned_type_index).clone(); let base_size = array_layout.base_size; let align = array_layout.align; let len_to_elems_delta = base_size.checked_sub(len_offset).unwrap(); // First, compute the array's total size from its base size, element // size, and length. - let size = emit_array_size(builder, array_layout, init); + let size = emit_array_size(func_env, builder, &array_layout, init); // Next, allocate the array. assert!(align.is_power_of_two()); diff --git a/crates/fuzzing/Cargo.toml b/crates/fuzzing/Cargo.toml index 91682ac86119..66d512a7c596 100644 --- a/crates/fuzzing/Cargo.toml +++ b/crates/fuzzing/Cargo.toml @@ -11,6 +11,9 @@ license = "Apache-2.0 WITH LLVM-exception" [lints] workspace = true +[build-dependencies] +wasmtime-wast-util = { path = '../wast-util' } + [dependencies] anyhow = { workspace = true } arbitrary = { workspace = true, features = ["derive"] } @@ -24,13 +27,14 @@ tempfile = "3.3.0" wasmparser = { workspace = true } wasmprinter = { workspace = true } wasmtime = { workspace = true, features = ['default', 'winch', 'gc', 'memory-protection-keys'] } -wasmtime-wast = { workspace = true } +wasmtime-wast = { workspace = true, features = ['component-model'] } wasm-encoder = { workspace = true } wasm-smith = { workspace = true } wasm-mutate = { workspace = true } wasm-spec-interpreter = { path = "./wasm-spec-interpreter", optional = true } wasmi = "0.39.1" futures = { workspace = true } +wasmtime-wast-util = { path = '../wast-util' } # We rely on precompiled v8 binaries, but rusty-v8 doesn't have a precompiled # binary for MinGW which is built on our CI. It does have one for Windows-msvc, diff --git a/crates/fuzzing/build.rs b/crates/fuzzing/build.rs index 316a15f21e5b..08b9bddfe3a0 100644 --- a/crates/fuzzing/build.rs +++ b/crates/fuzzing/build.rs @@ -4,39 +4,39 @@ use std::env; use std::path::PathBuf; +use wasmtime_wast_util::WastTest; fn main() { println!("cargo:rerun-if-changed=build.rs"); let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - let dirs = [ - "tests/spec_testsuite", - "tests/misc_testsuite", - "tests/misc_testsuite/multi-memory", - "tests/misc_testsuite/simd", - "tests/misc_testsuite/threads", - ]; let mut root = env::current_dir().unwrap(); root.pop(); // chop off 'fuzzing' root.pop(); // chop off 'crates' - let mut code = format!("static FILES: &[(&str, &str)] = &[\n"); - - let mut entries = Vec::new(); - for dir in dirs { - for entry in root.join(dir).read_dir().unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - if path.extension().and_then(|s| s.to_str()) == Some("wast") { - entries.push(path); - } - } - } - entries.sort(); - for path in entries { - let path = path.to_str().expect("path is not valid utf-8"); - code.push_str(&format!("({path:?}, include_str!({path:?})),\n")); + + let tests = wasmtime_wast_util::find_tests(&root).unwrap(); + + let mut code = format!("static FILES: &[fn() -> wasmtime_wast_util::WastTest] = &[\n"); + + for test in tests { + let WastTest { + path, + contents: _, + config, + } = test; + println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); + code.push_str(&format!( + "|| {{ + wasmtime_wast_util::WastTest {{ + path: {path:?}.into(), + contents: include_str!({path:?}).into(), + config: wasmtime_wast_util::{config:?}, + }} + }}," + )); } + code.push_str("];\n"); std::fs::write(out_dir.join("wasttests.rs"), code).unwrap(); } diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs index bfeb5718c434..18de6bfccfd9 100644 --- a/crates/fuzzing/src/generators/config.rs +++ b/crates/fuzzing/src/generators/config.rs @@ -10,6 +10,7 @@ use arbitrary::{Arbitrary, Unstructured}; use std::sync::Arc; use std::time::Duration; use wasmtime::{Engine, Module, MpkEnabled, Store}; +use wasmtime_wast_util::{limits, WastConfig, WastTest}; /// Configuration for `wasmtime::Config` and generated modules for a session of /// fuzzing. @@ -114,49 +115,111 @@ impl Config { self.module_config.generate(input, default_fuel) } - /// Tests whether this configuration is capable of running all wast tests. - pub fn is_wast_test_compliant(&self) -> bool { - let config = &self.module_config.config; - - // Check for wasm features that must be disabled to run spec tests - if config.memory64_enabled { - return false; + /// Updates this configuration to be able to run the `test` specified. + /// + /// This primarily updates `self.module_config` to ensure that it enables + /// all features and proposals necessary to execute the `test` specified. + /// This will additionally update limits in the pooling allocator to be able + /// to execute all tests. + pub fn make_wast_test_compliant(&mut self, test: &WastTest) -> WastConfig { + // Enable/disable some proposals that aren't configurable in wasm-smith + // but are configurable in Wasmtime. + self.module_config.extended_const_enabled = test.config.extended_const.unwrap_or(false); + self.module_config.function_references_enabled = test + .config + .function_references + .or(test.config.gc) + .unwrap_or(false); + self.module_config.component_model_more_flags = + test.config.component_model_more_flags.unwrap_or(false); + + // Enable/disable proposals that wasm-smith has knobs for which will be + // read when creating `wasmtime::Config`. + let config = &mut self.module_config.config; + config.bulk_memory_enabled = true; + config.multi_value_enabled = true; + config.simd_enabled = true; + config.wide_arithmetic_enabled = test.config.wide_arithmetic.unwrap_or(false); + config.memory64_enabled = test.config.memory64.unwrap_or(false); + config.tail_call_enabled = test.config.tail_call.unwrap_or(false); + config.custom_page_sizes_enabled = test.config.custom_page_sizes.unwrap_or(false); + config.threads_enabled = test.config.threads.unwrap_or(false); + config.gc_enabled = test.config.gc.unwrap_or(false); + config.reference_types_enabled = config.gc_enabled + || self.module_config.function_references_enabled + || test.config.reference_types.unwrap_or(false); + if test.config.multi_memory.unwrap_or(false) { + config.max_memories = limits::MEMORIES_PER_MODULE as usize; + } else { + config.max_memories = 1; } - // Check for wasm features that must be enabled to run spec tests - if !config.bulk_memory_enabled - || !config.reference_types_enabled - || !config.multi_value_enabled - || !config.simd_enabled - || !config.threads_enabled - || config.max_memories <= 1 - { - return false; + match &mut self.wasmtime.memory_config { + MemoryConfig::Normal(config) => { + if let Some(n) = &mut config.memory_reservation { + *n = (*n).max(limits::MEMORY_SIZE as u64); + } + } + MemoryConfig::CustomUnaligned => {} } - // Make sure the runtime limits allow for the instantiation of all spec - // tests. Note that the max memories must be precisely one since 0 won't - // instantiate spec tests and more than one is multi-memory which is - // disabled for spec tests. - if config.max_memories != 1 || config.max_tables < 5 { - return false; + // FIXME: it might be more ideal to avoid the need for this entirely + // and to just let the test fail. If a test fails due to a pooling + // allocator resource limit being met we could ideally detect that and + // let the fuzz test case pass. That would avoid the need to hardcode + // so much here and in theory wouldn't reduce the usefulness of fuzzers + // all that much. At this time though we can't easily test this configuration. + if let InstanceAllocationStrategy::Pooling(pooling) = &mut self.wasmtime.strategy { + // Clamp protection keys between 1 & 2 to reduce the number of + // slots and then multiply the total memories by the number of keys + // we have since a single store has access to only one key. + pooling.max_memory_protection_keys = pooling.max_memory_protection_keys.max(1).min(2); + pooling.total_memories = pooling + .total_memories + .max(limits::MEMORIES * (pooling.max_memory_protection_keys as u32)); + + // For other limits make sure they meet the minimum threshold + // required for our wast tests. + pooling.total_component_instances = pooling + .total_component_instances + .max(limits::COMPONENT_INSTANCES); + pooling.total_tables = pooling.total_tables.max(limits::TABLES); + pooling.max_tables_per_module = + pooling.max_tables_per_module.max(limits::TABLES_PER_MODULE); + pooling.max_memories_per_module = pooling + .max_memories_per_module + .max(limits::MEMORIES_PER_MODULE); + pooling.max_memories_per_component = pooling + .max_memories_per_component + .max(limits::MEMORIES_PER_MODULE); + pooling.total_core_instances = pooling.total_core_instances.max(limits::CORE_INSTANCES); + pooling.max_memory_size = pooling.max_memory_size.max(limits::MEMORY_SIZE); + pooling.table_elements = pooling.table_elements.max(limits::TABLE_ELEMENTS); + pooling.core_instance_size = pooling.core_instance_size.max(limits::CORE_INSTANCE_SIZE); + pooling.component_instance_size = pooling + .component_instance_size + .max(limits::CORE_INSTANCE_SIZE); } - if let InstanceAllocationStrategy::Pooling(pooling) = &self.wasmtime.strategy { - // Check to see if any item limit is less than the required - // threshold to execute the spec tests. - if pooling.total_memories < 1 - || pooling.total_tables < 5 - || pooling.table_elements < 1_000 - || pooling.max_memory_size < (900 << 16) - || pooling.total_core_instances < 500 - || pooling.core_instance_size < 64 * 1024 - { - return false; - } + // Return the test configuration that this fuzz configuration represents + // which is used afterwards to test if the `test` here is expected to + // fail or not. + WastConfig { + collector: match self.wasmtime.collector { + Collector::Null => wasmtime_wast_util::Collector::Null, + Collector::DeferredReferenceCounting => { + wasmtime_wast_util::Collector::DeferredReferenceCounting + } + }, + pooling: matches!( + self.wasmtime.strategy, + InstanceAllocationStrategy::Pooling(_) + ), + compiler: match self.wasmtime.compiler_strategy { + CompilerStrategy::Cranelift => wasmtime_wast_util::Compiler::Cranelift, + CompilerStrategy::Winch => wasmtime_wast_util::Compiler::Winch, + }, } - - true } /// Converts this to a `wasmtime::Config` object @@ -166,7 +229,7 @@ impl Config { let mut cfg = wasmtime::Config::new(); cfg.wasm_bulk_memory(true) - .wasm_reference_types(true) + .wasm_reference_types(self.module_config.config.reference_types_enabled) .wasm_multi_value(self.module_config.config.multi_value_enabled) .wasm_multi_memory(self.module_config.config.max_memories > 1) .wasm_simd(self.module_config.config.simd_enabled) @@ -174,10 +237,12 @@ impl Config { .wasm_tail_call(self.module_config.config.tail_call_enabled) .wasm_custom_page_sizes(self.module_config.config.custom_page_sizes_enabled) .wasm_threads(self.module_config.config.threads_enabled) - .wasm_function_references(self.module_config.config.gc_enabled) + .wasm_function_references(self.module_config.function_references_enabled) .wasm_gc(self.module_config.config.gc_enabled) .wasm_custom_page_sizes(self.module_config.config.custom_page_sizes_enabled) .wasm_wide_arithmetic(self.module_config.config.wide_arithmetic_enabled) + .wasm_extended_const(self.module_config.extended_const_enabled) + .wasm_component_model_more_flags(self.module_config.component_model_more_flags) .native_unwind_info(cfg!(target_os = "windows") || self.wasmtime.native_unwind_info) .cranelift_nan_canonicalization(self.wasmtime.canonicalize_nans) .cranelift_opt_level(self.wasmtime.opt_level.to_wasmtime()) diff --git a/crates/fuzzing/src/generators/module.rs b/crates/fuzzing/src/generators/module.rs index f3fcd4a51636..51ef41edae8b 100644 --- a/crates/fuzzing/src/generators/module.rs +++ b/crates/fuzzing/src/generators/module.rs @@ -10,6 +10,16 @@ use arbitrary::{Arbitrary, Unstructured}; pub struct ModuleConfig { #[allow(missing_docs)] pub config: wasm_smith::Config, + + // These knobs aren't exposed in `wasm-smith` at this time but are exposed + // in our `*.wast` testing so keep knobs here so they can be read during + // config-to-`wasmtime::Config` translation. + #[allow(missing_docs)] + pub extended_const_enabled: bool, + #[allow(missing_docs)] + pub function_references_enabled: bool, + #[allow(missing_docs)] + pub component_model_more_flags: bool, } impl<'a> Arbitrary<'a> for ModuleConfig { @@ -53,7 +63,12 @@ impl<'a> Arbitrary<'a> for ModuleConfig { // do that most of the time. config.disallow_traps = u.ratio(9, 10)?; - Ok(ModuleConfig { config }) + Ok(ModuleConfig { + extended_const_enabled: false, + component_model_more_flags: false, + function_references_enabled: config.gc_enabled, + config, + }) } } diff --git a/crates/fuzzing/src/generators/wast_test.rs b/crates/fuzzing/src/generators/wast_test.rs index f93a387861ec..63200b36fb9a 100644 --- a/crates/fuzzing/src/generators/wast_test.rs +++ b/crates/fuzzing/src/generators/wast_test.rs @@ -8,18 +8,16 @@ include!(concat!(env!("OUT_DIR"), "/wasttests.rs")); /// A wast test from this repository. #[derive(Debug)] pub struct WastTest { - /// The filename of the spec test - pub file: &'static str, - /// The `*.wast` contents of the spec test - pub contents: &'static str, + #[allow(missing_docs)] + pub test: wasmtime_wast_util::WastTest, } impl<'a> Arbitrary<'a> for WastTest { fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - // NB: this does get a uniform value in the provided range. - let i = u.int_in_range(0..=FILES.len() - 1)?; - let (file, contents) = FILES[i]; - Ok(WastTest { file, contents }) + log::debug!("{}", u.is_empty()); + Ok(WastTest { + test: u.choose(FILES)?(), + }) } fn size_hint(_depth: usize) -> (usize, Option) { diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs index ebbe9bceb77f..168c267ef49f 100644 --- a/crates/fuzzing/src/oracles.rs +++ b/crates/fuzzing/src/oracles.rs @@ -90,7 +90,7 @@ impl StoreLimits { // `memcpy`. As more data is added over time growths get more and // more expensive meaning that fuel may not be effective at limiting // execution time. - remaining_growths: AtomicUsize::new(100), + remaining_growths: AtomicUsize::new(1000), oom: AtomicBool::new(false), })) } @@ -640,9 +640,21 @@ pub fn make_api_calls(api: generators::api::ApiCalls) { /// Executes the wast `test` with the `config` specified. /// /// Ensures that wast tests pass regardless of the `Config`. -pub fn wast_test(fuzz_config: generators::Config, test: generators::WastTest) { +pub fn wast_test(mut fuzz_config: generators::Config, test: generators::WastTest) { crate::init_fuzzing(); - if !fuzz_config.is_wast_test_compliant() { + let test = &test.test; + + // Discard tests that allocate a lot of memory as we don't want to OOM the + // fuzzer and we also limit memory growth which would cause the test to + // fail. + if test.config.hogs_memory.unwrap_or(false) { + return; + } + + // Transform `fuzz_config` to be valid for `test` and make sure that this + // test is supposed to pass. + let wast_config = fuzz_config.make_wast_test_compliant(test); + if test.should_fail(&wast_config) { return; } @@ -654,16 +666,16 @@ pub fn wast_test(fuzz_config: generators::Config, test: generators::WastTest) { } } - log::debug!("running {:?}", test.file); + log::debug!("running {:?}", test.path); let mut wast_context = WastContext::new(fuzz_config.to_store()); wast_context .register_spectest(&wasmtime_wast::SpectestConfig { - use_shared_memory: false, + use_shared_memory: true, suppress_prints: true, }) .unwrap(); wast_context - .run_buffer(test.file, test.contents.as_bytes()) + .run_buffer(test.path.to_str().unwrap(), test.contents.as_bytes()) .unwrap(); } diff --git a/crates/wast-util/Cargo.toml b/crates/wast-util/Cargo.toml new file mode 100644 index 000000000000..a69da0a6b724 --- /dev/null +++ b/crates/wast-util/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "wasmtime-wast-util" +version.workspace = true +authors.workspace = true +edition.workspace = true +rust-version.workspace = true +publish = false +description = "Utilities for `*.wast` files and tests in Wasmtime" +license = "Apache-2.0 WITH LLVM-exception" +repository = "https://github.com/bytecodealliance/wasmtime" + +[dependencies] +anyhow = { workspace = true } +serde = { workspace = true } +serde_derive = { workspace = true } +toml = { workspace = true } + +[lints] +workspace = true diff --git a/crates/wast-util/src/lib.rs b/crates/wast-util/src/lib.rs new file mode 100644 index 000000000000..c1177ec099ec --- /dev/null +++ b/crates/wast-util/src/lib.rs @@ -0,0 +1,422 @@ +use anyhow::{Context, Result}; +use serde::de::DeserializeOwned; +use serde_derive::Deserialize; +use std::fmt; +use std::fs; +use std::path::Path; +use std::path::PathBuf; + +/// Limits for running wast tests. +/// +/// This is useful for sharing between `tests/wast.rs` and fuzzing, for +/// example, and is used as the minimum threshold for configuration when +/// fuzzing. +/// +/// Note that it's ok to increase these numbers if a test comes along and needs +/// it, they're just here as empirically found minimum thresholds so far and +/// they're not too scientific. +pub mod limits { + pub const MEMORY_SIZE: usize = 805 << 16; + pub const MEMORIES: u32 = 450; + pub const TABLES: u32 = 200; + pub const MEMORIES_PER_MODULE: u32 = 9; + pub const TABLES_PER_MODULE: u32 = 5; + pub const COMPONENT_INSTANCES: u32 = 50; + pub const CORE_INSTANCES: u32 = 900; + pub const TABLE_ELEMENTS: usize = 1000; + pub const CORE_INSTANCE_SIZE: usize = 64 * 1024; +} + +/// Local all `*.wast` tests under `root` which should be the path to the root +/// of the wasmtime repository. +pub fn find_tests(root: &Path) -> Result> { + let mut tests = Vec::new(); + add_tests(&mut tests, &root.join("tests/spec_testsuite"), false)?; + add_tests(&mut tests, &root.join("tests/misc_testsuite"), true)?; + Ok(tests) +} + +fn add_tests(tests: &mut Vec, path: &Path, has_config: bool) -> Result<()> { + for entry in path.read_dir().context("failed to read directory")? { + let entry = entry.context("failed to read directory entry")?; + let path = entry.path(); + if entry + .file_type() + .context("failed to get file type")? + .is_dir() + { + add_tests(tests, &path, has_config).context("failed to read sub-directory")?; + continue; + } + + if path.extension().and_then(|s| s.to_str()) != Some("wast") { + continue; + } + + let contents = + fs::read_to_string(&path).with_context(|| format!("failed to read test: {path:?}"))?; + let config = if has_config { + parse_test_config(&contents) + .with_context(|| format!("failed to parse test configuration: {path:?}"))? + } else { + spec_test_config(&path) + }; + tests.push(WastTest { + path, + contents, + config, + }) + } + Ok(()) +} + +fn spec_test_config(test: &Path) -> TestConfig { + let mut ret = TestConfig::default(); + match spec_proposal_from_path(test) { + Some("multi-memory") => { + ret.multi_memory = Some(true); + ret.reference_types = Some(true); + } + Some("wide-arithmetic") => { + ret.wide_arithmetic = Some(true); + } + Some("threads") => { + ret.threads = Some(true); + ret.reference_types = Some(false); + } + Some("tail-call") => { + ret.tail_call = Some(true); + ret.reference_types = Some(true); + } + Some("relaxed-simd") => { + ret.relaxed_simd = Some(true); + } + Some("memory64") => { + ret.memory64 = Some(true); + ret.tail_call = Some(true); + ret.gc = Some(true); + ret.extended_const = Some(true); + ret.multi_memory = Some(true); + ret.relaxed_simd = Some(true); + } + Some("extended-const") => { + ret.extended_const = Some(true); + ret.reference_types = Some(true); + } + Some("custom-page-sizes") => { + ret.custom_page_sizes = Some(true); + ret.multi_memory = Some(true); + } + Some("exception-handling") => { + ret.reference_types = Some(true); + } + Some("gc") => { + ret.gc = Some(true); + ret.tail_call = Some(true); + } + Some("function-references") => { + ret.function_references = Some(true); + ret.tail_call = Some(true); + } + Some("annotations") => {} + Some(proposal) => panic!("unsuported proposal {proposal:?}"), + None => ret.reference_types = Some(true), + } + + ret +} + +/// Parse test configuration from the specified test, comments starting with +/// `;;!`. +pub fn parse_test_config(wat: &str) -> Result +where + T: DeserializeOwned, +{ + // The test config source is the leading lines of the WAT file that are + // prefixed with `;;!`. + let config_lines: Vec<_> = wat + .lines() + .take_while(|l| l.starts_with(";;!")) + .map(|l| &l[3..]) + .collect(); + let config_text = config_lines.join("\n"); + + toml::from_str(&config_text).context("failed to parse the test configuration") +} + +/// A `*.wast` test with its path, contents, and configuration. +#[derive(Clone)] +pub struct WastTest { + pub path: PathBuf, + pub contents: String, + pub config: TestConfig, +} + +impl fmt::Debug for WastTest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("WastTest") + .field("path", &self.path) + .field("contents", &"...") + .field("config", &self.config) + .finish() + } +} + +/// Per-test configuration which is written down in the test file itself for +/// `misc_testsuite/**/*.wast` or in `spec_test_config` above for spec tests. +#[derive(Debug, PartialEq, Default, Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct TestConfig { + pub memory64: Option, + pub custom_page_sizes: Option, + pub multi_memory: Option, + pub threads: Option, + pub gc: Option, + pub function_references: Option, + pub relaxed_simd: Option, + pub reference_types: Option, + pub tail_call: Option, + pub extended_const: Option, + pub wide_arithmetic: Option, + pub hogs_memory: Option, + pub nan_canonicalization: Option, + pub component_model_more_flags: Option, +} + +/// Configuration that spec tests can run under. +pub struct WastConfig { + /// Compiler chosen to run this test. + pub compiler: Compiler, + /// Whether or not the pooling allocator is enabled. + pub pooling: bool, + /// What garbage collector is being used. + pub collector: Collector, +} + +#[derive(PartialEq, Debug, Copy, Clone)] +pub enum Compiler { + Cranelift, + Winch, +} + +#[derive(PartialEq, Debug, Copy, Clone)] +pub enum Collector { + Auto, + Null, + DeferredReferenceCounting, +} + +impl WastTest { + /// Returns whether this test exercises the GC types and might want to use + /// multiple different garbage collectors. + pub fn test_uses_gc_types(&self) -> bool { + self.config + .gc + .or(self.config.function_references) + .unwrap_or(false) + } + + /// Returns the optional spec proposal that this test is associated with. + pub fn spec_proposal(&self) -> Option<&str> { + spec_proposal_from_path(&self.path) + } + + /// Returns whether this test should fail under the specified extra + /// configuration. + pub fn should_fail(&self, config: &WastConfig) -> bool { + // Winch only supports x86_64 at this time. + if config.compiler == Compiler::Winch && !cfg!(target_arch = "x86_64") { + return true; + } + + // Disable spec tests for proposals that Winch does not implement yet. + if config.compiler == Compiler::Winch { + // A few proposals that winch has no support for. + if self.config.gc == Some(true) + || self.config.threads == Some(true) + || self.config.tail_call == Some(true) + || self.config.function_references == Some(true) + || self.config.gc == Some(true) + || self.config.relaxed_simd == Some(true) + { + return true; + } + + let unsupported = [ + // externref/reference-types related + "component-model/modules.wast", + "extended-const/elem.wast", + "extended-const/global.wast", + "memory64/threads.wast", + "misc_testsuite/externref-id-function.wast", + "misc_testsuite/externref-segment.wast", + "misc_testsuite/externref-segments.wast", + "misc_testsuite/externref-table-dropped-segment-issue-8281.wast", + "misc_testsuite/linking-errors.wast", + "misc_testsuite/many_table_gets_lead_to_gc.wast", + "misc_testsuite/mutable_externref_globals.wast", + "misc_testsuite/no-mixup-stack-maps.wast", + "misc_testsuite/no-panic.wast", + "misc_testsuite/simple_ref_is_null.wast", + "misc_testsuite/table_grow_with_funcref.wast", + "spec_testsuite/br_table.wast", + "spec_testsuite/data-invalid.wast", + "spec_testsuite/elem.wast", + "spec_testsuite/global.wast", + "spec_testsuite/linking.wast", + "spec_testsuite/ref_func.wast", + "spec_testsuite/ref_is_null.wast", + "spec_testsuite/ref_null.wast", + "spec_testsuite/select.wast", + "spec_testsuite/table-sub.wast", + "spec_testsuite/table_fill.wast", + "spec_testsuite/table_get.wast", + "spec_testsuite/table_grow.wast", + "spec_testsuite/table_set.wast", + "spec_testsuite/table_size.wast", + "spec_testsuite/unreached-invalid.wast", + "spec_testsuite/call_indirect.wast", + // simd-related failures + "annotations/simd_lane.wast", + "memory64/simd.wast", + "misc_testsuite/int-to-float-splat.wast", + "misc_testsuite/issue6562.wast", + "misc_testsuite/simd/almost-extmul.wast", + "misc_testsuite/simd/canonicalize-nan.wast", + "misc_testsuite/simd/cvt-from-uint.wast", + "misc_testsuite/simd/issue4807.wast", + "misc_testsuite/simd/issue6725-no-egraph-panic.wast", + "misc_testsuite/simd/issue_3327_bnot_lowering.wast", + "misc_testsuite/simd/load_splat_out_of_bounds.wast", + "misc_testsuite/simd/replace-lane-preserve.wast", + "misc_testsuite/simd/spillslot-size-fuzzbug.wast", + "misc_testsuite/simd/unaligned-load.wast", + "multi-memory/simd_memory-multi.wast", + "spec_testsuite/simd_align.wast", + "spec_testsuite/simd_bit_shift.wast", + "spec_testsuite/simd_bitwise.wast", + "spec_testsuite/simd_boolean.wast", + "spec_testsuite/simd_const.wast", + "spec_testsuite/simd_conversions.wast", + "spec_testsuite/simd_f32x4.wast", + "spec_testsuite/simd_f32x4_arith.wast", + "spec_testsuite/simd_f32x4_cmp.wast", + "spec_testsuite/simd_f32x4_pmin_pmax.wast", + "spec_testsuite/simd_f32x4_rounding.wast", + "spec_testsuite/simd_f64x2.wast", + "spec_testsuite/simd_f64x2_arith.wast", + "spec_testsuite/simd_f64x2_cmp.wast", + "spec_testsuite/simd_f64x2_pmin_pmax.wast", + "spec_testsuite/simd_f64x2_rounding.wast", + "spec_testsuite/simd_i16x8_arith.wast", + "spec_testsuite/simd_i16x8_arith2.wast", + "spec_testsuite/simd_i16x8_cmp.wast", + "spec_testsuite/simd_i16x8_extadd_pairwise_i8x16.wast", + "spec_testsuite/simd_i16x8_extmul_i8x16.wast", + "spec_testsuite/simd_i16x8_q15mulr_sat_s.wast", + "spec_testsuite/simd_i16x8_sat_arith.wast", + "spec_testsuite/simd_i32x4_arith.wast", + "spec_testsuite/simd_i32x4_arith2.wast", + "spec_testsuite/simd_i32x4_cmp.wast", + "spec_testsuite/simd_i32x4_dot_i16x8.wast", + "spec_testsuite/simd_i32x4_extadd_pairwise_i16x8.wast", + "spec_testsuite/simd_i32x4_extmul_i16x8.wast", + "spec_testsuite/simd_i32x4_trunc_sat_f32x4.wast", + "spec_testsuite/simd_i32x4_trunc_sat_f64x2.wast", + "spec_testsuite/simd_i64x2_arith.wast", + "spec_testsuite/simd_i64x2_arith2.wast", + "spec_testsuite/simd_i64x2_cmp.wast", + "spec_testsuite/simd_i64x2_extmul_i32x4.wast", + "spec_testsuite/simd_i8x16_arith.wast", + "spec_testsuite/simd_i8x16_arith2.wast", + "spec_testsuite/simd_i8x16_cmp.wast", + "spec_testsuite/simd_i8x16_sat_arith.wast", + "spec_testsuite/simd_int_to_int_extend.wast", + "spec_testsuite/simd_lane.wast", + "spec_testsuite/simd_load.wast", + "spec_testsuite/simd_load16_lane.wast", + "spec_testsuite/simd_load32_lane.wast", + "spec_testsuite/simd_load64_lane.wast", + "spec_testsuite/simd_load8_lane.wast", + "spec_testsuite/simd_load_extend.wast", + "spec_testsuite/simd_load_splat.wast", + "spec_testsuite/simd_load_zero.wast", + "spec_testsuite/simd_splat.wast", + "spec_testsuite/simd_store16_lane.wast", + "spec_testsuite/simd_store32_lane.wast", + "spec_testsuite/simd_store64_lane.wast", + "spec_testsuite/simd_store8_lane.wast", + ]; + + if unsupported.iter().any(|part| self.path.ends_with(part)) { + return true; + } + } + + for part in self.path.iter() { + // Not implemented in Wasmtime yet + if part == "exception-handling" { + return !self.path.ends_with("binary.wast"); + } + + if part == "memory64" { + if [ + // wasmtime doesn't implement exceptions yet + "imports.wast", + "ref_null.wast", + "exports.wast", + "throw.wast", + "throw_ref.wast", + "try_table.wast", + "tag.wast", + "instance.wast", + ] + .iter() + .any(|i| self.path.ends_with(i)) + { + return true; + } + } + } + + // Some tests are known to fail with the pooling allocator + if config.pooling { + let unsupported = [ + // allocates too much memory for the pooling configuration here + "misc_testsuite/memory64/more-than-4gb.wast", + // shared memories + pooling allocator aren't supported yet + "misc_testsuite/memory-combos.wast", + "misc_testsuite/threads/LB.wast", + "misc_testsuite/threads/LB_atomic.wast", + "misc_testsuite/threads/MP.wast", + "misc_testsuite/threads/MP_atomic.wast", + "misc_testsuite/threads/MP_wait.wast", + "misc_testsuite/threads/SB.wast", + "misc_testsuite/threads/SB_atomic.wast", + "misc_testsuite/threads/atomics_notify.wast", + "misc_testsuite/threads/atomics_wait_address.wast", + "misc_testsuite/threads/wait_notify.wast", + "spec_testsuite/proposals/threads/atomic.wast", + "spec_testsuite/proposals/threads/exports.wast", + "spec_testsuite/proposals/threads/memory.wast", + ]; + + if unsupported.iter().any(|part| self.path.ends_with(part)) { + return true; + } + } + + false + } +} + +fn spec_proposal_from_path(path: &Path) -> Option<&str> { + let mut iter = path.iter(); + loop { + match iter.next()?.to_str()? { + "proposals" => break, + _ => {} + } + } + Some(iter.next()?.to_str()?) +} diff --git a/tests/disas.rs b/tests/disas.rs index 8dabf786fd4c..4494ac6e7188 100644 --- a/tests/disas.rs +++ b/tests/disas.rs @@ -55,8 +55,6 @@ use tempfile::TempDir; use wasmtime::{Engine, OptLevel, Strategy}; use wasmtime_cli_flags::CommonOptions; -mod support; - fn main() -> Result<()> { if cfg!(miri) { return Ok(()); @@ -153,7 +151,7 @@ impl Test { fn new(path: &Path) -> Result { let contents = std::fs::read_to_string(path).with_context(|| format!("failed to read {path:?}"))?; - let config: TestConfig = support::parse_test_config(&contents) + let config: TestConfig = wasmtime_wast_util::parse_test_config(&contents) .context("failed to parse test configuration as TOML")?; let mut flags = vec!["wasmtime"]; match &config.flags { diff --git a/tests/misc_testsuite/memory64/more-than-4gb.wast b/tests/misc_testsuite/memory64/more-than-4gb.wast index 3a23af02a600..8c7f4057ba2e 100644 --- a/tests/misc_testsuite/memory64/more-than-4gb.wast +++ b/tests/misc_testsuite/memory64/more-than-4gb.wast @@ -1,4 +1,5 @@ ;;! memory64 = true +;;! hogs_memory = true ;; try to create as few 4gb memories as we can to reduce the memory consumption ;; of this test, so create one up front here and use it below. diff --git a/tests/support/mod.rs b/tests/support/mod.rs deleted file mode 100644 index 10d071d4fab1..000000000000 --- a/tests/support/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -use anyhow::{Context, Result}; -use serde::de::DeserializeOwned; - -/// Parse test configuration from the specified test, comments starting with -/// `;;!`. -pub fn parse_test_config(wat: &str) -> Result -where - T: DeserializeOwned, -{ - // The test config source is the leading lines of the WAT file that are - // prefixed with `;;!`. - let config_lines: Vec<_> = wat - .lines() - .take_while(|l| l.starts_with(";;!")) - .map(|l| &l[3..]) - .collect(); - let config_text = config_lines.join("\n"); - - toml::from_str(&config_text).context("failed to parse the test configuration") -} diff --git a/tests/wast.rs b/tests/wast.rs index 3f56eea02627..e7ff317849a7 100644 --- a/tests/wast.rs +++ b/tests/wast.rs @@ -1,58 +1,29 @@ use anyhow::{bail, Context}; use libtest_mimic::{Arguments, FormatSetting, Trial}; -use serde_derive::Deserialize; -use std::path::Path; use std::sync::{Condvar, LazyLock, Mutex}; use wasmtime::{ - Collector, Config, Engine, InstanceAllocationStrategy, MpkEnabled, PoolingAllocationConfig, - Store, Strategy, + Config, Engine, InstanceAllocationStrategy, MpkEnabled, PoolingAllocationConfig, Store, }; use wasmtime_environ::Memory; use wasmtime_wast::{SpectestConfig, WastContext}; - -mod support; +use wasmtime_wast_util::{limits, Collector, Compiler, WastConfig, WastTest}; fn main() { env_logger::init(); - let mut trials = Vec::new(); - if !cfg!(miri) { - add_tests(&mut trials, "tests/spec_testsuite".as_ref()); - add_tests(&mut trials, "tests/misc_testsuite".as_ref()); - } - - // There's a lot of tests so print only a `.` to keep the output a - // bit more terse by default. - let mut args = Arguments::from_args(); - if args.format.is_none() { - args.format = Some(FormatSetting::Terse); - } - libtest_mimic::run(&args, trials).exit() -} - -fn add_tests(trials: &mut Vec, path: &Path) { - for entry in path.read_dir().unwrap() { - let entry = entry.unwrap(); - let path = entry.path(); - if entry.file_type().unwrap().is_dir() { - add_tests(trials, &path); - continue; - } - - if path.extension().and_then(|s| s.to_str()) != Some("wast") { - continue; - } + let tests = if cfg!(miri) { + Vec::new() + } else { + wasmtime_wast_util::find_tests(".".as_ref()).unwrap() + }; - let test_uses_gc_types = path.iter().any(|part| { - part.to_str().map_or(false, |s| { - s.contains("gc") - || s.contains("function-references") - || s.contains("reference-types") - || s.contains("exception-handling") - }) - }); + let mut trials = Vec::new(); - for strategy in [Strategy::Cranelift, Strategy::Winch] { + // For each test generate a combinatorial matrix of all configurations to + // run this test in. + for test in tests { + let test_uses_gc_types = test.test_uses_gc_types(); + for compiler in [Compiler::Cranelift, Compiler::Winch] { for pooling in [true, false] { let collectors: &[_] = if !pooling && test_uses_gc_types { &[Collector::DeferredReferenceCounting, Collector::Null] @@ -63,22 +34,22 @@ fn add_tests(trials: &mut Vec, path: &Path) { for collector in collectors.iter().copied() { let trial = Trial::test( format!( - "{strategy:?}/{}{}{}", + "{compiler:?}/{}{}{}", if pooling { "pooling/" } else { "" }, if collector != Collector::Auto { format!("{collector:?}/") } else { String::new() }, - path.to_str().unwrap() + test.path.to_str().unwrap() ), { - let path = path.clone(); + let test = test.clone(); move || { run_wast( - &path, + &test, WastConfig { - strategy, + compiler, pooling, collector, }, @@ -92,306 +63,21 @@ fn add_tests(trials: &mut Vec, path: &Path) { } } } -} - -fn should_fail(test: &Path, wast_config: &WastConfig, test_config: &TestConfig) -> bool { - // Winch only supports x86_64 at this time. - if wast_config.strategy == Strategy::Winch && !cfg!(target_arch = "x86_64") { - return true; - } - - // Disable spec tests for proposals that Winch does not implement yet. - if wast_config.strategy == Strategy::Winch { - // A few proposals that winch has no support for. - if test_config.gc == Some(true) - || test_config.threads == Some(true) - || test_config.tail_call == Some(true) - || test_config.function_references == Some(true) - || test_config.gc == Some(true) - || test_config.relaxed_simd == Some(true) - { - return true; - } - - let unsupported = [ - // externref/reference-types related - "component-model/modules.wast", - "extended-const/elem.wast", - "extended-const/global.wast", - "memory64/threads.wast", - "misc_testsuite/externref-id-function.wast", - "misc_testsuite/externref-segment.wast", - "misc_testsuite/externref-segments.wast", - "misc_testsuite/externref-table-dropped-segment-issue-8281.wast", - "misc_testsuite/linking-errors.wast", - "misc_testsuite/many_table_gets_lead_to_gc.wast", - "misc_testsuite/mutable_externref_globals.wast", - "misc_testsuite/no-mixup-stack-maps.wast", - "misc_testsuite/no-panic.wast", - "misc_testsuite/simple_ref_is_null.wast", - "misc_testsuite/table_grow_with_funcref.wast", - "spec_testsuite/br_table.wast", - "spec_testsuite/data-invalid.wast", - "spec_testsuite/elem.wast", - "spec_testsuite/global.wast", - "spec_testsuite/linking.wast", - "spec_testsuite/ref_func.wast", - "spec_testsuite/ref_is_null.wast", - "spec_testsuite/ref_null.wast", - "spec_testsuite/select.wast", - "spec_testsuite/table-sub.wast", - "spec_testsuite/table_fill.wast", - "spec_testsuite/table_get.wast", - "spec_testsuite/table_grow.wast", - "spec_testsuite/table_set.wast", - "spec_testsuite/table_size.wast", - "spec_testsuite/unreached-invalid.wast", - "spec_testsuite/call_indirect.wast", - // simd-related failures - "annotations/simd_lane.wast", - "memory64/simd.wast", - "misc_testsuite/int-to-float-splat.wast", - "misc_testsuite/issue6562.wast", - "misc_testsuite/simd/almost-extmul.wast", - "misc_testsuite/simd/canonicalize-nan.wast", - "misc_testsuite/simd/cvt-from-uint.wast", - "misc_testsuite/simd/issue4807.wast", - "misc_testsuite/simd/issue6725-no-egraph-panic.wast", - "misc_testsuite/simd/issue_3327_bnot_lowering.wast", - "misc_testsuite/simd/load_splat_out_of_bounds.wast", - "misc_testsuite/simd/replace-lane-preserve.wast", - "misc_testsuite/simd/spillslot-size-fuzzbug.wast", - "misc_testsuite/simd/unaligned-load.wast", - "multi-memory/simd_memory-multi.wast", - "spec_testsuite/simd_align.wast", - "spec_testsuite/simd_bit_shift.wast", - "spec_testsuite/simd_bitwise.wast", - "spec_testsuite/simd_boolean.wast", - "spec_testsuite/simd_const.wast", - "spec_testsuite/simd_conversions.wast", - "spec_testsuite/simd_f32x4.wast", - "spec_testsuite/simd_f32x4_arith.wast", - "spec_testsuite/simd_f32x4_cmp.wast", - "spec_testsuite/simd_f32x4_pmin_pmax.wast", - "spec_testsuite/simd_f32x4_rounding.wast", - "spec_testsuite/simd_f64x2.wast", - "spec_testsuite/simd_f64x2_arith.wast", - "spec_testsuite/simd_f64x2_cmp.wast", - "spec_testsuite/simd_f64x2_pmin_pmax.wast", - "spec_testsuite/simd_f64x2_rounding.wast", - "spec_testsuite/simd_i16x8_arith.wast", - "spec_testsuite/simd_i16x8_arith2.wast", - "spec_testsuite/simd_i16x8_cmp.wast", - "spec_testsuite/simd_i16x8_extadd_pairwise_i8x16.wast", - "spec_testsuite/simd_i16x8_extmul_i8x16.wast", - "spec_testsuite/simd_i16x8_q15mulr_sat_s.wast", - "spec_testsuite/simd_i16x8_sat_arith.wast", - "spec_testsuite/simd_i32x4_arith.wast", - "spec_testsuite/simd_i32x4_arith2.wast", - "spec_testsuite/simd_i32x4_cmp.wast", - "spec_testsuite/simd_i32x4_dot_i16x8.wast", - "spec_testsuite/simd_i32x4_extadd_pairwise_i16x8.wast", - "spec_testsuite/simd_i32x4_extmul_i16x8.wast", - "spec_testsuite/simd_i32x4_trunc_sat_f32x4.wast", - "spec_testsuite/simd_i32x4_trunc_sat_f64x2.wast", - "spec_testsuite/simd_i64x2_arith.wast", - "spec_testsuite/simd_i64x2_arith2.wast", - "spec_testsuite/simd_i64x2_cmp.wast", - "spec_testsuite/simd_i64x2_extmul_i32x4.wast", - "spec_testsuite/simd_i8x16_arith.wast", - "spec_testsuite/simd_i8x16_arith2.wast", - "spec_testsuite/simd_i8x16_cmp.wast", - "spec_testsuite/simd_i8x16_sat_arith.wast", - "spec_testsuite/simd_int_to_int_extend.wast", - "spec_testsuite/simd_lane.wast", - "spec_testsuite/simd_load.wast", - "spec_testsuite/simd_load16_lane.wast", - "spec_testsuite/simd_load32_lane.wast", - "spec_testsuite/simd_load64_lane.wast", - "spec_testsuite/simd_load8_lane.wast", - "spec_testsuite/simd_load_extend.wast", - "spec_testsuite/simd_load_splat.wast", - "spec_testsuite/simd_load_zero.wast", - "spec_testsuite/simd_splat.wast", - "spec_testsuite/simd_store16_lane.wast", - "spec_testsuite/simd_store32_lane.wast", - "spec_testsuite/simd_store64_lane.wast", - "spec_testsuite/simd_store8_lane.wast", - ]; - - if unsupported.iter().any(|part| test.ends_with(part)) { - return true; - } - } - - for part in test.iter() { - // Not implemented in Wasmtime yet - if part == "exception-handling" { - return !test.ends_with("binary.wast"); - } - - if part == "memory64" { - if [ - // wasmtime doesn't implement exceptions yet - "imports.wast", - "ref_null.wast", - "exports.wast", - "throw.wast", - "throw_ref.wast", - "try_table.wast", - "tag.wast", - "instance.wast", - ] - .iter() - .any(|i| test.ends_with(i)) - { - return true; - } - } - } - // Some tests are known to fail with the pooling allocator - if wast_config.pooling { - let unsupported = [ - // allocates too much memory for the pooling configuration here - "misc_testsuite/memory64/more-than-4gb.wast", - // shared memories + pooling allocator aren't supported yet - "misc_testsuite/memory-combos.wast", - "misc_testsuite/threads/LB.wast", - "misc_testsuite/threads/LB_atomic.wast", - "misc_testsuite/threads/MP.wast", - "misc_testsuite/threads/MP_atomic.wast", - "misc_testsuite/threads/MP_wait.wast", - "misc_testsuite/threads/SB.wast", - "misc_testsuite/threads/SB_atomic.wast", - "misc_testsuite/threads/atomics_notify.wast", - "misc_testsuite/threads/atomics_wait_address.wast", - "misc_testsuite/threads/wait_notify.wast", - "spec_testsuite/proposals/threads/atomic.wast", - "spec_testsuite/proposals/threads/exports.wast", - "spec_testsuite/proposals/threads/memory.wast", - ]; - - if unsupported.iter().any(|part| test.ends_with(part)) { - return true; - } - } - - false -} - -/// Configuration where the main function will generate a combinatorial -/// matrix of these top-level configurations to run the entire test suite with -/// that configuration. -struct WastConfig { - strategy: Strategy, - pooling: bool, - collector: Collector, -} - -/// Per-test configuration which is written down in the test file itself for -/// `misc_testsuite/**/*.wast` or in `spec_test_config` below for spec tests. -#[derive(Debug, PartialEq, Default, Deserialize)] -#[serde(deny_unknown_fields)] -struct TestConfig { - memory64: Option, - custom_page_sizes: Option, - multi_memory: Option, - threads: Option, - gc: Option, - function_references: Option, - relaxed_simd: Option, - reference_types: Option, - tail_call: Option, - extended_const: Option, - wide_arithmetic: Option, - hogs_memory: Option, - nan_canonicalization: Option, - component_model_more_flags: Option, -} - -fn spec_test_config(wast: &Path) -> TestConfig { - let mut ret = TestConfig::default(); - - match wast.strip_prefix("proposals") { - // This lists the features require to run the various spec tests suites - // in their `proposals` folder. - Ok(rest) => { - let proposal = rest.iter().next().unwrap().to_str().unwrap(); - match proposal { - "multi-memory" => { - ret.multi_memory = Some(true); - ret.reference_types = Some(true); - } - "wide-arithmetic" => { - ret.wide_arithmetic = Some(true); - } - "threads" => { - ret.threads = Some(true); - ret.reference_types = Some(false); - } - "tail-call" => { - ret.tail_call = Some(true); - ret.reference_types = Some(true); - } - "relaxed-simd" => { - ret.relaxed_simd = Some(true); - } - "memory64" => { - ret.memory64 = Some(true); - ret.tail_call = Some(true); - ret.gc = Some(true); - ret.extended_const = Some(true); - ret.multi_memory = Some(true); - ret.relaxed_simd = Some(true); - } - "extended-const" => { - ret.extended_const = Some(true); - ret.reference_types = Some(true); - } - "custom-page-sizes" => { - ret.custom_page_sizes = Some(true); - ret.multi_memory = Some(true); - } - "exception-handling" => { - ret.reference_types = Some(true); - } - "gc" => { - ret.gc = Some(true); - ret.tail_call = Some(true); - } - "function-references" => { - ret.function_references = Some(true); - ret.tail_call = Some(true); - } - "annotations" => {} - _ => panic!("unsuported proposal {proposal:?}"), - } - } - - // This lists the features required to run the top-level of spec tests - // outside of the `proposals` directory. - Err(_) => { - ret.reference_types = Some(true); - } + // There's a lot of tests so print only a `.` to keep the output a + // bit more terse by default. + let mut args = Arguments::from_args(); + if args.format.is_none() { + args.format = Some(FormatSetting::Terse); } - ret + libtest_mimic::run(&args, trials).exit() } // Each of the tests included from `wast_testsuite_tests` will call this // function which actually executes the `wast` test suite given the `strategy` // to compile it. -fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { - let wast_contents = std::fs::read_to_string(wast) - .with_context(|| format!("failed to read `{}`", wast.display()))?; - - // If this is a spec test then the configuration for it is loaded via - // `spec_test_config`, but otherwise it's required to be listed in the top - // of the file as we control the contents of the file. - let mut test_config = match wast.strip_prefix("tests/spec_testsuite") { - Ok(test) => spec_test_config(test), - Err(_) => support::parse_test_config(&wast_contents)?, - }; +fn run_wast(test: &WastTest, config: WastConfig) -> anyhow::Result<()> { + let mut test_config = test.config.clone(); // FIXME: this is a bit of a hack to get Winch working here for now. Winch // passes some tests on aarch64 so returning `true` from `should_fail` @@ -410,9 +96,14 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { test_config.reference_types = Some(true); } - let should_fail = should_fail(wast, &config, &test_config); - - let wast = Path::new(wast); + // Determine whether this test is expected to fail or pass. Regardless the + // test is executed and the result of the execution is asserted to match + // this expectation. Note that this means that the test can't, for example, + // panic or segfault as a result. + // + // Updates to whether a test should pass or fail should be done in the + // `crates/wast-util/src/lib.rs` file. + let should_fail = test.should_fail(&config); // Note that all of these proposals/features are currently default-off to // ensure that we annotate all tests accurately with what features they @@ -445,8 +136,8 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { .or(test_config.gc) .unwrap_or(false); - let is_cranelift = match config.strategy { - Strategy::Cranelift => true, + let is_cranelift = match config.compiler { + Compiler::Cranelift => true, _ => false, }; @@ -463,8 +154,15 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { .wasm_extended_const(extended_const) .wasm_wide_arithmetic(wide_arithmetic) .wasm_component_model_more_flags(component_model_more_flags) - .strategy(config.strategy) - .collector(config.collector) + .strategy(match config.compiler { + Compiler::Cranelift => wasmtime::Strategy::Cranelift, + Compiler::Winch => wasmtime::Strategy::Winch, + }) + .collector(match config.collector { + Collector::Auto => wasmtime::Collector::Auto, + Collector::Null => wasmtime::Collector::Null, + Collector::DeferredReferenceCounting => wasmtime::Collector::DeferredReferenceCounting, + }) .cranelift_nan_canonicalization(nan_canonicalization); if is_cranelift { @@ -521,24 +219,23 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { // When multiple memories are used and are configured in the pool then // force the usage of static memories without guards to reduce the VM // impact. - let max_memory_size = 805 << 16; + let max_memory_size = limits::MEMORY_SIZE; if multi_memory { cfg.memory_reservation(max_memory_size as u64); cfg.memory_reservation_for_growth(0); cfg.memory_guard_size(0); } - // The limits here are crafted such that the wast tests should pass. - // However, these limits may become insufficient in the future as the - // wast tests change. If a wast test fails because of a limit being - // "exceeded" or if memory/table fails to grow, the values here will - // need to be adjusted. let mut pool = PoolingAllocationConfig::default(); - pool.total_memories(450 * 2) + pool.total_memories(limits::MEMORIES * 2) .max_memory_protection_keys(2) .max_memory_size(max_memory_size) - .max_memories_per_module(if multi_memory { 9 } else { 1 }) - .max_tables_per_module(5); + .max_memories_per_module(if multi_memory { + limits::MEMORIES_PER_MODULE + } else { + 1 + }) + .max_tables_per_module(limits::TABLES_PER_MODULE); // When testing, we may choose to start with MPK force-enabled to ensure // we use that functionality. @@ -573,7 +270,7 @@ fn run_wast(wast: &Path, config: WastConfig) -> anyhow::Result<()> { suppress_prints: true, })?; wast_context - .run_buffer(wast.to_str().unwrap(), wast_contents.as_bytes()) + .run_buffer(test.path.to_str().unwrap(), test.contents.as_bytes()) .with_context(|| format!("failed to run spec test with {desc} engine")) });