Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Parse WASM code only once for JIT #18

Merged
merged 4 commits into from
Aug 7, 2023
Merged

Parse WASM code only once for JIT #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0e694a9
parse WASM code only once for JIT
linh2931 Jul 27, 2023
c800ac7
use more constexpr and template type
linh2931 Jul 27, 2023
e88fd38
make code more concise and reserve vector size before use
linh2931 Jul 31, 2023
c28f152
remove reversed == operator as it is compiled into recursive function…
linh2931 Aug 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 19 additions & 25 deletions include/eosio/vm/allocator.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,35 +294,22 @@ namespace eosio { namespace vm {
// uses mmap for big memory allocation
heifner marked this conversation as resolved.
Show resolved Hide resolved
_base = (char*)mmap(NULL, max_memory_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "failed to mmap for default memory.");
_mmap_used = true;
_capacity = max_memory_size;
}

// size in bytes
void use_fixed_memory(bool is_jit, size_t size) {
void use_fixed_memory(size_t size) {
EOS_VM_ASSERT(0 < size && size <= max_memory_size, wasm_bad_alloc, "Too large or 0 fixed memory size");
EOS_VM_ASSERT(_base == nullptr, wasm_bad_alloc, "Fixed memory already allocated");

_is_jit = is_jit;
if (_is_jit) {
_base = (char*)std::calloc(size, sizeof(char));
EOS_VM_ASSERT(_base != nullptr, wasm_bad_alloc, "malloc in use_fixed_memory failed.");
_mmap_used = false;
} else {
_base = (char*)mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "mmap in use_fixed_memoryfailed.");
_mmap_used = true;
}
_base = (char*)mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "mmap in use_fixed_memory failed.");
_capacity = size;
}

~growable_allocator() {
if (_base != nullptr) {
if (_mmap_used) {
munmap(_base, _capacity);
} else {
std::free(_base);
}
munmap(_base, _capacity);
}
if (_is_jit && _code_base) {
jit_allocator::instance().free(_code_base);
Expand Down Expand Up @@ -373,11 +360,6 @@ namespace eosio { namespace vm {
}
}

void set_code_base_and_size(char* code_base, size_t code_size) {
_code_base = code_base;
_code_size = code_size;
}

// Sets protection on code pages to allow them to be executed.
void enable_code(bool is_jit) {
mprotect(_code_base, _code_size, is_jit?PROT_EXEC:(PROT_READ|PROT_WRITE));
Expand Down Expand Up @@ -409,18 +391,31 @@ namespace eosio { namespace vm {
* Finalize the memory by unmapping any excess pages, this means that the allocator will no longer grow
*/
void finalize() {
if(_mmap_used && _capacity != _offset) {
if(_capacity != _offset) {
std::size_t final_size = align_to_page(_offset);
if (final_size < _capacity) { // final_size can grow to _capacity after align_to_page.
// make sure no 0 size passed to munmap
EOS_VM_ASSERT(munmap(_base + final_size, _capacity - final_size) == 0, wasm_bad_alloc, "failed to finalize growable_allocator");
_capacity = _offset = final_size;
if (final_size == 0) {
// _base became invalid after munmap if final_size is 0 so
// set it to nullptr to avoid being used
_base = nullptr;
}
}
_capacity = _offset = final_size;
}
}

void free() { EOS_VM_ASSERT(false, wasm_bad_alloc, "unimplemented"); }

void release_base_memory()
{
if (_base != nullptr) {
EOS_VM_ASSERT(munmap(_base, _capacity) == 0, wasm_bad_alloc, "failed to release base memory");
_base = nullptr;
}
}

void reset() { _offset = 0; }

size_t _offset = 0;
Expand All @@ -430,7 +425,6 @@ namespace eosio { namespace vm {
char* _code_base = nullptr;
size_t _code_size = 0;
bool _is_jit = false;
bool _mmap_used = false;
};

template <typename T>
Expand Down
55 changes: 28 additions & 27 deletions include/eosio/vm/backend.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,14 @@ namespace eosio { namespace vm {
void construct(host_t* host=nullptr) {
mod.finalize();
ctx.set_wasm_allocator(memory_alloc);
// Now data required by JIT is finalized; create JIT module
// such that memory used in parsing can be released.
if constexpr (Impl::is_jit) {
mod.make_jit_module();

// Important. Release the memory used by parsing.
mod.allocator.release_base_memory();
}
ctx.initialize_globals();
if constexpr (!std::is_same_v<HostFunctions, std::nullptr_t>)
HostFunctions::resolve(mod);
Expand Down Expand Up @@ -101,7 +109,8 @@ namespace eosio { namespace vm {
}
// Leap:
// * Contract validation only needs single parsing as the instantiated module is not cached.
// * Contract execution requires two-passes parsing to prevent memory mappings exhaustion
// * JIT execution needs single parsing only.
// * Interpreter execution requires two-passes parsing to prevent memory mappings exhaustion
backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true)
: memory_alloc(alloc), ctx(parse_module2(ptr, sz, options, single_parsing), detail::get_max_call_depth(options)) {
ctx.set_max_pages(detail::get_max_pages(options));
Expand All @@ -117,34 +126,26 @@ namespace eosio { namespace vm {
if (single_parsing) {
mod.allocator.use_default_memory();
return parser_t{ mod.allocator, options }.parse_module2(ptr, sz, mod, debug);
}
} else {
// To prevent large number of memory mappings used, two-passes of
// parsing are performed.
wasm_code_ptr orig_ptr = ptr;
size_t largest_size = 0;

// First pass: finds max size of memory required by parsing.
{
// Memory used by this pass is freed when going out of the scope
module first_pass_module;
first_pass_module.allocator.use_default_memory();
parser_t{ first_pass_module.allocator, options }.parse_module2(ptr, sz, first_pass_module, debug);
first_pass_module.finalize();
largest_size = first_pass_module.allocator.largest_used_size();
}

// To prevent large number of memory mappings used, use two-passes parsing.
// The first pass finds max size of memory required for parsing;
// this memory is released after parsing.
// The second pass uses malloc with the required size of memory.
wasm_code_ptr orig_ptr = ptr;
size_t largest_size = 0;

// First pass: finds max size of memory required by parsing.
// Memory used by parsing will be freed when going out of the scope
{
module first_pass_module;
// For JIT, skips code generation as it is not needed and
// does not count the code memory size
detail::code_generate_mode code_gen_mode = Impl::is_jit ? detail::code_generate_mode::skip : detail::code_generate_mode::use_same_allocator;
first_pass_module.allocator.use_default_memory();
parser_t{ first_pass_module.allocator, options }.parse_module2(ptr, sz, first_pass_module, debug, code_gen_mode);
first_pass_module.finalize();
largest_size = first_pass_module.allocator.largest_used_size();
// Second pass: uses actual required memory for final parsing
mod.allocator.use_fixed_memory(largest_size);
return parser_t{ mod.allocator, options }.parse_module2(orig_ptr, sz, mod, debug);
}

// Second pass: uses largest_size of memory for actual parsing
mod.allocator.use_fixed_memory(Impl::is_jit, largest_size);
// For JIT, uses a seperate allocator for code generation as mod's memory
// does not include memory for code
detail::code_generate_mode code_gen_mode = Impl::is_jit ? detail::code_generate_mode::use_seperate_allocator : detail::code_generate_mode::use_same_allocator;
return parser_t{ mod.allocator, options }.parse_module2(orig_ptr, sz, mod, debug, code_gen_mode);
}

template <typename... Args>
Expand Down
86 changes: 53 additions & 33 deletions include/eosio/vm/execution_context.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,30 +80,49 @@ namespace eosio { namespace vm {
using host_invoker_t = typename host_invoker<HF>::type;
}

template<typename Derived, typename Host>
template<typename Derived, typename Host, bool IsJit>
class execution_context_base {
using host_type = detail::host_type_t<Host>;
public:
Derived& derived() { return static_cast<Derived&>(*this); }
execution_context_base(module& m) : _mod(m) {}

inline void initialize_globals() {
if constexpr (IsJit) {
return initialize_globals_impl(*_mod.jit_mod);
}
else {
return initialize_globals_impl(_mod);
}
}

template<typename Module>
inline void initialize_globals_impl(const Module& mod) {
EOS_VM_ASSERT(_globals.empty(), wasm_memory_exception, "initialize_globals called on non-empty _globals");
_globals.reserve(_mod.globals.size());
for (uint32_t i = 0; i < _mod.globals.size(); i++) {
_globals.emplace_back(_mod.globals[i].init);
_globals.reserve(mod.globals.size());
for (uint32_t i = 0; i < mod.globals.size(); i++) {
_globals.emplace_back(mod.globals[i].init);
}
}

inline int32_t grow_linear_memory(int32_t pages) {
if constexpr (IsJit) {
return grow_linear_memory_impl(*_mod.jit_mod, pages);
} else {
return grow_linear_memory_impl(_mod, pages);
}
}

template<typename Module>
inline int32_t grow_linear_memory_impl(const Module& mod, int32_t pages) {
const int32_t sz = _wasm_alloc->get_current_page();
if (pages < 0) {
if (sz + pages < 0)
return -1;
_wasm_alloc->free<char>(-pages);
} else {
if (!_mod.memories.size() || _max_pages - sz < static_cast<uint32_t>(pages) ||
(_mod.memories[0].limits.flags && (static_cast<int32_t>(_mod.memories[0].limits.maximum) - sz < pages)))
if (!mod.memories.size() || _max_pages - sz < static_cast<uint32_t>(pages) ||
(mod.memories[0].limits.flags && (static_cast<int32_t>(mod.memories[0].limits.maximum) - sz < pages)))
return -1;
_wasm_alloc->alloc<char>(pages);
}
Expand All @@ -128,35 +147,36 @@ namespace eosio { namespace vm {

inline std::error_code get_error_code() const { return _error_code; }

inline void reset() {
template<typename Module>
inline void reset(Module& mod) {
EOS_VM_ASSERT(_mod.error == nullptr, wasm_interpreter_exception, _mod.error);

// Reset the capacity of underlying memory used by operand stack if it is
// greater than initial_stack_size
_os.reset_capacity();

_linear_memory = _wasm_alloc->get_base_ptr<char>();
if(_mod.memories.size()) {
EOS_VM_ASSERT(_mod.memories[0].limits.initial <= _max_pages, wasm_bad_alloc, "Cannot allocate initial linear memory.");
_wasm_alloc->reset(_mod.memories[0].limits.initial);
if(mod.memories.size()) {
EOS_VM_ASSERT(mod.memories[0].limits.initial <= _max_pages, wasm_bad_alloc, "Cannot allocate initial linear memory.");
_wasm_alloc->reset(mod.memories[0].limits.initial);
} else
_wasm_alloc->reset();

for (uint32_t i = 0; i < _mod.data.size(); i++) {
const auto& data_seg = _mod.data[i];
for (uint32_t i = 0; i < mod.data.size(); i++) {
const auto& data_seg = mod.data[i];
uint32_t offset = data_seg.offset.value.i32; // force to unsigned
auto available_memory = _mod.memories[0].limits.initial * static_cast<uint64_t>(page_size);
auto available_memory = mod.memories[0].limits.initial * static_cast<uint64_t>(page_size);
auto required_memory = static_cast<uint64_t>(offset) + data_seg.data.size();
EOS_VM_ASSERT(required_memory <= available_memory, wasm_memory_exception, "data out of range");
auto addr = _linear_memory + offset;
memcpy((char*)(addr), data_seg.data.raw(), data_seg.data.size());
memcpy((char*)(addr), data_seg.data.data(), data_seg.data.size());
}

// reset the mutable globals
EOS_VM_ASSERT(_globals.size() == _mod.globals.size(), wasm_memory_exception, "number of globals in execution_context not equall to the one in module");
for (uint32_t i = 0; i < _mod.globals.size(); i++) {
if (_mod.globals[i].type.mutability) {
_globals[i] = _mod.globals[i].init;
EOS_VM_ASSERT(_globals.size() == mod.globals.size(), wasm_memory_exception, "number of globals in execution_context not equall to the one in module");
for (uint32_t i = 0; i < mod.globals.size(); i++) {
if (mod.globals[i].type.mutability) {
_globals[i] = mod.globals[i].init;
}
}
}
Expand All @@ -176,8 +196,8 @@ namespace eosio { namespace vm {

protected:

template<typename... Args>
static void type_check_args(const func_type& ft, Args&&...) {
template<typename Func_type, typename... Args>
static void type_check_args(const Func_type& ft, Args&&...) {
EOS_VM_ASSERT(sizeof...(Args) == ft.param_types.size(), wasm_interpreter_exception, "wrong number of arguments");
uint32_t i = 0;
EOS_VM_ASSERT((... && (to_wasm_type_v<detail::type_converter_t<Host>, Args> == ft.param_types.at(i++))), wasm_interpreter_exception, "unexpected argument type");
Expand Down Expand Up @@ -209,8 +229,8 @@ namespace eosio { namespace vm {
struct jit_visitor { template<typename T> jit_visitor(T&&) {} };

template<typename Host>
class null_execution_context : public execution_context_base<null_execution_context<Host>, Host> {
using base_type = execution_context_base<null_execution_context<Host>, Host>;
class null_execution_context : public execution_context_base<null_execution_context<Host>, Host, false> {
using base_type = execution_context_base<null_execution_context<Host>, Host, false>;
public:
null_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m) {}
};
Expand All @@ -224,8 +244,8 @@ namespace eosio { namespace vm {
};

template<typename Host, bool EnableBacktrace = false>
class jit_execution_context : public frame_info_holder<EnableBacktrace>, public execution_context_base<jit_execution_context<Host, EnableBacktrace>, Host> {
using base_type = execution_context_base<jit_execution_context<Host, EnableBacktrace>, Host>;
class jit_execution_context : public frame_info_holder<EnableBacktrace>, public execution_context_base<jit_execution_context<Host, EnableBacktrace>, Host, true> {
using base_type = execution_context_base<jit_execution_context<Host, EnableBacktrace>, Host, true>;
using host_type = detail::host_type_t<Host>;
public:
using base_type::execute;
Expand All @@ -246,7 +266,7 @@ namespace eosio { namespace vm {
}

inline native_value call_host_function(native_value* stack, uint32_t index) {
const auto& ft = _mod.get_function_type(index);
const auto& ft = _mod.jit_mod->get_function_type(index);
uint32_t num_params = ft.param_types.size();
#ifndef NDEBUG
uint32_t original_operands = get_operand_stack().size();
Expand All @@ -260,7 +280,7 @@ namespace eosio { namespace vm {
default: assert(!"Unexpected type in param_types.");
}
}
_rhf(_host, get_interface(), _mod.import_functions[index]);
_rhf(_host, get_interface(), _mod.jit_mod->import_functions[index]);
native_value result{uint64_t{0}};
// guarantee that the junk bits are zero, to avoid problems.
auto set_result = [&result](auto val) { std::memcpy(&result, &val, sizeof(val)); };
Expand All @@ -280,7 +300,7 @@ namespace eosio { namespace vm {
}

inline void reset() {
base_type::reset();
base_type::reset(*(_mod.jit_mod));
get_operand_stack().eat(0);
}

Expand All @@ -292,7 +312,7 @@ namespace eosio { namespace vm {

_host = host;

const func_type& ft = _mod.get_function_type(func_index);
const auto& ft = _mod.jit_mod->get_function_type(func_index);
this->type_check_args(ft, static_cast<Args&&>(args)...);
native_value result;

Expand All @@ -304,7 +324,7 @@ namespace eosio { namespace vm {
#pragma GCC diagnostic pop

try {
if (func_index < _mod.get_imported_functions_size()) {
if (func_index < _mod.jit_mod->get_imported_functions_size()) {
std::reverse(args_raw + 0, args_raw + sizeof...(Args));
result = call_host_function(args_raw, func_index);
} else {
Expand All @@ -317,7 +337,7 @@ namespace eosio { namespace vm {
if(stack) {
stack = static_cast<char*>(stack) - 24;
}
auto fn = reinterpret_cast<native_value (*)(void*, void*)>(_mod.code[func_index - _mod.get_imported_functions_size()].jit_code_offset + _mod.allocator._code_base);
auto fn = reinterpret_cast<native_value (*)(void*, void*)>(_mod.jit_mod->jit_code_offset[func_index - _mod.jit_mod->get_imported_functions_size()] + _mod.allocator._code_base);

if constexpr(EnableBacktrace) {
sigset_t block_mask;
Expand Down Expand Up @@ -534,8 +554,8 @@ namespace eosio { namespace vm {
};

template <typename Host>
class execution_context : public execution_context_base<execution_context<Host>, Host> {
using base_type = execution_context_base<execution_context<Host>, Host>;
class execution_context : public execution_context_base<execution_context<Host>, Host, false> {
using base_type = execution_context_base<execution_context<Host>, Host, false>;
using host_type = detail::host_type_t<Host>;
public:
using base_type::_mod;
Expand Down Expand Up @@ -712,7 +732,7 @@ namespace eosio { namespace vm {
}

inline void reset() {
base_type::reset();
base_type::reset(_mod);
_state = execution_state{};
get_operand_stack().eat(_state.os_index);
_as.eat(_state.as_index);
Expand Down
Loading