From 5dd5275b60b2c9d9e1a83d08e62eb0d06e88491d Mon Sep 17 00:00:00 2001 From: Areg Hayrapetian Date: Wed, 10 Aug 2022 18:15:38 -0700 Subject: [PATCH 01/36] Update submodule URLs --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index a1603de0..a2054d0f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3,4 +3,4 @@ url = https://github.com/catchorg/Catch2 [submodule "external/softfloat"] path = external/softfloat - url = https://github.com/eosnetworkfoundation/mandel-berkeley-softfloat-3 + url = https://github.com/AntelopeIO/berkeley-softfloat-3 From a0eafe5d9ed4d13519d660e14bd34fbe7158e520 Mon Sep 17 00:00:00 2001 From: Areg Hayrapetian Date: Wed, 10 Aug 2022 18:23:17 -0700 Subject: [PATCH 02/36] Update softfloat submodule commit --- external/softfloat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/softfloat b/external/softfloat index 94dac6e5..daac447a 160000 --- a/external/softfloat +++ b/external/softfloat @@ -1 +1 @@ -Subproject commit 94dac6e56c980a99e3e38bdff89a94600de0066d +Subproject commit daac447a38d39981f54c1b737842970879d58921 From f9c7d41f65e60c656af71e7dbbf0f40400a6174a Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 11 Aug 2022 16:33:21 -0400 Subject: [PATCH 03/36] Change eosnetworkfoundation/mandel-eos-vm to AntelopeIO/eos-vm --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index f8ae7c28..17281572 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -eosnetworkfoundation/mandel-eos-vm +AntelopeIO/eos-vm Copyright (c) 2021-2022 EOS Network Foundation (ENF) and its contributors. All rights reserved. This ENF software is based upon: From d3d2671a7d6ddb6eb932975e23cf10400395997d Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 15 Aug 2022 13:10:21 -0400 Subject: [PATCH 04/36] Remove EOSIO references --- CONTRIBUTING.md | 148 ------------------------------------------------ IMPORTANT.md | 27 --------- README.md | 15 +---- 3 files changed, 1 insertion(+), 189 deletions(-) delete mode 100644 CONTRIBUTING.md delete mode 100644 IMPORTANT.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 93135c15..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,148 +0,0 @@ -# Contributing to EOS-VM - -Interested in contributing? That's awesome! Here are some guidelines to get started quickly and easily: - -- [Reporting An Issue](#reporting-an-issue) - - [Bug Reports](#bug-reports) - - [Feature Requests](#feature-requests) - - [Change Requests](#change-requests) -- [Working on EOS-VM](#working-on-eos-vm) - - [Feature Branches](#feature-branches) - - [Submitting Pull Requests](#submitting-pull-requests) - - [Testing and Quality Assurance](#testing-and-quality-assurance) -- [Conduct](#conduct) -- [Contributor License & Acknowledgments](#contributor-license--acknowledgments) -- [References](#references) - -## Reporting An Issue - -If you're about to raise an issue because you think you've found a problem with EOS-VM, or you'd like to make a request for a new feature in the codebase, or any other reason… please read this first. - -The GitHub issue tracker is the preferred channel for [bug reports](#bug-reports), [feature requests](#feature-requests), and [submitting pull requests](#submitting-pull-requests), but please respect the following restrictions: - -* Please **search for existing issues**. Help us keep duplicate issues to a minimum by checking to see if someone has already reported your problem or requested your idea. - -* Please **be civil**. Keep the discussion on topic and respect the opinions of others. See also our [Contributor Code of Conduct](#conduct). - -### Bug Reports - -A bug is a _demonstrable problem_ that is caused by the code in the repository. Good bug reports are extremely helpful - thank you! - -Guidelines for bug reports: - -1. **Use the GitHub issue search** — check if the issue has already been - reported. - -1. **Check if the issue has been fixed** — look for [closed issues in the - current milestone](https://github.com/EOSIO/eos-vm/issues?q=is%3Aissue+is%3Aclosed) or try to reproduce it - using the latest `develop` branch. - -A good bug report shouldn't leave others needing to chase you up for more information. Be sure to include the details of your environment and relevant tests that demonstrate the failure. - -[Report a bug](https://github.com/EOSIO/eos-vm/issues/new?title=Bug%3A) - -### Feature Requests - -Feature requests are welcome. Before you submit one be sure to have: - -1. **Use the GitHub search** and check the feature hasn't already been requested. -1. Take a moment to think about whether your idea fits with the scope and aims of the project. -1. Remember, it's up to *you* to make a strong case to convince the project's leaders of the merits of this feature. Please provide as much detail and context as possible, this means explaining the use case and why it is likely to be common. - -### Change Requests - -Change requests cover both architectural and functional changes to how EOS-VM works. If you have an idea for a new or different dependency, a refactor, or an improvement to a feature, etc - please be sure to: - -1. **Use the GitHub search** and check someone else didn't get there first -1. Take a moment to think about the best way to make a case for, and explain what you're thinking. Are you sure this shouldn't really be - a [bug report](#bug-reports) or a [feature request](#feature-requests)? Is it really one idea or is it many? What's the context? What problem are you solving? Why is what you are suggesting better than what's already there? - -## Working on EOS-VM - -Code contributions are welcome and encouraged! If you are looking for a good place to start, check out the [good first issue](https://github.com/EOSIO/eos-vm/labels/good%20first%20issue) label in GitHub issues. - -Also, please follow these guidelines when submitting code: - -### Feature Branches - -To get it out of the way: - -- **[develop](https://github.com/EOSIO/eos-vm/tree/develop)** is the development branch. All work on the next release happens here so you should generally branch off `develop`. Do **NOT** use this branch for a production site. -- **[master](https://github.com/EOSIO/eos-vm/tree/master)** contains the latest release of EOS-VM. This branch may be used in production. Do **NOT** use this branch to work on EOS-VM's source. - -### Submitting Pull Requests - -*Not yet licensed as OSS and not ready for PRs.* Please [raise an issue](#reporting-an-issue) for now, especially if you find a bug. Later, this makes it more likely that there will be enough information available for later PRs to be properly investigated, tested, and merged. - -### Testing and Quality Assurance - -Never underestimate just how useful quality assurance is. If you're looking to get involved with the code base and don't know where to start, checking out and testing a pull request is one of the most useful things you could do. - -Essentially, [check out the latest develop branch](#working-on-eos-vm), take it for a spin, and if you find anything odd, please follow the [bug report guidelines](#bug-reports) and let us know! - -## Conduct - -While contributing, please be respectful and constructive, so that participation in our project is a positive experience for everyone. - -Examples of behavior that contributes to creating a positive environment include: -- Using welcoming and inclusive language -- Being respectful of differing viewpoints and experiences -- Gracefully accepting constructive criticism -- Focusing on what is best for the community -- Showing empathy towards other community members - -Examples of unacceptable behavior include: -- The use of sexualized language or imagery and unwelcome sexual attention or advances -- Trolling, insulting/derogatory comments, and personal or political attacks -- Public or private harassment -- Publishing others’ private information, such as a physical or electronic address, without explicit permission -- Other conduct which could reasonably be considered inappropriate in a professional setting - -## Contributor License & Acknowledgments - -Whenever you make a contribution to this project, you license your contribution under the terms of the [MIT LICENSE](https://github.com/EOSIO/eos/blob/master/LICENSE), and you represent and warrant that you have the right to license your contribution under those terms. Whenever you make a contribution to this project, you also certify in the terms of the Developer’s Certificate of Origin set out below: - -``` -Developer Certificate of Origin -Version 1.1 - -Copyright (C) 2004, 2006 The Linux Foundation and its contributors. -1 Letterman Drive -Suite D4700 -San Francisco, CA, 94129 - -Everyone is permitted to copy and distribute verbatim copies of this -license document, but changing it is not allowed. - - -Developer's Certificate of Origin 1.1 - -By making a contribution to this project, I certify that: - -(a) The contribution was created in whole or in part by me and I - have the right to submit it under the open source license - indicated in the file; or - -(b) The contribution is based upon previous work that, to the best - of my knowledge, is covered under an appropriate open source - license and I have the right under that license to submit that - work with modifications, whether created in whole or in part - by me, under the same open source license (unless I am - permitted to submit under a different license), as indicated - in the file; or - -(c) The contribution was provided directly to me by some other - person who certified (a), (b) or (c) and I have not modified - it. - -(d) I understand and agree that this project and the contribution - are public and that a record of the contribution (including all - personal information I submit with it, including my sign-off) is - maintained indefinitely and may be redistributed consistent with - this project or the open source license(s) involved. -``` - -## References - -* Overall CONTRIB adapted from https://github.com/mathjax/MathJax/blob/master/CONTRIBUTING.md -* Conduct section adapted from the Contributor Covenant, version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html diff --git a/IMPORTANT.md b/IMPORTANT.md deleted file mode 100644 index ed433799..00000000 --- a/IMPORTANT.md +++ /dev/null @@ -1,27 +0,0 @@ -# Important Notice - -We (block.one and its affiliates) make available EOSIO and other software, updates, patches and documentation (collectively, Software) on a voluntary basis as a member of the EOSIO community. A condition of you accessing any Software, websites, articles, media, publications, documents or other material (collectively, Material) is your acceptance of the terms of this important notice. - -## Software -We are not responsible for ensuring the overall performance of Software or any related applications. Any test results or performance figures are indicative and will not reflect performance under all conditions. Software may contain components that are open sourced and subject to their own licenses; you are responsible for ensuring your compliance with those licenses. - -We make no representation, warranty, guarantee or undertaking in respect of Software, whether expressed or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose and noninfringement. In no event shall we be liable for any claim, damages or other liability, whether in an action of contract, tort or otherwise, arising from, out of or in connection with the Software or the use or other dealings in the Software. - -Wallets and related components are complex software that require the highest levels of security. If incorrectly built or used, they may compromise users’ private keys and digital assets. Wallet applications and related components should undergo thorough security evaluations before being used. Only experienced developers should work with such Software. - -Material is not made available to any person or entity that is the subject of sanctions administered or enforced by any country or government or otherwise designated on any list of prohibited or restricted parties (including but not limited to the lists maintained by the United Nations Security Council, the U.S. Government, the European Union or its Member States, or other applicable government authority) or organized or resident in a country or territory that is the subject of country-wide or territory-wide sanctions. You represent and warrant that neither you nor any party having a direct or indirect beneficial interest in you or on whose behalf you are acting as agent or nominee is such a person or entity and you will comply with all applicable import, re-import, sanctions, anti-boycott, export, and re-export control laws and regulations. If this is not accurate or you do not agree, then you must immediately cease accessing our Material and delete all copies of Software. - -Any person using or offering Software in connection with providing software, goods or services to third parties shall advise such third parties of this important notice, including all limitations, restrictions and exclusions of liability. - -## Trademarks -Block.one, EOSIO, EOS, the heptahedron and associated logos and related marks are our trademarks. Other trademarks referenced in Material are the property of their respective owners. - -## Third parties -Any reference in Material to any third party or third-party product, resource or service is not an endorsement or recommendation by Block.one. We are not responsible for, and disclaim any and all responsibility and liability for, your use of or reliance on any of these resources. Third-party resources may be updated, changed or terminated at any time, so information in Material may be out of date or inaccurate. - -## Forward-looking statements -Please note that in making statements expressing Block.one’s vision, we do not guarantee anything, and all aspects of our vision are subject to change at any time and in all respects at Block.one’s sole discretion, with or without notice. We call these “forward-looking statements”, which includes statements on our website and in other Material, other than statements of historical facts, such as statements regarding EOSIO’s development, expected performance, and future features, or our business strategy, plans, prospects, developments and objectives. These statements are only predictions and reflect Block.one’s current beliefs and expectations with respect to future events; they are based on assumptions and are subject to risk, uncertainties and change at any time. - -We operate in a rapidly changing environment and new risks emerge from time to time. Given these risks and uncertainties, you are cautioned not to rely on these forward-looking statements. Actual results, performance or events may differ materially from what is predicted in the forward-looking statements. Some of the factors that could cause actual results, performance or events to differ materially from the forward-looking statements include, without limitation: technical feasibility and barriers; market trends and volatility; continued availability of capital, financing and personnel; product acceptance; the commercial success of any new products or technologies; competition; government regulation and laws; and general economic, market or business conditions. - -All statements are valid only as of the date of first posting and Block.one is under no obligation to, and expressly disclaims any obligation to, update or alter any statements, whether as a result of new information, subsequent events or otherwise. Nothing in any Material constitutes technological, financial, investment, legal or other advice, either in general or with regard to any particular situation or implementation. Please consult with experts in appropriate areas before implementing or utilizing anything contained in Material. diff --git a/README.md b/README.md index 1854eb20..12a2692a 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ EOS VM is designed from the ground up for the high demands of blockchain applica While EOS VM was designed for blockchain, we believe it is ideally suited for any application looking to embed a High Performance WebAssembly engine. -We designed EOS VM to meet the needs of EOSIO blockchains after using three of the most common WebAssembly engines: Binaryen, WABT, and WAVM. These WebAssembly engines were the single largest source of security issues impacting EOSIO blockchains. While WAVM provides extremely fast execution, it is not suited to running a live blockchain because it has extremely long and unpredictable compilation times and the need to recompile all contracts every time the process restarts. WABT was designed as a toolkit for manipulating WebAssembly first and as an execution engine second. +We designed EOS VM to meet the needs of [Antelope](https://github.com/AntelopeIO/) blockchains after using three of the most common WebAssembly engines: Binaryen, WABT, and WAVM. These WebAssembly engines were the single largest source of security issues impacting Antelope blockchains. While WAVM provides extremely fast execution, it is not suited to running a live blockchain because it has extremely long and unpredictable compilation times and the need to recompile all contracts every time the process restarts. WABT was designed as a toolkit for manipulating WebAssembly first and as an execution engine second. We considered the WebAssembly engines used by the largest browsers, but they all come with considerable overhead and assumptions which are inappropriate for a reusable library or to be embedded in a blockchain. It is our hope that one day major browsers will opt to switch to EOS VM. @@ -79,16 +79,3 @@ Extensions to Wasm itself can be made by simply defining the new section (aka C+ ## Using EOS-VM [Quick Overview](./docs/OVERVIEW.md) - - -## Contributing - -[Contributing Guide](./CONTRIBUTING.md) - -[Code of Conduct](./CONTRIBUTING.md#conduct) - -## Important - -See [LICENSE](./LICENSE) for copyright and license terms. - -All repositories and other materials are provided subject to the terms of this [IMPORTANT](./IMPORTANT.md) notice and you must familiarize yourself with its terms. The notice contains important information, limitations and restrictions relating to our software, publications, trademarks, third-party resources, and forward-looking statements. By accessing any of our repositories and other materials, you accept and agree to the terms of the notice. From 898eed354f5a56bb4cf485efd08326432d05d838 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Fri, 7 Oct 2022 07:34:01 -0500 Subject: [PATCH 05/36] fix is_callable_v --- include/eosio/vm/function_traits.hpp | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/include/eosio/vm/function_traits.hpp b/include/eosio/vm/function_traits.hpp index 76a88181..6cea1389 100644 --- a/include/eosio/vm/function_traits.hpp +++ b/include/eosio/vm/function_traits.hpp @@ -72,9 +72,6 @@ namespace eosio { namespace vm { inline constexpr U&& make_dependent(U&& u) { return static_cast(u); } } - template - inline constexpr static bool is_callable_v = EOS_VM_HAS_MEMBER(AUTO_PARAM_WORKAROUND(FN), operator()); - template constexpr bool is_callable(F&& fn) { return EOS_VM_HAS_MEMBER(fn, operator()); } @@ -102,7 +99,7 @@ namespace eosio { namespace vm { std::tuple, Args>...>>; template constexpr auto get_types(F&& fn) { - if constexpr (is_callable_v) + if constexpr (is_callable(fn)) return get_types(&F::operator()); else return get_types(fn); @@ -144,7 +141,7 @@ namespace eosio { namespace vm { constexpr auto parameters_from_impl(R(Cls::*)(Args...)const &&) -> pack_from_t; template constexpr auto parameters_from_impl(F&& fn) { - if constexpr (is_callable_v) + if constexpr (is_callable(fn)) return parameters_from_impl(&F::operator()); else return parameters_from_impl(fn); From 1b6c0964aaeb56600ddc76b27fdb0685f4af16d2 Mon Sep 17 00:00:00 2001 From: Scott Bailey Date: Fri, 28 Oct 2022 13:45:00 -0500 Subject: [PATCH 06/36] Calling execute() with no `args` (i.e. `execute(host_type,jit_visitor,uint32_t)`) results in a "statement has no effect [-Werror=unused-value]" warning. Dissable `unused-value` for this function. --- include/eosio/vm/execution_context.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index a6b98f13..1c2f80e6 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -268,6 +268,11 @@ namespace eosio { namespace vm { get_operand_stack().eat(0); } +// Calling execute() with no `args` (i.e. `execute(host_type,jit_visitor,uint32_t)`) results in a "statement has no effect +// [-Werror=unused-value]" warning. Dissable for this function. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-value" + template inline std::optional execute(host_type* host, jit_visitor, uint32_t func_index, Args... args) { auto saved_host = _host; @@ -333,6 +338,10 @@ namespace eosio { namespace vm { __builtin_unreachable(); } +// Re-enable unused value warning. +#pragma GCC diagnostic push + + #ifdef __x86_64__ int backtrace(void** out, int count, void* uc) const { static_assert(EnableBacktrace); From 398f0330e538d74a4f4b4a800951ad449ae4c7b3 Mon Sep 17 00:00:00 2001 From: Scott Bailey Date: Wed, 2 Nov 2022 14:02:56 -0500 Subject: [PATCH 07/36] Restrict warning ignore to a single line. --- include/eosio/vm/execution_context.hpp | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 1c2f80e6..4ba2e8a3 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -268,11 +268,6 @@ namespace eosio { namespace vm { get_operand_stack().eat(0); } -// Calling execute() with no `args` (i.e. `execute(host_type,jit_visitor,uint32_t)`) results in a "statement has no effect -// [-Werror=unused-value]" warning. Dissable for this function. -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-value" - template inline std::optional execute(host_type* host, jit_visitor, uint32_t func_index, Args... args) { auto saved_host = _host; @@ -284,7 +279,13 @@ namespace eosio { namespace vm { const func_type& ft = _mod.get_function_type(func_index); this->type_check_args(ft, static_cast(args)...); native_value result; - native_value args_raw[] = { transform_arg(static_cast(args))... }; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-value" + // Calling execute() with no `args` (i.e. `execute(host_type,jit_visitor,uint32_t)`) results in a "statement has no + // effect [-Werror=unused-value]" warning on this line. Dissable warning. + native_value args_raw[] = { transform_arg( static_cast(args))... }; +#pragma GCC diagnostic push try { if (func_index < _mod.get_imported_functions_size()) { @@ -338,10 +339,6 @@ namespace eosio { namespace vm { __builtin_unreachable(); } -// Re-enable unused value warning. -#pragma GCC diagnostic push - - #ifdef __x86_64__ int backtrace(void** out, int count, void* uc) const { static_assert(EnableBacktrace); From f490b330c41e766e4ab745e83ea36aaabe288d0d Mon Sep 17 00:00:00 2001 From: Scott Bailey Date: Thu, 3 Nov 2022 08:43:12 -0500 Subject: [PATCH 08/36] Address review comment: push should be pop. --- include/eosio/vm/execution_context.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 4ba2e8a3..7be6832e 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -285,7 +285,7 @@ namespace eosio { namespace vm { // Calling execute() with no `args` (i.e. `execute(host_type,jit_visitor,uint32_t)`) results in a "statement has no // effect [-Werror=unused-value]" warning on this line. Dissable warning. native_value args_raw[] = { transform_arg( static_cast(args))... }; -#pragma GCC diagnostic push +#pragma GCC diagnostic pop try { if (func_index < _mod.get_imported_functions_size()) { From 6765de272aa483407a82659d2e6966eae648a249 Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Fri, 21 Apr 2023 14:54:23 -0400 Subject: [PATCH 09/36] Remove unneeded declared constructor. In C++20, a struct with a declared constructor is not an aggregate anymore, so attempts to construct these with initializer_lists throughout the code fail. --- include/eosio/vm/opcodes_def.hpp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/include/eosio/vm/opcodes_def.hpp b/include/eosio/vm/opcodes_def.hpp index 2834441a..6c52a93b 100644 --- a/include/eosio/vm/opcodes_def.hpp +++ b/include/eosio/vm/opcodes_def.hpp @@ -308,7 +308,6 @@ #define EOS_VM_CREATE_EXIT_TYPE(name, code) \ struct EOS_VM_OPCODE_T(name) { \ - EOS_VM_OPCODE_T(name)() = default; \ uint32_t pc; \ static constexpr uint8_t opcode = code; \ }; @@ -328,7 +327,6 @@ #define EOS_VM_CREATE_BR_TABLE_TYPE(name, code) \ struct EOS_VM_OPCODE_T(name) { \ - EOS_VM_OPCODE_T(name)() = default; \ struct elem_t { uint32_t pc; uint32_t stack_pop; }; \ elem_t* table; \ uint32_t size; \ @@ -338,20 +336,17 @@ #define EOS_VM_CREATE_TYPES(name, code) \ struct EOS_VM_OPCODE_T(name) { \ - EOS_VM_OPCODE_T(name)() = default; \ static constexpr uint8_t opcode = code; \ }; #define EOS_VM_CREATE_CALL_TYPES(name, code) \ struct EOS_VM_OPCODE_T(name) { \ - EOS_VM_OPCODE_T(name)() = default; \ uint32_t index; \ static constexpr uint8_t opcode = code; \ }; #define EOS_VM_CREATE_CALL_IMM_TYPES(name, code) \ struct EOS_VM_OPCODE_T(name) { \ - EOS_VM_OPCODE_T(name)() = default; \ uint32_t index; \ uint16_t locals; \ uint16_t return_type; \ @@ -360,14 +355,12 @@ #define EOS_VM_CREATE_VARIABLE_ACCESS_TYPES(name, code) \ struct EOS_VM_OPCODE_T(name) { \ - EOS_VM_OPCODE_T(name)() = default; \ uint32_t index; \ static constexpr uint8_t opcode = code; \ }; #define EOS_VM_CREATE_MEMORY_TYPES(name, code) \ struct EOS_VM_OPCODE_T(name) { \ - EOS_VM_OPCODE_T(name)() = default; \ uint32_t flags_align; \ uint32_t offset; \ static constexpr uint8_t opcode = code; \ From 00a3b00825f2f1f9890876da4ab751cc7a6b05fe Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Wed, 14 Jun 2023 16:16:53 -0400 Subject: [PATCH 10/36] EOS VM memory issues fix --- include/eosio/vm/allocator.hpp | 110 ++++++++++++++++++------- include/eosio/vm/backend.hpp | 56 +++++++++++-- include/eosio/vm/constants.hpp | 7 +- include/eosio/vm/execution_context.hpp | 4 + include/eosio/vm/parser.hpp | 38 +++++++-- include/eosio/vm/types.hpp | 2 +- include/eosio/vm/wasm_stack.hpp | 11 +++ include/eosio/vm/x86_64.hpp | 15 ++-- tests/allocator_tests.cpp | 59 +++++++++++++ 9 files changed, 248 insertions(+), 54 deletions(-) diff --git a/include/eosio/vm/allocator.hpp b/include/eosio/vm/allocator.hpp index 0fd6c6ae..b338a59e 100644 --- a/include/eosio/vm/allocator.hpp +++ b/include/eosio/vm/allocator.hpp @@ -214,9 +214,13 @@ namespace eosio { namespace vm { blocks_by_size_t::iterator allocate_segment(std::size_t min_size) { std::size_t size = std::max(min_size, segment_size); - void* base = mmap(nullptr, size, PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - segment s{base, size}; + // To avoid additional memory mappings being created during permission changes of + // from PROT_EXEC to PROT_READ | PROT_WRITE, and back to PROT_EXEC, + // set permisions to PROT_READ | PROT_WRITE initially. + // The permission will be changed to PROT_EXEC after executible code is copied. + void* base = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); EOS_VM_ASSERT(base != MAP_FAILED, wasm_bad_alloc, "failed to allocate jit segment"); + segment s{base, size}; _segments.emplace_back(std::move(s)); bool success = false; auto guard_1 = scope_guard{[&] { if(!success) { _segments.pop_back(); } }}; @@ -267,7 +271,6 @@ namespace eosio { namespace vm { class growable_allocator { public: static constexpr size_t max_memory_size = 1024 * 1024 * 1024; // 1GB - static constexpr size_t chunk_size = 128 * 1024; // 128KB template static constexpr size_t align_offset(size_t offset) { return (offset + align_amt - 1) & ~(align_amt - 1); } @@ -277,22 +280,51 @@ namespace eosio { namespace vm { return (offset + pagesize - 1) & ~(pagesize - 1); } + growable_allocator() {} + // size in bytes - growable_allocator(size_t size) { + explicit growable_allocator(size_t size) { EOS_VM_ASSERT(size <= max_memory_size, wasm_bad_alloc, "Too large initial memory size"); - _base = (char*)mmap(NULL, max_memory_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "mmap failed."); - if (size != 0) { - size_t chunks_to_alloc = (align_offset(size) / chunk_size); - _size += (chunk_size * chunks_to_alloc); - mprotect((char*)_base, _size, PROT_READ | PROT_WRITE); - } + use_default_memory(); + } + + void use_default_memory() { + EOS_VM_ASSERT(_base == nullptr, wasm_bad_alloc, "default memory already allocated"); + + // uses mmap for big memory allocation + _base = (char*)mmap(NULL, max_memory_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "failed to mmap for default memory."); + _mmap_used = true; _capacity = max_memory_size; } + // size in bytes + void use_fixed_memory(bool is_jit, size_t size) { + EOS_VM_ASSERT(0 < size && size <= max_memory_size, wasm_bad_alloc, "Too large or 0 fixed memory size"); + EOS_VM_ASSERT(_base == nullptr, wasm_bad_alloc, "Fixed memory already allocated"); + + _is_jit = is_jit; + if (_is_jit) { + _base = (char*)std::calloc(size, sizeof(char)); + EOS_VM_ASSERT(_base != nullptr, wasm_bad_alloc, "malloc in use_fixed_memory failed."); + _mmap_used = false; + } else { + _base = (char*)mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "mmap in use_fixed_memoryfailed."); + _mmap_used = true; + } + _capacity = size; + } + ~growable_allocator() { - munmap(_base, _capacity); - if (is_jit) { + if (_base != nullptr) { + if (_mmap_used) { + munmap(_base, _capacity); + } else { + std::free(_base); + } + } + if (_is_jit && _code_base) { jit_allocator::instance().free(_code_base); } } @@ -301,19 +333,19 @@ namespace eosio { namespace vm { template T* alloc(size_t size = 0) { static_assert(max_memory_size % alignof(T) == 0, "alignment must divide max_memory_size."); + EOS_VM_ASSERT(_capacity % alignof(T) == 0, wasm_bad_alloc, "alignment must divide _capacity."); _offset = align_offset(_offset); // Evaluating the inequality in this form cannot cause integer overflow. // Once this assertion passes, the rest of the function is safe. - EOS_VM_ASSERT ((max_memory_size - _offset) / sizeof(T) >= size, wasm_bad_alloc, "Allocated too much memory"); + EOS_VM_ASSERT ((_capacity - _offset) / sizeof(T) >= size, wasm_bad_alloc, "Allocated too much memory"); size_t aligned = (sizeof(T) * size) + _offset; - if (aligned > _size) { - size_t chunks_to_alloc = align_offset(aligned - _size) / chunk_size; - mprotect((char*)_base + _size, (chunk_size * chunks_to_alloc), PROT_READ | PROT_WRITE); - _size += (chunk_size * chunks_to_alloc); - } + EOS_VM_ASSERT (aligned <= _capacity, wasm_bad_alloc, "Allocated too much memory after aligned"); T* ptr = (T*)(_base + _offset); _offset = aligned; + if (_offset > _largest_offset) { + _largest_offset = _offset; + } return ptr; } @@ -334,18 +366,24 @@ namespace eosio { namespace vm { int err = mprotect(executable_code, _code_size, PROT_READ | PROT_WRITE); EOS_VM_ASSERT(err == 0, wasm_bad_alloc, "mprotect failed"); std::memcpy(executable_code, _code_base, _code_size); - is_jit = true; _code_base = (char*)executable_code; + enable_code(IsJit); + _is_jit = true; _offset = (char*)code_base - _base; } - enable_code(IsJit); + } + + void set_code_base_and_size(char* code_base, size_t code_size) { + _code_base = code_base; + _code_size = code_size; } // Sets protection on code pages to allow them to be executed. void enable_code(bool is_jit) { mprotect(_code_base, _code_size, is_jit?PROT_EXEC:(PROT_READ|PROT_WRITE)); } - // Make code pages unexecutable + // Make code pages unexecutable so deadline timer can kill an + // execution (in both JIT and Interpreter) void disable_code() { mprotect(_code_base, _code_size, PROT_NONE); } @@ -363,14 +401,21 @@ namespace eosio { namespace vm { _offset = ((char*)ptr - _base); } + size_t largest_used_size() { + return align_to_page(_largest_offset); + } + /* * Finalize the memory by unmapping any excess pages, this means that the allocator will no longer grow */ void finalize() { - if(_capacity != _offset) { + if(_mmap_used && _capacity != _offset) { std::size_t final_size = align_to_page(_offset); - EOS_VM_ASSERT(munmap(_base + final_size, _capacity - final_size) == 0, wasm_bad_alloc, "failed to finalize growable_allocator"); - _capacity = _size = _offset = final_size; + if (final_size < _capacity) { // final_size can grow to _capacity after align_to_page. + // make sure no 0 size passed to munmap + EOS_VM_ASSERT(munmap(_base + final_size, _capacity - final_size) == 0, wasm_bad_alloc, "failed to finalize growable_allocator"); + } + _capacity = _offset = final_size; } } @@ -378,13 +423,14 @@ namespace eosio { namespace vm { void reset() { _offset = 0; } - size_t _offset = 0; - size_t _size = 0; - std::size_t _capacity = 0; - char* _base; - char* _code_base = nullptr; - size_t _code_size = 0; - bool is_jit = false; + size_t _offset = 0; + size_t _largest_offset = 0; + size_t _capacity = 0; + char* _base = nullptr; + char* _code_base = nullptr; + size_t _code_size = 0; + bool _is_jit = false; + bool _mmap_used = false; }; template diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 54a7e262..3172bd67 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -74,36 +74,78 @@ namespace eosio { namespace vm { } public: backend(wasm_code&& code, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parser_t{ mod.allocator, options }.parse_module(code, mod, debug), detail::get_max_call_depth(options)) { + : memory_alloc(alloc), ctx(parse_module(code, options), detail::get_max_call_depth(options)) { ctx.set_max_pages(detail::get_max_pages(options)); construct(&host); } backend(wasm_code&& code, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parser_t{ mod.allocator, options }.parse_module(code, mod, debug), detail::get_max_call_depth(options)) { + : memory_alloc(alloc), ctx(parse_module(code, options), detail::get_max_call_depth(options)) { ctx.set_max_pages(detail::get_max_pages(options)); construct(); } backend(wasm_code& code, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parser_t{ mod.allocator, options }.parse_module(code, mod, debug), detail::get_max_call_depth(options)) { + : memory_alloc(alloc), ctx(parse_module(code, options), detail::get_max_call_depth(options)) { ctx.set_max_pages(detail::get_max_pages(options)); construct(&host); } backend(wasm_code& code, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parser_t{ mod.allocator, options }.parse_module(code, mod, debug), detail::get_max_call_depth(options)) { + : memory_alloc(alloc), ctx(parse_module(code, options), detail::get_max_call_depth(options)) { ctx.set_max_pages(detail::get_max_pages(options)); construct(); } backend(wasm_code_ptr& ptr, size_t sz, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parser_t{ mod.allocator, options }.parse_module2(ptr, sz, mod, debug), detail::get_max_call_depth(options)) { + : memory_alloc(alloc), ctx(parse_module2(ptr, sz, options, true), detail::get_max_call_depth(options)) { // single parsing. original behavior ctx.set_max_pages(detail::get_max_pages(options)); construct(&host); } - backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parser_t{ mod.allocator, options }.parse_module2(ptr, sz, mod, debug), detail::get_max_call_depth(options)) { + // Leap: + // * Contract validation only needs single parsing as the instantiated module is not cached. + // * Contract execution requires two-passes parsing to prevent memory mappings exhaustion + backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true) + : memory_alloc(alloc), ctx(parse_module2(ptr, sz, options, single_parsing), detail::get_max_call_depth(options)) { ctx.set_max_pages(detail::get_max_pages(options)); construct(); } + module& parse_module(wasm_code& code, const Options& options) { + mod.allocator.use_default_memory(); + return parser_t{ mod.allocator, options }.parse_module(code, mod, debug); + } + + module& parse_module2(wasm_code_ptr& ptr, size_t sz, const Options& options, bool single_parsing) { + if (single_parsing) { + mod.allocator.use_default_memory(); + return parser_t{ mod.allocator, options }.parse_module2(ptr, sz, mod, debug); + } + + // To prevent large number of memory mappings used, use two-passes parsing. + // The first pass finds max size of memory required for parsing; + // this memory is released after parsing. + // The second pass uses malloc with the required size of memory. + wasm_code_ptr orig_ptr = ptr; + size_t largest_size = 0; + + // First pass: finds max size of memory required by parsing. + // Memory used by parsing will be freed when going out of the scope + { + module first_pass_module; + // For JIT, skips code generation as it is not needed and + // does not count the code memory size + detail::code_generate_mode code_gen_mode = Impl::is_jit ? detail::code_generate_mode::skip : detail::code_generate_mode::use_same_allocator; + first_pass_module.allocator.use_default_memory(); + parser_t{ first_pass_module.allocator, options }.parse_module2(ptr, sz, first_pass_module, debug, code_gen_mode); + first_pass_module.finalize(); + largest_size = first_pass_module.allocator.largest_used_size(); + } + + // Second pass: uses largest_size of memory for actual parsing + mod.allocator.use_fixed_memory(Impl::is_jit, largest_size); + // For JIT, uses a seperate allocator for code generation as mod's memory + // does not include memory for code + detail::code_generate_mode code_gen_mode = Impl::is_jit ? detail::code_generate_mode::use_seperate_allocator : detail::code_generate_mode::use_same_allocator; + return parser_t{ mod.allocator, options }.parse_module2(orig_ptr, sz, mod, debug, code_gen_mode); + } + template inline auto operator()(host_t& host, const std::string_view& mod, const std::string_view& func, Args... args) { return call(host, mod, func, args...); diff --git a/include/eosio/vm/constants.hpp b/include/eosio/vm/constants.hpp index 3ca10f14..b1106525 100644 --- a/include/eosio/vm/constants.hpp +++ b/include/eosio/vm/constants.hpp @@ -10,7 +10,12 @@ namespace eosio { namespace vm { id_size = sizeof(uint8_t), varuint32_size = 5, max_call_depth = 250, - initial_stack_size = 8*1024, + // initial_stack_size is used for reserving initial memory for operand stack. + // For JIT, operand stack is only used in calling host function calls, where + // number of elements required can never be big. + // For Interpreter, performance is not a concern. + // Intentionally set to a small number. + initial_stack_size = 8, initial_module_size = 1 * 1024 * 1024, max_memory = 4ull << 31, max_useable_memory = (1ull << 32), //4GiB diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index a6b98f13..e576bf64 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -123,6 +123,10 @@ namespace eosio { namespace vm { inline void reset() { EOS_VM_ASSERT(_mod.error == nullptr, wasm_interpreter_exception, _mod.error); + // Reset the capacity of underlying memory used by operand stack if it is + // greater than initial_stack_size + _os.reset_capacity(); + _linear_memory = _wasm_alloc->get_base_ptr(); if(_mod.memories.size()) { EOS_VM_ASSERT(_mod.memories[0].limits.initial <= _max_pages, wasm_bad_alloc, "Cannot allocate initial linear memory."); diff --git a/include/eosio/vm/parser.hpp b/include/eosio/vm/parser.hpp index 9789a48d..d35581c6 100644 --- a/include/eosio/vm/parser.hpp +++ b/include/eosio/vm/parser.hpp @@ -22,6 +22,12 @@ namespace eosio { namespace vm { namespace detail { + enum class code_generate_mode { + use_same_allocator = 0, // uses the same allocator as in module for code generation + use_seperate_allocator = 1, // uses a different temporary allocator for code generation + skip = 2 // skip code generation + }; + static constexpr unsigned get_size_for_type(uint8_t type) { switch(type) { case types::i32: @@ -295,12 +301,12 @@ namespace eosio { namespace vm { return mod; } - inline module& parse_module2(wasm_code_ptr& code_ptr, size_t sz, module& mod, DebugInfo& debug) { - parse_module(code_ptr, sz, mod, debug); + inline module& parse_module2(wasm_code_ptr& code_ptr, size_t sz, module& mod, DebugInfo& debug, detail::code_generate_mode mode = detail::code_generate_mode::use_same_allocator) { + parse_module(code_ptr, sz, mod, debug, mode); return mod; } - void parse_module(wasm_code_ptr& code_ptr, size_t sz, module& mod, DebugInfo& debug) { + void parse_module(wasm_code_ptr& code_ptr, size_t sz, module& mod, DebugInfo& debug, detail::code_generate_mode mode = detail::code_generate_mode::use_same_allocator) { _mod = &mod; EOS_VM_ASSERT(parse_magic(code_ptr) == constants::magic, wasm_parse_exception, "magic number did not match"); EOS_VM_ASSERT(parse_version(code_ptr) == constants::version, wasm_parse_exception, @@ -338,7 +344,7 @@ namespace eosio { namespace vm { case section_id::element_section: parse_section(code_ptr, mod.elements); break; - case section_id::code_section: parse_section(code_ptr, mod.code); break; + case section_id::code_section: parse_section(code_ptr, mod.code, mode); break; case section_id::data_section: parse_section(code_ptr, mod.data); break; default: EOS_VM_ASSERT(false, wasm_parse_exception, "error invalid section id"); } @@ -1309,12 +1315,31 @@ namespace eosio { namespace vm { } template inline void parse_section(wasm_code_ptr& code, - vec>& elems) { + vec>& elems, detail::code_generate_mode mode) { const void* code_start = code.raw() - code.offset(); parse_section_impl(code, elems, detail::get_max_function_section_elements(_options), [&](wasm_code_ptr& code, function_body& fb, std::size_t idx) { parse_function_body(code, fb, idx); }); EOS_VM_ASSERT( elems.size() == _mod->functions.size(), wasm_parse_exception, "code section must have the same size as the function section" ); - Writer code_writer(_allocator, code.bounds() - code.offset(), *_mod); + + if (mode == detail::code_generate_mode::skip) { + return; + } else if (mode == detail::code_generate_mode::use_seperate_allocator) { + // Leap: in 2-pass parsing, save temporary JIT executible in a + // seperate allocator so the executible will not be part of + // instantiated module's allocator and won't be cached. + growable_allocator allocator; + allocator.use_default_memory(); + write_code_out(allocator, code, code_start); + // pass the code base address and size to the main module's allocator + _mod->allocator.set_code_base_and_size(allocator._code_base, allocator._code_size); + allocator._code_base = nullptr; // make sure code_base won't be freed when going out of current scope by allocator's destructor + } else { + write_code_out(_allocator, code, code_start); + } + } + + void write_code_out(growable_allocator& allocator, wasm_code_ptr& code, const void* code_start) { + Writer code_writer(allocator, code.bounds() - code.offset(), *_mod); imap.on_code_start(code_writer.get_base_addr(), code_start); for (size_t i = 0; i < _function_bodies.size(); i++) { function_body& fb = _mod->code[i]; @@ -1328,6 +1353,7 @@ namespace eosio { namespace vm { } imap.on_code_end(code_writer.get_addr(), code.raw()); } + template inline void parse_section(wasm_code_ptr& code, vec>& elems) { diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index 2e12ddcb..149afc24 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -164,7 +164,7 @@ namespace eosio { namespace vm { }; struct module { - growable_allocator allocator = { constants::initial_module_size }; + growable_allocator allocator; uint32_t start = std::numeric_limits::max(); guarded_vector types = { allocator, 0 }; guarded_vector imports = { allocator, 0 }; diff --git a/include/eosio/vm/wasm_stack.hpp b/include/eosio/vm/wasm_stack.hpp index 0046d387..da161534 100644 --- a/include/eosio/vm/wasm_stack.hpp +++ b/include/eosio/vm/wasm_stack.hpp @@ -63,6 +63,17 @@ namespace eosio { namespace vm { size_t size() const { return _index; } size_t capacity() const { return _store.size(); } + // This is only applicable when underlying allocator is unmanaged_vector, + // which is std::vector + void reset_capacity() { + if constexpr (std::is_same_v) { + if (_store.capacity() > constants::initial_stack_size) { + _store.resize(constants::initial_stack_size); + _store.shrink_to_fit(); + } + } + } + private: using base_data_store_t = std::conditional_t, unmanaged_vector, managed_vector>; diff --git a/include/eosio/vm/x86_64.hpp b/include/eosio/vm/x86_64.hpp index d9b49458..b52a8cfa 100644 --- a/include/eosio/vm/x86_64.hpp +++ b/include/eosio/vm/x86_64.hpp @@ -34,9 +34,9 @@ namespace eosio { namespace vm { class machine_code_writer { public: machine_code_writer(growable_allocator& alloc, std::size_t source_bytes, module& mod) : - _mod(mod), _code_segment_base(alloc.start_code()) { + _mod(mod), _allocator(alloc), _code_segment_base(_allocator.start_code()) { const std::size_t code_size = 4 * 16; // 4 error handlers, each is 16 bytes. - _code_start = _mod.allocator.alloc(code_size); + _code_start = _allocator.alloc(code_size); _code_end = _code_start + code_size; code = _code_start; @@ -51,7 +51,7 @@ namespace eosio { namespace vm { // emit host functions const uint32_t num_imported = mod.get_imported_functions_size(); const std::size_t host_functions_size = (40 + 10 * Context::async_backtrace()) * num_imported; - _code_start = _mod.allocator.alloc(host_functions_size); + _code_start = _allocator.alloc(host_functions_size); _code_end = _code_start + host_functions_size; // code already set for(uint32_t i = 0; i < num_imported; ++i) { @@ -67,7 +67,7 @@ namespace eosio { namespace vm { // can use random access _table_element_size = 17; const std::size_t table_size = _table_element_size*_mod.tables[0].table.size(); - _code_start = _mod.allocator.alloc(table_size); + _code_start = _allocator.alloc(table_size); _code_end = _code_start + table_size; // code already set for(uint32_t i = 0; i < _mod.tables[0].table.size(); ++i) { @@ -96,7 +96,7 @@ namespace eosio { namespace vm { assert(code == _code_end); } } - ~machine_code_writer() { _mod.allocator.end_code(_code_segment_base); } + ~machine_code_writer() { _allocator.end_code(_code_segment_base); } static constexpr std::size_t max_prologue_size = 21; static constexpr std::size_t max_epilogue_size = 10; @@ -105,7 +105,7 @@ namespace eosio { namespace vm { // FIXME: This is not a tight upper bound const std::size_t instruction_size_ratio_upper_bound = use_softfloat?(Context::async_backtrace()?63:49):79; std::size_t code_size = max_prologue_size + _mod.code[funcnum].size * instruction_size_ratio_upper_bound + max_epilogue_size; - _code_start = _mod.allocator.alloc(code_size); + _code_start = _allocator.alloc(code_size); _code_end = _code_start + code_size; code = _code_start; start_function(code, funcnum + _mod.get_imported_functions_size()); @@ -2094,7 +2094,7 @@ namespace eosio { namespace vm { using fn_type = native_value(*)(void* context, void* memory); void finalize(function_body& body) { - _mod.allocator.reclaim(code, _code_end - code); + _allocator.reclaim(code, _code_end - code); body.jit_code_offset = _code_start - (unsigned char*)_code_segment_base; } @@ -2128,6 +2128,7 @@ namespace eosio { namespace vm { } module& _mod; + growable_allocator& _allocator; void * _code_segment_base; const func_type* _ft; unsigned char * _code_start; diff --git a/tests/allocator_tests.cpp b/tests/allocator_tests.cpp index bcea4639..fce8895c 100644 --- a/tests/allocator_tests.cpp +++ b/tests/allocator_tests.cpp @@ -97,3 +97,62 @@ TEST_CASE("Testing reclaim", "[growable_allocator]") { int * ptr2 = alloc.alloc(10); CHECK(ptr2 == ptr1 + 2); } + +TEST_CASE("Testing use_default_memory", "[growable_allocator]") { + growable_allocator alloc(1024); + // use_default_memory cannot be called when memory is already allocated by constructor + CHECK_THROWS_AS(alloc.use_default_memory(), wasm_bad_alloc); + + growable_allocator alloc1; + alloc1.use_default_memory(); + // use_default_memory cannot be called multiple times + CHECK_THROWS_AS(alloc1.use_default_memory(), wasm_bad_alloc); + + growable_allocator alloc3; + alloc3.use_default_memory(); + // can allocate as much as researved memory + alloc3.alloc(growable_allocator::max_memory_size); + // cannot allocate more than researved memory + CHECK_THROWS_AS(alloc3.alloc(1), wasm_bad_alloc); +} + +TEST_CASE("Testing use_fixed_memory", "[growable_allocator]") { + growable_allocator alloc(1024); + // use_fixed_memory cannot be called when memory is already allocated by constructor + CHECK_THROWS_AS(alloc.use_fixed_memory(false, 4096), wasm_bad_alloc); + + growable_allocator alloc1; + alloc1.use_fixed_memory(true, 1024); + // use_fixed_memory cannot be called multiple times + CHECK_THROWS_AS(alloc1.use_fixed_memory(true, 1024), wasm_bad_alloc); + + growable_allocator alloc2; + // fixed_memory size cannot be 0 + CHECK_THROWS_AS(alloc2.use_fixed_memory(true, 0), wasm_bad_alloc); + // fixed_memory size cannot be too big + CHECK_THROWS_AS(alloc2.use_fixed_memory(true, growable_allocator::max_memory_size + 1), wasm_bad_alloc); + // fixed_memory size can be growable_allocator::max_memory_size + alloc2.use_fixed_memory(true, growable_allocator::max_memory_size); + + growable_allocator alloc3; + // reserved 1024 bytes + alloc3.use_fixed_memory(true, 1024); + // can allocate less than researved memory + alloc3.alloc(1000); + // can allocate equal to researved memory ( 1000+24 == 1024) + alloc3.alloc(24); + // cannot allocate more than researved memory ( 1000+24+1 > 1024) + CHECK_THROWS_AS(alloc3.alloc(1), wasm_bad_alloc); +} + +TEST_CASE("Testing mixed use_fixed_memory and alloc2.use_default_memory", "[growable_allocator]") { + growable_allocator alloc1; + alloc1.use_default_memory(); + // use_fixed_memory and use_fixed_memory cannot be mixed + CHECK_THROWS_AS(alloc1.use_fixed_memory(true, 1024), wasm_bad_alloc); + + growable_allocator alloc2; + alloc2.use_fixed_memory(true, 1024); + // use_fixed_memory and use_default_memory cannot be mixed + CHECK_THROWS_AS(alloc2.use_default_memory(), wasm_bad_alloc); +} From 73a88d245a594f5a85510ed1dee81e2ac7f535aa Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 18 Jul 2023 16:36:00 -0400 Subject: [PATCH 11/36] store wasm globals in execution context such that a single parsed module can be shared by multiple threads --- include/eosio/vm/backend.hpp | 1 + include/eosio/vm/execution_context.hpp | 62 +++++++++++--- include/eosio/vm/parser.hpp | 1 - include/eosio/vm/types.hpp | 1 - include/eosio/vm/x86_64.hpp | 107 ++++++++++++++++++------- 5 files changed, 130 insertions(+), 42 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 3172bd67..9c607eb5 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -66,6 +66,7 @@ namespace eosio { namespace vm { void construct(host_t* host=nullptr) { mod.finalize(); ctx.set_wasm_allocator(memory_alloc); + ctx.initialize_globals(); if constexpr (!std::is_same_v) HostFunctions::resolve(mod); // FIXME: should not hard code knowledge of null_backend here diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index ba48f8c6..d99ca511 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -87,6 +87,12 @@ namespace eosio { namespace vm { Derived& derived() { return static_cast(*this); } execution_context_base(module& m) : _mod(m) {} + inline void initialize_globals() { + for (uint32_t i = 0; i < _mod.globals.size(); i++) { + _globals.emplace_back(_mod.globals[i].init); + } + } + inline int32_t grow_linear_memory(int32_t pages) { const int32_t sz = _wasm_alloc->get_current_page(); if (pages < 0) { @@ -146,8 +152,9 @@ namespace eosio { namespace vm { // reset the mutable globals for (uint32_t i = 0; i < _mod.globals.size(); i++) { - if (_mod.globals[i].type.mutability) - _mod.globals[i].current = _mod.globals[i].init; + if (_mod.globals[i].type.mutability) { + _globals[i] = _mod.globals[i].init; + } } } @@ -193,6 +200,7 @@ namespace eosio { namespace vm { detail::host_invoker_t _rhf; std::error_code _error_code; operand_stack _os; + std::vector _globals; }; struct jit_visitor { template jit_visitor(T&&) {} }; @@ -226,6 +234,7 @@ namespace eosio { namespace vm { using base_type::get_operand_stack; using base_type::linear_memory; using base_type::get_interface; + using base_type::_globals; jit_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m), _remaining_call_depth(max_call_depth) {} @@ -404,6 +413,38 @@ namespace eosio { namespace vm { static constexpr bool async_backtrace() { return EnableBacktrace; } #endif + inline int32_t get_global_i32(uint32_t index) { + return _globals[index].value.i32; + } + + inline int64_t get_global_i64(uint32_t index) { + return _globals[index].value.i64; + } + + inline uint32_t get_global_f32(uint32_t index) { + return _globals[index].value.f32; + } + + inline uint64_t get_global_f64(uint32_t index) { + return _globals[index].value.f64; + } + + inline void set_global_i32(uint32_t index, int32_t value) { + _globals[index].value.i32 = value; + } + + inline void set_global_i64(uint32_t index, int64_t value) { + _globals[index].value.i64 = value; + } + + inline void set_global_f32(uint32_t index, uint32_t value) { + _globals[index].value.f32 = value; + } + + inline void set_global_f64(uint32_t index, uint64_t value) { + _globals[index].value.f64 = value; + } + protected: template @@ -501,6 +542,7 @@ namespace eosio { namespace vm { using base_type::get_operand_stack; using base_type::linear_memory; using base_type::get_interface; + using base_type::_globals; execution_context(module& m, uint32_t max_call_depth) : base_type(m), _base_allocator{max_call_depth*sizeof(activation_frame)}, @@ -585,10 +627,10 @@ namespace eosio { namespace vm { EOS_VM_ASSERT(index < _mod.globals.size(), wasm_interpreter_exception, "global index out of range"); const auto& gl = _mod.globals[index]; switch (gl.type.content_type) { - case types::i32: return i32_const_t{ *(uint32_t*)&gl.current.value.i32 }; - case types::i64: return i64_const_t{ *(uint64_t*)&gl.current.value.i64 }; - case types::f32: return f32_const_t{ gl.current.value.f32 }; - case types::f64: return f64_const_t{ gl.current.value.f64 }; + case types::i32: return i32_const_t{ _globals[index].value.i32 }; + case types::i64: return i64_const_t{ _globals[index].value.i64 }; + case types::f32: return f32_const_t{ _globals[index].value.f32 }; + case types::f64: return f64_const_t{ _globals[index].value.f64 }; default: throw wasm_interpreter_exception{ "invalid global type" }; } } @@ -600,22 +642,22 @@ namespace eosio { namespace vm { visit(overloaded{ [&](const i32_const_t& i) { EOS_VM_ASSERT(gl.type.content_type == types::i32, wasm_interpreter_exception, "expected i32 global type"); - gl.current.value.i32 = i.data.ui; + _globals[index].value.i32 = i.data.ui; }, [&](const i64_const_t& i) { EOS_VM_ASSERT(gl.type.content_type == types::i64, wasm_interpreter_exception, "expected i64 global type"); - gl.current.value.i64 = i.data.ui; + _globals[index].value.i64 = i.data.ui; }, [&](const f32_const_t& f) { EOS_VM_ASSERT(gl.type.content_type == types::f32, wasm_interpreter_exception, "expected f32 global type"); - gl.current.value.f32 = f.data.ui; + _globals[index].value.f32 = f.data.ui; }, [&](const f64_const_t& f) { EOS_VM_ASSERT(gl.type.content_type == types::f64, wasm_interpreter_exception, "expected f64 global type"); - gl.current.value.f64 = f.data.ui; + _globals[index].value.f64 = f.data.ui; }, [](auto) { throw wasm_interpreter_exception{ "invalid global type" }; } }, el); diff --git a/include/eosio/vm/parser.hpp b/include/eosio/vm/parser.hpp index d35581c6..ab37f7b9 100644 --- a/include/eosio/vm/parser.hpp +++ b/include/eosio/vm/parser.hpp @@ -469,7 +469,6 @@ namespace eosio { namespace vm { if(gv.type.mutability) on_mutable_global(ct); parse_init_expr(code, gv.init, ct); - gv.current = gv.init; } void parse_memory_type(wasm_code_ptr& code, memory_type& mt) { diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index 149afc24..443d119a 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -74,7 +74,6 @@ namespace eosio { namespace vm { struct global_variable { global_type type; init_expr init; - init_expr current; }; struct table_type { diff --git a/include/eosio/vm/x86_64.hpp b/include/eosio/vm/x86_64.hpp index b52a8cfa..23ddf0a1 100644 --- a/include/eosio/vm/x86_64.hpp +++ b/include/eosio/vm/x86_64.hpp @@ -445,43 +445,64 @@ namespace eosio { namespace vm { } void emit_get_global(uint32_t globalidx) { - auto icount = variable_size_instr(13, 14); + auto icount = variable_size_instr(24, 42); // emit_setup_backtrace can be 0 or 9, and emit_restore_backtrace 0 or 9, the total of the rest 24 auto& gl = _mod.globals[globalidx]; - void *ptr = &gl.current.value; + emit_setup_backtrace(); + // pushq %rdi -- save %rdi content onto stack + emit_bytes(0x57); + // pushq %rsi -- save %rsi content onto stack + emit_bytes(0x56); + // movq $globalidx, %rsi -- pass globalidx to %rsi, the second argument + emit_bytes(0x48, 0xc7, 0xc6); + emit_operand32(globalidx); + // movabsq $get_global, %rax + emit_bytes(0x48, 0xb8); switch(gl.type.content_type) { - case types::i32: - case types::f32: - // movabsq $ptr, %rax - emit_bytes(0x48, 0xb8); - emit_operand_ptr(ptr); - // movl (%rax), eax - emit_bytes(0x8b, 0x00); - // push %rax - emit_bytes(0x50); - break; - case types::i64: - case types::f64: - // movabsq $ptr, %rax - emit_bytes(0x48, 0xb8); - emit_operand_ptr(ptr); - // movl (%rax), %rax - emit_bytes(0x48, 0x8b, 0x00); - // push %rax - emit_bytes(0x50); - break; + case types::i32: emit_operand_ptr(&get_global_i32); break; + case types::i64: emit_operand_ptr(&get_global_i64); break; + case types::f32: emit_operand_ptr(&get_global_f32); break; + case types::f64: emit_operand_ptr(&get_global_f64); break; } + // call *%rax + emit_bytes(0xff, 0xd0); + // pop %rsi + emit_bytes(0x5e); + // pop %rdi + emit_bytes(0x5f); + emit_restore_backtrace(); + // push %rax -- return result + emit_bytes(0x50); } + void emit_set_global(uint32_t globalidx) { - auto icount = fixed_size_instr(14); + auto icount = variable_size_instr(24, 42); // emit_setup_backtrace can be 0 or 9, and emit_restore_backtrace 0 or 9, the total of the rest 24 auto& gl = _mod.globals[globalidx]; - void *ptr = &gl.current.value; - // popq %rcx - emit_bytes(0x59); - // movabsq $ptr, %rax + // popq %rdx -- pass global value to %rdx, the third argument in set_global + emit_bytes(0x5a); + emit_setup_backtrace(); + // pushq %rdi -- save %rdi content onto stack + emit_bytes(0x57); + // pushq %rsi -- save %rsi content onto stack + emit_bytes(0x56); + // movq $globalidx, %rsi -- pass globalidx to %rsi, the second argument + emit_bytes(0x48, 0xc7, 0xc6); + emit_operand32(globalidx); + // movabsq $set_global, %rax emit_bytes(0x48, 0xb8); - emit_operand_ptr(ptr); - // movq %rcx, (%rax) - emit_bytes(0x48, 0x89, 0x08); + //emit_operand_ptr(&set_global); + switch(gl.type.content_type) { + case types::i32: emit_operand_ptr(&set_global_i32); break; + case types::i64: emit_operand_ptr(&set_global_i64); break; + case types::f32: emit_operand_ptr(&set_global_f32); break; + case types::f64: emit_operand_ptr(&set_global_f64); break; + } + // call *%rax + emit_bytes(0xff, 0xd0); + // pop %rsi + emit_bytes(0x5e); + // pop %rdi + emit_bytes(0x5f); + emit_restore_backtrace(); } void emit_i32_load(uint32_t /*alignment*/, uint32_t offset) { @@ -2692,6 +2713,32 @@ namespace eosio { namespace vm { return context->grow_linear_memory(pages); } + static int32_t get_global_i32(Context* context /*rdi*/, uint32_t index /*rsi*/) { + return context->get_global_i32(index); + } + static int64_t get_global_i64(Context* context /*rdi*/, uint32_t index /*rsi*/) { + return context->get_global_i64(index); + } + static uint32_t get_global_f32(Context* context /*rdi*/, uint32_t index /*rsi*/) { + return context->get_global_f32(index); + } + static uint64_t get_global_f64(Context* context /*rdi*/, uint32_t index /*rsi*/) { + return context->get_global_f64(index); + } + + static void set_global_i32(Context* context /*rdi*/, uint32_t index /*rsi*/, int32_t value /*rdx*/) { + context->set_global_i32(index, value); + } + static void set_global_i64(Context* context /*rdi*/, uint32_t index /*rsi*/, int64_t value /*rdx*/) { + context->set_global_i64(index, value); + } + static void set_global_f32(Context* context /*rdi*/, uint32_t index /*rsi*/, uint32_t value /*rdx*/) { + context->set_global_f32(index, value); + } + static void set_global_f64(Context* context /*rdi*/, uint32_t index /*rsi*/, uint64_t value /*rdx*/) { + context->set_global_f64(index, value); + } + static void on_unreachable() { vm::throw_( "unreachable" ); } static void on_fp_error() { vm::throw_( "floating point error" ); } static void on_call_indirect_error() { vm::throw_( "call_indirect out of range" ); } From a5aadacab5c0695a9f8c71890b9aed25e3c881ea Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Wed, 19 Jul 2023 14:59:15 -0400 Subject: [PATCH 12/36] add more EOS_VM_ASSERT protections --- include/eosio/vm/exceptions.hpp | 1 + include/eosio/vm/execution_context.hpp | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/include/eosio/vm/exceptions.hpp b/include/eosio/vm/exceptions.hpp index 125b2105..f4c68366 100644 --- a/include/eosio/vm/exceptions.hpp +++ b/include/eosio/vm/exceptions.hpp @@ -41,6 +41,7 @@ namespace eosio { namespace vm { DECLARE_EXCEPTION( guarded_ptr_exception, 4010000, "pointer out of bounds" ) DECLARE_EXCEPTION( timeout_exception, 4010001, "timeout" ) DECLARE_EXCEPTION( wasm_exit_exception, 4010002, "exit" ) + DECLARE_EXCEPTION( wasm_globals_oob_exception, 4010003, "globals index out of bounds" ) DECLARE_EXCEPTION( span_exception, 4020000, "span exception" ) DECLARE_EXCEPTION( profile_exception, 4030000, "profile exception" ) }} // eosio::vm diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index d99ca511..5252ed5b 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -88,6 +88,8 @@ namespace eosio { namespace vm { execution_context_base(module& m) : _mod(m) {} inline void initialize_globals() { + EOS_VM_ASSERT(_globals.empty(), wasm_memory_exception, "initialize_globals called on non-empty _globals"); + _globals.reserve(_mod.globals.size()); for (uint32_t i = 0; i < _mod.globals.size(); i++) { _globals.emplace_back(_mod.globals[i].init); } @@ -151,6 +153,7 @@ namespace eosio { namespace vm { } // reset the mutable globals + EOS_VM_ASSERT(_globals.size() == _mod.globals.size(), wasm_memory_exception, "number of globals in execution_context not equall to the one in module"); for (uint32_t i = 0; i < _mod.globals.size(); i++) { if (_mod.globals[i].type.mutability) { _globals[i] = _mod.globals[i].init; @@ -414,34 +417,42 @@ namespace eosio { namespace vm { #endif inline int32_t get_global_i32(uint32_t index) { + EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in get_global_i32"); return _globals[index].value.i32; } inline int64_t get_global_i64(uint32_t index) { + EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in get_global_i64"); return _globals[index].value.i64; } inline uint32_t get_global_f32(uint32_t index) { + EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in get_global_f32"); return _globals[index].value.f32; } inline uint64_t get_global_f64(uint32_t index) { + EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in get_global_f64"); return _globals[index].value.f64; } inline void set_global_i32(uint32_t index, int32_t value) { + EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in set_global_i32"); _globals[index].value.i32 = value; } inline void set_global_i64(uint32_t index, int64_t value) { + EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in set_global_i64"); _globals[index].value.i64 = value; } inline void set_global_f32(uint32_t index, uint32_t value) { + EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in set_global_f32"); _globals[index].value.f32 = value; } inline void set_global_f64(uint32_t index, uint64_t value) { + EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in set_global_f64"); _globals[index].value.f64 = value; } From eb7961543a6a1cf16fe66da824dc326e5daaecd8 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Wed, 19 Jul 2023 18:08:51 -0400 Subject: [PATCH 13/36] revert the index check (the previous code did not do that) --- include/eosio/vm/exceptions.hpp | 1 - include/eosio/vm/execution_context.hpp | 8 -------- 2 files changed, 9 deletions(-) diff --git a/include/eosio/vm/exceptions.hpp b/include/eosio/vm/exceptions.hpp index f4c68366..125b2105 100644 --- a/include/eosio/vm/exceptions.hpp +++ b/include/eosio/vm/exceptions.hpp @@ -41,7 +41,6 @@ namespace eosio { namespace vm { DECLARE_EXCEPTION( guarded_ptr_exception, 4010000, "pointer out of bounds" ) DECLARE_EXCEPTION( timeout_exception, 4010001, "timeout" ) DECLARE_EXCEPTION( wasm_exit_exception, 4010002, "exit" ) - DECLARE_EXCEPTION( wasm_globals_oob_exception, 4010003, "globals index out of bounds" ) DECLARE_EXCEPTION( span_exception, 4020000, "span exception" ) DECLARE_EXCEPTION( profile_exception, 4030000, "profile exception" ) }} // eosio::vm diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 5252ed5b..a3621292 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -417,42 +417,34 @@ namespace eosio { namespace vm { #endif inline int32_t get_global_i32(uint32_t index) { - EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in get_global_i32"); return _globals[index].value.i32; } inline int64_t get_global_i64(uint32_t index) { - EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in get_global_i64"); return _globals[index].value.i64; } inline uint32_t get_global_f32(uint32_t index) { - EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in get_global_f32"); return _globals[index].value.f32; } inline uint64_t get_global_f64(uint32_t index) { - EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in get_global_f64"); return _globals[index].value.f64; } inline void set_global_i32(uint32_t index, int32_t value) { - EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in set_global_i32"); _globals[index].value.i32 = value; } inline void set_global_i64(uint32_t index, int64_t value) { - EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in set_global_i64"); _globals[index].value.i64 = value; } inline void set_global_f32(uint32_t index, uint32_t value) { - EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in set_global_f32"); _globals[index].value.f32 = value; } inline void set_global_f64(uint32_t index, uint64_t value) { - EOS_VM_ASSERT(index < _globals.size(), wasm_globals_oob_exception, "global index out of range in set_global_f64"); _globals[index].value.f64 = value; } From 65568f5e5ee0d79aeb2a382a58d0596ae28d0434 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 20 Jul 2023 21:01:37 -0400 Subject: [PATCH 14/36] add EOS_VM_ASSERT for interpreter set_global --- include/eosio/vm/execution_context.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index a3621292..60489d48 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -640,6 +640,7 @@ namespace eosio { namespace vm { inline void set_global(uint32_t index, const operand_stack_elem& el) { EOS_VM_ASSERT(index < _mod.globals.size(), wasm_interpreter_exception, "global index out of range"); + EOS_VM_ASSERT(index < _globals.size(), wasm_interpreter_exception, "index for _globals out of range"); auto& gl = _mod.globals[index]; EOS_VM_ASSERT(gl.type.mutability, wasm_interpreter_exception, "global is not mutable"); visit(overloaded{ [&](const i32_const_t& i) { From 0e694a978635f8ddef9c2e592635471d5387746a Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 27 Jul 2023 10:15:38 -0400 Subject: [PATCH 15/36] parse WASM code only once for JIT --- include/eosio/vm/allocator.hpp | 44 ++++---- include/eosio/vm/backend.hpp | 52 +++++----- include/eosio/vm/execution_context.hpp | 81 +++++++++------ include/eosio/vm/host_function.hpp | 24 +++-- include/eosio/vm/parser.hpp | 32 ++---- include/eosio/vm/types.hpp | 138 ++++++++++++++++++++++++- include/eosio/vm/vector.hpp | 1 + tests/allocator_tests.cpp | 28 ++--- 8 files changed, 269 insertions(+), 131 deletions(-) diff --git a/include/eosio/vm/allocator.hpp b/include/eosio/vm/allocator.hpp index b338a59e..d39206b6 100644 --- a/include/eosio/vm/allocator.hpp +++ b/include/eosio/vm/allocator.hpp @@ -294,35 +294,22 @@ namespace eosio { namespace vm { // uses mmap for big memory allocation _base = (char*)mmap(NULL, max_memory_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "failed to mmap for default memory."); - _mmap_used = true; _capacity = max_memory_size; } // size in bytes - void use_fixed_memory(bool is_jit, size_t size) { + void use_fixed_memory(size_t size) { EOS_VM_ASSERT(0 < size && size <= max_memory_size, wasm_bad_alloc, "Too large or 0 fixed memory size"); EOS_VM_ASSERT(_base == nullptr, wasm_bad_alloc, "Fixed memory already allocated"); - _is_jit = is_jit; - if (_is_jit) { - _base = (char*)std::calloc(size, sizeof(char)); - EOS_VM_ASSERT(_base != nullptr, wasm_bad_alloc, "malloc in use_fixed_memory failed."); - _mmap_used = false; - } else { - _base = (char*)mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "mmap in use_fixed_memoryfailed."); - _mmap_used = true; - } + _base = (char*)mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "mmap in use_fixed_memoryfailed."); _capacity = size; } ~growable_allocator() { if (_base != nullptr) { - if (_mmap_used) { - munmap(_base, _capacity); - } else { - std::free(_base); - } + munmap(_base, _capacity); } if (_is_jit && _code_base) { jit_allocator::instance().free(_code_base); @@ -373,11 +360,6 @@ namespace eosio { namespace vm { } } - void set_code_base_and_size(char* code_base, size_t code_size) { - _code_base = code_base; - _code_size = code_size; - } - // Sets protection on code pages to allow them to be executed. void enable_code(bool is_jit) { mprotect(_code_base, _code_size, is_jit?PROT_EXEC:(PROT_READ|PROT_WRITE)); @@ -409,18 +391,31 @@ namespace eosio { namespace vm { * Finalize the memory by unmapping any excess pages, this means that the allocator will no longer grow */ void finalize() { - if(_mmap_used && _capacity != _offset) { + if(_capacity != _offset) { std::size_t final_size = align_to_page(_offset); if (final_size < _capacity) { // final_size can grow to _capacity after align_to_page. // make sure no 0 size passed to munmap EOS_VM_ASSERT(munmap(_base + final_size, _capacity - final_size) == 0, wasm_bad_alloc, "failed to finalize growable_allocator"); + _capacity = _offset = final_size; + if (final_size == 0) { + // _base became invalid after munmap if final_size is 0 so + // set it to nullptr to avoid being used + _base = nullptr; + } } - _capacity = _offset = final_size; } } void free() { EOS_VM_ASSERT(false, wasm_bad_alloc, "unimplemented"); } + void release_base_memory() + { + if (_base != nullptr) { + EOS_VM_ASSERT(munmap(_base, _capacity) == 0, wasm_bad_alloc, "failed to release base memory"); + _base = nullptr; + } + } + void reset() { _offset = 0; } size_t _offset = 0; @@ -430,7 +425,6 @@ namespace eosio { namespace vm { char* _code_base = nullptr; size_t _code_size = 0; bool _is_jit = false; - bool _mmap_used = false; }; template diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 9c607eb5..75e85ae8 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -66,6 +66,11 @@ namespace eosio { namespace vm { void construct(host_t* host=nullptr) { mod.finalize(); ctx.set_wasm_allocator(memory_alloc); + // Now data required by JIT is finalized; create JIT module + // such that memory used in parsing can be released. + if (Impl::is_jit) { + mod.make_jit_module(); + } ctx.initialize_globals(); if constexpr (!std::is_same_v) HostFunctions::resolve(mod); @@ -101,7 +106,8 @@ namespace eosio { namespace vm { } // Leap: // * Contract validation only needs single parsing as the instantiated module is not cached. - // * Contract execution requires two-passes parsing to prevent memory mappings exhaustion + // * JIT execution needs single parsing only. + // * Interpreter execution requires two-passes parsing to prevent memory mappings exhaustion backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true) : memory_alloc(alloc), ctx(parse_module2(ptr, sz, options, single_parsing), detail::get_max_call_depth(options)) { ctx.set_max_pages(detail::get_max_pages(options)); @@ -117,34 +123,26 @@ namespace eosio { namespace vm { if (single_parsing) { mod.allocator.use_default_memory(); return parser_t{ mod.allocator, options }.parse_module2(ptr, sz, mod, debug); - } - - // To prevent large number of memory mappings used, use two-passes parsing. - // The first pass finds max size of memory required for parsing; - // this memory is released after parsing. - // The second pass uses malloc with the required size of memory. - wasm_code_ptr orig_ptr = ptr; - size_t largest_size = 0; + } else { + // To prevent large number of memory mappings used, two-passes of + // parsing are performed. + wasm_code_ptr orig_ptr = ptr; + size_t largest_size = 0; + + // First pass: finds max size of memory required by parsing. + { + // Memory used by this pass is freed when going out of the scope + module first_pass_module; + first_pass_module.allocator.use_default_memory(); + parser_t{ first_pass_module.allocator, options }.parse_module2(ptr, sz, first_pass_module, debug); + first_pass_module.finalize(); + largest_size = first_pass_module.allocator.largest_used_size(); + } - // First pass: finds max size of memory required by parsing. - // Memory used by parsing will be freed when going out of the scope - { - module first_pass_module; - // For JIT, skips code generation as it is not needed and - // does not count the code memory size - detail::code_generate_mode code_gen_mode = Impl::is_jit ? detail::code_generate_mode::skip : detail::code_generate_mode::use_same_allocator; - first_pass_module.allocator.use_default_memory(); - parser_t{ first_pass_module.allocator, options }.parse_module2(ptr, sz, first_pass_module, debug, code_gen_mode); - first_pass_module.finalize(); - largest_size = first_pass_module.allocator.largest_used_size(); + // Second pass: uses actual required memory for final parsing + mod.allocator.use_fixed_memory(largest_size); + return parser_t{ mod.allocator, options }.parse_module2(orig_ptr, sz, mod, debug); } - - // Second pass: uses largest_size of memory for actual parsing - mod.allocator.use_fixed_memory(Impl::is_jit, largest_size); - // For JIT, uses a seperate allocator for code generation as mod's memory - // does not include memory for code - detail::code_generate_mode code_gen_mode = Impl::is_jit ? detail::code_generate_mode::use_seperate_allocator : detail::code_generate_mode::use_same_allocator; - return parser_t{ mod.allocator, options }.parse_module2(orig_ptr, sz, mod, debug, code_gen_mode); } template diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 60489d48..739ccc37 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -85,25 +85,44 @@ namespace eosio { namespace vm { using host_type = detail::host_type_t; public: Derived& derived() { return static_cast(*this); } - execution_context_base(module& m) : _mod(m) {} + execution_context_base(module& m, bool is_jit) : _mod(m), _is_jit(is_jit) {} inline void initialize_globals() { + if (_is_jit) { + return initialize_globals_impl(*_mod.jit_mod); + } + else { + return initialize_globals_impl(_mod); + } + } + + template + inline void initialize_globals_impl(const Module& mod) { EOS_VM_ASSERT(_globals.empty(), wasm_memory_exception, "initialize_globals called on non-empty _globals"); - _globals.reserve(_mod.globals.size()); - for (uint32_t i = 0; i < _mod.globals.size(); i++) { - _globals.emplace_back(_mod.globals[i].init); + _globals.reserve(mod.globals.size()); + for (uint32_t i = 0; i < mod.globals.size(); i++) { + _globals.emplace_back(mod.globals[i].init); } } inline int32_t grow_linear_memory(int32_t pages) { + if (_is_jit) { + return grow_linear_memory_impl(*_mod.jit_mod, pages); + } else { + return grow_linear_memory_impl(_mod, pages); + } + } + + template + inline int32_t grow_linear_memory_impl(const Module& mod, int32_t pages) { const int32_t sz = _wasm_alloc->get_current_page(); if (pages < 0) { if (sz + pages < 0) return -1; _wasm_alloc->free(-pages); } else { - if (!_mod.memories.size() || _max_pages - sz < static_cast(pages) || - (_mod.memories[0].limits.flags && (static_cast(_mod.memories[0].limits.maximum) - sz < pages))) + if (!mod.memories.size() || _max_pages - sz < static_cast(pages) || + (mod.memories[0].limits.flags && (static_cast(mod.memories[0].limits.maximum) - sz < pages))) return -1; _wasm_alloc->alloc(pages); } @@ -128,7 +147,8 @@ namespace eosio { namespace vm { inline std::error_code get_error_code() const { return _error_code; } - inline void reset() { + template + inline void reset(Module& mod) { EOS_VM_ASSERT(_mod.error == nullptr, wasm_interpreter_exception, _mod.error); // Reset the capacity of underlying memory used by operand stack if it is @@ -136,27 +156,27 @@ namespace eosio { namespace vm { _os.reset_capacity(); _linear_memory = _wasm_alloc->get_base_ptr(); - if(_mod.memories.size()) { - EOS_VM_ASSERT(_mod.memories[0].limits.initial <= _max_pages, wasm_bad_alloc, "Cannot allocate initial linear memory."); - _wasm_alloc->reset(_mod.memories[0].limits.initial); + if(mod.memories.size()) { + EOS_VM_ASSERT(mod.memories[0].limits.initial <= _max_pages, wasm_bad_alloc, "Cannot allocate initial linear memory."); + _wasm_alloc->reset(mod.memories[0].limits.initial); } else _wasm_alloc->reset(); - for (uint32_t i = 0; i < _mod.data.size(); i++) { - const auto& data_seg = _mod.data[i]; + for (uint32_t i = 0; i < mod.data.size(); i++) { + const auto& data_seg = mod.data[i]; uint32_t offset = data_seg.offset.value.i32; // force to unsigned - auto available_memory = _mod.memories[0].limits.initial * static_cast(page_size); + auto available_memory = mod.memories[0].limits.initial * static_cast(page_size); auto required_memory = static_cast(offset) + data_seg.data.size(); EOS_VM_ASSERT(required_memory <= available_memory, wasm_memory_exception, "data out of range"); auto addr = _linear_memory + offset; - memcpy((char*)(addr), data_seg.data.raw(), data_seg.data.size()); + memcpy((char*)(addr), data_seg.data.data(), data_seg.data.size()); } // reset the mutable globals - EOS_VM_ASSERT(_globals.size() == _mod.globals.size(), wasm_memory_exception, "number of globals in execution_context not equall to the one in module"); - for (uint32_t i = 0; i < _mod.globals.size(); i++) { - if (_mod.globals[i].type.mutability) { - _globals[i] = _mod.globals[i].init; + EOS_VM_ASSERT(_globals.size() == mod.globals.size(), wasm_memory_exception, "number of globals in execution_context not equall to the one in module"); + for (uint32_t i = 0; i < mod.globals.size(); i++) { + if (mod.globals[i].type.mutability) { + _globals[i] = mod.globals[i].init; } } } @@ -176,8 +196,8 @@ namespace eosio { namespace vm { protected: - template - static void type_check_args(const func_type& ft, Args&&...) { + template + static void type_check_args(const Func_type& ft, Args&&...) { EOS_VM_ASSERT(sizeof...(Args) == ft.param_types.size(), wasm_interpreter_exception, "wrong number of arguments"); uint32_t i = 0; EOS_VM_ASSERT((... && (to_wasm_type_v, Args> == ft.param_types.at(i++))), wasm_interpreter_exception, "unexpected argument type"); @@ -204,6 +224,7 @@ namespace eosio { namespace vm { std::error_code _error_code; operand_stack _os; std::vector _globals; + bool _is_jit = false; }; struct jit_visitor { template jit_visitor(T&&) {} }; @@ -212,7 +233,7 @@ namespace eosio { namespace vm { class null_execution_context : public execution_context_base, Host> { using base_type = execution_context_base, Host>; public: - null_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m) {} + null_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m, false) {} }; template @@ -239,14 +260,14 @@ namespace eosio { namespace vm { using base_type::get_interface; using base_type::_globals; - jit_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m), _remaining_call_depth(max_call_depth) {} + jit_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m, true), _remaining_call_depth(max_call_depth) {} void set_max_call_depth(std::uint32_t max_call_depth) { _remaining_call_depth = max_call_depth; } inline native_value call_host_function(native_value* stack, uint32_t index) { - const auto& ft = _mod.get_function_type(index); + const auto& ft = _mod.jit_mod->get_function_type(index); uint32_t num_params = ft.param_types.size(); #ifndef NDEBUG uint32_t original_operands = get_operand_stack().size(); @@ -260,7 +281,7 @@ namespace eosio { namespace vm { default: assert(!"Unexpected type in param_types."); } } - _rhf(_host, get_interface(), _mod.import_functions[index]); + _rhf(_host, get_interface(), _mod.jit_mod->import_functions[index]); native_value result{uint64_t{0}}; // guarantee that the junk bits are zero, to avoid problems. auto set_result = [&result](auto val) { std::memcpy(&result, &val, sizeof(val)); }; @@ -280,7 +301,7 @@ namespace eosio { namespace vm { } inline void reset() { - base_type::reset(); + base_type::reset(*(_mod.jit_mod)); get_operand_stack().eat(0); } @@ -292,7 +313,7 @@ namespace eosio { namespace vm { _host = host; - const func_type& ft = _mod.get_function_type(func_index); + const auto& ft = _mod.jit_mod->get_function_type(func_index); this->type_check_args(ft, static_cast(args)...); native_value result; @@ -304,7 +325,7 @@ namespace eosio { namespace vm { #pragma GCC diagnostic pop try { - if (func_index < _mod.get_imported_functions_size()) { + if (func_index < _mod.jit_mod->get_imported_functions_size()) { std::reverse(args_raw + 0, args_raw + sizeof...(Args)); result = call_host_function(args_raw, func_index); } else { @@ -317,7 +338,7 @@ namespace eosio { namespace vm { if(stack) { stack = static_cast(stack) - 24; } - auto fn = reinterpret_cast(_mod.code[func_index - _mod.get_imported_functions_size()].jit_code_offset + _mod.allocator._code_base); + auto fn = reinterpret_cast(_mod.jit_mod->code_offset[func_index - _mod.jit_mod->get_imported_functions_size()] + _mod.allocator._code_base); if constexpr(EnableBacktrace) { sigset_t block_mask; @@ -548,7 +569,7 @@ namespace eosio { namespace vm { using base_type::_globals; execution_context(module& m, uint32_t max_call_depth) - : base_type(m), _base_allocator{max_call_depth*sizeof(activation_frame)}, + : base_type(m, false), _base_allocator{max_call_depth*sizeof(activation_frame)}, _as{max_call_depth, _base_allocator}, _halt(exit_t{}) {} void set_max_call_depth(uint32_t max_call_depth) { @@ -712,7 +733,7 @@ namespace eosio { namespace vm { } inline void reset() { - base_type::reset(); + base_type::reset(_mod); _state = execution_state{}; get_operand_stack().eat(_state.os_index); _as.eat(_state.as_index); diff --git a/include/eosio/vm/host_function.hpp b/include/eosio/vm/host_function.hpp index c54f8daa..1f4c2806 100644 --- a/include/eosio/vm/host_function.hpp +++ b/include/eosio/vm/host_function.hpp @@ -361,9 +361,10 @@ namespace eosio { namespace vm { std::vector ret; }; - inline bool operator==(const host_function& lhs, const func_type& rhs) { + template + inline bool operator==(const host_function& lhs, const Func_type& rhs) { return lhs.params.size() == rhs.param_types.size() && - std::equal(lhs.params.begin(), lhs.params.end(), rhs.param_types.raw()) && + std::equal(lhs.params.begin(), lhs.params.end(), rhs.param_types.data()) && lhs.ret.size() == rhs.return_count && (lhs.ret.size() == 0 || lhs.ret[0] == rhs.return_type); } @@ -452,20 +453,27 @@ namespace eosio { namespace vm { mappings::get().template add_mapping(mod, name); } + static void resolve(module& mod) { + if (mod.jit_mod != nullptr) { + resolve_impl(*mod.jit_mod); + } else { + resolve_impl(mod); + } + } + template - static void resolve(Module& mod) { + static void resolve_impl(Module& mod) { auto& imports = mod.import_functions; auto& current_mappings = mappings::get(); for (std::size_t i = 0; i < mod.imports.size(); i++) { std::string mod_name = - std::string((char*)mod.imports[i].module_str.raw(), mod.imports[i].module_str.size()); - std::string fn_name = std::string((char*)mod.imports[i].field_str.raw(), mod.imports[i].field_str.size()); + std::string((char*)mod.imports[i].module_str.data(), mod.imports[i].module_str.size()); + std::string fn_name = std::string((char*)mod.imports[i].field_str.data(), mod.imports[i].field_str.size()); EOS_VM_ASSERT(current_mappings.named_mapping.count({ mod_name, fn_name }), wasm_link_exception, std::string("no mapping for imported function ") + fn_name); imports[i] = current_mappings.named_mapping[{ mod_name, fn_name }]; - const import_entry& entry = mod.imports[i]; - EOS_VM_ASSERT(entry.kind == Function, wasm_link_exception, std::string("importing non-function ") + fn_name); - EOS_VM_ASSERT(current_mappings.host_functions[imports[i]] == mod.types[entry.type.func_t], wasm_link_exception, std::string("wrong type for imported function ") + fn_name); + EOS_VM_ASSERT(mod.imports[i].kind == Function, wasm_link_exception, std::string("importing non-function ") + fn_name); + EOS_VM_ASSERT(current_mappings.host_functions[imports[i]] == mod.types[mod.imports[i].type.func_t], wasm_link_exception, std::string("wrong type for imported function ") + fn_name); } } diff --git a/include/eosio/vm/parser.hpp b/include/eosio/vm/parser.hpp index ab37f7b9..9dc35bee 100644 --- a/include/eosio/vm/parser.hpp +++ b/include/eosio/vm/parser.hpp @@ -22,12 +22,6 @@ namespace eosio { namespace vm { namespace detail { - enum class code_generate_mode { - use_same_allocator = 0, // uses the same allocator as in module for code generation - use_seperate_allocator = 1, // uses a different temporary allocator for code generation - skip = 2 // skip code generation - }; - static constexpr unsigned get_size_for_type(uint8_t type) { switch(type) { case types::i32: @@ -301,12 +295,12 @@ namespace eosio { namespace vm { return mod; } - inline module& parse_module2(wasm_code_ptr& code_ptr, size_t sz, module& mod, DebugInfo& debug, detail::code_generate_mode mode = detail::code_generate_mode::use_same_allocator) { - parse_module(code_ptr, sz, mod, debug, mode); + inline module& parse_module2(wasm_code_ptr& code_ptr, size_t sz, module& mod, DebugInfo& debug) { + parse_module(code_ptr, sz, mod, debug); return mod; } - void parse_module(wasm_code_ptr& code_ptr, size_t sz, module& mod, DebugInfo& debug, detail::code_generate_mode mode = detail::code_generate_mode::use_same_allocator) { + void parse_module(wasm_code_ptr& code_ptr, size_t sz, module& mod, DebugInfo& debug) { _mod = &mod; EOS_VM_ASSERT(parse_magic(code_ptr) == constants::magic, wasm_parse_exception, "magic number did not match"); EOS_VM_ASSERT(parse_version(code_ptr) == constants::version, wasm_parse_exception, @@ -344,7 +338,7 @@ namespace eosio { namespace vm { case section_id::element_section: parse_section(code_ptr, mod.elements); break; - case section_id::code_section: parse_section(code_ptr, mod.code, mode); break; + case section_id::code_section: parse_section(code_ptr, mod.code); break; case section_id::data_section: parse_section(code_ptr, mod.data); break; default: EOS_VM_ASSERT(false, wasm_parse_exception, "error invalid section id"); } @@ -1314,27 +1308,13 @@ namespace eosio { namespace vm { } template inline void parse_section(wasm_code_ptr& code, - vec>& elems, detail::code_generate_mode mode) { + vec>& elems) { const void* code_start = code.raw() - code.offset(); parse_section_impl(code, elems, detail::get_max_function_section_elements(_options), [&](wasm_code_ptr& code, function_body& fb, std::size_t idx) { parse_function_body(code, fb, idx); }); EOS_VM_ASSERT( elems.size() == _mod->functions.size(), wasm_parse_exception, "code section must have the same size as the function section" ); - if (mode == detail::code_generate_mode::skip) { - return; - } else if (mode == detail::code_generate_mode::use_seperate_allocator) { - // Leap: in 2-pass parsing, save temporary JIT executible in a - // seperate allocator so the executible will not be part of - // instantiated module's allocator and won't be cached. - growable_allocator allocator; - allocator.use_default_memory(); - write_code_out(allocator, code, code_start); - // pass the code base address and size to the main module's allocator - _mod->allocator.set_code_base_and_size(allocator._code_base, allocator._code_size); - allocator._code_base = nullptr; // make sure code_base won't be freed when going out of current scope by allocator's destructor - } else { - write_code_out(_allocator, code, code_start); - } + write_code_out(_allocator, code, code_start); } void write_code_out(growable_allocator& allocator, wasm_code_ptr& code, const void* code_start) { diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index 443d119a..9b404980 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -189,11 +189,140 @@ namespace eosio { namespace vm { // to memory with static storage duration. const char * error = nullptr; + // Stores data needed by JIT execution, in memory managed by standard + // C++ vectors, not growable_allocator used by guarded_vector, + // such that growable_allocator can be released after parsing. + // This is to make it possible parsing WASM only once for JIT. + struct jit_mod_t { + struct jit_func_type { + value_type form; + std::vector param_types; + uint8_t return_count; + value_type return_type; + }; + struct jit_import_type { + uint32_t func_t; + }; + struct jit_import_entry { + std::vector module_str; + std::vector field_str; + external_kind kind; + jit_import_type type; + }; + struct jit_export_entry { + std::vector field_str; + external_kind kind; + uint32_t index; + }; + struct jit_data_segment { + uint32_t index; + init_expr offset; + std::vector data; + }; + + std::vector types; + std::vector imports; + std::vector functions; + std::vector memories; + std::vector globals; + std::vector exports; + std::vector code_offset; + std::vector data; + std::vector import_functions; + + auto& get_function_type(uint32_t index) const { + if (index < get_imported_functions_size()) + return types[imports[index].type.func_t]; + return types[functions[index - get_imported_functions_size()]]; + } + uint32_t get_imported_functions_size() const { + return get_imported_functions_size_impl(imports); + } + }; + + // The memory storing module data for JIT + std::unique_ptr jit_mod; + + // Constructs data for JIT execution. + // Called from backend::construct() after parsing is finalized. + void make_jit_module() { + jit_mod = std::make_unique(); + + for (uint32_t i = 0; i < types.size(); ++i) { + const auto& type = types[i]; + jit_mod_t::jit_func_type func_type { + type.form, + {type.param_types.data(), type.param_types.data() + type.param_types.size()}, + type.return_count, + type.return_type + }; + jit_mod->types.emplace_back(func_type); + } + + for (uint32_t i = 0; i < imports.size(); ++i) { + const auto& entry = imports[i]; + jit_mod_t::jit_import_entry import_entry { + {entry.module_str.data(), entry.module_str.data() + entry.module_str.size()}, + {entry.field_str.data(), entry.field_str.data() + entry.field_str.size()}, + entry.kind, + {entry.type.func_t} + }; + jit_mod->imports.emplace_back(import_entry); + } + + if (memories.size() > 0) { + jit_mod->memories.emplace_back(memories[0]); // memories has one element only + } + + if (functions.size() > 0) { + jit_mod->functions.assign(functions.data(), functions.data() + functions.size()); + } + + if (globals.size() > 0) { + jit_mod->globals.assign(globals.raw(), globals.raw() + globals.size()); + } + + for (uint32_t i = 0; i < exports.size(); ++i) { + const auto& entry = exports[i]; + jit_mod_t::jit_export_entry export_entry { + {entry.field_str.data(), entry.field_str.data() + entry.field_str.size()}, + entry.kind, + entry.index + }; + jit_mod->exports.emplace_back(export_entry); + } + + for (uint32_t i = 0; i < code.size(); ++i) { + jit_mod->code_offset.emplace_back(code[i].jit_code_offset); + } + + for (uint32_t i = 0; i < data.size(); ++i) { + const auto& data_seg = data[i]; + jit_mod_t::jit_data_segment jit_mod_seg{ + data_seg.index, + data_seg.offset, + {data_seg.data.data(), data_seg.data.data() + data_seg.data.size()} + }; + jit_mod->data.emplace_back(jit_mod_seg); + } + + if (import_functions.size() > 0) { + jit_mod->import_functions.assign(import_functions.data(), import_functions.data() + import_functions.size()); + } + + // Important. Release the memory used during parsing. + allocator.release_base_memory(); + } + void finalize() { import_functions.resize(get_imported_functions_size()); allocator.finalize(); } uint32_t get_imported_functions_size() const { + return get_imported_functions_size_impl(imports); + } + template + static uint32_t get_imported_functions_size_impl(const Imports& imports) { uint32_t number_of_imports = 0; for (uint32_t i = 0; i < imports.size(); i++) { if (imports[i].kind == external_kind::Function) @@ -223,11 +352,18 @@ namespace eosio { namespace vm { return types[functions[index - get_imported_functions_size()]]; } + // When jit_mod is available, this function executes on jit_mod, + // otherwise on module itself. uint32_t get_exported_function(const std::string_view str) { + return (jit_mod) ? get_exported_function_impl(jit_mod->exports, str) : get_exported_function_impl(exports, str); + } + + template + static uint32_t get_exported_function_impl(const Exports& exports, const std::string_view str) { uint32_t index = std::numeric_limits::max(); for (uint32_t i = 0; i < exports.size(); i++) { if (exports[i].kind == external_kind::Function && exports[i].field_str.size() == str.size() && - memcmp((const char*)str.data(), (const char*)exports[i].field_str.raw(), exports[i].field_str.size()) == + memcmp((const char*)str.data(), (const char*)exports[i].field_str.data(), exports[i].field_str.size()) == 0) { index = exports[i].index; break; diff --git a/include/eosio/vm/vector.hpp b/include/eosio/vm/vector.hpp index df505be4..60015de8 100644 --- a/include/eosio/vm/vector.hpp +++ b/include/eosio/vm/vector.hpp @@ -75,6 +75,7 @@ namespace eosio { namespace vm { constexpr inline T& operator[] (size_t i) const { return at(i); } constexpr inline T& operator[] (size_t i) { return at(i); } constexpr inline T* raw() const { return _data; } + constexpr inline T* data() const { return _data; } constexpr inline size_t size() const { return _size; } constexpr inline void set( T* data, size_t size, size_t index=-1 ) { _size = size; _data = data; _index = index == -1 ? size - 1 : index; } constexpr inline void copy( T* data, size_t size ) { diff --git a/tests/allocator_tests.cpp b/tests/allocator_tests.cpp index fce8895c..b3747ec1 100644 --- a/tests/allocator_tests.cpp +++ b/tests/allocator_tests.cpp @@ -110,38 +110,38 @@ TEST_CASE("Testing use_default_memory", "[growable_allocator]") { growable_allocator alloc3; alloc3.use_default_memory(); - // can allocate as much as researved memory + // can allocate as much as reserved memory alloc3.alloc(growable_allocator::max_memory_size); - // cannot allocate more than researved memory + // cannot allocate more than reserved memory CHECK_THROWS_AS(alloc3.alloc(1), wasm_bad_alloc); } TEST_CASE("Testing use_fixed_memory", "[growable_allocator]") { growable_allocator alloc(1024); // use_fixed_memory cannot be called when memory is already allocated by constructor - CHECK_THROWS_AS(alloc.use_fixed_memory(false, 4096), wasm_bad_alloc); + CHECK_THROWS_AS(alloc.use_fixed_memory(4096), wasm_bad_alloc); growable_allocator alloc1; - alloc1.use_fixed_memory(true, 1024); + alloc1.use_fixed_memory(1024); // use_fixed_memory cannot be called multiple times - CHECK_THROWS_AS(alloc1.use_fixed_memory(true, 1024), wasm_bad_alloc); + CHECK_THROWS_AS(alloc1.use_fixed_memory(1024), wasm_bad_alloc); growable_allocator alloc2; // fixed_memory size cannot be 0 - CHECK_THROWS_AS(alloc2.use_fixed_memory(true, 0), wasm_bad_alloc); + CHECK_THROWS_AS(alloc2.use_fixed_memory(0), wasm_bad_alloc); // fixed_memory size cannot be too big - CHECK_THROWS_AS(alloc2.use_fixed_memory(true, growable_allocator::max_memory_size + 1), wasm_bad_alloc); + CHECK_THROWS_AS(alloc2.use_fixed_memory(growable_allocator::max_memory_size + 1), wasm_bad_alloc); // fixed_memory size can be growable_allocator::max_memory_size - alloc2.use_fixed_memory(true, growable_allocator::max_memory_size); + alloc2.use_fixed_memory(growable_allocator::max_memory_size); growable_allocator alloc3; // reserved 1024 bytes - alloc3.use_fixed_memory(true, 1024); - // can allocate less than researved memory + alloc3.use_fixed_memory(1024); + // can allocate less than reserved memory alloc3.alloc(1000); - // can allocate equal to researved memory ( 1000+24 == 1024) + // can allocate equal to reserved memory ( 1000+24 == 1024) alloc3.alloc(24); - // cannot allocate more than researved memory ( 1000+24+1 > 1024) + // cannot allocate more than reserved memory ( 1000+24+1 > 1024) CHECK_THROWS_AS(alloc3.alloc(1), wasm_bad_alloc); } @@ -149,10 +149,10 @@ TEST_CASE("Testing mixed use_fixed_memory and alloc2.use_default_memory", "[grow growable_allocator alloc1; alloc1.use_default_memory(); // use_fixed_memory and use_fixed_memory cannot be mixed - CHECK_THROWS_AS(alloc1.use_fixed_memory(true, 1024), wasm_bad_alloc); + CHECK_THROWS_AS(alloc1.use_fixed_memory(1024), wasm_bad_alloc); growable_allocator alloc2; - alloc2.use_fixed_memory(true, 1024); + alloc2.use_fixed_memory(1024); // use_fixed_memory and use_default_memory cannot be mixed CHECK_THROWS_AS(alloc2.use_default_memory(), wasm_bad_alloc); } From c800ac796c25034fe5aa73ef2939f98401765a44 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 27 Jul 2023 16:18:49 -0400 Subject: [PATCH 16/36] use more constexpr and template type --- include/eosio/vm/allocator.hpp | 2 +- include/eosio/vm/backend.hpp | 5 ++++- include/eosio/vm/execution_context.hpp | 27 +++++++++++++------------- include/eosio/vm/types.hpp | 3 --- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/include/eosio/vm/allocator.hpp b/include/eosio/vm/allocator.hpp index d39206b6..6ed2669a 100644 --- a/include/eosio/vm/allocator.hpp +++ b/include/eosio/vm/allocator.hpp @@ -303,7 +303,7 @@ namespace eosio { namespace vm { EOS_VM_ASSERT(_base == nullptr, wasm_bad_alloc, "Fixed memory already allocated"); _base = (char*)mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "mmap in use_fixed_memoryfailed."); + EOS_VM_ASSERT(_base != MAP_FAILED, wasm_bad_alloc, "mmap in use_fixed_memory failed."); _capacity = size; } diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 75e85ae8..c90c1875 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -68,8 +68,11 @@ namespace eosio { namespace vm { ctx.set_wasm_allocator(memory_alloc); // Now data required by JIT is finalized; create JIT module // such that memory used in parsing can be released. - if (Impl::is_jit) { + if constexpr (Impl::is_jit) { mod.make_jit_module(); + + // Important. Release the memory used by parsing. + mod.allocator.release_base_memory(); } ctx.initialize_globals(); if constexpr (!std::is_same_v) diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 739ccc37..0c0eff21 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -80,15 +80,15 @@ namespace eosio { namespace vm { using host_invoker_t = typename host_invoker::type; } - template + template class execution_context_base { using host_type = detail::host_type_t; public: Derived& derived() { return static_cast(*this); } - execution_context_base(module& m, bool is_jit) : _mod(m), _is_jit(is_jit) {} + execution_context_base(module& m) : _mod(m) {} inline void initialize_globals() { - if (_is_jit) { + if constexpr (IsJit) { return initialize_globals_impl(*_mod.jit_mod); } else { @@ -106,7 +106,7 @@ namespace eosio { namespace vm { } inline int32_t grow_linear_memory(int32_t pages) { - if (_is_jit) { + if constexpr (IsJit) { return grow_linear_memory_impl(*_mod.jit_mod, pages); } else { return grow_linear_memory_impl(_mod, pages); @@ -224,16 +224,15 @@ namespace eosio { namespace vm { std::error_code _error_code; operand_stack _os; std::vector _globals; - bool _is_jit = false; }; struct jit_visitor { template jit_visitor(T&&) {} }; template - class null_execution_context : public execution_context_base, Host> { - using base_type = execution_context_base, Host>; + class null_execution_context : public execution_context_base, Host, false> { + using base_type = execution_context_base, Host, false>; public: - null_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m, false) {} + null_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m) {} }; template @@ -245,8 +244,8 @@ namespace eosio { namespace vm { }; template - class jit_execution_context : public frame_info_holder, public execution_context_base, Host> { - using base_type = execution_context_base, Host>; + class jit_execution_context : public frame_info_holder, public execution_context_base, Host, true> { + using base_type = execution_context_base, Host, true>; using host_type = detail::host_type_t; public: using base_type::execute; @@ -260,7 +259,7 @@ namespace eosio { namespace vm { using base_type::get_interface; using base_type::_globals; - jit_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m, true), _remaining_call_depth(max_call_depth) {} + jit_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m), _remaining_call_depth(max_call_depth) {} void set_max_call_depth(std::uint32_t max_call_depth) { _remaining_call_depth = max_call_depth; @@ -555,8 +554,8 @@ namespace eosio { namespace vm { }; template - class execution_context : public execution_context_base, Host> { - using base_type = execution_context_base, Host>; + class execution_context : public execution_context_base, Host, false> { + using base_type = execution_context_base, Host, false>; using host_type = detail::host_type_t; public: using base_type::_mod; @@ -569,7 +568,7 @@ namespace eosio { namespace vm { using base_type::_globals; execution_context(module& m, uint32_t max_call_depth) - : base_type(m, false), _base_allocator{max_call_depth*sizeof(activation_frame)}, + : base_type(m), _base_allocator{max_call_depth*sizeof(activation_frame)}, _as{max_call_depth, _base_allocator}, _halt(exit_t{}) {} void set_max_call_depth(uint32_t max_call_depth) { diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index 9b404980..f06590a2 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -309,9 +309,6 @@ namespace eosio { namespace vm { if (import_functions.size() > 0) { jit_mod->import_functions.assign(import_functions.data(), import_functions.data() + import_functions.size()); } - - // Important. Release the memory used during parsing. - allocator.release_base_memory(); } void finalize() { From e88fd38c92ddae10b9b9075e12eea8a5af749bf2 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Mon, 31 Jul 2023 16:15:29 -0400 Subject: [PATCH 17/36] make code more concise and reserve vector size before use --- include/eosio/vm/execution_context.hpp | 2 +- include/eosio/vm/types.hpp | 90 +++++++++++++++----------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 0c0eff21..ea938a73 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -337,7 +337,7 @@ namespace eosio { namespace vm { if(stack) { stack = static_cast(stack) - 24; } - auto fn = reinterpret_cast(_mod.jit_mod->code_offset[func_index - _mod.jit_mod->get_imported_functions_size()] + _mod.allocator._code_base); + auto fn = reinterpret_cast(_mod.jit_mod->jit_code_offset[func_index - _mod.jit_mod->get_imported_functions_size()] + _mod.allocator._code_base); if constexpr(EnableBacktrace) { sigset_t block_mask; diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index f06590a2..fad99f9d 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -192,6 +192,8 @@ namespace eosio { namespace vm { // Stores data needed by JIT execution, in memory managed by standard // C++ vectors, not growable_allocator used by guarded_vector, // such that growable_allocator can be released after parsing. + // All growable_allocators (including recursively) used in module + // are redefined by std::vector in jit_mod_t. // This is to make it possible parsing WASM only once for JIT. struct jit_mod_t { struct jit_func_type { @@ -223,12 +225,15 @@ namespace eosio { namespace vm { std::vector types; std::vector imports; std::vector functions; + // tables not needed during JIT execution std::vector memories; std::vector globals; std::vector exports; - std::vector code_offset; + // elements not needed during JIT execution + std::vector jit_code_offset; std::vector data; std::vector import_functions; + // type_aliases and fast_functions not needed during JIT execution auto& get_function_type(uint32_t index) const { if (index < get_imported_functions_size()) @@ -248,26 +253,30 @@ namespace eosio { namespace vm { void make_jit_module() { jit_mod = std::make_unique(); - for (uint32_t i = 0; i < types.size(); ++i) { - const auto& type = types[i]; - jit_mod_t::jit_func_type func_type { - type.form, - {type.param_types.data(), type.param_types.data() + type.param_types.size()}, - type.return_count, - type.return_type - }; - jit_mod->types.emplace_back(func_type); + if (auto types_size = types.size(); types_size > 0) { + jit_mod->types.reserve(types_size); + for (uint32_t i = 0; i < types_size; ++i) { + const auto& type = types[i]; + jit_mod->types.emplace_back(jit_mod_t::jit_func_type{ + type.form, + {type.param_types.data(), type.param_types.data() + type.param_types.size()}, + type.return_count, + type.return_type + }); + } } - for (uint32_t i = 0; i < imports.size(); ++i) { - const auto& entry = imports[i]; - jit_mod_t::jit_import_entry import_entry { - {entry.module_str.data(), entry.module_str.data() + entry.module_str.size()}, - {entry.field_str.data(), entry.field_str.data() + entry.field_str.size()}, - entry.kind, - {entry.type.func_t} - }; - jit_mod->imports.emplace_back(import_entry); + if (auto imports_size = imports.size(); imports_size > 0) { + jit_mod->imports.reserve(imports_size); + for (uint32_t i = 0; i < imports_size; ++i) { + const auto& entry = imports[i]; + jit_mod->imports.emplace_back(jit_mod_t::jit_import_entry{ + {entry.module_str.data(), entry.module_str.data() + entry.module_str.size()}, + {entry.field_str.data(), entry.field_str.data() + entry.field_str.size()}, + entry.kind, + {entry.type.func_t} + }); + } } if (memories.size() > 0) { @@ -282,28 +291,35 @@ namespace eosio { namespace vm { jit_mod->globals.assign(globals.raw(), globals.raw() + globals.size()); } - for (uint32_t i = 0; i < exports.size(); ++i) { - const auto& entry = exports[i]; - jit_mod_t::jit_export_entry export_entry { - {entry.field_str.data(), entry.field_str.data() + entry.field_str.size()}, - entry.kind, - entry.index - }; - jit_mod->exports.emplace_back(export_entry); + if (auto exports_size = exports.size(); exports_size > 0) { + jit_mod->exports.reserve(exports_size); + for (uint32_t i = 0; i < exports_size; ++i) { + const auto& entry = exports[i]; + jit_mod->exports.emplace_back(jit_mod_t::jit_export_entry{ + {entry.field_str.data(), entry.field_str.data() + entry.field_str.size()}, + entry.kind, + entry.index + }); + } } - for (uint32_t i = 0; i < code.size(); ++i) { - jit_mod->code_offset.emplace_back(code[i].jit_code_offset); + if (auto code_size = code.size(); code_size > 0) { + jit_mod->jit_code_offset.reserve(code_size); + for (uint32_t i = 0; i < code_size; ++i) { + jit_mod->jit_code_offset.emplace_back(code[i].jit_code_offset); + } } - for (uint32_t i = 0; i < data.size(); ++i) { - const auto& data_seg = data[i]; - jit_mod_t::jit_data_segment jit_mod_seg{ - data_seg.index, - data_seg.offset, - {data_seg.data.data(), data_seg.data.data() + data_seg.data.size()} - }; - jit_mod->data.emplace_back(jit_mod_seg); + if (auto data_size = data.size(); data_size > 0) { + jit_mod->data.reserve(data_size); + for (uint32_t i = 0; i < data_size; ++i) { + const auto& data_seg = data[i]; + jit_mod->data.emplace_back(jit_mod_t::jit_data_segment{ + data_seg.index, + data_seg.offset, + {data_seg.data.data(), data_seg.data.data() + data_seg.data.size()} + }); + } } if (import_functions.size() > 0) { From c28f152bf2fc3ceacdf6ae1e5c83aea1b7edcd48 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 1 Aug 2023 16:09:06 -0400 Subject: [PATCH 18/36] remove reversed == operator as it is compiled into recursive function calling itself --- include/eosio/vm/host_function.hpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/include/eosio/vm/host_function.hpp b/include/eosio/vm/host_function.hpp index 1f4c2806..5ef8fc4c 100644 --- a/include/eosio/vm/host_function.hpp +++ b/include/eosio/vm/host_function.hpp @@ -368,9 +368,6 @@ namespace eosio { namespace vm { lhs.ret.size() == rhs.return_count && (lhs.ret.size() == 0 || lhs.ret[0] == rhs.return_type); } - inline bool operator==(const func_type& lhs, const host_function& rhs) { - return rhs == lhs; - } template void get_args(value_type*& out, std::index_sequence) { From 380396e4b08ec121a9eba27328b67bea76f1496d Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Fri, 4 Aug 2023 17:43:02 -0400 Subject: [PATCH 19/36] provide capability to use a single execution context per thread --- include/eosio/vm/backend.hpp | 111 ++++++++++++++++--------- include/eosio/vm/execution_context.hpp | 109 +++++++++++++----------- 2 files changed, 131 insertions(+), 89 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index c90c1875..d5ce7c3a 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -65,7 +65,9 @@ namespace eosio { namespace vm { using parser_t = typename Impl::template parser; void construct(host_t* host=nullptr) { mod.finalize(); - ctx.set_wasm_allocator(memory_alloc); + if (owns_exec_ctx) { + ctx->set_wasm_allocator(memory_alloc); + } // Now data required by JIT is finalized; create JIT module // such that memory used in parsing can be released. if constexpr (Impl::is_jit) { @@ -74,49 +76,64 @@ namespace eosio { namespace vm { // Important. Release the memory used by parsing. mod.allocator.release_base_memory(); } - ctx.initialize_globals(); + if (owns_exec_ctx) { + ctx->initialize_globals(); + } if constexpr (!std::is_same_v) HostFunctions::resolve(mod); // FIXME: should not hard code knowledge of null_backend here - if constexpr (!std::is_same_v) - initialize(host); + if (owns_exec_ctx) { + if constexpr (!std::is_same_v) + initialize(host); + } } public: backend(wasm_code&& code, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parse_module(code, options), detail::get_max_call_depth(options)) { - ctx.set_max_pages(detail::get_max_pages(options)); + : memory_alloc(alloc), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { + ctx->set_max_pages(detail::get_max_pages(options)); construct(&host); } backend(wasm_code&& code, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parse_module(code, options), detail::get_max_call_depth(options)) { - ctx.set_max_pages(detail::get_max_pages(options)); + : memory_alloc(alloc), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { + ctx->set_max_pages(detail::get_max_pages(options)); construct(); } backend(wasm_code& code, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parse_module(code, options), detail::get_max_call_depth(options)) { - ctx.set_max_pages(detail::get_max_pages(options)); + : memory_alloc(alloc), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { + ctx->set_max_pages(detail::get_max_pages(options)); construct(&host); } backend(wasm_code& code, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parse_module(code, options), detail::get_max_call_depth(options)) { - ctx.set_max_pages(detail::get_max_pages(options)); + : memory_alloc(alloc), ctx(new context_t{(parse_module(code, options)), detail::get_max_call_depth(options)}) { + ctx->set_max_pages(detail::get_max_pages(options)); construct(); } backend(wasm_code_ptr& ptr, size_t sz, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(parse_module2(ptr, sz, options, true), detail::get_max_call_depth(options)) { // single parsing. original behavior - ctx.set_max_pages(detail::get_max_pages(options)); + : memory_alloc(alloc), ctx(new context_t{parse_module2(ptr, sz, options, true), detail::get_max_call_depth(options)}) { // single parsing. original behavior { + ctx->set_max_pages(detail::get_max_pages(options)); construct(&host); } // Leap: // * Contract validation only needs single parsing as the instantiated module is not cached. // * JIT execution needs single parsing only. // * Interpreter execution requires two-passes parsing to prevent memory mappings exhaustion - backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true) - : memory_alloc(alloc), ctx(parse_module2(ptr, sz, options, single_parsing), detail::get_max_call_depth(options)) { - ctx.set_max_pages(detail::get_max_pages(options)); + backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true, bool backend_owns_exec_ctx = true) + : memory_alloc(alloc), owns_exec_ctx(backend_owns_exec_ctx), initial_max_call_depth(detail::get_max_call_depth(options)), initial_max_pages(detail::get_max_pages(options)) { + if (owns_exec_ctx) { + ctx = new context_t{parse_module2(ptr, sz, options, single_parsing), initial_max_call_depth}; + ctx->set_max_pages(initial_max_pages); + } else { + parse_module2(ptr, sz, options, single_parsing); + } construct(); } + ~backend() { + if (owns_exec_ctx && ctx) { + delete ctx; + } + } + module& parse_module(wasm_code& code, const Options& options) { mod.allocator.use_default_memory(); return parser_t{ mod.allocator, options }.parse_module(code, mod, debug); @@ -148,6 +165,18 @@ namespace eosio { namespace vm { } } + void set_context(context_t* ctx_ptr) { + ctx = ctx_ptr; + } + + inline void reset_max_call_depth() { + ctx->set_max_call_depth(initial_max_call_depth); + } + + inline void reset_max_pages() { + ctx->set_max_pages(initial_max_pages); + } + template inline auto operator()(host_t& host, const std::string_view& mod, const std::string_view& func, Args... args) { return call(host, mod, func, args...); @@ -160,16 +189,16 @@ namespace eosio { namespace vm { // Only dynamic options matter. Parser options will be ignored. inline backend& initialize(host_t* host, const Options& new_options) { - ctx.set_max_call_depth(detail::get_max_call_depth(new_options)); - ctx.set_max_pages(detail::get_max_pages(new_options)); + ctx->set_max_call_depth(detail::get_max_call_depth(new_options)); + ctx->set_max_pages(detail::get_max_pages(new_options)); initialize(host); return *this; } inline backend& initialize(host_t* host=nullptr) { if (memory_alloc) { - ctx.reset(); - ctx.execute_start(host, interpret_visitor(ctx)); + ctx->reset(); + ctx->execute_start(host, interpret_visitor(*ctx)); } return *this; } @@ -178,12 +207,13 @@ namespace eosio { namespace vm { return initialize(&host); } + template inline bool call_indirect(host_t* host, uint32_t func_index, Args... args) { if constexpr (eos_vm_debug) { - ctx.execute_func_table(host, debug_visitor(ctx), func_index, args...); + ctx->execute_func_table(host, debug_visitor(*ctx), func_index, args...); } else { - ctx.execute_func_table(host, interpret_visitor(ctx), func_index, args...); + ctx->execute_func_table(host, interpret_visitor(*ctx), func_index, args...); } return true; } @@ -191,9 +221,9 @@ namespace eosio { namespace vm { template inline bool call(host_t* host, uint32_t func_index, Args... args) { if constexpr (eos_vm_debug) { - ctx.execute(host, debug_visitor(ctx), func_index, args...); + ctx->execute(host, debug_visitor(*ctx), func_index, args...); } else { - ctx.execute(host, interpret_visitor(ctx), func_index, args...); + ctx->execute(host, interpret_visitor(*ctx), func_index, args...); } return true; } @@ -201,9 +231,9 @@ namespace eosio { namespace vm { template inline bool call(host_t& host, const std::string_view& mod, const std::string_view& func, Args... args) { if constexpr (eos_vm_debug) { - ctx.execute(&host, debug_visitor(ctx), func, args...); + ctx->execute(&host, debug_visitor(*ctx), func, args...); } else { - ctx.execute(&host, interpret_visitor(ctx), func, args...); + ctx->execute(&host, interpret_visitor(*ctx), func, args...); } return true; } @@ -211,9 +241,9 @@ namespace eosio { namespace vm { template inline bool call(const std::string_view& mod, const std::string_view& func, Args... args) { if constexpr (eos_vm_debug) { - ctx.execute(nullptr, debug_visitor(ctx), func, args...); + ctx->execute(nullptr, debug_visitor(*ctx), func, args...); } else { - ctx.execute(nullptr, interpret_visitor(ctx), func, args...); + ctx->execute(nullptr, interpret_visitor(*ctx), func, args...); } return true; } @@ -221,18 +251,18 @@ namespace eosio { namespace vm { template inline auto call_with_return(host_t& host, const std::string_view& mod, const std::string_view& func, Args... args ) { if constexpr (eos_vm_debug) { - return ctx.execute(&host, debug_visitor(ctx), func, args...); + return ctx->execute(&host, debug_visitor(*ctx), func, args...); } else { - return ctx.execute(&host, interpret_visitor(ctx), func, args...); + return ctx->execute(&host, interpret_visitor(*ctx), func, args...); } } template inline auto call_with_return(const std::string_view& mod, const std::string_view& func, Args... args) { if constexpr (eos_vm_debug) { - return ctx.execute(nullptr, debug_visitor(ctx), func, args...); + return ctx->execute(nullptr, debug_visitor(*ctx), func, args...); } else { - return ctx.execute(nullptr, interpret_visitor(ctx), func, args...); + return ctx->execute(nullptr, interpret_visitor(*ctx), func, args...); } } @@ -265,7 +295,7 @@ namespace eosio { namespace vm { for (int i = 0; i < mod.exports.size(); i++) { if (mod.exports[i].kind == external_kind::Function) { std::string s{ (const char*)mod.exports[i].field_str.raw(), mod.exports[i].field_str.size() }; - ctx.execute(host, interpret_visitor(ctx), s); + ctx->execute(host, interpret_visitor(*ctx), s); } } }); @@ -277,7 +307,7 @@ namespace eosio { namespace vm { for (int i = 0; i < mod.exports.size(); i++) { if (mod.exports[i].kind == external_kind::Function) { std::string s{ (const char*)mod.exports[i].field_str.raw(), mod.exports[i].field_str.size() }; - ctx.execute(nullptr, interpret_visitor(ctx), s); + ctx->execute(nullptr, interpret_visitor(*ctx), s); } } }); @@ -285,13 +315,13 @@ namespace eosio { namespace vm { inline void set_wasm_allocator(wasm_allocator* alloc) { memory_alloc = alloc; - ctx.set_wasm_allocator(memory_alloc); + ctx->set_wasm_allocator(memory_alloc); } inline wasm_allocator* get_wasm_allocator() { return memory_alloc; } inline module& get_module() { return mod; } - inline void exit(const std::error_code& ec) { ctx.exit(ec); } - inline auto& get_context() { return ctx; } + inline void exit(const std::error_code& ec) { ctx->exit(ec); } + inline auto& get_context() { return *ctx; } const DebugInfo& get_debug() const { return debug; } @@ -299,6 +329,9 @@ namespace eosio { namespace vm { wasm_allocator* memory_alloc = nullptr; // non owning pointer module mod; DebugInfo debug; - context_t ctx; + context_t* ctx = nullptr; + bool owns_exec_ctx = true; + uint32_t initial_max_call_depth; + uint32_t initial_max_pages; }; }} // namespace eosio::vm diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index ea938a73..be9128b4 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -85,14 +85,15 @@ namespace eosio { namespace vm { using host_type = detail::host_type_t; public: Derived& derived() { return static_cast(*this); } - execution_context_base(module& m) : _mod(m) {} + execution_context_base() {} + execution_context_base(module* m) : _mod(m) {} inline void initialize_globals() { if constexpr (IsJit) { - return initialize_globals_impl(*_mod.jit_mod); + return initialize_globals_impl(*_mod->jit_mod); } else { - return initialize_globals_impl(_mod); + return initialize_globals_impl(*_mod); } } @@ -107,9 +108,9 @@ namespace eosio { namespace vm { inline int32_t grow_linear_memory(int32_t pages) { if constexpr (IsJit) { - return grow_linear_memory_impl(*_mod.jit_mod, pages); + return grow_linear_memory_impl(*_mod->jit_mod, pages); } else { - return grow_linear_memory_impl(_mod, pages); + return grow_linear_memory_impl(*_mod, pages); } } @@ -136,7 +137,8 @@ namespace eosio { namespace vm { throw wasm_exit_exception{"Exiting"}; } - inline module& get_module() { return _mod; } + inline void set_module(module* mod) { _mod = mod; } + inline module& get_module() { return *_mod; } inline void set_wasm_allocator(wasm_allocator* alloc) { _wasm_alloc = alloc; } inline auto get_wasm_allocator() { return _wasm_alloc; } inline char* linear_memory() { return _linear_memory; } @@ -149,7 +151,7 @@ namespace eosio { namespace vm { template inline void reset(Module& mod) { - EOS_VM_ASSERT(_mod.error == nullptr, wasm_interpreter_exception, _mod.error); + EOS_VM_ASSERT(_mod->error == nullptr, wasm_interpreter_exception, _mod->error); // Reset the capacity of underlying memory used by operand stack if it is // greater than initial_stack_size @@ -172,26 +174,26 @@ namespace eosio { namespace vm { memcpy((char*)(addr), data_seg.data.data(), data_seg.data.size()); } - // reset the mutable globals - EOS_VM_ASSERT(_globals.size() == mod.globals.size(), wasm_memory_exception, "number of globals in execution_context not equall to the one in module"); + // Globals are different from contract to contract. + // Need to clear _globals in execution context. + _globals.clear(); + _globals.reserve(mod.globals.size()); for (uint32_t i = 0; i < mod.globals.size(); i++) { - if (mod.globals[i].type.mutability) { - _globals[i] = mod.globals[i].init; - } + _globals.emplace_back(mod.globals[i].init); } } template inline std::optional execute(host_type* host, Visitor&& visitor, const std::string_view func, Args... args) { - uint32_t func_index = _mod.get_exported_function(func); + uint32_t func_index = _mod->get_exported_function(func); return derived().execute(host, std::forward(visitor), func_index, std::forward(args)...); } template inline void execute_start(host_type* host, Visitor&& visitor) { - if (_mod.start != std::numeric_limits::max()) - derived().execute(host, std::forward(visitor), _mod.start); + if (_mod->start != std::numeric_limits::max()) + derived().execute(host, std::forward(visitor), _mod->start); } protected: @@ -217,7 +219,7 @@ namespace eosio { namespace vm { } char* _linear_memory = nullptr; - module& _mod; + module* _mod = nullptr; wasm_allocator* _wasm_alloc; uint32_t _max_pages = max_pages; detail::host_invoker_t _rhf; @@ -232,7 +234,8 @@ namespace eosio { namespace vm { class null_execution_context : public execution_context_base, Host, false> { using base_type = execution_context_base, Host, false>; public: - null_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m) {} + null_execution_context() {} + null_execution_context(module& m, std::uint32_t max_call_depth) : base_type(&m) {} }; template @@ -259,14 +262,16 @@ namespace eosio { namespace vm { using base_type::get_interface; using base_type::_globals; - jit_execution_context(module& m, std::uint32_t max_call_depth) : base_type(m), _remaining_call_depth(max_call_depth) {} + jit_execution_context() {} + + jit_execution_context(module& m, std::uint32_t max_call_depth) : base_type(&m), _remaining_call_depth(max_call_depth) {} void set_max_call_depth(std::uint32_t max_call_depth) { _remaining_call_depth = max_call_depth; } inline native_value call_host_function(native_value* stack, uint32_t index) { - const auto& ft = _mod.jit_mod->get_function_type(index); + const auto& ft = _mod->jit_mod->get_function_type(index); uint32_t num_params = ft.param_types.size(); #ifndef NDEBUG uint32_t original_operands = get_operand_stack().size(); @@ -280,7 +285,7 @@ namespace eosio { namespace vm { default: assert(!"Unexpected type in param_types."); } } - _rhf(_host, get_interface(), _mod.jit_mod->import_functions[index]); + _rhf(_host, get_interface(), _mod->jit_mod->import_functions[index]); native_value result{uint64_t{0}}; // guarantee that the junk bits are zero, to avoid problems. auto set_result = [&result](auto val) { std::memcpy(&result, &val, sizeof(val)); }; @@ -300,7 +305,7 @@ namespace eosio { namespace vm { } inline void reset() { - base_type::reset(*(_mod.jit_mod)); + base_type::reset(*(_mod->jit_mod)); get_operand_stack().eat(0); } @@ -312,7 +317,7 @@ namespace eosio { namespace vm { _host = host; - const auto& ft = _mod.jit_mod->get_function_type(func_index); + const auto& ft = _mod->jit_mod->get_function_type(func_index); this->type_check_args(ft, static_cast(args)...); native_value result; @@ -324,12 +329,12 @@ namespace eosio { namespace vm { #pragma GCC diagnostic pop try { - if (func_index < _mod.jit_mod->get_imported_functions_size()) { + if (func_index < _mod->jit_mod->get_imported_functions_size()) { std::reverse(args_raw + 0, args_raw + sizeof...(Args)); result = call_host_function(args_raw, func_index); } else { std::size_t maximum_stack_usage = - (_mod.maximum_stack + 2 /*frame ptr + return ptr*/) * (_remaining_call_depth + 1) + + (_mod->maximum_stack + 2 /*frame ptr + return ptr*/) * (_remaining_call_depth + 1) + sizeof...(Args) + 4 /* scratch space */; stack_allocator alt_stack(maximum_stack_usage * sizeof(native_value)); // reserve 24 bytes for data accessed by inline assembly @@ -337,7 +342,7 @@ namespace eosio { namespace vm { if(stack) { stack = static_cast(stack) - 24; } - auto fn = reinterpret_cast(_mod.jit_mod->jit_code_offset[func_index - _mod.jit_mod->get_imported_functions_size()] + _mod.allocator._code_base); + auto fn = reinterpret_cast(_mod->jit_mod->jit_code_offset[func_index - _mod->jit_mod->get_imported_functions_size()] + _mod->allocator._code_base); if constexpr(EnableBacktrace) { sigset_t block_mask; @@ -402,8 +407,8 @@ namespace eosio { namespace vm { out[i++] = rip; // If we were interrupted in the function prologue or epilogue, // avoid dropping the parent frame. - auto code_base = reinterpret_cast(_mod.allocator.get_code_start()); - auto code_end = code_base + _mod.allocator._code_size; + auto code_base = reinterpret_cast(_mod->allocator.get_code_start()); + auto code_end = code_base + _mod->allocator._code_size; if(rip >= code_base && rip < code_end && count > 1) { // function prologue if(*reinterpret_cast(rip) == 0x55) { @@ -567,8 +572,11 @@ namespace eosio { namespace vm { using base_type::get_interface; using base_type::_globals; + execution_context() + : base_type(), _halt(exit_t{}) {} + execution_context(module& m, uint32_t max_call_depth) - : base_type(m), _base_allocator{max_call_depth*sizeof(activation_frame)}, + : base_type(&m), _base_allocator{max_call_depth*sizeof(activation_frame)}, _as{max_call_depth, _base_allocator}, _halt(exit_t{}) {} void set_max_call_depth(uint32_t max_call_depth) { @@ -585,20 +593,20 @@ namespace eosio { namespace vm { inline void call(uint32_t index) { // TODO validate index is valid - if (index < _mod.get_imported_functions_size()) { + if (index < _mod->get_imported_functions_size()) { // TODO validate only importing functions - const auto& ft = _mod.types[_mod.imports[index].type.func_t]; + const auto& ft = _mod->types[_mod->imports[index].type.func_t]; type_check(ft); inc_pc(); push_call( activation_frame{ nullptr, 0 } ); - _rhf(_state.host, get_interface(), _mod.import_functions[index]); + _rhf(_state.host, get_interface(), _mod->import_functions[index]); pop_call(); } else { - // const auto& ft = _mod.types[_mod.functions[index - _mod.get_imported_functions_size()]]; + // const auto& ft = _mod->types[_mod->functions[index - _mod->get_imported_functions_size()]]; // type_check(ft); push_call(index); setup_locals(index); - set_pc( _mod.get_function_pc(index) ); + set_pc( _mod->get_function_pc(index) ); } } @@ -615,7 +623,7 @@ namespace eosio { namespace vm { std::cout << " }\n"; } - inline uint32_t table_elem(uint32_t i) { return _mod.tables[0].table[i]; } + inline uint32_t table_elem(uint32_t i) { return _mod->tables[0].table[i]; } inline void push_operand(operand_stack_elem el) { get_operand_stack().push(std::move(el)); } inline operand_stack_elem get_operand(uint32_t index) const { return get_operand_stack().get(_last_op_index + index); } inline void eat_operands(uint32_t index) { get_operand_stack().eat(index); } @@ -632,7 +640,7 @@ namespace eosio { namespace vm { return_pc = _state.pc + 1; _as.push( activation_frame{ return_pc, _last_op_index } ); - _last_op_index = get_operand_stack().size() - _mod.get_function_type(index).param_types.size(); + _last_op_index = get_operand_stack().size() - _mod->get_function_type(index).param_types.size(); } inline void apply_pop_call(uint32_t num_locals, uint16_t return_count) { @@ -647,8 +655,9 @@ namespace eosio { namespace vm { inline operand_stack_elem pop_operand() { return get_operand_stack().pop(); } inline operand_stack_elem& peek_operand(size_t i = 0) { return get_operand_stack().peek(i); } inline operand_stack_elem get_global(uint32_t index) { - EOS_VM_ASSERT(index < _mod.globals.size(), wasm_interpreter_exception, "global index out of range"); - const auto& gl = _mod.globals[index]; + EOS_VM_ASSERT(index < _mod->globals.size(), wasm_interpreter_exception, "global index out of range"); + EOS_VM_ASSERT(index < _globals.size(), wasm_interpreter_exception, "index for _globals out of range in get_global for interpreter"); + const auto& gl = _mod->globals[index]; switch (gl.type.content_type) { case types::i32: return i32_const_t{ _globals[index].value.i32 }; case types::i64: return i64_const_t{ _globals[index].value.i64 }; @@ -659,9 +668,9 @@ namespace eosio { namespace vm { } inline void set_global(uint32_t index, const operand_stack_elem& el) { - EOS_VM_ASSERT(index < _mod.globals.size(), wasm_interpreter_exception, "global index out of range"); + EOS_VM_ASSERT(index < _mod->globals.size(), wasm_interpreter_exception, "global index out of range"); EOS_VM_ASSERT(index < _globals.size(), wasm_interpreter_exception, "index for _globals out of range"); - auto& gl = _mod.globals[index]; + auto& gl = _mod->globals[index]; EOS_VM_ASSERT(gl.type.mutability, wasm_interpreter_exception, "global is not mutable"); visit(overloaded{ [&](const i32_const_t& i) { EOS_VM_ASSERT(gl.type.content_type == types::i32, wasm_interpreter_exception, @@ -721,7 +730,7 @@ namespace eosio { namespace vm { inline opcode* get_pc() const { return _state.pc; } inline void set_relative_pc(uint32_t pc_offset) { - _state.pc = _mod.code[0].code + pc_offset; + _state.pc = _mod->code[0].code + pc_offset; } inline void set_pc(opcode* pc) { _state.pc = pc; } inline void inc_pc(uint32_t offset=1) { _state.pc += offset; } @@ -732,7 +741,7 @@ namespace eosio { namespace vm { } inline void reset() { - base_type::reset(_mod); + base_type::reset(*_mod); _state = execution_state{}; get_operand_stack().eat(_state.os_index); _as.eat(_state.as_index); @@ -747,14 +756,14 @@ namespace eosio { namespace vm { template inline std::optional execute(host_type* host, Visitor&& visitor, const std::string_view func, Args... args) { - uint32_t func_index = _mod.get_exported_function(func); + uint32_t func_index = _mod->get_exported_function(func); return execute(host, std::forward(visitor), func_index, std::forward(args)...); } template inline void execute_start(host_type* host, Visitor&& visitor) { - if (_mod.start != std::numeric_limits::max()) - execute(host, std::forward(visitor), _mod.start); + if (_mod->start != std::numeric_limits::max()) + execute(host, std::forward(visitor), _mod->start); } template @@ -779,21 +788,21 @@ namespace eosio { namespace vm { _last_op_index = last_last_op_index; }); - this->type_check_args(_mod.get_function_type(func_index), static_cast(args)...); + this->type_check_args(_mod->get_function_type(func_index), static_cast(args)...); push_args(args...); push_call(func_index); - if (func_index < _mod.get_imported_functions_size()) { - _rhf(_state.host, get_interface(), _mod.import_functions[func_index]); + if (func_index < _mod->get_imported_functions_size()) { + _rhf(_state.host, get_interface(), _mod->import_functions[func_index]); } else { - _state.pc = _mod.get_function_pc(func_index); + _state.pc = _mod->get_function_pc(func_index); setup_locals(func_index); vm::invoke_with_signal_handler([&]() { execute(visitor); }, &handle_signal); } - if (_mod.get_function_type(func_index).return_count && !_state.exiting) { + if (_mod->get_function_type(func_index).return_count && !_state.exiting) { return pop_operand(); } else { return {}; @@ -834,7 +843,7 @@ namespace eosio { namespace vm { } inline void setup_locals(uint32_t index) { - const auto& fn = _mod.code[index - _mod.get_imported_functions_size()]; + const auto& fn = _mod->code[index - _mod->get_imported_functions_size()]; for (uint32_t i = 0; i < fn.locals.size(); i++) { for (uint32_t j = 0; j < fn.locals[i].count; j++) switch (fn.locals[i].type) { From 9c5f5489ad4d1ba7b0243b83834455d2d31c4c13 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Tue, 8 Aug 2023 16:23:31 -0400 Subject: [PATCH 20/36] use a more descriptive name for exec_ctx_created_by_backend and add more comments --- include/eosio/vm/backend.hpp | 25 ++++++++++++++++--------- include/eosio/vm/execution_context.hpp | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index d5ce7c3a..58354b87 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -65,7 +65,7 @@ namespace eosio { namespace vm { using parser_t = typename Impl::template parser; void construct(host_t* host=nullptr) { mod.finalize(); - if (owns_exec_ctx) { + if (exec_ctx_created_by_backend) { ctx->set_wasm_allocator(memory_alloc); } // Now data required by JIT is finalized; create JIT module @@ -76,13 +76,13 @@ namespace eosio { namespace vm { // Important. Release the memory used by parsing. mod.allocator.release_base_memory(); } - if (owns_exec_ctx) { + if (exec_ctx_created_by_backend) { ctx->initialize_globals(); } if constexpr (!std::is_same_v) HostFunctions::resolve(mod); // FIXME: should not hard code knowledge of null_backend here - if (owns_exec_ctx) { + if (exec_ctx_created_by_backend) { if constexpr (!std::is_same_v) initialize(host); } @@ -117,9 +117,11 @@ namespace eosio { namespace vm { // * Contract validation only needs single parsing as the instantiated module is not cached. // * JIT execution needs single parsing only. // * Interpreter execution requires two-passes parsing to prevent memory mappings exhaustion - backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true, bool backend_owns_exec_ctx = true) - : memory_alloc(alloc), owns_exec_ctx(backend_owns_exec_ctx), initial_max_call_depth(detail::get_max_call_depth(options)), initial_max_pages(detail::get_max_pages(options)) { - if (owns_exec_ctx) { + // * Leap reuses execution context per thread; is_exec_ctx_created_by_backend is set + // to false when a backend is constructued + backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true, bool is_exec_ctx_created_by_backend = true) + : memory_alloc(alloc), exec_ctx_created_by_backend(is_exec_ctx_created_by_backend), initial_max_call_depth(detail::get_max_call_depth(options)), initial_max_pages(detail::get_max_pages(options)) { + if (exec_ctx_created_by_backend) { ctx = new context_t{parse_module2(ptr, sz, options, single_parsing), initial_max_call_depth}; ctx->set_max_pages(initial_max_pages); } else { @@ -129,9 +131,11 @@ namespace eosio { namespace vm { } ~backend() { - if (owns_exec_ctx && ctx) { + if (exec_ctx_created_by_backend && ctx) { + // delete only if the context was created by the backend delete ctx; } + ctx = nullptr; } module& parse_module(wasm_code& code, const Options& options) { @@ -166,14 +170,18 @@ namespace eosio { namespace vm { } void set_context(context_t* ctx_ptr) { + // execution context can be only set when it is not already created by the backend + assert(!exec_ctx_created_by_backend); ctx = ctx_ptr; } inline void reset_max_call_depth() { + assert(!exec_ctx_created_by_backend); ctx->set_max_call_depth(initial_max_call_depth); } inline void reset_max_pages() { + assert(!exec_ctx_created_by_backend); ctx->set_max_pages(initial_max_pages); } @@ -207,7 +215,6 @@ namespace eosio { namespace vm { return initialize(&host); } - template inline bool call_indirect(host_t* host, uint32_t func_index, Args... args) { if constexpr (eos_vm_debug) { @@ -330,7 +337,7 @@ namespace eosio { namespace vm { module mod; DebugInfo debug; context_t* ctx = nullptr; - bool owns_exec_ctx = true; + bool exec_ctx_created_by_backend = true; // true if execution context is created by backend (legacy behavior), false if provided by users (Leap uses this) uint32_t initial_max_call_depth; uint32_t initial_max_pages; }; diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index be9128b4..0989c23b 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -175,7 +175,7 @@ namespace eosio { namespace vm { } // Globals are different from contract to contract. - // Need to clear _globals in execution context. + // Need to clear _globals at the start of an execution. _globals.clear(); _globals.reserve(mod.globals.size()); for (uint32_t i = 0; i < mod.globals.size(); i++) { From def032fc63ea66258f5a8d5ec186fa0767e956f8 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 10 Aug 2023 14:34:21 -0400 Subject: [PATCH 21/36] Incorporate review comments --- include/eosio/vm/backend.hpp | 5 ++--- include/eosio/vm/execution_context.hpp | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 58354b87..0eac1445 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -135,7 +135,6 @@ namespace eosio { namespace vm { // delete only if the context was created by the backend delete ctx; } - ctx = nullptr; } module& parse_module(wasm_code& code, const Options& options) { @@ -338,7 +337,7 @@ namespace eosio { namespace vm { DebugInfo debug; context_t* ctx = nullptr; bool exec_ctx_created_by_backend = true; // true if execution context is created by backend (legacy behavior), false if provided by users (Leap uses this) - uint32_t initial_max_call_depth; - uint32_t initial_max_pages; + uint32_t initial_max_call_depth = 0; + uint32_t initial_max_pages = 0; }; }} // namespace eosio::vm diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 0989c23b..7bdf8d13 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -174,7 +174,7 @@ namespace eosio { namespace vm { memcpy((char*)(addr), data_seg.data.data(), data_seg.data.size()); } - // Globals are different from contract to contract. + // Globals can be different from one WASM code to another. // Need to clear _globals at the start of an execution. _globals.clear(); _globals.reserve(mod.globals.size()); @@ -555,7 +555,7 @@ namespace eosio { namespace vm { #endif host_type * _host = nullptr; - uint32_t _remaining_call_depth; + uint32_t _remaining_call_depth = 0; }; template From 7d184fb359683ff8653ea0ac37f4fc10df8f2fe8 Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Wed, 23 Aug 2023 12:05:20 -0400 Subject: [PATCH 22/36] support single wasm interface --- include/eosio/vm/backend.hpp | 70 +++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 0eac1445..1e2069ec 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -64,23 +64,23 @@ namespace eosio { namespace vm { using context_t = typename Impl::template context; using parser_t = typename Impl::template parser; void construct(host_t* host=nullptr) { - mod.finalize(); + mod->finalize(); if (exec_ctx_created_by_backend) { ctx->set_wasm_allocator(memory_alloc); } // Now data required by JIT is finalized; create JIT module // such that memory used in parsing can be released. if constexpr (Impl::is_jit) { - mod.make_jit_module(); + mod->make_jit_module(); // Important. Release the memory used by parsing. - mod.allocator.release_base_memory(); + mod->allocator.release_base_memory(); } if (exec_ctx_created_by_backend) { ctx->initialize_globals(); } if constexpr (!std::is_same_v) - HostFunctions::resolve(mod); + HostFunctions::resolve(*mod); // FIXME: should not hard code knowledge of null_backend here if (exec_ctx_created_by_backend) { if constexpr (!std::is_same_v) @@ -88,28 +88,29 @@ namespace eosio { namespace vm { } } public: + backend() {} backend(wasm_code&& code, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { ctx->set_max_pages(detail::get_max_pages(options)); construct(&host); } backend(wasm_code&& code, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { ctx->set_max_pages(detail::get_max_pages(options)); construct(); } backend(wasm_code& code, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { ctx->set_max_pages(detail::get_max_pages(options)); construct(&host); } backend(wasm_code& code, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(new context_t{(parse_module(code, options)), detail::get_max_call_depth(options)}) { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{(parse_module(code, options)), detail::get_max_call_depth(options)}) { ctx->set_max_pages(detail::get_max_pages(options)); construct(); } backend(wasm_code_ptr& ptr, size_t sz, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), ctx(new context_t{parse_module2(ptr, sz, options, true), detail::get_max_call_depth(options)}) { // single parsing. original behavior { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module2(ptr, sz, options, true), detail::get_max_call_depth(options)}) { // single parsing. original behavior { ctx->set_max_pages(detail::get_max_pages(options)); construct(&host); } @@ -119,8 +120,8 @@ namespace eosio { namespace vm { // * Interpreter execution requires two-passes parsing to prevent memory mappings exhaustion // * Leap reuses execution context per thread; is_exec_ctx_created_by_backend is set // to false when a backend is constructued - backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true, bool is_exec_ctx_created_by_backend = true) - : memory_alloc(alloc), exec_ctx_created_by_backend(is_exec_ctx_created_by_backend), initial_max_call_depth(detail::get_max_call_depth(options)), initial_max_pages(detail::get_max_pages(options)) { + backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true, bool exec_ctx_by_backend = true) + : memory_alloc(alloc), mod(std::make_shared()), exec_ctx_created_by_backend(exec_ctx_by_backend), initial_max_call_depth(detail::get_max_call_depth(options)), initial_max_pages(detail::get_max_pages(options)) { if (exec_ctx_created_by_backend) { ctx = new context_t{parse_module2(ptr, sz, options, single_parsing), initial_max_call_depth}; ctx->set_max_pages(initial_max_pages); @@ -130,6 +131,16 @@ namespace eosio { namespace vm { construct(); } + backend& operator=(const backend& other) { + if (this != &other) { + mod = other.mod; + exec_ctx_created_by_backend = other.exec_ctx_created_by_backend; + initial_max_call_depth = other.initial_max_call_depth; + initial_max_pages = other.initial_max_pages; + } + return *this; + } + ~backend() { if (exec_ctx_created_by_backend && ctx) { // delete only if the context was created by the backend @@ -138,14 +149,14 @@ namespace eosio { namespace vm { } module& parse_module(wasm_code& code, const Options& options) { - mod.allocator.use_default_memory(); - return parser_t{ mod.allocator, options }.parse_module(code, mod, debug); + mod->allocator.use_default_memory(); + return parser_t{ mod->allocator, options }.parse_module(code, *mod, debug); } module& parse_module2(wasm_code_ptr& ptr, size_t sz, const Options& options, bool single_parsing) { if (single_parsing) { - mod.allocator.use_default_memory(); - return parser_t{ mod.allocator, options }.parse_module2(ptr, sz, mod, debug); + mod->allocator.use_default_memory(); + return parser_t{ mod->allocator, options }.parse_module2(ptr, sz, *mod, debug); } else { // To prevent large number of memory mappings used, two-passes of // parsing are performed. @@ -163,23 +174,25 @@ namespace eosio { namespace vm { } // Second pass: uses actual required memory for final parsing - mod.allocator.use_fixed_memory(largest_size); - return parser_t{ mod.allocator, options }.parse_module2(orig_ptr, sz, mod, debug); + mod->allocator.use_fixed_memory(largest_size); + return parser_t{ mod->allocator, options }.parse_module2(orig_ptr, sz, *mod, debug); } } void set_context(context_t* ctx_ptr) { - // execution context can be only set when it is not already created by the backend + // ctx cannot be set if it is created by the backend assert(!exec_ctx_created_by_backend); ctx = ctx_ptr; } inline void reset_max_call_depth() { + // max_call_depth cannot be reset if ctx is created by the backend assert(!exec_ctx_created_by_backend); ctx->set_max_call_depth(initial_max_call_depth); } inline void reset_max_pages() { + // max_pages cannot be reset if ctx is created by the backend assert(!exec_ctx_created_by_backend); ctx->set_max_pages(initial_max_pages); } @@ -277,13 +290,13 @@ namespace eosio { namespace vm { std::atomic _timed_out = false; auto reenable_code = scope_guard{[&](){ if (_timed_out) { - mod.allocator.enable_code(Impl::is_jit); + mod->allocator.enable_code(Impl::is_jit); } }}; try { auto wd_guard = wd.scoped_run([this,&_timed_out]() { _timed_out = true; - mod.allocator.disable_code(); + mod->allocator.disable_code(); }); static_cast(f)(); } catch(wasm_memory_exception&) { @@ -298,9 +311,9 @@ namespace eosio { namespace vm { template inline void execute_all(Watchdog&& wd, host_t& host) { timed_run(static_cast(wd), [&]() { - for (int i = 0; i < mod.exports.size(); i++) { - if (mod.exports[i].kind == external_kind::Function) { - std::string s{ (const char*)mod.exports[i].field_str.raw(), mod.exports[i].field_str.size() }; + for (int i = 0; i < mod->exports.size(); i++) { + if (mod->exports[i].kind == external_kind::Function) { + std::string s{ (const char*)mod->exports[i].field_str.raw(), mod->exports[i].field_str.size() }; ctx->execute(host, interpret_visitor(*ctx), s); } } @@ -310,9 +323,9 @@ namespace eosio { namespace vm { template inline void execute_all(Watchdog&& wd) { timed_run(static_cast(wd), [&]() { - for (int i = 0; i < mod.exports.size(); i++) { - if (mod.exports[i].kind == external_kind::Function) { - std::string s{ (const char*)mod.exports[i].field_str.raw(), mod.exports[i].field_str.size() }; + for (int i = 0; i < mod->exports.size(); i++) { + if (mod->exports[i].kind == external_kind::Function) { + std::string s{ (const char*)mod->exports[i].field_str.raw(), mod->exports[i].field_str.size() }; ctx->execute(nullptr, interpret_visitor(*ctx), s); } } @@ -324,8 +337,7 @@ namespace eosio { namespace vm { ctx->set_wasm_allocator(memory_alloc); } - inline wasm_allocator* get_wasm_allocator() { return memory_alloc; } - inline module& get_module() { return mod; } + inline module& get_module() { return *mod; } inline void exit(const std::error_code& ec) { ctx->exit(ec); } inline auto& get_context() { return *ctx; } @@ -333,7 +345,7 @@ namespace eosio { namespace vm { private: wasm_allocator* memory_alloc = nullptr; // non owning pointer - module mod; + std::shared_ptr mod = nullptr; DebugInfo debug; context_t* ctx = nullptr; bool exec_ctx_created_by_backend = true; // true if execution context is created by backend (legacy behavior), false if provided by users (Leap uses this) From b79fd167ef39dc147e148ad1d1604e0cb82e9dab Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Sat, 26 Aug 2023 11:55:21 -0400 Subject: [PATCH 23/36] remove copy constructor from backend and use a share method instead --- include/eosio/vm/backend.hpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 1e2069ec..ec662f60 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -131,15 +131,6 @@ namespace eosio { namespace vm { construct(); } - backend& operator=(const backend& other) { - if (this != &other) { - mod = other.mod; - exec_ctx_created_by_backend = other.exec_ctx_created_by_backend; - initial_max_call_depth = other.initial_max_call_depth; - initial_max_pages = other.initial_max_pages; - } - return *this; - } ~backend() { if (exec_ctx_created_by_backend && ctx) { @@ -179,6 +170,13 @@ namespace eosio { namespace vm { } } + void share(const backend& from) { + mod = from.mod; + exec_ctx_created_by_backend = from.exec_ctx_created_by_backend; + initial_max_call_depth = from.initial_max_call_depth; + initial_max_pages = from.initial_max_pages; + } + void set_context(context_t* ctx_ptr) { // ctx cannot be set if it is created by the backend assert(!exec_ctx_created_by_backend); From 23d897d38b8de8db59e164d762ede935255f990e Mon Sep 17 00:00:00 2001 From: Lin Huang Date: Thu, 31 Aug 2023 21:21:20 -0400 Subject: [PATCH 24/36] prevent invalid sharing of compiled mod in backend --- include/eosio/vm/backend.hpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index ec662f60..25dcb91a 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -90,27 +90,27 @@ namespace eosio { namespace vm { public: backend() {} backend(wasm_code&& code, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}), mod_sharable{true} { ctx->set_max_pages(detail::get_max_pages(options)); construct(&host); } backend(wasm_code&& code, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}), mod_sharable{true} { ctx->set_max_pages(detail::get_max_pages(options)); construct(); } backend(wasm_code& code, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}) { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module(code, options), detail::get_max_call_depth(options)}), mod_sharable{true} { ctx->set_max_pages(detail::get_max_pages(options)); construct(&host); } backend(wasm_code& code, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{(parse_module(code, options)), detail::get_max_call_depth(options)}) { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{(parse_module(code, options)), detail::get_max_call_depth(options)}), mod_sharable{true} { ctx->set_max_pages(detail::get_max_pages(options)); construct(); } backend(wasm_code_ptr& ptr, size_t sz, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) - : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module2(ptr, sz, options, true), detail::get_max_call_depth(options)}) { // single parsing. original behavior { + : memory_alloc(alloc), mod(std::make_shared()), ctx(new context_t{parse_module2(ptr, sz, options, true), detail::get_max_call_depth(options)}), mod_sharable{true} { // single parsing. original behavior { ctx->set_max_pages(detail::get_max_pages(options)); construct(&host); } @@ -121,7 +121,7 @@ namespace eosio { namespace vm { // * Leap reuses execution context per thread; is_exec_ctx_created_by_backend is set // to false when a backend is constructued backend(wasm_code_ptr& ptr, size_t sz, wasm_allocator* alloc, const Options& options = Options{}, bool single_parsing = true, bool exec_ctx_by_backend = true) - : memory_alloc(alloc), mod(std::make_shared()), exec_ctx_created_by_backend(exec_ctx_by_backend), initial_max_call_depth(detail::get_max_call_depth(options)), initial_max_pages(detail::get_max_pages(options)) { + : memory_alloc(alloc), mod(std::make_shared()), exec_ctx_created_by_backend(exec_ctx_by_backend), mod_sharable{true}, initial_max_call_depth(detail::get_max_call_depth(options)), initial_max_pages(detail::get_max_pages(options)) { if (exec_ctx_created_by_backend) { ctx = new context_t{parse_module2(ptr, sz, options, single_parsing), initial_max_call_depth}; ctx->set_max_pages(initial_max_pages); @@ -170,8 +170,12 @@ namespace eosio { namespace vm { } } + // Shares compiled module with another backend which never compiles + // module itself. void share(const backend& from) { - mod = from.mod; + assert(from.mod_sharable); // `from` backend's mod is sharable + assert(!mod_sharable); // `to` backend's mod must not be sharable + mod = from.mod; exec_ctx_created_by_backend = from.exec_ctx_created_by_backend; initial_max_call_depth = from.initial_max_call_depth; initial_max_pages = from.initial_max_pages; @@ -347,6 +351,7 @@ namespace eosio { namespace vm { DebugInfo debug; context_t* ctx = nullptr; bool exec_ctx_created_by_backend = true; // true if execution context is created by backend (legacy behavior), false if provided by users (Leap uses this) + bool mod_sharable = false; // true if mod is sharable (compiled by the backend) uint32_t initial_max_call_depth = 0; uint32_t initial_max_pages = 0; }; From f7426423c9751ad9fcc6d450b5175e4a3e2c2294 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 5 Sep 2023 14:59:12 -0600 Subject: [PATCH 25/36] elems are needed because tables are now mutable --- include/eosio/vm/types.hpp | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index 7244e33c..b4d905fc 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -120,6 +120,7 @@ namespace eosio { namespace vm { uint32_t index; init_expr offset; elem_mode mode; + // TODO: move mutable data to execution_context bool dropped; guarded_vector elems; }; @@ -239,6 +240,14 @@ namespace eosio { namespace vm { external_kind kind; uint32_t index; }; + struct jit_elem_segment { + uint32_t index; + init_expr offset; + elem_mode mode; + // TODO: move mutable data to execution_context + bool dropped; + std::vector elems; + }; struct jit_data_segment { uint32_t index; init_expr offset; @@ -252,7 +261,7 @@ namespace eosio { namespace vm { std::vector memories; std::vector globals; std::vector exports; - // elements not needed during JIT execution + std::vector elements; std::vector jit_code_offset; std::vector data; std::vector import_functions; @@ -326,6 +335,20 @@ namespace eosio { namespace vm { } } + if (auto elem_size = elements.size(); elem_size > 0) + { + jit_mod->elements.reserve(elem_size); + for (uint32_t i = 0; i < elem_size; ++i) + { + const auto& elem_seg = elements[i]; + jit_mod->elements.emplace_back(jit_mod_t::jit_elem_segment{ + .index = elem_seg.index, + .offset = elem_seg.offset, + .elems = {elem_seg.elems.data(), elem_seg.elems.data() + elem_seg.elems.size()} + }); + } + } + if (auto code_size = code.size(); code_size > 0) { jit_mod->jit_code_offset.reserve(code_size); for (uint32_t i = 0; i < code_size; ++i) { From 5f10fe33cc3a4c9764a94d73650e09a3d18ee47a Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Wed, 6 Sep 2023 15:29:23 -0600 Subject: [PATCH 26/36] Fix compile errors --- include/eosio/vm/backend.hpp | 4 +- include/eosio/vm/execution_context.hpp | 78 +++++++++++++++----------- include/eosio/vm/types.hpp | 25 +++++++-- 3 files changed, 67 insertions(+), 40 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index ca744967..282bd106 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -219,8 +219,8 @@ namespace eosio { namespace vm { inline backend& initialize(stack_manager& alt_stack, host_t* host=nullptr) { if (memory_alloc) { - ctx.reset(); - ctx.execute_start(alt_stack, host, interpret_visitor(ctx)); + ctx->reset(); + ctx->execute_start(alt_stack, host, interpret_visitor(*ctx)); } return *this; } diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index f9165344..743b4738 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -122,10 +122,18 @@ namespace eosio { namespace vm { using host_type = detail::host_type_t; public: Derived& derived() { return static_cast(*this); } + auto& resolve_module() { + if constexpr (IsJit) { + return *_mod->jit_mod; + } else { + return *_mod; + } + } execution_context_base() {} execution_context_base(module* m) : _mod(m) { - if (_mod.indirect_table(0)) { - _alt_table.reset(new table_entry[_mod.tables[0].limits.initial]); + if (_mod->indirect_table(0)) { + _alt_table.reset(new table_entry[_mod->tables[0].limits.initial]); + } } inline void initialize_globals() { @@ -214,7 +222,7 @@ namespace eosio { namespace vm { auto required_memory = static_cast(offset) + data_seg.data.size(); EOS_VM_ASSERT(required_memory <= available_memory, wasm_memory_exception, "data out of range"); auto addr = _linear_memory + offset; - memcpy((char*)(addr), data_seg.data.raw(), data_seg.data.size()); + memcpy((char*)(addr), data_seg.data.data(), data_seg.data.size()); data_seg.dropped = true; } } @@ -228,26 +236,26 @@ namespace eosio { namespace vm { } // reset the table - if (_mod.tables.size() != 0) { + if (mod.tables.size() != 0) { char* table_location = _linear_memory + wasm_allocator::table_offset(); table_entry* table_start; - if (_mod.indirect_table(0)) { + if (_mod->indirect_table(0)) { table_start = _alt_table.get(); std::memcpy(table_location, &table_start, sizeof(table_start)); } else { - table_start = new (table_location) table_entry[_mod.tables[0].limits.initial]; + table_start = new (table_location) table_entry[mod.tables[0].limits.initial]; } - std::memset(table_start, 0xff, _mod.tables[0].limits.initial * sizeof(table_entry)); - for (uint32_t i = 0; i < _mod.elements.size(); ++i) { - auto& elem_seg = _mod.elements[i]; + std::memset(table_start, 0xff, mod.tables[0].limits.initial * sizeof(table_entry)); + for (uint32_t i = 0; i < mod.elements.size(); ++i) { + auto& elem_seg = mod.elements[i]; if (elem_seg.mode == elem_mode::passive) { elem_seg.dropped = false; } else if (elem_seg.mode == elem_mode::declarative) { elem_seg.dropped = true; } else { uint32_t offset = elem_seg.offset.value.i32; - EOS_VM_ASSERT(static_cast(offset) + elem_seg.elems.size() <= _mod.tables[0].limits.initial, wasm_memory_exception, "elem out of range"); - std::memcpy(table_start + offset, elem_seg.elems.raw(), elem_seg.elems.size() * sizeof(table_entry)); + EOS_VM_ASSERT(static_cast(offset) + elem_seg.elems.size() <= mod.tables[0].limits.initial, wasm_memory_exception, "elem out of range"); + std::memcpy(table_start + offset, elem_seg.elems.data(), elem_seg.elems.size() * sizeof(table_entry)); elem_seg.dropped = true; } } @@ -255,23 +263,26 @@ namespace eosio { namespace vm { } void init_linear_memory(uint32_t x, uint32_t d, uint32_t s, uint32_t n) { - assert(x < _mod.data.size()); - const auto& data_seg = _mod.data[x]; + auto& mod = resolve_module(); + assert(x < mod.data.size()); + const auto& data_seg = mod.data[x]; auto data_len = data_seg.dropped? 0 : data_seg.data.size(); if (std::uint64_t{s} + n > data_len) throw_("data out of range"); void* dest = get_interface().template validate_pointer(d, n); - std::memcpy(dest, data_seg.data.raw() + s, n); + std::memcpy(dest, data_seg.data.data() + s, n); } void drop_data(uint32_t x) { - assert(x < _mod.data.size()); - auto& data_seg = _mod.data[x]; + auto& mod = resolve_module(); + assert(x < mod.data.size()); + auto& data_seg = mod.data[x]; data_seg.dropped = true; } table_entry* get_table_base() { - if (_mod.indirect_table(0)) { + auto& mod = resolve_module(); + if (_mod->indirect_table(0)) { return (*reinterpret_cast(linear_memory() + wasm_allocator::table_offset())); } else { return reinterpret_cast(linear_memory() + wasm_allocator::table_offset()); @@ -279,24 +290,27 @@ namespace eosio { namespace vm { } void init_table(uint32_t x, uint32_t d, uint32_t s, uint32_t n) { - assert(x < _mod.elements.size()); - const auto& elem_seg = _mod.elements[x]; + auto& mod = resolve_module(); + assert(x < mod.elements.size()); + const auto& elem_seg = mod.elements[x]; auto elem_len = elem_seg.dropped? 0 : elem_seg.elems.size(); if (std::uint64_t{s} + n > elem_len) throw_("elem out of range"); - if (std::uint64_t{d} + n > _mod.tables[0].limits.initial) + if (std::uint64_t{d} + n > mod.tables[0].limits.initial) throw_("wasm memory out-of-bounds"); - std::memcpy(get_table_base() + d, elem_seg.elems.raw() + s, n * sizeof(table_entry)); + std::memcpy(get_table_base() + d, elem_seg.elems.data() + s, n * sizeof(table_entry)); } void drop_elem(uint32_t x) { - assert(x < _mod.elements.size()); - auto& elem_seg = _mod.elements[x]; + auto& mod = resolve_module(); + assert(x < mod.elements.size()); + auto& elem_seg = mod.elements[x]; elem_seg.dropped = true; } table_entry* get_table_ptr(uint32_t base, uint32_t size) { - if (std::uint64_t{base} + size > _mod.tables[0].limits.initial) + auto& mod = resolve_module(); + if (std::uint64_t{base} + size > mod.tables[0].limits.initial) throw_("table out of range"); return get_table_base() + base; } @@ -311,7 +325,7 @@ namespace eosio { namespace vm { template inline std::optional execute(stack_manager& alt_stack, host_type* host, Visitor&& visitor, const std::string_view func, Args... args) { - uint32_t func_index = _mod.get_exported_function(func); + uint32_t func_index = _mod->get_exported_function(func); return derived().execute(alt_stack, host, std::forward(visitor), func_index, std::forward(args)...); } @@ -323,8 +337,8 @@ namespace eosio { namespace vm { template inline void execute_start(stack_manager& alt_stack, host_type* host, Visitor&& visitor) { - if (_mod.start != std::numeric_limits::max()) - derived().execute(alt_stack, host, std::forward(visitor), _mod.start); + if (_mod->start != std::numeric_limits::max()) + derived().execute(alt_stack, host, std::forward(visitor), _mod->start); } protected: @@ -450,13 +464,13 @@ namespace eosio { namespace vm { std::size_t get_maximum_stack_size() { - if (_mod.stack_limit_is_bytes) + if (_mod->stack_limit_is_bytes) { return this->_remaining_call_depth * 2; } else { - return (_mod.maximum_stack + 2 /*frame ptr + return ptr*/) * (this->_remaining_call_depth + 1) * sizeof(native_value); + return (_mod->maximum_stack + 2 /*frame ptr + return ptr*/) * (this->_remaining_call_depth + 1) * sizeof(native_value); } } @@ -718,7 +732,7 @@ namespace eosio { namespace vm { const auto& ft = _mod->types[_mod->imports[index].type.func_t]; type_check(ft); inc_pc(); - std::uint32_t frame_size = _mod.get_function_stack_size(index); + std::uint32_t frame_size = _mod->get_function_stack_size(index); EOS_VM_ASSERT (frame_size <= _remaining_call_depth, wasm_interpreter_exception, "stack overflow"); _remaining_call_depth -= frame_size; push_call( activation_frame{ nullptr, 0 } ); @@ -767,7 +781,7 @@ namespace eosio { namespace vm { return_pc = _state.pc + 1; { - std::uint32_t frame_size = _mod.get_function_stack_size(index); + std::uint32_t frame_size = _mod->get_function_stack_size(index); EOS_VM_ASSERT (frame_size <= _remaining_call_depth, wasm_interpreter_exception, "stack overflow"); _remaining_call_depth -= frame_size; } @@ -832,7 +846,7 @@ namespace eosio { namespace vm { [&](const v128_const_t& v) { EOS_VM_ASSERT(gl.type.content_type == types::v128, wasm_interpreter_exception, "expected v128 global type"); - gl.current.value.v128 = v.data; + _globals[index].value.v128 = v.data; }, [](auto) { throw wasm_interpreter_exception{ "invalid global type" }; } }, el); diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index b4d905fc..a308f0fd 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -251,13 +251,15 @@ namespace eosio { namespace vm { struct jit_data_segment { uint32_t index; init_expr offset; + bool passive; + bool dropped; std::vector data; }; std::vector types; std::vector imports; std::vector functions; - // tables not needed during JIT execution + std::vector tables; std::vector memories; std::vector globals; std::vector exports; @@ -319,6 +321,10 @@ namespace eosio { namespace vm { jit_mod->functions.assign(functions.data(), functions.data() + functions.size()); } + if (tables.size() > 0) { + jit_mod->tables.assign(tables.data(), tables.data() + tables.size()); + } + if (globals.size() > 0) { jit_mod->globals.assign(globals.raw(), globals.raw() + globals.size()); } @@ -344,6 +350,7 @@ namespace eosio { namespace vm { jit_mod->elements.emplace_back(jit_mod_t::jit_elem_segment{ .index = elem_seg.index, .offset = elem_seg.offset, + .mode = elem_seg.mode, .elems = {elem_seg.elems.data(), elem_seg.elems.data() + elem_seg.elems.size()} }); } @@ -361,9 +368,10 @@ namespace eosio { namespace vm { for (uint32_t i = 0; i < data_size; ++i) { const auto& data_seg = data[i]; jit_mod->data.emplace_back(jit_mod_t::jit_data_segment{ - data_seg.index, - data_seg.offset, - {data_seg.data.data(), data_seg.data.data() + data_seg.data.size()} + .index = data_seg.index, + .offset = data_seg.offset, + .passive = data_seg.passive, + .data = {data_seg.data.data(), data_seg.data.data() + data_seg.data.size()} }); } } @@ -441,8 +449,13 @@ namespace eosio { namespace vm { return index; } - bool indirect_table(std::size_t i) { - return i < tables.size() && (tables[i].limits.initial * sizeof(table_entry) > wasm_allocator::table_size()); + bool indirect_table(std::size_t i) + { + return jit_mod? indirect_table_impl(*jit_mod, i) : indirect_table_impl(*this, i); + } + + static bool indirect_table_impl(auto& mod, std::size_t i) { + return i < mod.tables.size() && (mod.tables[i].limits.initial * sizeof(table_entry) > wasm_allocator::table_size()); } void normalize_types() { From 034dba5f3885efda6d79851624d85af7e6ed14ee Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Thu, 7 Sep 2023 10:02:17 -0600 Subject: [PATCH 27/36] Fix [get|set]_global for v128 --- include/eosio/vm/execution_context.hpp | 8 ++++++++ include/eosio/vm/x86_64.hpp | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 743b4738..25e31fec 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -636,6 +636,10 @@ namespace eosio { namespace vm { return _globals[index].value.f64; } + inline v128_t get_global_v128(uint32_t index) { + return _globals[index].value.v128; + } + inline void set_global_i32(uint32_t index, int32_t value) { _globals[index].value.i32 = value; } @@ -652,6 +656,10 @@ namespace eosio { namespace vm { _globals[index].value.f64 = value; } + inline void set_global_v128(uint32_t index, v128_t value) { + _globals[index].value.v128 = value; + } + protected: template diff --git a/include/eosio/vm/x86_64.hpp b/include/eosio/vm/x86_64.hpp index 0e5f6a5d..3278d192 100644 --- a/include/eosio/vm/x86_64.hpp +++ b/include/eosio/vm/x86_64.hpp @@ -592,6 +592,8 @@ namespace eosio { namespace vm { case types::i64: emit_operand_ptr(&get_global_i64); break; case types::f32: emit_operand_ptr(&get_global_f32); break; case types::f64: emit_operand_ptr(&get_global_f64); break; + case types::v128: emit_operand_ptr(&get_global_v128); break; + default: assert(!"Unknown global type"); } // call *%rax emit_bytes(0xff, 0xd0); @@ -601,6 +603,9 @@ namespace eosio { namespace vm { emit_bytes(0x5f); emit_restore_backtrace(); // push %rax -- return result + if (gl.type.content_type == types::v128) { + emit_push(rdx); + } emit_bytes(0x50); } @@ -609,6 +614,9 @@ namespace eosio { namespace vm { auto& gl = _mod.globals[globalidx]; // popq %rdx -- pass global value to %rdx, the third argument in set_global emit_bytes(0x5a); + if (gl.type.content_type == types::v128) { + emit_pop(rcx); + } emit_setup_backtrace(); // pushq %rdi -- save %rdi content onto stack emit_bytes(0x57); @@ -625,6 +633,8 @@ namespace eosio { namespace vm { case types::i64: emit_operand_ptr(&set_global_i64); break; case types::f32: emit_operand_ptr(&set_global_f32); break; case types::f64: emit_operand_ptr(&set_global_f64); break; + case types::v128: emit_operand_ptr(&set_global_v128); break; + default: assert(!"Unknown global type"); } // call *%rax emit_bytes(0xff, 0xd0); @@ -6282,6 +6292,9 @@ namespace eosio { namespace vm { static uint64_t get_global_f64(Context* context /*rdi*/, uint32_t index /*rsi*/) { return context->get_global_f64(index); } + static v128_t get_global_v128(Context* context /*rdi*/, uint32_t index /*rsi*/) { + return context->get_global_v128(index); + } static void set_global_i32(Context* context /*rdi*/, uint32_t index /*rsi*/, int32_t value /*rdx*/) { context->set_global_i32(index, value); @@ -6295,6 +6308,9 @@ namespace eosio { namespace vm { static void set_global_f64(Context* context /*rdi*/, uint32_t index /*rsi*/, uint64_t value /*rdx*/) { context->set_global_f64(index, value); } + static void set_global_v128(Context* context /*rdi*/, uint32_t index /*rsi*/, v128_t value /*rdx+rcx*/) { + context->set_global_v128(index, value); + } static void on_unreachable() { vm::throw_( "unreachable" ); } static void on_fp_error() { vm::throw_( "floating point error" ); } From bf71f3a2abccc5915c832f9decc49b6dc11af78b Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Thu, 7 Sep 2023 12:20:32 -0600 Subject: [PATCH 28/36] If the table is dynamically allocated, it needs to be reset when the module is changed. --- include/eosio/vm/execution_context.hpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 25e31fec..089fe1e0 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -130,11 +130,7 @@ namespace eosio { namespace vm { } } execution_context_base() {} - execution_context_base(module* m) : _mod(m) { - if (_mod->indirect_table(0)) { - _alt_table.reset(new table_entry[_mod->tables[0].limits.initial]); - } - } + execution_context_base(module* m) : _mod(m) {} inline void initialize_globals() { if constexpr (IsJit) { @@ -185,7 +181,10 @@ namespace eosio { namespace vm { throw wasm_exit_exception{"Exiting"}; } - inline void set_module(module* mod) { _mod = mod; } + inline void set_module(module* mod) { + _mod = mod; + _alt_table.reset(); + } inline module& get_module() { return *_mod; } inline void set_wasm_allocator(wasm_allocator* alloc) { _wasm_alloc = alloc; } inline auto get_wasm_allocator() { return _wasm_alloc; } @@ -240,6 +239,9 @@ namespace eosio { namespace vm { char* table_location = _linear_memory + wasm_allocator::table_offset(); table_entry* table_start; if (_mod->indirect_table(0)) { + if (!_alt_table) { + _alt_table.reset(new table_entry[mod.tables[0].limits.initial]); + } table_start = _alt_table.get(); std::memcpy(table_location, &table_start, sizeof(table_start)); } else { From f9ee616098e4b568d5611b681650723ce27ed090 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Thu, 7 Sep 2023 13:36:33 -0600 Subject: [PATCH 29/36] Fix psibase compile errors --- include/eosio/vm/backend.hpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 282bd106..3a704fc0 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -110,6 +110,12 @@ namespace eosio { namespace vm { ctx->set_max_pages(detail::get_max_pages(options)); construct(); } + template + backend(wasm_code& code, wasm_allocator* alloc, const Options& options, XDebugInfo& debug) + : memory_alloc(alloc), ctx(new context_t{(parse_module(code, options, debug)), detail::choose_stack_limit(options)}) { + ctx->set_max_pages(detail::get_max_pages(options)); + construct(); + } backend(wasm_code_ptr& ptr, size_t sz, host_t& host, wasm_allocator* alloc, const Options& options = Options{}) : memory_alloc(alloc), ctx(new context_t{parse_module2(ptr, sz, options, true), detail::choose_stack_limit(options)}) { // single parsing. original behavior { ctx->set_max_pages(detail::get_max_pages(options)); @@ -144,6 +150,12 @@ namespace eosio { namespace vm { return parser_t{ mod.allocator, options }.parse_module(code, mod, debug); } + template + module& parse_module(wasm_code& code, const Options& options, XDebugInfo& debug) { + mod.allocator.use_default_memory(); + return parser_tpl{ mod.allocator, options }.parse_module(code, mod, debug); + } + module& parse_module2(wasm_code_ptr& ptr, size_t sz, const Options& options, bool single_parsing) { if (single_parsing) { mod.allocator.use_default_memory(); @@ -252,9 +264,9 @@ namespace eosio { namespace vm { template inline bool call(stack_manager& alt_stack, host_t& host, const std::string_view& mod, const std::string_view& func, Args... args) { if constexpr (eos_vm_debug) { - ctx.execute(alt_stack, &host, debug_visitor(ctx), func, args...); + ctx->execute(alt_stack, &host, debug_visitor(*ctx), func, args...); } else { - ctx.execute(alt_stack, &host, interpret_visitor(ctx), func, args...); + ctx->execute(alt_stack, &host, interpret_visitor(*ctx), func, args...); } return true; } From 9ec7735cfd13f8eef782de261e95892ee4c0cf26 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Thu, 7 Sep 2023 16:03:38 -0600 Subject: [PATCH 30/36] Move mutable dropped field of elem and data into execution_context --- include/eosio/vm/execution_context.hpp | 23 ++++++++++++----------- include/eosio/vm/types.hpp | 6 ------ 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 089fe1e0..20959f1e 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -211,18 +211,17 @@ namespace eosio { namespace vm { } else _wasm_alloc->reset(); + _dropped_data.assign(mod.data.size(), false); for (uint32_t i = 0; i < mod.data.size(); i++) { auto& data_seg = mod.data[i]; uint32_t offset = data_seg.offset.value.i32; // force to unsigned - if (data_seg.passive) { - data_seg.dropped = false; - } else { + if (!data_seg.passive) { auto available_memory = mod.memories[0].limits.initial * static_cast(page_size); auto required_memory = static_cast(offset) + data_seg.data.size(); EOS_VM_ASSERT(required_memory <= available_memory, wasm_memory_exception, "data out of range"); auto addr = _linear_memory + offset; memcpy((char*)(addr), data_seg.data.data(), data_seg.data.size()); - data_seg.dropped = true; + _dropped_data[i] = true; } } @@ -248,17 +247,17 @@ namespace eosio { namespace vm { table_start = new (table_location) table_entry[mod.tables[0].limits.initial]; } std::memset(table_start, 0xff, mod.tables[0].limits.initial * sizeof(table_entry)); + _dropped_elems.assign(mod.elements.size(), false); for (uint32_t i = 0; i < mod.elements.size(); ++i) { auto& elem_seg = mod.elements[i]; if (elem_seg.mode == elem_mode::passive) { - elem_seg.dropped = false; } else if (elem_seg.mode == elem_mode::declarative) { - elem_seg.dropped = true; + _dropped_elems[i] = true; } else { uint32_t offset = elem_seg.offset.value.i32; EOS_VM_ASSERT(static_cast(offset) + elem_seg.elems.size() <= mod.tables[0].limits.initial, wasm_memory_exception, "elem out of range"); std::memcpy(table_start + offset, elem_seg.elems.data(), elem_seg.elems.size() * sizeof(table_entry)); - elem_seg.dropped = true; + _dropped_elems[i] = true; } } } @@ -268,7 +267,7 @@ namespace eosio { namespace vm { auto& mod = resolve_module(); assert(x < mod.data.size()); const auto& data_seg = mod.data[x]; - auto data_len = data_seg.dropped? 0 : data_seg.data.size(); + auto data_len = _dropped_data[x]? 0 : data_seg.data.size(); if (std::uint64_t{s} + n > data_len) throw_("data out of range"); void* dest = get_interface().template validate_pointer(d, n); @@ -279,7 +278,7 @@ namespace eosio { namespace vm { auto& mod = resolve_module(); assert(x < mod.data.size()); auto& data_seg = mod.data[x]; - data_seg.dropped = true; + _dropped_data[x] = true; } table_entry* get_table_base() { @@ -295,7 +294,7 @@ namespace eosio { namespace vm { auto& mod = resolve_module(); assert(x < mod.elements.size()); const auto& elem_seg = mod.elements[x]; - auto elem_len = elem_seg.dropped? 0 : elem_seg.elems.size(); + auto elem_len = _dropped_elems[x]? 0 : elem_seg.elems.size(); if (std::uint64_t{s} + n > elem_len) throw_("elem out of range"); if (std::uint64_t{d} + n > mod.tables[0].limits.initial) @@ -307,7 +306,7 @@ namespace eosio { namespace vm { auto& mod = resolve_module(); assert(x < mod.elements.size()); auto& elem_seg = mod.elements[x]; - elem_seg.dropped = true; + _dropped_elems[x] = true; } table_entry* get_table_ptr(uint32_t base, uint32_t size) { @@ -374,6 +373,8 @@ namespace eosio { namespace vm { operand_stack _os; std::unique_ptr _alt_table; std::vector _globals; + std::vector _dropped_elems; + std::vector _dropped_data; }; struct jit_visitor { template jit_visitor(T&&) {} }; diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index a308f0fd..1080dd7f 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -120,8 +120,6 @@ namespace eosio { namespace vm { uint32_t index; init_expr offset; elem_mode mode; - // TODO: move mutable data to execution_context - bool dropped; guarded_vector elems; }; @@ -159,7 +157,6 @@ namespace eosio { namespace vm { uint32_t index; init_expr offset; bool passive; - bool dropped; guarded_vector data; }; @@ -244,15 +241,12 @@ namespace eosio { namespace vm { uint32_t index; init_expr offset; elem_mode mode; - // TODO: move mutable data to execution_context - bool dropped; std::vector elems; }; struct jit_data_segment { uint32_t index; init_expr offset; bool passive; - bool dropped; std::vector data; }; From 5d88ced7e3b00e802fd3c0a267b58a4c3ce6e2b7 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Mon, 11 Sep 2023 09:01:48 -0600 Subject: [PATCH 31/36] Since we're copying to std::vector anyway, just use std::vector from the start. --- include/eosio/vm/backend.hpp | 4 +- include/eosio/vm/bitcode_writer.hpp | 4 +- include/eosio/vm/execution_context.hpp | 3 +- include/eosio/vm/null_writer.hpp | 4 +- include/eosio/vm/parser.hpp | 102 ++++++++++++++----------- include/eosio/vm/types.hpp | 66 ++++++++-------- include/eosio/vm/x86_64.hpp | 6 +- tools/addr2line.cpp | 6 +- 8 files changed, 101 insertions(+), 94 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 3a704fc0..0c86e533 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -337,7 +337,7 @@ namespace eosio { namespace vm { timed_run(static_cast(wd), [&]() { for (int i = 0; i < mod.exports.size(); i++) { if (mod.exports[i].kind == external_kind::Function) { - std::string s{ (const char*)mod.exports[i].field_str.raw(), mod.exports[i].field_str.size() }; + std::string s{ (const char*)mod.exports[i].field_str.data(), mod.exports[i].field_str.size() }; ctx->execute(host, interpret_visitor(*ctx), s); } } @@ -349,7 +349,7 @@ namespace eosio { namespace vm { timed_run(static_cast(wd), [&]() { for (int i = 0; i < mod.exports.size(); i++) { if (mod.exports[i].kind == external_kind::Function) { - std::string s{ (const char*)mod.exports[i].field_str.raw(), mod.exports[i].field_str.size() }; + std::string s{ (const char*)mod.exports[i].field_str.data(), mod.exports[i].field_str.size() }; ctx->execute(nullptr, interpret_visitor(*ctx), s); } } diff --git a/include/eosio/vm/bitcode_writer.hpp b/include/eosio/vm/bitcode_writer.hpp index 9b2b907b..cf1e48cb 100644 --- a/include/eosio/vm/bitcode_writer.hpp +++ b/include/eosio/vm/bitcode_writer.hpp @@ -364,12 +364,12 @@ namespace eosio { namespace vm { void emit_error() { fb[op_index++] = error_t{}; } void fix_branch(uint32_t* branch, uint32_t target) { if(branch) *branch = _base_offset + target; } - void emit_prologue(const func_type& ft, const guarded_vector&, uint32_t idx) { + void emit_prologue(const func_type& ft, const std::vector&, uint32_t idx) { op_index = 0; // pre-allocate for the function body code, so we have a big blob of memory to work with during function code parsing fb = guarded_vector{_allocator, _mod->code[idx].size }; } - void emit_epilogue(const func_type& ft, const guarded_vector& locals, uint32_t idx) { + void emit_epilogue(const func_type& ft, const std::vector& locals, uint32_t idx) { fb.resize(op_index + 1); uint32_t locals_count = 0; for(uint32_t i = 0; i < locals.size(); ++i) { diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index 20959f1e..abad759a 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -216,6 +216,7 @@ namespace eosio { namespace vm { auto& data_seg = mod.data[i]; uint32_t offset = data_seg.offset.value.i32; // force to unsigned if (!data_seg.passive) { + assert(!mod.memories.empty() && "Validation should ensure that an active data segment has a valid memory"); auto available_memory = mod.memories[0].limits.initial * static_cast(page_size); auto required_memory = static_cast(offset) + data_seg.data.size(); EOS_VM_ASSERT(required_memory <= available_memory, wasm_memory_exception, "data out of range"); @@ -773,7 +774,7 @@ namespace eosio { namespace vm { } inline uint32_t table_elem(uint32_t i) { - EOS_VM_ASSERT(i < _mod->tables[0].limits.initial, wasm_interpreter_exception, "table index out of range"); + EOS_VM_ASSERT(i < _mod->tables.at(0).limits.initial, wasm_interpreter_exception, "table index out of range"); return this->get_table_base()[i].index; } inline void push_operand(operand_stack_elem el) { get_operand_stack().push(std::move(el)); } diff --git a/include/eosio/vm/null_writer.hpp b/include/eosio/vm/null_writer.hpp index 05561154..afff0fcf 100644 --- a/include/eosio/vm/null_writer.hpp +++ b/include/eosio/vm/null_writer.hpp @@ -459,8 +459,8 @@ class null_writer { void emit_table_copy() {} void fix_branch(branch_t, label_t) {} - void emit_prologue(const func_type& /*ft*/, const guarded_vector& /*locals*/, uint32_t /*idx*/) {} - void emit_epilogue(const func_type& /*ft*/, const guarded_vector& /*locals*/, uint32_t /*idx*/) {} + void emit_prologue(const func_type& /*ft*/, const std::vector& /*locals*/, uint32_t /*idx*/) {} + void emit_epilogue(const func_type& /*ft*/, const std::vector& /*locals*/, uint32_t /*idx*/) {} void finalize(function_body& /*body*/) {} const void* get_addr() const { return nullptr; } diff --git a/include/eosio/vm/parser.hpp b/include/eosio/vm/parser.hpp index e9388bd0..7c505f28 100644 --- a/include/eosio/vm/parser.hpp +++ b/include/eosio/vm/parser.hpp @@ -315,12 +315,11 @@ namespace eosio { namespace vm { } } - guarded_vector parse_utf8_string(wasm_code_ptr& code, std::uint32_t max_size) { + std::vector parse_utf8_string(wasm_code_ptr& code, std::uint32_t max_size) { auto len = parse_varuint32(code); EOS_VM_ASSERT(len <= max_size, wasm_parse_exception, "name too long"); auto guard = code.scoped_shrink_bounds(len); - auto result = guarded_vector{ _allocator, len }; - result.copy(code.raw(), len); + auto result = std::vector(code.raw(), code.raw() + len); validate_utf8_string(code, len); return result; } @@ -432,7 +431,7 @@ namespace eosio { namespace vm { inline void parse_custom(wasm_code_ptr& code) { auto section_name = parse_utf8_string(code, 0xFFFFFFFFu); // ignored, but needs to be validated if(detail::get_parse_custom_section_name(_options) && - section_name.size() == 4 && std::memcmp(section_name.raw(), "name", 4) == 0) { + section_name.size() == 4 && std::memcmp(section_name.data(), "name", 4) == 0) { parse_name_section(code); } else { // skip to the end of the section @@ -440,7 +439,7 @@ namespace eosio { namespace vm { } } - inline void parse_name_map(wasm_code_ptr& code, guarded_vector& map) { + inline void parse_name_map(wasm_code_ptr& code, std::vector& map) { for(uint32_t i = 0; i < map.size(); ++i) { map[i].idx = parse_varuint32(code); map[i].name = parse_utf8_string(code, 0xFFFFFFFFu); @@ -448,22 +447,19 @@ namespace eosio { namespace vm { } inline void parse_name_section(wasm_code_ptr& code) { - _mod->names = _allocator.alloc(1); - new (_mod->names) name_section; + _mod->names.emplace(); if(code.bounds() == code.offset()) return; if(*code == 0) { ++code; auto subsection_guard = code.scoped_consume_items(parse_varuint32(code)); - _mod->names->module_name = _allocator.alloc>(1); - new (_mod->names->module_name) guarded_vector(parse_utf8_string(code, 0xFFFFFFFFu)); + _mod->names->module_name.emplace(parse_utf8_string(code, 0xFFFFFFFFu)); } if(code.bounds() == code.offset()) return; if(*code == 1) { ++code; auto subsection_guard = code.scoped_consume_items(parse_varuint32(code)); uint32_t size = parse_varuint32(code); - _mod->names->function_names = _allocator.alloc>(1); - new (_mod->names->function_names) guarded_vector(_allocator, size); + _mod->names->function_names.emplace(size); parse_name_map(code, *_mod->names->function_names); } if(code.bounds() == code.offset()) return; @@ -471,13 +467,12 @@ namespace eosio { namespace vm { ++code; auto subsection_guard = code.scoped_consume_items(parse_varuint32(code)); uint32_t size = parse_varuint32(code); - _mod->names->local_names = _allocator.alloc>(1); - new (_mod->names->local_names) guarded_vector(_allocator, size); + _mod->names->local_names.emplace(size); for(uint32_t i = 0; i < size; ++i) { auto& [idx,namemap] = (*_mod->names->local_names)[i]; idx = parse_varuint32(code); uint32_t local_size = parse_varuint32(code); - namemap = guarded_vector(_allocator, local_size); + namemap = std::vector(local_size); parse_name_map(code, namemap); } } @@ -566,7 +561,7 @@ namespace eosio { namespace vm { void parse_func_type(wasm_code_ptr& code, func_type& ft) { ft.form = *code++; EOS_VM_ASSERT(ft.form == 0x60, wasm_parse_exception, "invalid function type"); - decltype(ft.param_types) param_types = { _allocator, parse_varuint32(code) }; + decltype(ft.param_types) param_types(parse_varuint32(code)); for (size_t i = 0; i < param_types.size(); i++) { uint8_t pt = *code++; param_types.at(i) = pt; @@ -617,7 +612,7 @@ namespace eosio { namespace vm { EOS_VM_ASSERT(!tt || tt->element_type == types::anyfunc, wasm_parse_exception, "elem type does not match table type"); uint32_t size = parse_varuint32(code); EOS_VM_ASSERT(size <= detail::get_max_element_segment_elements(_options), wasm_parse_exception, "elem segment too large"); - decltype(es.elems) elems = { _allocator, size }; + decltype(es.elems) elems(size); if (flags & 4) { for (uint32_t i = 0; i < size; i++) { parse_elem_expr(code, elems.at(i)); @@ -692,7 +687,7 @@ namespace eosio { namespace vm { const auto& local_cnt = parse_varuint32(code); _current_function_index++; EOS_VM_ASSERT(local_cnt <= detail::get_max_local_sets(_options), wasm_parse_exception, "Number of local sets exceeds limit"); - decltype(fb.locals) locals = { _allocator, local_cnt }; + decltype(fb.locals) locals(local_cnt); func_type& ft = _mod->types.at(_mod->functions.at(idx)); detail::max_func_local_bytes_checker local_checker(_options, ft); // parse the local entries @@ -821,7 +816,7 @@ namespace eosio { namespace vm { }; struct local_types_t { - local_types_t(const func_type& ft, const guarded_vector& locals_arg) : + local_types_t(const func_type& ft, const std::vector& locals_arg) : _ft(ft), _locals(locals_arg) { uint32_t count = ft.param_types.size(); _boundaries.push_back(count); @@ -845,7 +840,7 @@ namespace eosio { namespace vm { return total - _ft.param_types.size(); } const func_type& _ft; - const guarded_vector& _locals; + const std::vector& _locals; std::vector _boundaries; }; @@ -1872,8 +1867,7 @@ namespace eosio { namespace vm { EOS_VM_ASSERT(static_cast(static_cast(ds.offset.value.i32)) + len <= detail::get_max_linear_memory_init(_options), wasm_parse_exception, "out-of-bounds data section"); auto guard = code.scoped_shrink_bounds(len); - ds.data = decltype(ds.data){ _allocator, len}; - ds.data.copy(code.raw(), len); + ds.data.assign(code.raw(), code.raw() + len); code += len; } @@ -1885,48 +1879,62 @@ namespace eosio { namespace vm { for (size_t i = 0; i < count; i++) { elem_parse(code, elems.at(i), i); } } + template + inline void parse_section_impl(wasm_code_ptr& code, std::vector& elems, std::uint32_t max_elements, ParseFunc&& elem_parse) { + auto count = parse_varuint32(code); + EOS_VM_ASSERT(count <= max_elements, wasm_parse_exception, "number of section elements exceeded limit"); + elems.resize(count); + for (size_t i = 0; i < count; i++) { elem_parse(code, elems.at(i), i); } + } + template - inline void parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::type_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { parse_section_impl(code, elems, detail::get_max_type_section_elements(_options), [&](wasm_code_ptr& code, func_type& ft, std::size_t /*idx*/) { parse_func_type(code, ft); }); } template - inline void parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::import_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { parse_section_impl(code, elems, detail::get_max_import_section_elements(_options), [&](wasm_code_ptr& code, import_entry& ie, std::size_t /*idx*/) { parse_import_entry(code, ie); }); } template - inline void parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::function_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { parse_section_impl(code, elems, detail::get_max_function_section_elements(_options), [&](wasm_code_ptr& code, uint32_t& elem, std::size_t /*idx*/) { elem = parse_varuint32(code); }); } template - inline void parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::table_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { parse_section_impl(code, elems, 1, [&](wasm_code_ptr& code, table_type& tt, std::size_t /*idx*/) { parse_table_type(code, tt); }); } template - inline void parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::memory_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { parse_section_impl(code, elems, 1, [&](wasm_code_ptr& code, memory_type& mt, std::size_t idx) { EOS_VM_ASSERT(idx == 0, wasm_parse_exception, "only one memory is permitted"); parse_memory_type(code, mt); }); } template - inline void - parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::global_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { parse_section_impl(code, elems, detail::get_max_global_section_elements(_options), [&](wasm_code_ptr& code, global_variable& gv, std::size_t /*idx*/) { parse_global_variable(code, gv); }); } template - inline void parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::export_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { parse_section_impl(code, elems, detail::get_max_export_section_elements(_options), [&](wasm_code_ptr& code, export_entry& ee, std::size_t /*idx*/) { parse_export_entry(code, ee); }); } @@ -1938,15 +1946,16 @@ namespace eosio { namespace vm { EOS_VM_ASSERT(ft.return_count == 0 && ft.param_types.size() == 0, wasm_parse_exception, "wrong type for start"); } template - inline void - parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::element_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { parse_section_impl(code, elems, detail::get_max_element_section_elements(_options), [&](wasm_code_ptr& code, elem_segment& es, std::size_t /*idx*/) { parse_elem_segment(code, es); }); } template - inline void parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::code_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { const void* code_start = code.raw() - code.offset(); parse_section_impl(code, elems, detail::get_max_function_section_elements(_options), [&](wasm_code_ptr& code, function_body& fb, std::size_t idx) { parse_function_body(code, fb, idx); }); @@ -1973,8 +1982,9 @@ namespace eosio { namespace vm { } template - inline void parse_section(wasm_code_ptr& code, - vec>& elems) { + requires (id == section_id::data_section) + inline void parse_section(wasm_code_ptr& code, + std::vector& elems) { parse_section_impl(code, elems, detail::get_max_data_section_elements(_options), [&](wasm_code_ptr& code, data_segment& ds, std::size_t /*idx*/) { parse_data_segment(code, ds); }); if (_datacount) { @@ -2007,16 +2017,16 @@ namespace eosio { namespace vm { } void validate_exports() const { - std::vector*> export_names; + std::vector*> export_names; export_names.reserve(_mod->exports.size()); for (uint32_t i = 0; i < _mod->exports.size(); ++i) { export_names.push_back(&_mod->exports[i].field_str); } std::sort(export_names.begin(), export_names.end(), [](auto* lhs, auto* rhs) { - return std::lexicographical_compare(lhs->raw(), lhs->raw() + lhs->size(), rhs->raw(), rhs->raw() + rhs->size()); + return *lhs < *rhs; }); auto it = std::adjacent_find(export_names.begin(), export_names.end(), [](auto* lhs, auto* rhs) { - return lhs->size() == rhs->size() && std::equal(lhs->raw(), lhs->raw() + lhs->size(), rhs->raw()); + return *lhs == *rhs; }); EOS_VM_ASSERT(it == export_names.end(), wasm_parse_exception, "duplicate export name"); } diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index 1080dd7f..bc9a4cf9 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -41,7 +41,7 @@ namespace eosio { namespace vm { struct func_type { value_type form; // value for the func type constructor - guarded_vector param_types; + std::vector param_types; uint8_t return_count; value_type return_type; }; @@ -49,7 +49,7 @@ namespace eosio { namespace vm { inline bool operator==(const func_type& lhs, const func_type& rhs) { return lhs.form == rhs.form && lhs.param_types.size() == rhs.param_types.size() && - std::equal(lhs.param_types.raw(), lhs.param_types.raw() + lhs.param_types.size(), rhs.param_types.raw()) && + std::equal(lhs.param_types.data(), lhs.param_types.data() + lhs.param_types.size(), rhs.param_types.data()) && lhs.return_count == rhs.return_count && (lhs.return_count || lhs.return_type == rhs.return_type); } @@ -102,14 +102,14 @@ namespace eosio { namespace vm { }; struct import_entry { - guarded_vector module_str; - guarded_vector field_str; + std::vector module_str; + std::vector field_str; external_kind kind; import_type type; }; struct export_entry { - guarded_vector field_str; + std::vector field_str; external_kind kind; uint32_t index; }; @@ -120,7 +120,7 @@ namespace eosio { namespace vm { uint32_t index; init_expr offset; elem_mode mode; - guarded_vector elems; + std::vector elems; }; struct local_entry { @@ -147,7 +147,7 @@ namespace eosio { namespace vm { struct function_body { uint32_t size; - guarded_vector locals; + std::vector locals; opcode* code; std::size_t jit_code_offset; std::uint32_t stack_size = 0; @@ -157,7 +157,7 @@ namespace eosio { namespace vm { uint32_t index; init_expr offset; bool passive; - guarded_vector data; + std::vector data; }; using wasm_code = std::vector; @@ -167,37 +167,37 @@ namespace eosio { namespace vm { struct name_assoc { std::uint32_t idx; - guarded_vector name; + std::vector name; }; struct indirect_name_assoc { std::uint32_t idx; - guarded_vector namemap; + std::vector namemap; }; struct name_section { - guarded_vector* module_name = nullptr; - guarded_vector* function_names = nullptr; - guarded_vector* local_names = nullptr; + std::optional> module_name; + std::optional> function_names; + std::optional> local_names; }; struct module { growable_allocator allocator; uint32_t start = std::numeric_limits::max(); - guarded_vector types = { allocator, 0 }; - guarded_vector imports = { allocator, 0 }; - guarded_vector functions = { allocator, 0 }; - guarded_vector tables = { allocator, 0 }; - guarded_vector memories = { allocator, 0 }; - guarded_vector globals = { allocator, 0 }; - guarded_vector exports = { allocator, 0 }; - guarded_vector elements = { allocator, 0 }; - guarded_vector code = { allocator, 0 }; - guarded_vector data = { allocator, 0 }; + std::vector types; + std::vector imports; + std::vector functions; + std::vector tables; + std::vector memories; + std::vector globals; + std::vector exports; + std::vector elements; + std::vector code; + std::vector data; // Custom sections: - name_section* names = nullptr; + std::optional names; // not part of the spec for WASM - guarded_vector import_functions = { allocator, 0 }; + std::vector import_functions; guarded_vector type_aliases = { allocator, 0 }; guarded_vector fast_functions = { allocator, 0 }; uint64_t maximum_stack = 0; @@ -266,7 +266,7 @@ namespace eosio { namespace vm { auto& get_function_type(uint32_t index) const { if (index < get_imported_functions_size()) return types[imports[index].type.func_t]; - return types[functions[index - get_imported_functions_size()]]; + return types.at(functions[index - get_imported_functions_size()]); } uint32_t get_imported_functions_size() const { return get_imported_functions_size_impl(imports); @@ -320,7 +320,7 @@ namespace eosio { namespace vm { } if (globals.size() > 0) { - jit_mod->globals.assign(globals.raw(), globals.raw() + globals.size()); + jit_mod->globals.assign(globals.data(), globals.data() + globals.size()); } if (auto exports_size = exports.size(); exports_size > 0) { @@ -395,22 +395,18 @@ namespace eosio { namespace vm { inline uint32_t get_functions_total() const { return get_imported_functions_size() + get_functions_size(); } inline opcode* get_function_pc( uint32_t fidx ) const { EOS_VM_ASSERT( fidx >= get_imported_functions_size(), wasm_interpreter_exception, "trying to get the PC of an imported function" ); - return code[fidx-get_imported_functions_size()].code; - } - - inline auto& get_opcode(uint32_t pc) const { - return ((opcode*)&code[0].code[0])[pc]; + return code.at(fidx-get_imported_functions_size()).code; } inline uint32_t get_function_locals_size(uint32_t index) const { EOS_VM_ASSERT(index >= get_imported_functions_size(), wasm_interpreter_exception, "imported functions do not have locals"); - return code[index - get_imported_functions_size()].locals.size(); + return code.at(index - get_imported_functions_size()).locals.size(); } auto& get_function_type(uint32_t index) const { if (index < get_imported_functions_size()) return types[imports[index].type.func_t]; - return types[functions[index - get_imported_functions_size()]]; + return types.at(functions.at(index - get_imported_functions_size())); } uint32_t get_function_stack_size(uint32_t index) const { @@ -419,7 +415,7 @@ namespace eosio { namespace vm { } else if (index < get_imported_functions_size()) { return 0; } else { - return code[index - get_imported_functions_size()].stack_size; + return code.at(index - get_imported_functions_size()).stack_size; } } diff --git a/include/eosio/vm/x86_64.hpp b/include/eosio/vm/x86_64.hpp index 3278d192..dddabc6f 100644 --- a/include/eosio/vm/x86_64.hpp +++ b/include/eosio/vm/x86_64.hpp @@ -187,7 +187,7 @@ namespace eosio { namespace vm { static constexpr std::size_t max_prologue_size = 33; static constexpr std::size_t max_epilogue_size = 16; - void emit_prologue(const func_type& /*ft*/, const guarded_vector& locals, uint32_t funcnum) { + void emit_prologue(const func_type& /*ft*/, const std::vector& locals, uint32_t funcnum) { _ft = &_mod.types[_mod.functions[funcnum]]; _params = function_parameters{_ft}; _locals = function_locals{locals}; @@ -245,7 +245,7 @@ namespace eosio { namespace vm { } assert((char*)code <= (char*)_code_start + max_prologue_size); } - void emit_epilogue(const func_type& ft, const guarded_vector& locals, uint32_t /*funcnum*/) { + void emit_epilogue(const func_type& ft, const std::vector& locals, uint32_t /*funcnum*/) { #ifndef NDEBUG void * epilogue_start = code; #endif @@ -5274,7 +5274,7 @@ namespace eosio { namespace vm { struct function_locals { function_locals() = default; - function_locals(const guarded_vector& params) { + function_locals(const std::vector& params) { uint32_t offset = 0; int32_t frame_offset = 0; for(uint32_t i = 0; i < params.size(); ++i) { diff --git a/tools/addr2line.cpp b/tools/addr2line.cpp index 0c54722c..b2acb8d0 100644 --- a/tools/addr2line.cpp +++ b/tools/addr2line.cpp @@ -27,7 +27,7 @@ struct nm_debug_info { std::vector function_offsets; }; -eosio::vm::guarded_vector* find_export_name(eosio::vm::module& mod, uint32_t idx) { +std::vector* find_export_name(eosio::vm::module& mod, uint32_t idx) { if(mod.names && mod.names->function_names) { for(uint32_t i = 0; i < mod.names->function_names->size(); ++i) { if((*mod.names->function_names)[i].idx == idx) { @@ -101,8 +101,8 @@ int main(int argc, const char** argv) { parser.parse_module(code, mod, info); { for(std::size_t i = 0; i < info.function_offsets.size(); ++i) { - if(guarded_vector* name = find_export_name(mod, i + mod.get_imported_functions_size())) { - function_names.push_back(std::string(reinterpret_cast(name->raw()), name->size())); + if(std::vector* name = find_export_name(mod, i + mod.get_imported_functions_size())) { + function_names.push_back(std::string(reinterpret_cast(name->data()), name->size())); } else { function_names.push_back("fn" + std::to_string( i + mod.get_imported_functions_size())); } From 711c1a4c1b19c81ef03f24ac128f7176f04a3d86 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Mon, 11 Sep 2023 12:22:42 -0600 Subject: [PATCH 32/36] Move type_aliases and fast_functions to parser --- include/eosio/vm/parser.hpp | 32 ++++++++++++++++++++++++++++---- include/eosio/vm/types.hpp | 24 ------------------------ include/eosio/vm/x86_64.hpp | 4 ++-- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/include/eosio/vm/parser.hpp b/include/eosio/vm/parser.hpp index 7c505f28..59ce8517 100644 --- a/include/eosio/vm/parser.hpp +++ b/include/eosio/vm/parser.hpp @@ -389,7 +389,7 @@ namespace eosio { namespace vm { case section_id::import_section: parse_section(code_ptr, mod.imports); break; case section_id::function_section: parse_section(code_ptr, mod.functions); - mod.normalize_types(); + normalize_types(); break; case section_id::table_section: parse_section(code_ptr, mod.tables); break; case section_id::memory_section: @@ -579,6 +579,28 @@ namespace eosio { namespace vm { } } + void normalize_types() { + type_aliases.resize(_mod->types.size()); + for (uint32_t i = 0; i < _mod->types.size(); ++i) { + uint32_t j = 0; + for (; j < i; ++j) { + if (_mod->types[j] == _mod->types[i]) { + break; + } + } + type_aliases[i] = j; + } + + uint32_t imported_functions_size = _mod->get_imported_functions_size(); + fast_functions.resize(_mod->functions.size() + imported_functions_size); + for (uint32_t i = 0; i < imported_functions_size; ++i) { + fast_functions[i] = type_aliases.at(_mod->imports[i].type.func_t); + } + for (uint32_t i = 0; i < _mod->functions.size(); ++i) { + fast_functions[i + imported_functions_size] = type_aliases.at(_mod->functions[i]); + } + } + void parse_elem_segment(wasm_code_ptr& code, elem_segment& es) { table_type* tt = nullptr; std::uint32_t flags = parse_varuint32(code); @@ -620,7 +642,7 @@ namespace eosio { namespace vm { } else { for (uint32_t i = 0; i < size; i++) { uint32_t index = parse_varuint32(code); - elems.at(i).type = _mod->fast_functions[index]; + elems.at(i).type = fast_functions.at(index); elems.at(i).index = index; EOS_VM_ASSERT(index < _mod->get_functions_total(), wasm_parse_exception, "elem for undefined function"); } @@ -639,7 +661,7 @@ namespace eosio { namespace vm { case opcodes::ref_func: te.index = parse_varuint32(code); EOS_VM_ASSERT(te.index < _mod->get_functions_total(), wasm_parse_exception, "elem for undefined function"); - te.type = _mod->fast_functions[te.index]; + te.type = fast_functions.at(te.index); break; default: EOS_VM_ASSERT(false, wasm_parse_exception, @@ -1053,7 +1075,7 @@ namespace eosio { namespace vm { EOS_VM_ASSERT(ft.return_count <= 1, wasm_parse_exception, "unsupported"); if(ft.return_count) op_stack.push(ft.return_type); - code_writer.emit_call_indirect(ft, functypeidx); + code_writer.emit_call_indirect(ft, type_aliases[functypeidx]); EOS_VM_ASSERT(*code == 0, wasm_parse_exception, "call_indirect must end with 0x00."); code++; // 0x00 break; @@ -2042,5 +2064,7 @@ namespace eosio { namespace vm { detail::eosio_max_nested_structures_checker _nested_checker; std::optional _datacount; typename DebugInfo::builder imap; + std::vector type_aliases; + std::vector fast_functions; }; }} // namespace eosio::vm diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index bc9a4cf9..720ee39d 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -198,8 +198,6 @@ namespace eosio { namespace vm { // not part of the spec for WASM std::vector import_functions; - guarded_vector type_aliases = { allocator, 0 }; - guarded_vector fast_functions = { allocator, 0 }; uint64_t maximum_stack = 0; // The stack limit can be tracked as either frames or bytes bool stack_limit_is_bytes = false; @@ -447,27 +445,5 @@ namespace eosio { namespace vm { static bool indirect_table_impl(auto& mod, std::size_t i) { return i < mod.tables.size() && (mod.tables[i].limits.initial * sizeof(table_entry) > wasm_allocator::table_size()); } - - void normalize_types() { - type_aliases.resize(types.size()); - for (uint32_t i = 0; i < types.size(); ++i) { - uint32_t j = 0; - for (; j < i; ++j) { - if (types[j] == types[i]) { - break; - } - } - type_aliases[i] = j; - } - - uint32_t imported_functions_size = get_imported_functions_size(); - fast_functions.resize(functions.size() + imported_functions_size); - for (uint32_t i = 0; i < imported_functions_size; ++i) { - fast_functions[i] = type_aliases[imports[i].type.func_t]; - } - for (uint32_t i = 0; i < functions.size(); ++i) { - fast_functions[i + imported_functions_size] = type_aliases[functions[i]]; - } - } }; }} // namespace eosio::vm diff --git a/include/eosio/vm/x86_64.hpp b/include/eosio/vm/x86_64.hpp index dddabc6f..a8327828 100644 --- a/include/eosio/vm/x86_64.hpp +++ b/include/eosio/vm/x86_64.hpp @@ -102,10 +102,11 @@ namespace eosio { namespace vm { } ~machine_code_writer() { _allocator.end_code(_code_segment_base); + auto num_functions = _mod.get_functions_total(); for (auto& elem : _mod.elements) { for (auto& entry : elem.elems) { void* addr; - if (entry.index < _mod.fast_functions.size()) { + if (entry.index < num_functions) { addr = std::get(_function_relocations[entry.index]); } else { addr = call_indirect_handler; @@ -457,7 +458,6 @@ namespace eosio { namespace vm { auto icount = variable_size_instr(37, 64); emit_check_call_depth(); std::uint32_t table_size = _mod.tables[0].limits.initial; - functypeidx = _mod.type_aliases[functypeidx]; emit_pop(rax); emit_cmp(table_size, eax); fix_branch(emit_branchcc32(JAE), call_indirect_handler); From 1a51bd604739e11233a569dab8de7a9186ca05e5 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Mon, 11 Sep 2023 13:58:58 -0600 Subject: [PATCH 33/36] Remove jit_mod now that it's no longer substantially different from the module. --- include/eosio/vm/backend.hpp | 5 +- include/eosio/vm/execution_context.hpp | 31 ++--- include/eosio/vm/host_function.hpp | 6 +- include/eosio/vm/types.hpp | 171 +------------------------ 4 files changed, 13 insertions(+), 200 deletions(-) diff --git a/include/eosio/vm/backend.hpp b/include/eosio/vm/backend.hpp index 0c86e533..afc1263c 100644 --- a/include/eosio/vm/backend.hpp +++ b/include/eosio/vm/backend.hpp @@ -73,10 +73,7 @@ namespace eosio { namespace vm { // Now data required by JIT is finalized; create JIT module // such that memory used in parsing can be released. if constexpr (Impl::is_jit) { - mod.make_jit_module(); - - // Important. Release the memory used by parsing. - mod.allocator.release_base_memory(); + assert(mod.allocator._base == nullptr); } if (exec_ctx_created_by_backend) { ctx->initialize_globals(); diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index abad759a..e2a76294 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -123,22 +123,13 @@ namespace eosio { namespace vm { public: Derived& derived() { return static_cast(*this); } auto& resolve_module() { - if constexpr (IsJit) { - return *_mod->jit_mod; - } else { - return *_mod; - } + return *_mod; } execution_context_base() {} execution_context_base(module* m) : _mod(m) {} inline void initialize_globals() { - if constexpr (IsJit) { - return initialize_globals_impl(*_mod->jit_mod); - } - else { - return initialize_globals_impl(*_mod); - } + return initialize_globals_impl(*_mod); } template @@ -151,11 +142,7 @@ namespace eosio { namespace vm { } inline int32_t grow_linear_memory(int32_t pages) { - if constexpr (IsJit) { - return grow_linear_memory_impl(*_mod->jit_mod, pages); - } else { - return grow_linear_memory_impl(*_mod, pages); - } + return grow_linear_memory_impl(*_mod, pages); } template @@ -428,7 +415,7 @@ namespace eosio { namespace vm { std::uint32_t get_remaining_call_depth() const { return this->_remaining_call_depth; } inline native_value call_host_function(native_value* stack, uint32_t index) { - const auto& ft = _mod->jit_mod->get_function_type(index); + const auto& ft = _mod->get_function_type(index); uint32_t num_params = ft.param_types.size(); #ifndef NDEBUG uint32_t original_operands = get_operand_stack().size(); @@ -442,7 +429,7 @@ namespace eosio { namespace vm { default: assert(!"Unexpected type in param_types."); } } - _rhf(_host, get_interface(), _mod->jit_mod->import_functions[index]); + _rhf(_host, get_interface(), _mod->import_functions[index]); native_value result{uint64_t{0}}; // guarantee that the junk bits are zero, to avoid problems. auto set_result = [&result](auto val) { std::memcpy(&result, &val, sizeof(val)); }; @@ -462,7 +449,7 @@ namespace eosio { namespace vm { } inline void reset() { - base_type::reset(*(_mod->jit_mod)); + base_type::reset(*_mod); get_operand_stack().eat(0); } @@ -500,7 +487,7 @@ namespace eosio { namespace vm { _host = host; - const auto& ft = _mod->jit_mod->get_function_type(func_index); + const auto& ft = _mod->get_function_type(func_index); this->type_check_args(ft, static_cast(args)...); native_value_extended result; @@ -515,7 +502,7 @@ namespace eosio { namespace vm { #pragma GCC diagnostic pop try { - if (func_index < _mod->jit_mod->get_imported_functions_size()) { + if (func_index < _mod->get_imported_functions_size()) { std::reverse(args_raw + 0, args_raw + args_count); result.scalar = call_host_function(args_raw, func_index); } else { @@ -524,7 +511,7 @@ namespace eosio { namespace vm { if(stack) { stack = static_cast(stack) - 24; } - auto fn = reinterpret_cast(_mod->jit_mod->jit_code_offset[func_index - _mod->jit_mod->get_imported_functions_size()] + _mod->allocator._code_base); + auto fn = reinterpret_cast(_mod->code[func_index - _mod->get_imported_functions_size()].jit_code_offset + _mod->allocator._code_base); if constexpr(EnableBacktrace) { sigset_t block_mask; diff --git a/include/eosio/vm/host_function.hpp b/include/eosio/vm/host_function.hpp index 1f411159..404d7ee9 100644 --- a/include/eosio/vm/host_function.hpp +++ b/include/eosio/vm/host_function.hpp @@ -455,11 +455,7 @@ namespace eosio { namespace vm { } static void resolve(module& mod) { - if (mod.jit_mod != nullptr) { - resolve_impl(*mod.jit_mod); - } else { - resolve_impl(mod); - } + resolve_impl(mod); } template diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index 720ee39d..c4376f6d 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -208,171 +208,6 @@ namespace eosio { namespace vm { // to memory with static storage duration. const char * error = nullptr; - // Stores data needed by JIT execution, in memory managed by standard - // C++ vectors, not growable_allocator used by guarded_vector, - // such that growable_allocator can be released after parsing. - // All growable_allocators (including recursively) used in module - // are redefined by std::vector in jit_mod_t. - // This is to make it possible parsing WASM only once for JIT. - struct jit_mod_t { - struct jit_func_type { - value_type form; - std::vector param_types; - uint8_t return_count; - value_type return_type; - }; - struct jit_import_type { - uint32_t func_t; - }; - struct jit_import_entry { - std::vector module_str; - std::vector field_str; - external_kind kind; - jit_import_type type; - }; - struct jit_export_entry { - std::vector field_str; - external_kind kind; - uint32_t index; - }; - struct jit_elem_segment { - uint32_t index; - init_expr offset; - elem_mode mode; - std::vector elems; - }; - struct jit_data_segment { - uint32_t index; - init_expr offset; - bool passive; - std::vector data; - }; - - std::vector types; - std::vector imports; - std::vector functions; - std::vector tables; - std::vector memories; - std::vector globals; - std::vector exports; - std::vector elements; - std::vector jit_code_offset; - std::vector data; - std::vector import_functions; - // type_aliases and fast_functions not needed during JIT execution - - auto& get_function_type(uint32_t index) const { - if (index < get_imported_functions_size()) - return types[imports[index].type.func_t]; - return types.at(functions[index - get_imported_functions_size()]); - } - uint32_t get_imported_functions_size() const { - return get_imported_functions_size_impl(imports); - } - }; - - // The memory storing module data for JIT - std::unique_ptr jit_mod; - - // Constructs data for JIT execution. - // Called from backend::construct() after parsing is finalized. - void make_jit_module() { - jit_mod = std::make_unique(); - - if (auto types_size = types.size(); types_size > 0) { - jit_mod->types.reserve(types_size); - for (uint32_t i = 0; i < types_size; ++i) { - const auto& type = types[i]; - jit_mod->types.emplace_back(jit_mod_t::jit_func_type{ - type.form, - {type.param_types.data(), type.param_types.data() + type.param_types.size()}, - type.return_count, - type.return_type - }); - } - } - - if (auto imports_size = imports.size(); imports_size > 0) { - jit_mod->imports.reserve(imports_size); - for (uint32_t i = 0; i < imports_size; ++i) { - const auto& entry = imports[i]; - jit_mod->imports.emplace_back(jit_mod_t::jit_import_entry{ - {entry.module_str.data(), entry.module_str.data() + entry.module_str.size()}, - {entry.field_str.data(), entry.field_str.data() + entry.field_str.size()}, - entry.kind, - {entry.type.func_t} - }); - } - } - - if (memories.size() > 0) { - jit_mod->memories.emplace_back(memories[0]); // memories has one element only - } - - if (functions.size() > 0) { - jit_mod->functions.assign(functions.data(), functions.data() + functions.size()); - } - - if (tables.size() > 0) { - jit_mod->tables.assign(tables.data(), tables.data() + tables.size()); - } - - if (globals.size() > 0) { - jit_mod->globals.assign(globals.data(), globals.data() + globals.size()); - } - - if (auto exports_size = exports.size(); exports_size > 0) { - jit_mod->exports.reserve(exports_size); - for (uint32_t i = 0; i < exports_size; ++i) { - const auto& entry = exports[i]; - jit_mod->exports.emplace_back(jit_mod_t::jit_export_entry{ - {entry.field_str.data(), entry.field_str.data() + entry.field_str.size()}, - entry.kind, - entry.index - }); - } - } - - if (auto elem_size = elements.size(); elem_size > 0) - { - jit_mod->elements.reserve(elem_size); - for (uint32_t i = 0; i < elem_size; ++i) - { - const auto& elem_seg = elements[i]; - jit_mod->elements.emplace_back(jit_mod_t::jit_elem_segment{ - .index = elem_seg.index, - .offset = elem_seg.offset, - .mode = elem_seg.mode, - .elems = {elem_seg.elems.data(), elem_seg.elems.data() + elem_seg.elems.size()} - }); - } - } - - if (auto code_size = code.size(); code_size > 0) { - jit_mod->jit_code_offset.reserve(code_size); - for (uint32_t i = 0; i < code_size; ++i) { - jit_mod->jit_code_offset.emplace_back(code[i].jit_code_offset); - } - } - - if (auto data_size = data.size(); data_size > 0) { - jit_mod->data.reserve(data_size); - for (uint32_t i = 0; i < data_size; ++i) { - const auto& data_seg = data[i]; - jit_mod->data.emplace_back(jit_mod_t::jit_data_segment{ - .index = data_seg.index, - .offset = data_seg.offset, - .passive = data_seg.passive, - .data = {data_seg.data.data(), data_seg.data.data() + data_seg.data.size()} - }); - } - } - - if (import_functions.size() > 0) { - jit_mod->import_functions.assign(import_functions.data(), import_functions.data() + import_functions.size()); - } - } - void finalize() { import_functions.resize(get_imported_functions_size()); allocator.finalize(); @@ -417,10 +252,8 @@ namespace eosio { namespace vm { } } - // When jit_mod is available, this function executes on jit_mod, - // otherwise on module itself. uint32_t get_exported_function(const std::string_view str) { - return (jit_mod) ? get_exported_function_impl(jit_mod->exports, str) : get_exported_function_impl(exports, str); + return get_exported_function_impl(exports, str); } template @@ -439,7 +272,7 @@ namespace eosio { namespace vm { bool indirect_table(std::size_t i) { - return jit_mod? indirect_table_impl(*jit_mod, i) : indirect_table_impl(*this, i); + return indirect_table_impl(*this, i); } static bool indirect_table_impl(auto& mod, std::size_t i) { From 3248f0060def41391039644defe0435242510540 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 12 Sep 2023 12:41:14 -0600 Subject: [PATCH 34/36] Make opcodes relocatable --- include/eosio/vm/bitcode_writer.hpp | 6 +++--- include/eosio/vm/interpret_visitor.hpp | 3 ++- include/eosio/vm/opcodes_def.hpp | 2 -- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/include/eosio/vm/bitcode_writer.hpp b/include/eosio/vm/bitcode_writer.hpp index cf1e48cb..5d21462f 100644 --- a/include/eosio/vm/bitcode_writer.hpp +++ b/include/eosio/vm/bitcode_writer.hpp @@ -61,12 +61,12 @@ namespace eosio { namespace vm { _this{ &base }, _i{ 0 } { br_table_t& bt = _this->append_instr(br_table_t{}); - bt.offset = static_cast(((table_size * sizeof(br_table_t::elem_t))/sizeof(opcode))+2); + auto offset = static_cast(((table_size * sizeof(br_table_t::elem_t))/sizeof(opcode))+2); // point the branch table data to after the br_table instruction - _br_tab = bt.table = reinterpret_cast(&_this->fb[_this->op_index]); + _br_tab = reinterpret_cast(&_this->fb[_this->op_index]); - _this->op_index += bt.offset; + _this->op_index += offset; // canary to throw if we have overbounded our allocated memory _this->fb[_this->op_index] = error_t{}; diff --git a/include/eosio/vm/interpret_visitor.hpp b/include/eosio/vm/interpret_visitor.hpp index 55313576..afae164f 100644 --- a/include/eosio/vm/interpret_visitor.hpp +++ b/include/eosio/vm/interpret_visitor.hpp @@ -77,7 +77,8 @@ namespace eosio { namespace vm { } [[gnu::always_inline]] inline void operator()(const br_table_t& op) { const auto& in = context.pop_operand().to_ui32(); - const auto& entry = op.table[std::min(in, op.size)]; + const auto* table = reinterpret_cast(context.get_pc() + 1); + const auto& entry = table[std::min(in, op.size)]; context.jump(entry.stack_pop, entry.pc); } [[gnu::always_inline]] inline void operator()(const call_t& op) { diff --git a/include/eosio/vm/opcodes_def.hpp b/include/eosio/vm/opcodes_def.hpp index 9894ce0e..9395c8f7 100644 --- a/include/eosio/vm/opcodes_def.hpp +++ b/include/eosio/vm/opcodes_def.hpp @@ -609,9 +609,7 @@ #define EOS_VM_CREATE_BR_TABLE_TYPE(name, code) \ struct EOS_VM_OPCODE_T(name) { \ struct elem_t { uint32_t pc; uint32_t stack_pop; }; \ - elem_t* table; \ uint32_t size; \ - uint32_t offset; \ static constexpr uint8_t opcode = code; \ }; From f5aff5fa4f43879c72a310c86fead76863848805 Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 12 Sep 2023 14:51:22 -0600 Subject: [PATCH 35/36] Put pointer to globals in linear memory prefix region --- include/eosio/vm/allocator.hpp | 3 + include/eosio/vm/execution_context.hpp | 4 + include/eosio/vm/types.hpp | 7 +- include/eosio/vm/x86_64.hpp | 107 ++++++++++++------------- 4 files changed, 65 insertions(+), 56 deletions(-) diff --git a/include/eosio/vm/allocator.hpp b/include/eosio/vm/allocator.hpp index 00682ab8..e69229de 100644 --- a/include/eosio/vm/allocator.hpp +++ b/include/eosio/vm/allocator.hpp @@ -482,6 +482,9 @@ namespace eosio { namespace vm { static std::size_t table_size() { return syspagesize(); } + static std::int32_t globals_end() { + return -static_cast(syspagesize()); + } template void alloc(size_t size = 1 /*in pages*/) { if (size == 0) return; diff --git a/include/eosio/vm/execution_context.hpp b/include/eosio/vm/execution_context.hpp index e2a76294..55bc18a6 100644 --- a/include/eosio/vm/execution_context.hpp +++ b/include/eosio/vm/execution_context.hpp @@ -220,6 +220,10 @@ namespace eosio { namespace vm { for (uint32_t i = 0; i < mod.globals.size(); i++) { _globals.emplace_back(mod.globals[i].init); } + // Write a pointer to the globals into the context page + auto* globals_start = _globals.data(); + char* globals_location = _linear_memory + wasm_allocator::globals_end(); + std::memcpy(globals_location - sizeof(globals_start), &globals_start, sizeof(globals_start)); // reset the table if (mod.tables.size() != 0) { diff --git a/include/eosio/vm/types.hpp b/include/eosio/vm/types.hpp index c4376f6d..bf1b85f8 100644 --- a/include/eosio/vm/types.hpp +++ b/include/eosio/vm/types.hpp @@ -270,13 +270,18 @@ namespace eosio { namespace vm { return index; } + static std::size_t get_global_offset(std::uint32_t idx) + { + return idx * sizeof(init_expr) + offsetof(init_expr, value); + } + bool indirect_table(std::size_t i) { return indirect_table_impl(*this, i); } static bool indirect_table_impl(auto& mod, std::size_t i) { - return i < mod.tables.size() && (mod.tables[i].limits.initial * sizeof(table_entry) > wasm_allocator::table_size()); + return i < mod.tables.size() && (mod.tables[i].limits.initial * sizeof(table_entry) > (wasm_allocator::table_size()) - sizeof(void*)); } }; }} // namespace eosio::vm diff --git a/include/eosio/vm/x86_64.hpp b/include/eosio/vm/x86_64.hpp index a8327828..335766a5 100644 --- a/include/eosio/vm/x86_64.hpp +++ b/include/eosio/vm/x86_64.hpp @@ -574,75 +574,72 @@ namespace eosio { namespace vm { } } + auto emit_global_loc(uint32_t globalidx) + { + auto& gl = _mod.globals[globalidx]; + + auto offset = _mod.get_global_offset(globalidx); + emit_mov(*(rsi + (wasm_allocator::globals_end() - 8)), rcx); + if (offset > 0x7fffffff) { + // This isn't quite optimal, but realistically, no one should + // ever have this many globals + emit_mov(static_cast(offset), rdx); + emit_add(rdx, rcx); + offset = 0; + } + return *(rcx + static_cast(offset)); + } + void emit_get_global(uint32_t globalidx) { - auto icount = variable_size_instr(24, 42); // emit_setup_backtrace can be 0 or 9, and emit_restore_backtrace 0 or 9, the total of the rest 24 + COUNT_INSTR(); + auto icount = variable_size_instr(11, 36); + auto& gl = _mod.globals[globalidx]; - emit_setup_backtrace(); - // pushq %rdi -- save %rdi content onto stack - emit_bytes(0x57); - // pushq %rsi -- save %rsi content onto stack - emit_bytes(0x56); - // movq $globalidx, %rsi -- pass globalidx to %rsi, the second argument - emit_bytes(0x48, 0xc7, 0xc6); - emit_operand32(globalidx); - // movabsq $get_global, %rax - emit_bytes(0x48, 0xb8); + auto loc = emit_global_loc(globalidx); switch(gl.type.content_type) { - case types::i32: emit_operand_ptr(&get_global_i32); break; - case types::i64: emit_operand_ptr(&get_global_i64); break; - case types::f32: emit_operand_ptr(&get_global_f32); break; - case types::f64: emit_operand_ptr(&get_global_f64); break; - case types::v128: emit_operand_ptr(&get_global_v128); break; + case types::i32: + emit_mov(loc, eax); // min = op + modr/m + disp8 = 3 + emit_push(rax); + break; + case types::i64: + emit_mov(loc, rax); + emit_push(rax); + break; + case types::f32: + emit_mov(loc, eax); + emit_push(rax); + break; + case types::f64: + emit_mov(loc, rax); + emit_push(rax); + break; + case types::v128: + emit_vmovdqu(loc, xmm0); // 3 + disp32 = 7 + emit_push_v128(xmm0); // 4 + 5 = 9 + break; default: assert(!"Unknown global type"); } - // call *%rax - emit_bytes(0xff, 0xd0); - // pop %rsi - emit_bytes(0x5e); - // pop %rdi - emit_bytes(0x5f); - emit_restore_backtrace(); - // push %rax -- return result - if (gl.type.content_type == types::v128) { - emit_push(rdx); - } - emit_bytes(0x50); } void emit_set_global(uint32_t globalidx) { - auto icount = variable_size_instr(24, 42); // emit_setup_backtrace can be 0 or 9, and emit_restore_backtrace 0 or 9, the total of the rest 24 + COUNT_INSTR(); + auto icount = variable_size_instr(11, 36); + auto& gl = _mod.globals[globalidx]; - // popq %rdx -- pass global value to %rdx, the third argument in set_global - emit_bytes(0x5a); if (gl.type.content_type == types::v128) { - emit_pop(rcx); + emit_pop_v128(xmm0); + } else { + emit_pop(rax); } - emit_setup_backtrace(); - // pushq %rdi -- save %rdi content onto stack - emit_bytes(0x57); - // pushq %rsi -- save %rsi content onto stack - emit_bytes(0x56); - // movq $globalidx, %rsi -- pass globalidx to %rsi, the second argument - emit_bytes(0x48, 0xc7, 0xc6); - emit_operand32(globalidx); - // movabsq $set_global, %rax - emit_bytes(0x48, 0xb8); - //emit_operand_ptr(&set_global); + auto loc = emit_global_loc(globalidx); switch(gl.type.content_type) { - case types::i32: emit_operand_ptr(&set_global_i32); break; - case types::i64: emit_operand_ptr(&set_global_i64); break; - case types::f32: emit_operand_ptr(&set_global_f32); break; - case types::f64: emit_operand_ptr(&set_global_f64); break; - case types::v128: emit_operand_ptr(&set_global_v128); break; + case types::i32: emit_mov(eax, loc); break; + case types::i64: emit_mov(rax, loc); break; + case types::f32: emit_mov(eax, loc); break; + case types::f64: emit_mov(rax, loc); break; + case types::v128: emit_vmovdqu(xmm0, loc); break; default: assert(!"Unknown global type"); } - // call *%rax - emit_bytes(0xff, 0xd0); - // pop %rsi - emit_bytes(0x5e); - // pop %rdi - emit_bytes(0x5f); - emit_restore_backtrace(); } void emit_i32_load(uint32_t /*alignment*/, uint32_t offset) { From ef99468c57f88726ecbe97900fc361a28f267b3e Mon Sep 17 00:00:00 2001 From: Steven Watanabe Date: Tue, 12 Sep 2023 15:34:43 -0600 Subject: [PATCH 36/36] Remove dead code --- include/eosio/vm/x86_64.hpp | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/include/eosio/vm/x86_64.hpp b/include/eosio/vm/x86_64.hpp index 335766a5..edcef8e4 100644 --- a/include/eosio/vm/x86_64.hpp +++ b/include/eosio/vm/x86_64.hpp @@ -6277,38 +6277,6 @@ namespace eosio { namespace vm { static void on_memory_error() { throw_("wasm memory out-of-bounds"); } - static int32_t get_global_i32(Context* context /*rdi*/, uint32_t index /*rsi*/) { - return context->get_global_i32(index); - } - static int64_t get_global_i64(Context* context /*rdi*/, uint32_t index /*rsi*/) { - return context->get_global_i64(index); - } - static uint32_t get_global_f32(Context* context /*rdi*/, uint32_t index /*rsi*/) { - return context->get_global_f32(index); - } - static uint64_t get_global_f64(Context* context /*rdi*/, uint32_t index /*rsi*/) { - return context->get_global_f64(index); - } - static v128_t get_global_v128(Context* context /*rdi*/, uint32_t index /*rsi*/) { - return context->get_global_v128(index); - } - - static void set_global_i32(Context* context /*rdi*/, uint32_t index /*rsi*/, int32_t value /*rdx*/) { - context->set_global_i32(index, value); - } - static void set_global_i64(Context* context /*rdi*/, uint32_t index /*rsi*/, int64_t value /*rdx*/) { - context->set_global_i64(index, value); - } - static void set_global_f32(Context* context /*rdi*/, uint32_t index /*rsi*/, uint32_t value /*rdx*/) { - context->set_global_f32(index, value); - } - static void set_global_f64(Context* context /*rdi*/, uint32_t index /*rsi*/, uint64_t value /*rdx*/) { - context->set_global_f64(index, value); - } - static void set_global_v128(Context* context /*rdi*/, uint32_t index /*rsi*/, v128_t value /*rdx+rcx*/) { - context->set_global_v128(index, value); - } - static void on_unreachable() { vm::throw_( "unreachable" ); } static void on_fp_error() { vm::throw_( "floating point error" ); } static void on_call_indirect_error() { vm::throw_( "call_indirect out of range" ); }