Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

assigner: add trace match check #226

Merged
merged 1 commit into from
Jan 7, 2025
Merged

assigner: add trace match check #226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions proof-producer/libs/assigner/include/nil/proof-generator/assigner/assigner.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,25 @@

#include <nil/proof-generator/preset/preset.hpp>

#include <nil/proof-generator/assigner/options.hpp>
#include <nil/proof-generator/assigner/bytecode.hpp>
#include <nil/proof-generator/assigner/rw.hpp>
#include <nil/proof-generator/assigner/copy.hpp>
#include <nil/proof-generator/assigner/zkevm.hpp>
#include <nil/proof-generator/assigner/trace_parser.hpp>


namespace nil {
namespace proof_generator {

using AssignmentTableFiller = std::function<std::optional<std::string>(
crypto3::zk::snark::plonk_assignment_table<nil::crypto3::algebra::fields::pallas_fq>& assignment_table,
const boost::filesystem::path& trace_base_path,
const AssignerOptions& options)
>;

template<typename BlueprintFieldType>
std::map<const std::string, std::function<std::optional<std::string>(
nil::crypto3::zk::snark::plonk_assignment_table<BlueprintFieldType>& assignment_table,
const boost::filesystem::path& trace_base_path)>> circuit_selector = {
std::map<const std::string, AssignmentTableFiller> circuit_selector = {
{circuits::BYTECODE, fill_bytecode_assignment_table<BlueprintFieldType>},
{circuits::RW, fill_rw_assignment_table<BlueprintFieldType>},
{circuits::ZKEVM, fill_zkevm_assignment_table<BlueprintFieldType>},
Expand Down Expand Up @@ -60,15 +66,16 @@ namespace nil {
}

template<typename BlueprintFieldType>
std::optional<std::string> fill_assignment_table_single_thread(nil::crypto3::zk::snark::plonk_assignment_table<BlueprintFieldType>& assignment_table,
nil::crypto3::zk::snark::plonk_table_description<BlueprintFieldType>& desc,
std::optional<std::string> fill_assignment_table_single_thread(crypto3::zk::snark::plonk_assignment_table<BlueprintFieldType>& assignment_table,
crypto3::zk::snark::plonk_table_description<BlueprintFieldType>& desc,
const std::string& circuit_name,
const boost::filesystem::path& trace_base_path) {
const boost::filesystem::path& trace_base_path,
const AssignerOptions& options = {}) {
auto find_it = circuit_selector<BlueprintFieldType>.find(circuit_name);
if (find_it == circuit_selector<BlueprintFieldType>.end()) {
return "Unknown circuit name " + circuit_name;
}
const auto err = find_it->second(assignment_table, trace_base_path);
const auto err = find_it->second(assignment_table, trace_base_path, options);
if (err) {
return err;
}
Expand Down
8 changes: 5 additions & 3 deletions proof-producer/libs/assigner/include/nil/proof-generator/assigner/bytecode.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include <nil/crypto3/zk/snark/arithmetization/plonk/assignment.hpp>
#include <nil/blueprint/zkevm_bbf/bytecode.hpp>
#include <nil/proof-generator/assigner/trace_parser.hpp>
#include <nil/proof-generator/assigner/options.hpp>
#include <nil/proof-generator/preset/limits.hpp>

namespace nil {
Expand All @@ -16,7 +17,8 @@ namespace nil {
/// @brief Fill assignment table
template<typename BlueprintFieldType>
std::optional<std::string> fill_bytecode_assignment_table(nil::crypto3::zk::snark::plonk_assignment_table<BlueprintFieldType>& assignment_table,
const boost::filesystem::path& trace_base_path) {
const boost::filesystem::path& trace_base_path,
const AssignerOptions& options) {
// TODO: print full name here, not only base
BOOST_LOG_TRIVIAL(debug) << "fill bytecode table from " << trace_base_path << "\n";

Expand All @@ -28,12 +30,12 @@ namespace nil {
input.rlc_challenge = limits::RLC_CHALLENGE;

const auto bytecode_trace_path = get_bytecode_trace_path(trace_base_path);
const auto contract_bytecodes = deserialize_bytecodes_from_file(bytecode_trace_path);
const auto contract_bytecodes = deserialize_bytecodes_from_file(bytecode_trace_path, options);
if (!contract_bytecodes) {
return "can't read bytecode trace from file: " + bytecode_trace_path.string();
}

for (const auto& bytecode_it : contract_bytecodes.value()) {
for (const auto& bytecode_it : contract_bytecodes->value) {
const auto raw_bytecode = string_to_bytes(bytecode_it.second);
input.bytecodes.new_buffer(raw_bytecode);
input.keccak_buffers.new_buffer(raw_bytecode);
Expand Down
16 changes: 9 additions & 7 deletions proof-producer/libs/assigner/include/nil/proof-generator/assigner/copy.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

#include <nil/crypto3/zk/snark/arithmetization/plonk/assignment.hpp>
#include <nil/blueprint/zkevm_bbf/copy.hpp>
#include <nil/proof-generator/assigner/options.hpp>
#include <nil/proof-generator/assigner/trace_parser.hpp>
#include <nil/proof-generator/preset/limits.hpp>

Expand All @@ -19,7 +20,8 @@ namespace nil {
/// @brief Fill assignment table
template<typename BlueprintFieldType>
std::optional<std::string> fill_copy_events_assignment_table(nil::crypto3::zk::snark::plonk_assignment_table<BlueprintFieldType>& assignment_table,
const boost::filesystem::path& trace_base_path) {
const boost::filesystem::path& trace_base_path,
const AssignerOptions& options) {
BOOST_LOG_TRIVIAL(debug) << "fill copy table from " << trace_base_path << "\n";

using ComponentType = nil::blueprint::bbf::copy<BlueprintFieldType, nil::blueprint::bbf::GenerationStage::ASSIGNMENT>;
Expand All @@ -30,29 +32,29 @@ namespace nil {
input.rlc_challenge = limits::RLC_CHALLENGE;

const auto copy_trace_path = get_copy_trace_path(trace_base_path);
auto copy_events = deserialize_copy_events_from_file(copy_trace_path);
auto copy_events = deserialize_copy_events_from_file(copy_trace_path, options);
if (!copy_events) {
return "can't read copy events from file: " + copy_trace_path.string();
}
input.copy_events = std::move(copy_events.value());
input.copy_events = std::move(copy_events->value);

const auto bytecode_trace_path = get_bytecode_trace_path(trace_base_path);
const auto contract_bytecodes = deserialize_bytecodes_from_file(bytecode_trace_path);
const auto contract_bytecodes = deserialize_bytecodes_from_file(bytecode_trace_path, options, copy_events->index);
if (!contract_bytecodes) {
return "can't read bytecode trace from file: " + bytecode_trace_path.string();
}
for (const auto& bytecode_it : contract_bytecodes.value()) {
for (const auto& bytecode_it : contract_bytecodes->value) {
const auto raw_bytecode = string_to_bytes(bytecode_it.second);
input.bytecodes.new_buffer(raw_bytecode);
input.keccak_buffers.new_buffer(raw_bytecode);
}

const auto rw_trace_path = get_rw_trace_path(trace_base_path);
auto rw_operations = deserialize_rw_traces_from_file(rw_trace_path);
auto rw_operations = deserialize_rw_traces_from_file(rw_trace_path, options, copy_events->index);
if (!rw_operations) {
return "can't read rw operations trace from file: " + rw_trace_path.string();
}
input.rw_operations = std::move(rw_operations.value());
input.rw_operations = std::move(rw_operations->value);

auto start = std::chrono::high_resolution_clock::now();
ComponentType instance(
Expand Down
15 changes: 15 additions & 0 deletions proof-producer/libs/assigner/include/nil/proof-generator/assigner/options.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
#ifndef PROOF_GENERATOR_LIBS_ASSIGNER_OPTIONS_HPP_
#define PROOF_GENERATOR_LIBS_ASSIGNER_OPTIONS_HPP_

namespace nil {
namespace proof_generator {

struct AssignerOptions {
bool ignore_index_mismatch{};
oclaw marked this conversation as resolved.
Show resolved Hide resolved
};


} // proof_generator
} // namespace nil

#endif // PROOF_GENERATOR_LIBS_ASSIGNER_OPTIONS_HPP_
8 changes: 5 additions & 3 deletions proof-producer/libs/assigner/include/nil/proof-generator/assigner/rw.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include <boost/filesystem.hpp>
#include <nil/crypto3/zk/snark/arithmetization/plonk/assignment.hpp>
#include <nil/blueprint/zkevm_bbf/rw.hpp>
#include <nil/proof-generator/assigner/options.hpp>
#include <nil/proof-generator/assigner/trace_parser.hpp>
#include <nil/proof-generator/preset/limits.hpp>

Expand All @@ -17,21 +18,22 @@ namespace nil {
/// @brief Fill assignment table
template<typename BlueprintFieldType>
std::optional<std::string> fill_rw_assignment_table(nil::crypto3::zk::snark::plonk_assignment_table<BlueprintFieldType>& assignment_table,
const boost::filesystem::path& trace_base_path) {
const boost::filesystem::path& trace_base_path,
const AssignerOptions& options) {
BOOST_LOG_TRIVIAL(debug) << "fill rw table from " << trace_base_path << "\n";

using ComponentType = nil::blueprint::bbf::rw<BlueprintFieldType, nil::blueprint::bbf::GenerationStage::ASSIGNMENT>;

typename nil::blueprint::bbf::context<BlueprintFieldType, nil::blueprint::bbf::GenerationStage::ASSIGNMENT> context_object(assignment_table, limits::max_rows);

const auto rw_trace_path = get_rw_trace_path(trace_base_path);
auto input = deserialize_rw_traces_from_file(rw_trace_path);
auto input = deserialize_rw_traces_from_file(rw_trace_path, options);
if (!input) {
return "can't read rw from file: " + rw_trace_path.string();
}

auto start = std::chrono::high_resolution_clock::now();
ComponentType instance(context_object, input.value(), limits::max_rw_size, limits::max_mpt_size);
ComponentType instance(context_object, input->value, limits::max_rw_size, limits::max_mpt_size);
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - start);
std::cout << "FILL ASSIGNMENT TABLE: " << duration.count() << "\n";
return {};
Expand Down
111 changes: 100 additions & 11 deletions proof-producer/libs/assigner/include/nil/proof-generator/assigner/trace_parser.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
#ifndef PROOF_GENERATOR_LIBS_ASSIGNER_TRACE_PARSER_HPP_
#define PROOF_GENERATOR_LIBS_ASSIGNER_TRACE_PARSER_HPP_

#include <exception>
#include <iterator>
#include <optional>
#include <utility>
#include <vector>
#include <string>
#include <cstdint>
Expand All @@ -13,8 +17,10 @@
#include <nil/blueprint/zkevm_bbf/types/rw_operation.hpp>
#include <nil/blueprint/zkevm_bbf/types/zkevm_state.hpp>
#include <nil/blueprint/zkevm_bbf/types/copy_event.hpp>
#include <nil/blueprint/assert.hpp>

#include <nil/proof-generator/assigner/trace.pb.h>
#include <nil/proof-generator/assigner/options.hpp>

namespace nil {
namespace proof_generator {
Expand Down Expand Up @@ -137,11 +143,51 @@ namespace nil {
return res;
}

[[nodiscard]] std::optional<std::unordered_map<std::string, std::string>> deserialize_bytecodes_from_file(const boost::filesystem::path& bytecode_trace_path) {
[[nodiscard]] std::string get_trace_extension(const boost::filesystem::path& trace_path) {
return trace_path.extension().string();
}

using TraceIndex = uint64_t; // value expected to be the same for all traces from the same set
using TraceIndexOpt = std::optional<TraceIndex>;

inline bool check_trace_index(const AssignerOptions& options, TraceIndexOpt base, TraceIndex index) {
if (base.has_value() && index != *base) {
BOOST_LOG_TRIVIAL(warning) << "Trace index mismatch: expected " << *base << ", got " << index;
if (!options.ignore_index_mismatch) {
return false;
}
}
return true;
}


template <typename TraceType>
struct DeserializeResult {
TraceType value;
TraceIndex index;
};

template <typename T>
using DeserializeResultOpt = std::optional<DeserializeResult<T>>;

using BytecodeTraces = std::unordered_map<std::string, std::string>; // contract address -> bytecode
using RWTraces = blueprint::bbf::rw_operations_vector;
using ZKEVMTraces = std::vector<blueprint::bbf::zkevm_state>;
using CopyEvents = std::vector<blueprint::bbf::copy_event>;
using ExpTraces = std::vector<exp_input>;

[[nodiscard]] DeserializeResultOpt<BytecodeTraces> deserialize_bytecodes_from_file(
const boost::filesystem::path& bytecode_trace_path,
const AssignerOptions& opts,
TraceIndexOpt base_index = {}
) {
const auto pb_traces = read_pb_traces_from_file<executionproofs::BytecodeTraces>(bytecode_trace_path);
if (!pb_traces) {
return std::nullopt;
}
if (!check_trace_index(opts, base_index, pb_traces->trace_idx())) {
return std::nullopt;
}

// Read executed op codes
std::unordered_map<std::string, std::string> contract_bytecodes;
Expand All @@ -150,14 +196,24 @@ namespace nil {
contract_bytecodes.emplace(bytecode.first, bytecode.second);
}

return contract_bytecodes;
return DeserializeResult<BytecodeTraces>{
std::move(contract_bytecodes),
pb_traces->trace_idx()
};
}

[[nodiscard]] std::optional<blueprint::bbf::rw_operations_vector> deserialize_rw_traces_from_file(const boost::filesystem::path& rw_traces_path) {
[[nodiscard]] DeserializeResultOpt<RWTraces> deserialize_rw_traces_from_file(
const boost::filesystem::path& rw_traces_path,
const AssignerOptions& opts,
TraceIndexOpt base_index = {}
) {
const auto pb_traces = read_pb_traces_from_file<executionproofs::RWTraces>(rw_traces_path);
if (!pb_traces) {
return std::nullopt;
}
if (!check_trace_index(opts, base_index, pb_traces->trace_idx())) {
return std::nullopt;
}

blueprint::bbf::rw_operations_vector rw_traces;
rw_traces.reserve(pb_traces->stack_ops_size() + pb_traces->memory_ops_size() + pb_traces->storage_ops_size() + 1); // +1 slot for start op
Expand Down Expand Up @@ -208,14 +264,24 @@ namespace nil {
<< "memory " << pb_traces->memory_ops_size() << "\n"
<< "storage " << pb_traces->storage_ops_size() << "\n";

return rw_traces;
return DeserializeResult<RWTraces>{
std::move(rw_traces),
pb_traces->trace_idx()
};
}

[[nodiscard]] std::optional<std::vector<blueprint::bbf::zkevm_state>> deserialize_zkevm_state_traces_from_file(const boost::filesystem::path& zkevm_traces_path) {
[[nodiscard]] DeserializeResultOpt<ZKEVMTraces> deserialize_zkevm_state_traces_from_file(
const boost::filesystem::path& zkevm_traces_path,
const AssignerOptions& opts,
TraceIndexOpt base_index = {}
) {
const auto pb_traces = read_pb_traces_from_file<executionproofs::ZKEVMTraces>(zkevm_traces_path);
if (!pb_traces) {
return std::nullopt;
}
if (!check_trace_index(opts, base_index, pb_traces->trace_idx())) {
return std::nullopt;
}

std::vector<blueprint::bbf::zkevm_state> zkevm_states;
zkevm_states.reserve(pb_traces->zkevm_states_size());
Expand Down Expand Up @@ -247,14 +313,24 @@ namespace nil {
zkevm_states.back().error_opcode = static_cast<uint64_t>(pb_state.error_opcode());
}

return zkevm_states;
return DeserializeResult<ZKEVMTraces>{
std::move(zkevm_states),
pb_traces->trace_idx()
};
}

[[nodiscard]] std::optional<std::vector<blueprint::bbf::copy_event>> deserialize_copy_events_from_file(const boost::filesystem::path& copy_traces_file) {
[[nodiscard]] DeserializeResultOpt<CopyEvents> deserialize_copy_events_from_file(
const boost::filesystem::path& copy_traces_file,
const AssignerOptions& opts,
TraceIndexOpt base_index = {}
) {
const auto pb_traces = read_pb_traces_from_file<executionproofs::CopyTraces>(copy_traces_file);
if (!pb_traces) {
return std::nullopt;
}
if (!check_trace_index(opts, base_index, pb_traces->trace_idx())) {
return std::nullopt;
}

namespace bbf = blueprint::bbf;

Expand Down Expand Up @@ -286,26 +362,39 @@ namespace nil {
copy_events.push_back(std::move(event));
}

return copy_events;
return DeserializeResult<CopyEvents>{
std::move(copy_events),
pb_traces->trace_idx()
};
}

[[nodiscard]] std::optional<std::vector<exp_input>> deserialize_exp_traces_from_file(const boost::filesystem::path& exp_traces_path) {
[[nodiscard]] DeserializeResultOpt<ExpTraces> deserialize_exp_traces_from_file(
const boost::filesystem::path& exp_traces_path,
const AssignerOptions& opts,
TraceIndexOpt base_index = {}
) {
const auto pb_traces = read_pb_traces_from_file<executionproofs::ExpTraces>(exp_traces_path);
if (!pb_traces) {
return std::nullopt;
}
if (!check_trace_index(opts, base_index, pb_traces->trace_idx())) {
return std::nullopt;
}

std::vector<exp_input> exps;
exps.reserve(pb_traces->exp_ops_size());
for (const auto& pb_exp_op : pb_traces->exp_ops()) {
std::cout << "base: " << proto_uint256_to_zkevm_word(pb_exp_op.base()) << " , exponent: " << proto_uint256_to_zkevm_word(pb_exp_op.exponent()) << std::endl;
BOOST_LOG_TRIVIAL(trace) << "base: " << proto_uint256_to_zkevm_word(pb_exp_op.base()) << " , exponent: " << proto_uint256_to_zkevm_word(pb_exp_op.exponent()) << std::endl;
exps.emplace_back(
proto_uint256_to_zkevm_word(pb_exp_op.base()),
proto_uint256_to_zkevm_word(pb_exp_op.exponent())
);
}

return exps;
return DeserializeResult<ExpTraces>{
std::move(exps),
pb_traces->trace_idx()
};
}
} // namespace proof_generator
} // namespace nil
Expand Down
Loading
Loading