diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index ea3cce166092a..c607839a8d259 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -13,6 +13,7 @@ if(WIN32) endif() add_subdirectory(io_exerciser) +add_subdirectory(json) add_subdirectory(options) set(common_srcs diff --git a/src/common/io_exerciser/CMakeLists.txt b/src/common/io_exerciser/CMakeLists.txt index 07091df86e100..ab2e64fc22230 100644 --- a/src/common/io_exerciser/CMakeLists.txt +++ b/src/common/io_exerciser/CMakeLists.txt @@ -5,9 +5,11 @@ add_library(object_io_exerciser STATIC Model.cc ObjectModel.cc RadosIo.cc + EcIoSequence.cc ) target_link_libraries(object_io_exerciser - librados + librados global + json_structures ) \ No newline at end of file diff --git a/src/common/io_exerciser/DataGenerator.cc b/src/common/io_exerciser/DataGenerator.cc index 9aa77eeb6e98b..701c32fa9ec21 100644 --- a/src/common/io_exerciser/DataGenerator.cc +++ b/src/common/io_exerciser/DataGenerator.cc @@ -2,32 +2,28 @@ // vim: ts=8 sw=2 smarttab #include "DataGenerator.h" -#include "ObjectModel.h" +#include +#include +#include +#include "ObjectModel.h" #include "common/debug.h" #include "common/dout.h" - #include "fmt/format.h" #include "fmt/ranges.h" -#include -#include -#include - #define dout_subsys ceph_subsys_rados #define dout_context g_ceph_context using DataGenerator = ceph::io_exerciser::data_generation::DataGenerator; -using SeededRandomGenerator = ceph::io_exerciser::data_generation - ::SeededRandomGenerator; -using HeaderedSeededRandomGenerator = ceph::io_exerciser::data_generation - ::HeaderedSeededRandomGenerator; +using SeededRandomGenerator = + ceph::io_exerciser::data_generation ::SeededRandomGenerator; +using HeaderedSeededRandomGenerator = + ceph::io_exerciser::data_generation ::HeaderedSeededRandomGenerator; std::unique_ptr DataGenerator::create_generator( - GenerationType generationType, const ObjectModel& model) -{ - switch(generationType) - { + GenerationType generationType, const ObjectModel& model) { + switch (generationType) { case GenerationType::SeededRandom: return std::make_unique(model); case GenerationType::HeaderedSeededRandom: @@ -39,28 +35,25 @@ std::unique_ptr DataGenerator::create_generator( return nullptr; } -bufferlist DataGenerator::generate_wrong_data(uint64_t offset, uint64_t length) -{ +bufferlist DataGenerator::generate_wrong_data(uint64_t offset, + uint64_t length) { bufferlist retlist; uint64_t block_size = m_model.get_block_size(); char buffer[block_size]; - for (uint64_t block_offset = offset; - block_offset < offset + length; - block_offset++) - { + for (uint64_t block_offset = offset; block_offset < offset + length; + block_offset++) { std::memset(buffer, 0, block_size); retlist.append(ceph::bufferptr(buffer, block_size)); } return retlist; } -bool DataGenerator::validate(bufferlist& bufferlist, uint64_t offset, uint64_t length) -{ +bool DataGenerator::validate(bufferlist& bufferlist, uint64_t offset, + uint64_t length) { return bufferlist.contents_equal(generate_data(offset, length)); } -ceph::bufferptr SeededRandomGenerator::generate_block(uint64_t block_offset) -{ +ceph::bufferptr SeededRandomGenerator::generate_block(uint64_t block_offset) { uint64_t block_size = m_model.get_block_size(); char buffer[block_size]; @@ -70,29 +63,26 @@ ceph::bufferptr SeededRandomGenerator::generate_block(uint64_t block_offset) constexpr size_t generation_length = sizeof(uint64_t); - for (uint64_t i = 0; i < block_size; i+=(2*generation_length), rand1++, rand2--) - { + for (uint64_t i = 0; i < block_size; + i += (2 * generation_length), rand1++, rand2--) { std::memcpy(buffer + i, &rand1, generation_length); std::memcpy(buffer + i + generation_length, &rand2, generation_length); } size_t remainingBytes = block_size % (generation_length * 2); - if (remainingBytes > generation_length) - { + if (remainingBytes > generation_length) { size_t remainingBytes2 = remainingBytes - generation_length; std::memcpy(buffer + block_size - remainingBytes, &rand1, remainingBytes); std::memcpy(buffer + block_size - remainingBytes2, &rand2, remainingBytes2); - } - else if (remainingBytes > 0) - { + } else if (remainingBytes > 0) { std::memcpy(buffer + block_size - remainingBytes, &rand1, remainingBytes); } return ceph::bufferptr(buffer, block_size); } -ceph::bufferptr SeededRandomGenerator::generate_wrong_block(uint64_t block_offset) -{ +ceph::bufferptr SeededRandomGenerator::generate_wrong_block( + uint64_t block_offset) { uint64_t block_size = m_model.get_block_size(); char buffer[block_size]; @@ -102,141 +92,134 @@ ceph::bufferptr SeededRandomGenerator::generate_wrong_block(uint64_t block_offse constexpr size_t generation_length = sizeof(uint64_t); - for (uint64_t i = 0; i < block_size; i+=(2*generation_length), rand1++, rand2--) - { + for (uint64_t i = 0; i < block_size; + i += (2 * generation_length), rand1++, rand2--) { std::memcpy(buffer + i, &rand1, generation_length); std::memcpy(buffer + i + generation_length, &rand2, generation_length); } size_t remainingBytes = block_size % (generation_length * 2); - if (remainingBytes > generation_length) - { + if (remainingBytes > generation_length) { size_t remainingBytes2 = remainingBytes - generation_length; std::memcpy(buffer + block_size - remainingBytes, &rand1, remainingBytes); std::memcpy(buffer + block_size - remainingBytes2, &rand2, remainingBytes2); - } - else if (remainingBytes > 0) - { + } else if (remainingBytes > 0) { std::memcpy(buffer + block_size - remainingBytes, &rand1, remainingBytes); } return ceph::bufferptr(buffer, block_size); } -bufferlist SeededRandomGenerator::generate_data(uint64_t offset, uint64_t length) -{ +bufferlist SeededRandomGenerator::generate_data(uint64_t offset, + uint64_t length) { bufferlist retlist; - for (uint64_t block_offset = offset; block_offset < offset + length; block_offset++) - { + for (uint64_t block_offset = offset; block_offset < offset + length; + block_offset++) { retlist.append(generate_block(block_offset)); } return retlist; } -bufferlist SeededRandomGenerator::generate_wrong_data(uint64_t offset, uint64_t length) -{ +bufferlist SeededRandomGenerator::generate_wrong_data(uint64_t offset, + uint64_t length) { bufferlist retlist; - for (uint64_t block_offset = offset; block_offset < offset + length; block_offset++) - { + for (uint64_t block_offset = offset; block_offset < offset + length; + block_offset++) { retlist.append(generate_wrong_block(block_offset)); } return retlist; } -HeaderedSeededRandomGenerator - ::HeaderedSeededRandomGenerator(const ObjectModel& model, - std::optional unique_run_id) : - SeededRandomGenerator(model), - unique_run_id(unique_run_id.value_or(generate_unique_run_id())) -{ - -} +HeaderedSeededRandomGenerator ::HeaderedSeededRandomGenerator( + const ObjectModel& model, std::optional unique_run_id) + : SeededRandomGenerator(model), + unique_run_id(unique_run_id.value_or(generate_unique_run_id())) {} -uint64_t HeaderedSeededRandomGenerator::generate_unique_run_id() -{ +uint64_t HeaderedSeededRandomGenerator::generate_unique_run_id() { std::mt19937_64 random_generator = - std::mt19937_64(duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count()); + std::mt19937_64(duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count()); - return random_generator(); + return random_generator(); } -ceph::bufferptr HeaderedSeededRandomGenerator::generate_block(uint64_t block_offset) -{ +ceph::bufferptr HeaderedSeededRandomGenerator::generate_block( + uint64_t block_offset) { SeedBytes seed = m_model.get_seed(block_offset); - TimeBytes current_time = duration_cast( - std::chrono::system_clock::now().time_since_epoch()).count(); + TimeBytes current_time = + duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); - ceph::bufferptr bufferptr = SeededRandomGenerator::generate_block(block_offset); + ceph::bufferptr bufferptr = + SeededRandomGenerator::generate_block(block_offset); - std::memcpy(bufferptr.c_str() + uniqueIdStart(), &unique_run_id, uniqueIdLength()); + std::memcpy(bufferptr.c_str() + uniqueIdStart(), &unique_run_id, + uniqueIdLength()); std::memcpy(bufferptr.c_str() + seedStart(), &seed, seedLength()); std::memcpy(bufferptr.c_str() + timeStart(), ¤t_time, timeLength()); return bufferptr; } -ceph::bufferptr HeaderedSeededRandomGenerator::generate_wrong_block(uint64_t block_offset) -{ +ceph::bufferptr HeaderedSeededRandomGenerator::generate_wrong_block( + uint64_t block_offset) { return HeaderedSeededRandomGenerator::generate_block(block_offset % 8); } const HeaderedSeededRandomGenerator::UniqueIdBytes - HeaderedSeededRandomGenerator::readUniqueRunId(uint64_t block_offset, - const bufferlist& bufferlist) -{ +HeaderedSeededRandomGenerator::readUniqueRunId(uint64_t block_offset, + const bufferlist& bufferlist) { UniqueIdBytes read_unique_run_id = 0; - std::memcpy(&read_unique_run_id, - &bufferlist[(block_offset * m_model.get_block_size()) + uniqueIdStart()], - uniqueIdLength()); + std::memcpy( + &read_unique_run_id, + &bufferlist[(block_offset * m_model.get_block_size()) + uniqueIdStart()], + uniqueIdLength()); return read_unique_run_id; } const HeaderedSeededRandomGenerator::SeedBytes - HeaderedSeededRandomGenerator::readSeed(uint64_t block_offset, - const bufferlist& bufferlist) -{ +HeaderedSeededRandomGenerator::readSeed(uint64_t block_offset, + const bufferlist& bufferlist) { SeedBytes read_seed = 0; - std::memcpy(&read_seed, - &bufferlist[(block_offset * m_model.get_block_size()) + seedStart()], - seedLength()); + std::memcpy( + &read_seed, + &bufferlist[(block_offset * m_model.get_block_size()) + seedStart()], + seedLength()); return read_seed; } const HeaderedSeededRandomGenerator::TimeBytes - HeaderedSeededRandomGenerator::readDateTime(uint64_t block_offset, - const bufferlist& bufferlist) -{ +HeaderedSeededRandomGenerator::readDateTime(uint64_t block_offset, + const bufferlist& bufferlist) { TimeBytes read_time = 0; - std::memcpy(&read_time, - &bufferlist[(block_offset * m_model.get_block_size()) + timeStart()], - timeLength()); + std::memcpy( + &read_time, + &bufferlist[(block_offset * m_model.get_block_size()) + timeStart()], + timeLength()); return read_time; } bool HeaderedSeededRandomGenerator::validate(bufferlist& bufferlist, - uint64_t offset, uint64_t length) -{ + uint64_t offset, uint64_t length) { std::vector invalid_block_offsets; - for (uint64_t block_offset = offset; block_offset < offset + length; block_offset++) - { - bool valid_block - = validate_block(block_offset, - (bufferlist.c_str() + ((block_offset - offset) * - m_model.get_block_size()))); - if (!valid_block) - { + for (uint64_t block_offset = offset; block_offset < offset + length; + block_offset++) { + bool valid_block = validate_block( + block_offset, (bufferlist.c_str() + + ((block_offset - offset) * m_model.get_block_size()))); + if (!valid_block) { invalid_block_offsets.push_back(block_offset); } } - if (!invalid_block_offsets.empty()) - { + if (!invalid_block_offsets.empty()) { printDebugInformationForOffsets(offset, invalid_block_offsets, bufferlist); } @@ -244,59 +227,51 @@ bool HeaderedSeededRandomGenerator::validate(bufferlist& bufferlist, } bool HeaderedSeededRandomGenerator::validate_block(uint64_t block_offset, - const char* buffer_start) -{ + const char* buffer_start) { // We validate the block matches what we generate byte for byte // however we ignore the time section of the header ceph::bufferptr bufferptr = generate_block(block_offset); bool valid = strncmp(bufferptr.c_str(), buffer_start, timeStart()) == 0; - valid = valid ? strncmp(bufferptr.c_str() + timeEnd(), - buffer_start + timeEnd(), - m_model.get_block_size() - timeEnd()) == 0 : valid; + valid = valid + ? strncmp(bufferptr.c_str() + timeEnd(), buffer_start + timeEnd(), + m_model.get_block_size() - timeEnd()) == 0 + : valid; return valid; } const HeaderedSeededRandomGenerator::ErrorType - HeaderedSeededRandomGenerator::getErrorTypeForBlock(uint64_t read_offset, - uint64_t block_offset, - const bufferlist& bufferlist) -{ - try - { - UniqueIdBytes read_unique_run_id = readUniqueRunId(block_offset - read_offset, - bufferlist); - if (unique_run_id != read_unique_run_id) - { +HeaderedSeededRandomGenerator::getErrorTypeForBlock( + uint64_t read_offset, uint64_t block_offset, const bufferlist& bufferlist) { + try { + UniqueIdBytes read_unique_run_id = + readUniqueRunId(block_offset - read_offset, bufferlist); + if (unique_run_id != read_unique_run_id) { return ErrorType::RUN_ID_MISMATCH; } SeedBytes read_seed = readSeed(block_offset - read_offset, bufferlist); - if (m_model.get_seed(block_offset) != read_seed) - { + if (m_model.get_seed(block_offset) != read_seed) { return ErrorType::SEED_MISMATCH; } if (std::strncmp(&bufferlist[((block_offset - read_offset) * - m_model.get_block_size()) + bodyStart()], + m_model.get_block_size()) + + bodyStart()], generate_block(block_offset).c_str() + bodyStart(), - m_model.get_block_size() - bodyStart()) != 0) - { + m_model.get_block_size() - bodyStart()) != 0) { return ErrorType::DATA_MISMATCH; } - } - catch(const std::exception& e) - { + } catch (const std::exception& e) { return ErrorType::DATA_NOT_FOUND; } return ErrorType::UNKNOWN; } -void HeaderedSeededRandomGenerator - ::printDebugInformationForBlock(uint64_t read_offset, uint64_t block_offset, - const bufferlist& bufferlist) -{ - ErrorType blockError = getErrorTypeForBlock(read_offset, block_offset, bufferlist); +void HeaderedSeededRandomGenerator ::printDebugInformationForBlock( + uint64_t read_offset, uint64_t block_offset, const bufferlist& bufferlist) { + ErrorType blockError = + getErrorTypeForBlock(read_offset, block_offset, bufferlist); TimeBytes read_time = 0; std::time_t ttp; @@ -304,433 +279,361 @@ void HeaderedSeededRandomGenerator char read_bytes[m_model.get_block_size()]; char generated_bytes[m_model.get_block_size()]; - if (blockError == ErrorType::DATA_MISMATCH || blockError == ErrorType::UNKNOWN) - { + if (blockError == ErrorType::DATA_MISMATCH || + blockError == ErrorType::UNKNOWN) { read_time = readDateTime(block_offset - read_offset, bufferlist); - std::chrono::system_clock::time_point time_point{std::chrono::milliseconds{read_time}}; + std::chrono::system_clock::time_point time_point{ + std::chrono::milliseconds{read_time}}; ttp = std::chrono::system_clock::to_time_t(time_point); - std::memcpy(&read_bytes, - &bufferlist[((block_offset - read_offset) * m_model.get_block_size())], - m_model.get_block_size() - bodyStart()); - std::memcpy(&generated_bytes, - generate_block(block_offset).c_str(), + std::memcpy( + &read_bytes, + &bufferlist[((block_offset - read_offset) * m_model.get_block_size())], + m_model.get_block_size() - bodyStart()); + std::memcpy(&generated_bytes, generate_block(block_offset).c_str(), m_model.get_block_size() - bodyStart()); } std::string error_string; - switch(blockError) - { - case ErrorType::RUN_ID_MISMATCH: - { - UniqueIdBytes read_unique_run_id = readUniqueRunId((block_offset - read_offset), - bufferlist); - error_string = fmt::format("Header (Run ID) mismatch detected at block {} " - "(byte offset {}) Header expected run id {} but found id {}. " - "Block data corrupt or not written from this instance of this application.", - block_offset, - block_offset * m_model.get_block_size(), - unique_run_id, - read_unique_run_id); - } - break; - - case ErrorType::SEED_MISMATCH: - { + switch (blockError) { + case ErrorType::RUN_ID_MISMATCH: { + UniqueIdBytes read_unique_run_id = + readUniqueRunId((block_offset - read_offset), bufferlist); + error_string = fmt::format( + "Header (Run ID) mismatch detected at block {} " + "(byte offset {}) Header expected run id {} but found id {}. " + "Block data corrupt or not written from this instance of this " + "application.", + block_offset, block_offset * m_model.get_block_size(), unique_run_id, + read_unique_run_id); + } break; + + case ErrorType::SEED_MISMATCH: { SeedBytes read_seed = readSeed((block_offset - read_offset), bufferlist); - if (m_model.get_seed_offsets(read_seed).size() == 0) - { - error_string = fmt::format("Data (Seed) mismatch detected at block {}" - " (byte offset {}). Header expected seed {} but found seed {}. " - "Read data was not from any other recognised block in the object.", - block_offset, - block_offset * m_model.get_block_size(), - m_model.get_seed(block_offset), - read_seed); - } - else - { + if (m_model.get_seed_offsets(read_seed).size() == 0) { + error_string = fmt::format( + "Data (Seed) mismatch detected at block {}" + " (byte offset {}). Header expected seed {} but found seed {}. " + "Read data was not from any other recognised block in the object.", + block_offset, block_offset * m_model.get_block_size(), + m_model.get_seed(block_offset), read_seed); + } else { std::vector seed_offsets = m_model.get_seed_offsets(read_seed); - error_string = fmt::format("Data (Seed) mismatch detected at block {}" - " (byte offset {}). Header expected seed {} but found seed {}." - " Read data was from a different block(s): {}", - block_offset, - block_offset * m_model.get_block_size(), - m_model.get_seed(block_offset), - read_seed, + error_string = fmt::format( + "Data (Seed) mismatch detected at block {}" + " (byte offset {}). Header expected seed {} but found seed {}." + " Read data was from a different block(s): {}", + block_offset, block_offset * m_model.get_block_size(), + m_model.get_seed(block_offset), read_seed, fmt::join(seed_offsets.begin(), seed_offsets.end(), "")); } - } - break; - - case ErrorType::DATA_MISMATCH: - { - error_string = fmt::format("Data (Body) mismatch detected at block {}" - " (byte offset {}). Header data matches, data body does not." - " Data written at {}\nExpected data: \n{:02x}\nRead data:{:02x}", - block_offset, - block_offset * m_model.get_block_size(), + } break; + + case ErrorType::DATA_MISMATCH: { + error_string = fmt::format( + "Data (Body) mismatch detected at block {}" + " (byte offset {}). Header data matches, data body does not." + " Data written at {}\nExpected data: \n{:02x}\nRead data:{:02x}", + block_offset, block_offset * m_model.get_block_size(), std::ctime(&ttp), - fmt::join(generated_bytes, generated_bytes + m_model.get_block_size(), ""), + fmt::join(generated_bytes, generated_bytes + m_model.get_block_size(), + ""), fmt::join(read_bytes, read_bytes + m_model.get_block_size(), "")); - } - break; + } break; - case ErrorType::DATA_NOT_FOUND: - { + case ErrorType::DATA_NOT_FOUND: { uint64_t bufferlist_length = bufferlist.to_str().size(); - error_string = fmt::format("Data (Body) could not be read at block {}" - " (byte offset {}) offset in bufferlist returned from read: {}" - " ({} bytes). Returned bufferlist length: {}.", - block_offset, - block_offset * m_model.get_block_size(), + error_string = fmt::format( + "Data (Body) could not be read at block {}" + " (byte offset {}) offset in bufferlist returned from read: {}" + " ({} bytes). Returned bufferlist length: {}.", + block_offset, block_offset * m_model.get_block_size(), (block_offset - read_offset), (block_offset - read_offset) * m_model.get_block_size(), bufferlist_length); - } - break; + } break; case ErrorType::UNKNOWN: - [[ fallthrough ]]; - - default: - { - error_string = fmt::format("Data mismatch detected at block {}" - " (byte offset {}).\nExpected data:\n{:02x}\nRead data:\n{:02x}", - block_offset, - block_offset * m_model.get_block_size(), - fmt::join(generated_bytes, generated_bytes + m_model.get_block_size(), ""), + [[fallthrough]]; + + default: { + error_string = fmt::format( + "Data mismatch detected at block {}" + " (byte offset {}).\nExpected data:\n{:02x}\nRead data:\n{:02x}", + block_offset, block_offset * m_model.get_block_size(), + fmt::join(generated_bytes, generated_bytes + m_model.get_block_size(), + ""), fmt::join(read_bytes, read_bytes + m_model.get_block_size(), "")); - } - break; + } break; } dout(0) << error_string << dendl; } -void HeaderedSeededRandomGenerator - ::printDebugInformationForRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - ErrorType rangeError, - const bufferlist& bufferlist) -{ - switch(rangeError) - { - case ErrorType::RUN_ID_MISMATCH: - printDebugInformationForRunIdMismatchRange(read_offset, start_block_offset, - range_length_in_blocks, bufferlist); - break; - case ErrorType::SEED_MISMATCH: - printDebugInformationForSeedMismatchRange(read_offset, start_block_offset, - range_length_in_blocks, bufferlist); - break; - case ErrorType::DATA_MISMATCH: - printDebugInformationDataBodyMismatchRange(read_offset, start_block_offset, - range_length_in_blocks, bufferlist); - break; - case ErrorType::DATA_NOT_FOUND: - printDebugInformationDataNotFoundRange(read_offset, start_block_offset, - range_length_in_blocks, bufferlist); - break; - case ErrorType::UNKNOWN: - [[ fallthrough ]]; - default: - printDebugInformationCorruptRange(read_offset, start_block_offset, - range_length_in_blocks, bufferlist); - break; +void HeaderedSeededRandomGenerator ::printDebugInformationForRange( + uint64_t read_offset, uint64_t start_block_offset, + uint64_t range_length_in_blocks, ErrorType rangeError, + const bufferlist& bufferlist) { + switch (rangeError) { + case ErrorType::RUN_ID_MISMATCH: + printDebugInformationForRunIdMismatchRange( + read_offset, start_block_offset, range_length_in_blocks, bufferlist); + break; + case ErrorType::SEED_MISMATCH: + printDebugInformationForSeedMismatchRange( + read_offset, start_block_offset, range_length_in_blocks, bufferlist); + break; + case ErrorType::DATA_MISMATCH: + printDebugInformationDataBodyMismatchRange( + read_offset, start_block_offset, range_length_in_blocks, bufferlist); + break; + case ErrorType::DATA_NOT_FOUND: + printDebugInformationDataNotFoundRange( + read_offset, start_block_offset, range_length_in_blocks, bufferlist); + break; + case ErrorType::UNKNOWN: + [[fallthrough]]; + default: + printDebugInformationCorruptRange(read_offset, start_block_offset, + range_length_in_blocks, bufferlist); + break; } } -void HeaderedSeededRandomGenerator - ::printDebugInformationForRunIdMismatchRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist) -{ +void HeaderedSeededRandomGenerator ::printDebugInformationForRunIdMismatchRange( + uint64_t read_offset, uint64_t start_block_offset, + uint64_t range_length_in_blocks, const bufferlist& bufferlist) { uint64_t range_start = start_block_offset; uint64_t range_length = 0; - UniqueIdBytes initial_read_unique_run_id = readUniqueRunId(start_block_offset - read_offset, - bufferlist); + UniqueIdBytes initial_read_unique_run_id = + readUniqueRunId(start_block_offset - read_offset, bufferlist); for (uint64_t i = start_block_offset; - i < start_block_offset + range_length_in_blocks; i++) - { - ceph_assert(getErrorTypeForBlock(read_offset, i, bufferlist) - == ErrorType::RUN_ID_MISMATCH); + i < start_block_offset + range_length_in_blocks; i++) { + ceph_assert(getErrorTypeForBlock(read_offset, i, bufferlist) == + ErrorType::RUN_ID_MISMATCH); - UniqueIdBytes read_unique_run_id = readUniqueRunId(i - read_offset, bufferlist); + UniqueIdBytes read_unique_run_id = + readUniqueRunId(i - read_offset, bufferlist); if (initial_read_unique_run_id != read_unique_run_id || - i == (start_block_offset + range_length_in_blocks - 1)) - { - if (range_length == 1) - { + i == (start_block_offset + range_length_in_blocks - 1)) { + if (range_length == 1) { printDebugInformationForBlock(read_offset, i, bufferlist); - } - else if (range_length > 1) - { - dout(0) << fmt::format("Data (Run ID) Mismatch detected from block {} ({} bytes)" - " and spanning a range of {} blocks ({} bytes). " - "Expected run id {} for range but found id {}" - " for all blocks in range. " - "Block data corrupt or not written from this instance of this application.", - range_start, - range_start * m_model.get_block_size(), - range_length, - range_length * m_model.get_block_size(), - unique_run_id, - initial_read_unique_run_id) << dendl; + } else if (range_length > 1) { + dout(0) + << fmt::format( + "Data (Run ID) Mismatch detected from block {} ({} bytes)" + " and spanning a range of {} blocks ({} bytes). " + "Expected run id {} for range but found id {}" + " for all blocks in range. " + "Block data corrupt or not written from this instance of " + "this application.", + range_start, range_start * m_model.get_block_size(), + range_length, range_length * m_model.get_block_size(), + unique_run_id, initial_read_unique_run_id) + << dendl; } range_start = i; range_length = 1; initial_read_unique_run_id = read_unique_run_id; - } - else - { + } else { range_length++; } } - if (range_length == 1) - { - printDebugInformationForBlock(read_offset, - start_block_offset + range_length_in_blocks - 1, - bufferlist); - } - else if (range_length > 1) - { - dout(0) << fmt::format("Data (Run ID) Mismatch detected from block {}" - " ({} bytes) and spanning a range of {} blocks ({} bytes). " - "Expected run id {} for range but found id for all blocks in range. " - "Block data corrupt or not written from this instance of this application.", - range_start, - range_start * m_model.get_block_size(), - range_length, - range_length * m_model.get_block_size(), - unique_run_id, - initial_read_unique_run_id) + if (range_length == 1) { + printDebugInformationForBlock( + read_offset, start_block_offset + range_length_in_blocks - 1, + bufferlist); + } else if (range_length > 1) { + dout(0) << fmt::format( + "Data (Run ID) Mismatch detected from block {}" + " ({} bytes) and spanning a range of {} blocks ({} bytes). " + "Expected run id {} for range but found id for all blocks " + "in range. " + "Block data corrupt or not written from this instance of " + "this application.", + range_start, range_start * m_model.get_block_size(), + range_length, range_length * m_model.get_block_size(), + unique_run_id, initial_read_unique_run_id) << dendl; } } -void HeaderedSeededRandomGenerator - ::printDebugInformationForSeedMismatchRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist) -{ +void HeaderedSeededRandomGenerator ::printDebugInformationForSeedMismatchRange( + uint64_t read_offset, uint64_t start_block_offset, + uint64_t range_length_in_blocks, const bufferlist& bufferlist) { uint64_t range_start = start_block_offset; uint64_t range_length = 0; // Assert here if needed, as we can't support values // that can't be converted to a signed integer. - ceph_assert(m_model.get_block_size() < (std::numeric_limits::max() / 2)); + ceph_assert(m_model.get_block_size() < + (std::numeric_limits::max() / 2)); std::optional range_offset = 0; for (uint64_t i = start_block_offset; - i < start_block_offset + range_length_in_blocks; i++) - { - ceph_assert(getErrorTypeForBlock(read_offset, i, bufferlist) - == ErrorType::SEED_MISMATCH); + i < start_block_offset + range_length_in_blocks; i++) { + ceph_assert(getErrorTypeForBlock(read_offset, i, bufferlist) == + ErrorType::SEED_MISMATCH); SeedBytes read_seed = readSeed(i - read_offset, bufferlist); std::vector seed_found_offsets = m_model.get_seed_offsets(read_seed); if ((seed_found_offsets.size() == 1 && - (static_cast(seed_found_offsets.front() - i) == range_offset)) || - range_length == 0) - { - if (range_length == 0) - { + (static_cast(seed_found_offsets.front() - i) == + range_offset)) || + range_length == 0) { + if (range_length == 0) { range_start = i; - if (seed_found_offsets.size() > 0) - { + if (seed_found_offsets.size() > 0) { range_offset = seed_found_offsets.front() - i; - } - else - { + } else { range_offset = std::nullopt; } } range_length++; - } - else - { - if (range_length == 1) - { + } else { + if (range_length == 1) { printDebugInformationForBlock(read_offset, i - 1, bufferlist); - } - else if (range_length > 1 && range_offset.has_value()) - { - dout(0) << fmt::format("Data (Seed) Mismatch detected from block {}" - " ({} bytes) and spanning a range of {} blocks ({} bytes). " - "Returned data located starting from block {} ({} bytes) " - "and spanning a range of {} blocks ({} bytes).", - range_start, - range_start * m_model.get_block_size(), - range_length, range_length * m_model.get_block_size(), - static_cast(*range_offset) + range_start, - (static_cast(*range_offset) + range_start) - * m_model.get_block_size(), - range_length, - range_length * m_model.get_block_size()) - << dendl; - } - else - { - dout(0) << fmt::format("Data (Seed) Mismatch detected from block {}" - " ({} bytes) and spanning a range of {} blocks ({} bytes). " - "Data seed mismatch spanning a range of {} blocks ({} bytes).", - range_start, - range_start * m_model.get_block_size(), - range_length, range_length * m_model.get_block_size(), - range_length, - range_length * m_model.get_block_size()) - << dendl; + } else if (range_length > 1 && range_offset.has_value()) { + dout(0) + << fmt::format( + "Data (Seed) Mismatch detected from block {}" + " ({} bytes) and spanning a range of {} blocks ({} bytes). " + "Returned data located starting from block {} ({} bytes) " + "and spanning a range of {} blocks ({} bytes).", + range_start, range_start * m_model.get_block_size(), + range_length, range_length * m_model.get_block_size(), + static_cast(*range_offset) + range_start, + (static_cast(*range_offset) + range_start) * + m_model.get_block_size(), + range_length, range_length * m_model.get_block_size()) + << dendl; + } else { + dout(0) + << fmt::format( + "Data (Seed) Mismatch detected from block {}" + " ({} bytes) and spanning a range of {} blocks ({} bytes). " + "Data seed mismatch spanning a range of {} blocks ({} " + "bytes).", + range_start, range_start * m_model.get_block_size(), + range_length, range_length * m_model.get_block_size(), + range_length, range_length * m_model.get_block_size()) + << dendl; } range_length = 1; range_start = i; - if (seed_found_offsets.size() > 0) - { + if (seed_found_offsets.size() > 0) { range_offset = seed_found_offsets.front() - i; - } - else - { + } else { range_offset = std::nullopt; } } } - if (range_length == 1) - { - printDebugInformationForBlock(read_offset, - start_block_offset + range_length_in_blocks - 1, - bufferlist); - } - else if (range_length > 1 && range_offset.has_value()) - { - dout(0) << fmt::format("Data (Seed) Mismatch detected from block {} ({} bytes) " - "and spanning a range of {} blocks ({} bytes). " - "Returned data located starting from block {} ({} bytes) " - "and spanning a range of {} blocks ({} bytes).", - range_start, - range_start * m_model.get_block_size(), - range_length, - range_length * m_model.get_block_size(), - *range_offset + range_start, - (*range_offset + range_start) * m_model.get_block_size(), - range_length, - range_length * m_model.get_block_size()) + if (range_length == 1) { + printDebugInformationForBlock( + read_offset, start_block_offset + range_length_in_blocks - 1, + bufferlist); + } else if (range_length > 1 && range_offset.has_value()) { + dout(0) << fmt::format( + "Data (Seed) Mismatch detected from block {} ({} bytes) " + "and spanning a range of {} blocks ({} bytes). " + "Returned data located starting from block {} ({} bytes) " + "and spanning a range of {} blocks ({} bytes).", + range_start, range_start * m_model.get_block_size(), + range_length, range_length * m_model.get_block_size(), + *range_offset + range_start, + (*range_offset + range_start) * m_model.get_block_size(), + range_length, range_length * m_model.get_block_size()) << dendl; - } - else - { - dout(0) << fmt::format("Data (Seed) Mismatch detected from block {} ({} bytes) " - "and spanning a range of {} blocks ({} bytes). " - "and spanning a range of {} blocks ({} bytes).", - range_start, - range_start * m_model.get_block_size(), - range_length, - range_length * m_model.get_block_size(), - range_length, - range_length * m_model.get_block_size()) + } else { + dout(0) << fmt::format( + "Data (Seed) Mismatch detected from block {} ({} bytes) " + "and spanning a range of {} blocks ({} bytes). " + "and spanning a range of {} blocks ({} bytes).", + range_start, range_start * m_model.get_block_size(), + range_length, range_length * m_model.get_block_size(), + range_length, range_length * m_model.get_block_size()) << dendl; } } -void HeaderedSeededRandomGenerator -::printDebugInformationDataBodyMismatchRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist) -{ - dout(0) << fmt::format("Data Mismatch detected in blocks from {} to {}. " - "Headers look as expected for range, " - "but generated data body does not match. " - "More information given for individual blocks below.", - start_block_offset, - start_block_offset + range_length_in_blocks - 1) +void HeaderedSeededRandomGenerator ::printDebugInformationDataBodyMismatchRange( + uint64_t read_offset, uint64_t start_block_offset, + uint64_t range_length_in_blocks, const bufferlist& bufferlist) { + dout(0) << fmt::format( + "Data Mismatch detected in blocks from {} to {}. " + "Headers look as expected for range, " + "but generated data body does not match. " + "More information given for individual blocks below.", + start_block_offset, + start_block_offset + range_length_in_blocks - 1) << dendl; for (uint64_t i = start_block_offset; - i < start_block_offset + range_length_in_blocks; i++) - { + i < start_block_offset + range_length_in_blocks; i++) { printDebugInformationForBlock(read_offset, i, bufferlist); } } -void HeaderedSeededRandomGenerator - ::printDebugInformationCorruptRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist) -{ - dout(0) << fmt::format("Data Mismatch detected in blocks from {} to {}. " - "Headers look as expected for range, " - "but generated data body does not match. " - "More information given for individual blocks below.", - start_block_offset, - start_block_offset + range_length_in_blocks - 1) +void HeaderedSeededRandomGenerator ::printDebugInformationCorruptRange( + uint64_t read_offset, uint64_t start_block_offset, + uint64_t range_length_in_blocks, const bufferlist& bufferlist) { + dout(0) << fmt::format( + "Data Mismatch detected in blocks from {} to {}. " + "Headers look as expected for range, " + "but generated data body does not match. " + "More information given for individual blocks below.", + start_block_offset, + start_block_offset + range_length_in_blocks - 1) << dendl; for (uint64_t i = start_block_offset; - i < start_block_offset + range_length_in_blocks; i++) - { + i < start_block_offset + range_length_in_blocks; i++) { printDebugInformationForBlock(read_offset, i, bufferlist); } } -void HeaderedSeededRandomGenerator - ::printDebugInformationDataNotFoundRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist) -{ - dout(0) << fmt::format("Data not found for blocks from {} to {}. " - "More information given for individual blocks below.", - start_block_offset, - start_block_offset + range_length_in_blocks - 1) +void HeaderedSeededRandomGenerator ::printDebugInformationDataNotFoundRange( + uint64_t read_offset, uint64_t start_block_offset, + uint64_t range_length_in_blocks, const bufferlist& bufferlist) { + dout(0) << fmt::format( + "Data not found for blocks from {} to {}. " + "More information given for individual blocks below.", + start_block_offset, + start_block_offset + range_length_in_blocks - 1) << dendl; - for (uint64_t i = start_block_offset; i < start_block_offset + range_length_in_blocks; i++) - { + for (uint64_t i = start_block_offset; + i < start_block_offset + range_length_in_blocks; i++) { printDebugInformationForBlock(read_offset, i, bufferlist); } } -void HeaderedSeededRandomGenerator - ::printDebugInformationForOffsets(uint64_t read_offset, - std::vector offsets, - const bufferlist& bufferlist) -{ +void HeaderedSeededRandomGenerator ::printDebugInformationForOffsets( + uint64_t read_offset, std::vector offsets, + const bufferlist& bufferlist) { uint64_t range_start = 0; uint64_t range_length = 0; ErrorType rangeError = ErrorType::UNKNOWN; - for (const uint64_t& block_offset : offsets) - { - ErrorType blockError = getErrorTypeForBlock(read_offset, block_offset, - bufferlist); + for (const uint64_t& block_offset : offsets) { + ErrorType blockError = + getErrorTypeForBlock(read_offset, block_offset, bufferlist); - if (range_start == 0 && range_length == 0) - { + if (range_start == 0 && range_length == 0) { range_start = block_offset; range_length = 1; rangeError = blockError; - } - else if (blockError == rangeError && - range_start + range_length == block_offset) -{ + } else if (blockError == rangeError && + range_start + range_length == block_offset) { range_length++; - } - else - { - if (range_length == 1) - { + } else { + if (range_length == 1) { printDebugInformationForBlock(read_offset, range_start, bufferlist); - } - else if (range_length > 1) - { + } else if (range_length > 1) { printDebugInformationForRange(read_offset, range_start, range_length, rangeError, bufferlist); } @@ -741,12 +644,9 @@ void HeaderedSeededRandomGenerator } } - if (range_length == 1) - { + if (range_length == 1) { printDebugInformationForBlock(read_offset, range_start, bufferlist); - } - else if (range_length > 1) - { + } else if (range_length > 1) { printDebugInformationForRange(read_offset, range_start, range_length, rangeError, bufferlist); } diff --git a/src/common/io_exerciser/DataGenerator.h b/src/common/io_exerciser/DataGenerator.h index 1e5784a54ccd7..c497c78ed614a 100644 --- a/src/common/io_exerciser/DataGenerator.h +++ b/src/common/io_exerciser/DataGenerator.h @@ -3,8 +3,8 @@ #include #include -#include "include/buffer.h" #include "ObjectModel.h" +#include "include/buffer.h" /* Overview * @@ -23,149 +23,139 @@ * * class HeaderedSeededRandomGenerator * Inherits from SeededDataGenerator. Generates entirely random patterns - * based on the seed retrieved by the model, however also appends a + * based on the seed retrieved by the model, however also appends a * header to the start of each block. This generator also provides * a range of verbose debug options to help disagnose a miscompare * whenever it detects unexpected data. */ namespace ceph { - namespace io_exerciser { - namespace data_generation { - enum class GenerationType { - SeededRandom, - HeaderedSeededRandom - // CompressedGenerator - // MixedGenerator - }; - - class DataGenerator { - public: - virtual ~DataGenerator() = default; - static std::unique_ptr - create_generator(GenerationType generatorType, - const ObjectModel& model); - virtual bufferlist generate_data(uint64_t length, uint64_t offset)=0; - virtual bool validate(bufferlist& bufferlist, uint64_t offset, - uint64_t length); - - // Used for testing debug outputs from data generation - virtual bufferlist generate_wrong_data(uint64_t offset, uint64_t length); - - protected: - const ObjectModel& m_model; - - DataGenerator(const ObjectModel& model) : m_model(model) {} - }; - - class SeededRandomGenerator : public DataGenerator - { - public: - SeededRandomGenerator(const ObjectModel& model) - : DataGenerator(model) {} - - virtual bufferptr generate_block(uint64_t offset); - virtual bufferlist generate_data(uint64_t length, uint64_t offset); - virtual bufferptr generate_wrong_block(uint64_t offset); - virtual bufferlist generate_wrong_data(uint64_t offset, uint64_t length) override; - }; - - class HeaderedSeededRandomGenerator : public SeededRandomGenerator - { - public: - HeaderedSeededRandomGenerator(const ObjectModel& model, - std::optional unique_run_id = std::nullopt); - - bufferptr generate_block(uint64_t offset) override; - bufferptr generate_wrong_block(uint64_t offset) override; - bool validate(bufferlist& bufferlist, uint64_t offset, - uint64_t length) override; - - private: - using UniqueIdBytes = uint64_t; - using SeedBytes = int; - using TimeBytes = uint64_t; - - enum class ErrorType { - RUN_ID_MISMATCH, - SEED_MISMATCH, - DATA_MISMATCH, - DATA_NOT_FOUND, - UNKNOWN - }; - - constexpr uint8_t headerStart() const - { return 0; }; - constexpr uint8_t uniqueIdStart() const - { return headerStart(); }; - constexpr uint8_t uniqueIdLength() const - { return sizeof(UniqueIdBytes); }; - constexpr uint8_t seedStart() const - { return uniqueIdStart() + uniqueIdLength(); }; - constexpr uint8_t seedLength() const - { return sizeof(SeedBytes); }; - constexpr uint8_t timeStart() const - { return seedStart() + seedLength(); }; - constexpr uint8_t timeLength() const - { return sizeof(TimeBytes); }; - constexpr uint8_t timeEnd() const - { return timeStart() + timeLength(); }; - constexpr uint8_t headerLength() const - { return uniqueIdLength() + seedLength() + timeLength(); }; - constexpr uint8_t bodyStart() const - { return headerStart() + headerLength(); }; - - const UniqueIdBytes readUniqueRunId(uint64_t block_offset, - const bufferlist& bufferlist); - const SeedBytes readSeed(uint64_t block_offset, - const bufferlist& bufferlist); - const TimeBytes readDateTime(uint64_t block_offset, +namespace io_exerciser { +namespace data_generation { +enum class GenerationType { + SeededRandom, + HeaderedSeededRandom + // CompressedGenerator + // MixedGenerator +}; + +class DataGenerator { + public: + virtual ~DataGenerator() = default; + static std::unique_ptr create_generator( + GenerationType generatorType, const ObjectModel& model); + virtual bufferlist generate_data(uint64_t length, uint64_t offset) = 0; + virtual bool validate(bufferlist& bufferlist, uint64_t offset, + uint64_t length); + + // Used for testing debug outputs from data generation + virtual bufferlist generate_wrong_data(uint64_t offset, uint64_t length); + + protected: + const ObjectModel& m_model; + + DataGenerator(const ObjectModel& model) : m_model(model) {} +}; + +class SeededRandomGenerator : public DataGenerator { + public: + SeededRandomGenerator(const ObjectModel& model) : DataGenerator(model) {} + + virtual bufferptr generate_block(uint64_t offset); + bufferlist generate_data(uint64_t length, uint64_t offset) override; + virtual bufferptr generate_wrong_block(uint64_t offset); + bufferlist generate_wrong_data(uint64_t offset, + uint64_t length) override; +}; + +class HeaderedSeededRandomGenerator : public SeededRandomGenerator { + public: + HeaderedSeededRandomGenerator( + const ObjectModel& model, + std::optional unique_run_id = std::nullopt); + + bufferptr generate_block(uint64_t offset) override; + bufferptr generate_wrong_block(uint64_t offset) override; + bool validate(bufferlist& bufferlist, uint64_t offset, + uint64_t length) override; + + private: + using UniqueIdBytes = uint64_t; + using SeedBytes = int; + using TimeBytes = uint64_t; + + enum class ErrorType { + RUN_ID_MISMATCH, + SEED_MISMATCH, + DATA_MISMATCH, + DATA_NOT_FOUND, + UNKNOWN + }; + + constexpr uint8_t headerStart() const { return 0; }; + constexpr uint8_t uniqueIdStart() const { return headerStart(); }; + constexpr uint8_t uniqueIdLength() const { return sizeof(UniqueIdBytes); }; + constexpr uint8_t seedStart() const { + return uniqueIdStart() + uniqueIdLength(); + }; + constexpr uint8_t seedLength() const { return sizeof(SeedBytes); }; + constexpr uint8_t timeStart() const { return seedStart() + seedLength(); }; + constexpr uint8_t timeLength() const { return sizeof(TimeBytes); }; + constexpr uint8_t timeEnd() const { return timeStart() + timeLength(); }; + constexpr uint8_t headerLength() const { + return uniqueIdLength() + seedLength() + timeLength(); + }; + constexpr uint8_t bodyStart() const { + return headerStart() + headerLength(); + }; + + const UniqueIdBytes readUniqueRunId(uint64_t block_offset, + const bufferlist& bufferlist); + const SeedBytes readSeed(uint64_t block_offset, const bufferlist& bufferlist); + const TimeBytes readDateTime(uint64_t block_offset, + const bufferlist& bufferlist); + + const UniqueIdBytes unique_run_id; + + uint64_t generate_unique_run_id(); + + bool validate_block(uint64_t block_offset, const char* buffer_start); + + const ErrorType getErrorTypeForBlock(uint64_t read_offset, + uint64_t block_offset, const bufferlist& bufferlist); - const UniqueIdBytes unique_run_id; - - uint64_t generate_unique_run_id(); - - bool validate_block(uint64_t block_offset, const char* buffer_start); - - const ErrorType getErrorTypeForBlock(uint64_t read_offset, - uint64_t block_offset, - const bufferlist& bufferlist); - - void printDebugInformationForBlock(uint64_t read_offset, - uint64_t block_offset, - const bufferlist& bufferlist); - void printDebugInformationForRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - ErrorType rangeError, - const bufferlist& bufferlist); - - void printDebugInformationForRunIdMismatchRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist); - void printDebugInformationForSeedMismatchRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist); - void printDebugInformationDataBodyMismatchRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist); - void printDebugInformationDataNotFoundRange(uint64_t ßread_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist); - void printDebugInformationCorruptRange(uint64_t read_offset, - uint64_t start_block_offset, - uint64_t range_length_in_blocks, - const bufferlist& bufferlist); - - void printDebugInformationForOffsets(uint64_t read_offset, - std::vector offsets, - const bufferlist& bufferlist); - }; - } - } -} + void printDebugInformationForBlock(uint64_t read_offset, + uint64_t block_offset, + const bufferlist& bufferlist); + void printDebugInformationForRange(uint64_t read_offset, + uint64_t start_block_offset, + uint64_t range_length_in_blocks, + ErrorType rangeError, + const bufferlist& bufferlist); + + void printDebugInformationForRunIdMismatchRange( + uint64_t read_offset, uint64_t start_block_offset, + uint64_t range_length_in_blocks, const bufferlist& bufferlist); + void printDebugInformationForSeedMismatchRange( + uint64_t read_offset, uint64_t start_block_offset, + uint64_t range_length_in_blocks, const bufferlist& bufferlist); + void printDebugInformationDataBodyMismatchRange( + uint64_t read_offset, uint64_t start_block_offset, + uint64_t range_length_in_blocks, const bufferlist& bufferlist); + void printDebugInformationDataNotFoundRange(uint64_t ßread_offset, + uint64_t start_block_offset, + uint64_t range_length_in_blocks, + const bufferlist& bufferlist); + void printDebugInformationCorruptRange(uint64_t read_offset, + uint64_t start_block_offset, + uint64_t range_length_in_blocks, + const bufferlist& bufferlist); + + void printDebugInformationForOffsets(uint64_t read_offset, + std::vector offsets, + const bufferlist& bufferlist); +}; +} // namespace data_generation +} // namespace io_exerciser +} // namespace ceph diff --git a/src/common/io_exerciser/EcIoSequence.cc b/src/common/io_exerciser/EcIoSequence.cc new file mode 100644 index 0000000000000..611920c96e0f6 --- /dev/null +++ b/src/common/io_exerciser/EcIoSequence.cc @@ -0,0 +1,267 @@ +#include "EcIoSequence.h" + +#include + +using IoOp = ceph::io_exerciser::IoOp; +using Sequence = ceph::io_exerciser::Sequence; +using IoSequence = ceph::io_exerciser::IoSequence; +using EcIoSequence = ceph::io_exerciser::EcIoSequence; +using ReadInjectSequence = ceph::io_exerciser::ReadInjectSequence; + +bool EcIoSequence::is_supported(Sequence sequence) const { return true; } + +std::unique_ptr EcIoSequence::generate_sequence( + Sequence sequence, std::pair obj_size_range, int k, int m, + int seed) { + switch (sequence) { + case Sequence::SEQUENCE_SEQ0: + [[fallthrough]]; + case Sequence::SEQUENCE_SEQ1: + [[fallthrough]]; + case Sequence::SEQUENCE_SEQ2: + [[fallthrough]]; + case Sequence::SEQUENCE_SEQ3: + [[fallthrough]]; + case Sequence::SEQUENCE_SEQ4: + [[fallthrough]]; + case Sequence::SEQUENCE_SEQ5: + [[fallthrough]]; + case Sequence::SEQUENCE_SEQ6: + [[fallthrough]]; + case Sequence::SEQUENCE_SEQ7: + [[fallthrough]]; + case Sequence::SEQUENCE_SEQ8: + [[fallthrough]]; + case Sequence::SEQUENCE_SEQ9: + return std::make_unique(obj_size_range, seed, + sequence, k, m); + case Sequence::SEQUENCE_SEQ10: + return std::make_unique(obj_size_range, seed, k, m); + default: + ceph_abort_msg("Unrecognised sequence"); + } +} + +EcIoSequence::EcIoSequence(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed), + setup_inject(false), + clear_inject(false), + shard_to_inject(std::nullopt) {} + +void EcIoSequence::select_random_data_shard_to_inject_read_error(int k, int m) { + shard_to_inject = rng(k - 1); + setup_inject = true; +} + +void EcIoSequence::select_random_data_shard_to_inject_write_error(int k, + int m) { + // Write errors do not support injecting to the primary OSD + shard_to_inject = rng(1, k - 1); + setup_inject = true; +} + +void EcIoSequence::select_random_shard_to_inject_read_error(int k, int m) { + shard_to_inject = rng(k + m - 1); + setup_inject = true; +} + +void EcIoSequence::select_random_shard_to_inject_write_error(int k, int m) { + // Write errors do not support injecting to the primary OSD + shard_to_inject = rng(1, k + m - 1); + setup_inject = true; +} + +void EcIoSequence::generate_random_read_inject_type() { + inject_op_type = static_cast( + rng(static_cast(InjectOpType::ReadEIO), + static_cast(InjectOpType::ReadMissingShard))); +} + +void EcIoSequence::generate_random_write_inject_type() { + inject_op_type = static_cast( + rng(static_cast(InjectOpType::WriteFailAndRollback), + static_cast(InjectOpType::WriteOSDAbort))); +} + +ceph::io_exerciser::ReadInjectSequence::ReadInjectSequence( + std::pair obj_size_range, int seed, Sequence s, int k, int m) + : EcIoSequence(obj_size_range, seed) { + child_sequence = IoSequence::generate_sequence(s, obj_size_range, seed); + select_random_data_shard_to_inject_read_error(k, m); + generate_random_read_inject_type(); +} + +Sequence ceph::io_exerciser::ReadInjectSequence::get_id() const { + return child_sequence->get_id(); +} + +std::string ceph::io_exerciser::ReadInjectSequence::get_name() const { + return child_sequence->get_name() + + " running with read errors injected on shard " + + std::to_string(*shard_to_inject); +} + +std::unique_ptr ReadInjectSequence::next() { + step++; + + if (nextOp) { + std::unique_ptr retOp = nullptr; + nextOp.swap(retOp); + return retOp; + } + + std::unique_ptr childOp = child_sequence->next(); + + switch (childOp->getOpType()) { + case OpType::Remove: + nextOp.swap(childOp); + switch (inject_op_type) { + case InjectOpType::ReadEIO: + return ClearReadErrorInjectOp::generate(*shard_to_inject, 0); + case InjectOpType::ReadMissingShard: + return ClearReadErrorInjectOp::generate(*shard_to_inject, 1); + case InjectOpType::WriteFailAndRollback: + return ClearWriteErrorInjectOp::generate(*shard_to_inject, 0); + case InjectOpType::WriteOSDAbort: + return ClearWriteErrorInjectOp::generate(*shard_to_inject, 3); + case InjectOpType::None: + [[fallthrough]]; + default: + ceph_abort_msg("Unsupported operation"); + } + break; + case OpType::Create: + switch (inject_op_type) { + case InjectOpType::ReadEIO: + nextOp = InjectReadErrorOp::generate( + *shard_to_inject, 0, 0, std::numeric_limits::max()); + break; + case InjectOpType::ReadMissingShard: + nextOp = InjectReadErrorOp::generate( + *shard_to_inject, 1, 0, std::numeric_limits::max()); + break; + case InjectOpType::WriteFailAndRollback: + nextOp = InjectWriteErrorOp::generate( + *shard_to_inject, 0, 0, std::numeric_limits::max()); + break; + case InjectOpType::WriteOSDAbort: + nextOp = InjectWriteErrorOp::generate( + *shard_to_inject, 3, 0, std::numeric_limits::max()); + break; + case InjectOpType::None: + [[fallthrough]]; + default: + ceph_abort_msg("Unsupported operation"); + } + break; + default: + // Do nothing in default case + break; + } + + return childOp; +} + +std::unique_ptr +ceph::io_exerciser::ReadInjectSequence::_next() { + ceph_abort_msg( + "Should not reach this point, " + "this sequence should only consume complete sequences"); + + return DoneOp::generate(); +} + +ceph::io_exerciser::Seq10::Seq10(std::pair obj_size_range, int seed, + int k, int m) + : EcIoSequence(obj_size_range, seed), + offset(0), + length(1), + inject_error_done(false), + failed_write_done(false), + read_done(false), + successful_write_done(false), + test_all_lengths(false), // Only test length(1) due to time constraints + test_all_sizes( + false) // Only test obj_size(rand()) due to time constraints +{ + select_random_shard_to_inject_write_error(k, m); + // We will inject specifically as part of our sequence in this sequence + setup_inject = false; + if (!test_all_sizes) { + select_random_object_size(); + } +} + +Sequence ceph::io_exerciser::Seq10::get_id() const { + return Sequence::SEQUENCE_SEQ10; +} + +std::string ceph::io_exerciser::Seq10::get_name() const { + return "Sequential writes of length " + std::to_string(length) + + " with queue depth 1" + " first injecting a failed write and read it to ensure it rolls back, " + "then" + " successfully writing the data and reading the write the ensure it " + "is applied"; +} + +std::unique_ptr ceph::io_exerciser::Seq10::_next() { + if (!inject_error_done) { + inject_error_done = true; + return InjectWriteErrorOp::generate(*shard_to_inject, 0, 0, + std::numeric_limits::max()); + } else if (!failed_write_done) { + failed_write_done = true; + read_done = false; + barrier = true; + return SingleFailedWriteOp::generate(offset, length); + } else if (failed_write_done && !read_done) { + read_done = true; + barrier = true; + return SingleReadOp::generate(offset, length); + } else if (!clear_inject_done) { + clear_inject_done = true; + return ClearWriteErrorInjectOp::generate(*shard_to_inject, 0); + } else if (!successful_write_done) { + successful_write_done = true; + read_done = false; + barrier = true; + return SingleWriteOp::generate(offset, length); + } else if (successful_write_done && !read_done) { + read_done = true; + return SingleReadOp::generate(offset, length); + } else if (successful_write_done && read_done) { + offset++; + inject_error_done = false; + failed_write_done = false; + read_done = false; + clear_inject_done = false; + successful_write_done = false; + + if (offset + length >= obj_size) { + if (!test_all_lengths) { + remove = true; + done = true; + return BarrierOp::generate(); + } + + offset = 0; + length++; + if (length > obj_size) { + if (!test_all_sizes) { + remove = true; + done = true; + return BarrierOp::generate(); + } + + length = 1; + return increment_object_size(); + } + } + + return BarrierOp::generate(); + } else { + ceph_abort_msg("Sequence in undefined state. Aborting"); + return DoneOp::generate(); + } +} \ No newline at end of file diff --git a/src/common/io_exerciser/EcIoSequence.h b/src/common/io_exerciser/EcIoSequence.h new file mode 100644 index 0000000000000..37283b3906bda --- /dev/null +++ b/src/common/io_exerciser/EcIoSequence.h @@ -0,0 +1,65 @@ +#include "IoSequence.h" + +namespace ceph { +namespace io_exerciser { +class EcIoSequence : public IoSequence { + public: + virtual bool is_supported(Sequence sequence) const override; + static std::unique_ptr generate_sequence( + Sequence s, std::pair obj_size_range, int k, int m, int seed); + + protected: + bool setup_inject; + bool clear_inject; + std::optional shard_to_inject; + InjectOpType inject_op_type; + + EcIoSequence(std::pair obj_size_range, int seed); + + // Writes cannot be sent to injected on shard zero, so selections seperated + // out + void select_random_data_shard_to_inject_read_error(int k, int m); + void select_random_data_shard_to_inject_write_error(int k, int m); + void select_random_shard_to_inject_read_error(int k, int m); + void select_random_shard_to_inject_write_error(int k, int m); + void generate_random_read_inject_type(); + void generate_random_write_inject_type(); +}; + +class ReadInjectSequence : public EcIoSequence { + public: + ReadInjectSequence(std::pair obj_size_range, int seed, Sequence s, + int k, int m); + + Sequence get_id() const override; + std::string get_name() const override; + virtual std::unique_ptr next() override; + std::unique_ptr _next() override; + + private: + std::unique_ptr child_sequence; + std::unique_ptr nextOp; +}; + +class Seq10 : public EcIoSequence { + public: + Seq10(std::pair obj_size_range, int seed, int k, int m); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset; + uint64_t length; + + bool inject_error_done; + bool failed_write_done; + bool read_done; + bool clear_inject_done; + bool successful_write_done; + bool test_all_lengths; + bool test_all_sizes; +}; +} // namespace io_exerciser +} // namespace ceph \ No newline at end of file diff --git a/src/common/io_exerciser/IoOp.cc b/src/common/io_exerciser/IoOp.cc index cd855ba6fff8a..493d1f435b48a 100644 --- a/src/common/io_exerciser/IoOp.cc +++ b/src/common/io_exerciser/IoOp.cc @@ -1,188 +1,316 @@ #include "IoOp.h" -using IoOp = ceph::io_exerciser::IoOp; +#include "fmt/format.h" +#include "include/ceph_assert.h" -IoOp::IoOp( OpType op, - uint64_t offset1, uint64_t length1, - uint64_t offset2, uint64_t length2, - uint64_t offset3, uint64_t length3) : - op(op), - offset1(offset1), length1(length1), - offset2(offset2), length2(length2), - offset3(offset3), length3(length3) -{ +using IoOp = ceph::io_exerciser::IoOp; +using OpType = ceph::io_exerciser::OpType; -} +using DoneOp = ceph::io_exerciser::DoneOp; +using BarrierOp = ceph::io_exerciser::BarrierOp; +using CreateOp = ceph::io_exerciser::CreateOp; +using RemoveOp = ceph::io_exerciser::RemoveOp; +using SingleReadOp = ceph::io_exerciser::SingleReadOp; +using DoubleReadOp = ceph::io_exerciser::DoubleReadOp; +using TripleReadOp = ceph::io_exerciser::TripleReadOp; +using SingleWriteOp = ceph::io_exerciser::SingleWriteOp; +using DoubleWriteOp = ceph::io_exerciser::DoubleWriteOp; +using TripleWriteOp = ceph::io_exerciser::TripleWriteOp; +using SingleFailedWriteOp = ceph::io_exerciser::SingleFailedWriteOp; +using DoubleFailedWriteOp = ceph::io_exerciser::DoubleFailedWriteOp; +using TripleFailedWriteOp = ceph::io_exerciser::TripleFailedWriteOp; -std::string IoOp::value_to_string(uint64_t v) const -{ +namespace { +std::string value_to_string(uint64_t v) { if (v < 1024 || (v % 1024) != 0) { return std::to_string(v); - }else if (v < 1024*1024 || (v % (1024 * 1024)) != 0 ) { + } else if (v < 1024 * 1024 || (v % (1024 * 1024)) != 0) { return std::to_string(v / 1024) + "K"; - }else{ + } else { return std::to_string(v / 1024 / 1024) + "M"; } } +} // namespace -std::unique_ptr IoOp - ::generate_done() { +IoOp::IoOp() {} - return std::make_unique(OpType::Done); -} +template +ceph::io_exerciser::TestOp::TestOp() : IoOp() {} + +DoneOp::DoneOp() : TestOp() {} -std::unique_ptr IoOp - ::generate_barrier() { +std::string DoneOp::to_string(uint64_t block_size) const { return "Done"; } - return std::make_unique(OpType::BARRIER); +std::unique_ptr DoneOp::generate() { + return std::make_unique(); } -std::unique_ptr IoOp - ::generate_create(uint64_t size) { +BarrierOp::BarrierOp() : TestOp() {} - return std::make_unique(OpType::CREATE,0,size); +std::unique_ptr BarrierOp::generate() { + return std::make_unique(); } -std::unique_ptr IoOp - ::generate_remove() { - - return std::make_unique(OpType::REMOVE); +std::string BarrierOp::to_string(uint64_t block_size) const { + return "Barrier"; } -std::unique_ptr IoOp - ::generate_read(uint64_t offset, uint64_t length) { +CreateOp::CreateOp(uint64_t size) : TestOp(), size(size) {} - return std::make_unique(OpType::READ, offset, length); +std::unique_ptr CreateOp::generate(uint64_t size) { + return std::make_unique(size); } -std::unique_ptr IoOp - ::generate_read2(uint64_t offset1, uint64_t length1, - uint64_t offset2, uint64_t length2) { +std::string CreateOp::to_string(uint64_t block_size) const { + return "Create (size=" + value_to_string(size * block_size) + ")"; +} - if (offset1 < offset2) { - ceph_assert( offset1 + length1 <= offset2 ); - } else { - ceph_assert( offset2 + length2 <= offset1 ); - } +RemoveOp::RemoveOp() : TestOp() {} - return std::make_unique(OpType::READ2, - offset1, length1, - offset2, length2); +std::unique_ptr RemoveOp::generate() { + return std::make_unique(); } -std::unique_ptr IoOp - ::generate_read3(uint64_t offset1, uint64_t length1, - uint64_t offset2, uint64_t length2, - uint64_t offset3, uint64_t length3) { +std::string RemoveOp::to_string(uint64_t block_size) const { return "Remove"; } - if (offset1 < offset2) { - ceph_assert( offset1 + length1 <= offset2 ); - } else { - ceph_assert( offset2 + length2 <= offset1 ); +template +ceph::io_exerciser::ReadWriteOp::ReadWriteOp( + std::array&& offset, + std::array&& length) + : TestOp(), offset(offset), length(length) { + auto compare = [](uint64_t offset1, uint64_t length1, uint64_t offset2, + uint64_t length2) { + if (offset1 < offset2) { + ceph_assert(offset1 + length1 <= offset2); + } else { + ceph_assert(offset2 + length2 <= offset1); + } + }; + + if (numIOs > 1) { + for (int i = 0; i < numIOs - 1; i++) { + for (int j = i + 1; j < numIOs; j++) { + compare(offset[i], length[i], offset[j], length[j]); + } + } } - if (offset1 < offset3) { - ceph_assert( offset1 + length1 <= offset3 ); - } else { - ceph_assert( offset3 + length3 <= offset1 ); +} + +template +std::string ceph::io_exerciser::ReadWriteOp::to_string( + uint64_t block_size) const { + std::string offset_length_desc; + if (numIOs > 0) { + offset_length_desc += fmt::format( + "offset1={}", value_to_string(this->offset[0] * block_size)); + offset_length_desc += fmt::format( + ",length1={}", value_to_string(this->length[0] * block_size)); + for (int i = 1; i < numIOs; i++) { + offset_length_desc += fmt::format( + ",offset{}={}", i + 1, value_to_string(this->offset[i] * block_size)); + offset_length_desc += fmt::format( + ",length{}={}", i + 1, value_to_string(this->length[i] * block_size)); + } } - if (offset2 < offset3) { - ceph_assert( offset2 + length2 <= offset3 ); - } else { - ceph_assert( offset3 + length3 <= offset2 ); + switch (opType) { + case OpType::Read: + [[fallthrough]]; + case OpType::Read2: + [[fallthrough]]; + case OpType::Read3: + return fmt::format("Read{} ({})", numIOs, offset_length_desc); + case OpType::Write: + [[fallthrough]]; + case OpType::Write2: + [[fallthrough]]; + case OpType::Write3: + return fmt::format("Write{} ({})", numIOs, offset_length_desc); + case OpType::FailedWrite: + [[fallthrough]]; + case OpType::FailedWrite2: + [[fallthrough]]; + case OpType::FailedWrite3: + return fmt::format("FailedWrite{} ({})", numIOs, offset_length_desc); + default: + ceph_abort_msg( + fmt::format("Unsupported op type by ReadWriteOp ({})", opType)); } - return std::make_unique(OpType::READ3, - offset1, length1, - offset2, length2, - offset3, length3); } -std::unique_ptr IoOp::generate_write(uint64_t offset, uint64_t length) { - return std::make_unique(OpType::WRITE, offset, length); +SingleReadOp::SingleReadOp(uint64_t offset, uint64_t length) + : ReadWriteOp({offset}, {length}) {} + +std::unique_ptr SingleReadOp::generate(uint64_t offset, + uint64_t length) { + return std::make_unique(offset, length); } -std::unique_ptr IoOp::generate_write2(uint64_t offset1, uint64_t length1, - uint64_t offset2, uint64_t length2) { - if (offset1 < offset2) { - ceph_assert( offset1 + length1 <= offset2 ); - } else { - ceph_assert( offset2 + length2 <= offset1 ); - } - return std::make_unique(OpType::WRITE2, - offset1, length1, - offset2, length2); +DoubleReadOp::DoubleReadOp(uint64_t offset1, uint64_t length1, uint64_t offset2, + uint64_t length2) + : ReadWriteOp({offset1, offset2}, {length1, length2}) {} + +std::unique_ptr DoubleReadOp::generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2) { + return std::make_unique(offset1, length1, offset2, length2); } -std::unique_ptr IoOp::generate_write3(uint64_t offset1, uint64_t length1, - uint64_t offset2, uint64_t length2, - uint64_t offset3, uint64_t length3) { - if (offset1 < offset2) { - ceph_assert( offset1 + length1 <= offset2 ); - } else { - ceph_assert( offset2 + length2 <= offset1 ); - } - if (offset1 < offset3) { - ceph_assert( offset1 + length1 <= offset3 ); - } else { - ceph_assert( offset3 + length3 <= offset1 ); - } - if (offset2 < offset3) { - ceph_assert( offset2 + length2 <= offset3 ); - } else { - ceph_assert( offset3 + length3 <= offset2 ); - } - return std::make_unique(OpType::WRITE3, - offset1, length1, - offset2, length2, - offset3, length3); -} - -bool IoOp::done() { - return (op == OpType::Done); -} - -std::string IoOp::to_string(uint64_t block_size) const -{ - switch (op) { - case OpType::Done: - return "Done"; - case OpType::BARRIER: - return "Barrier"; - case OpType::CREATE: - return "Create (size=" + value_to_string(length1 * block_size) + ")"; - case OpType::REMOVE: - return "Remove"; - case OpType::READ: - return "Read (offset=" + value_to_string(offset1 * block_size) + - ",length=" + value_to_string(length1 * block_size) + ")"; - case OpType::READ2: - return "Read2 (offset1=" + value_to_string(offset1 * block_size) + - ",length1=" + value_to_string(length1 * block_size) + - ",offset2=" + value_to_string(offset2 * block_size) + - ",length2=" + value_to_string(length2 * block_size) + ")"; - case OpType::READ3: - return "Read3 (offset1=" + value_to_string(offset1 * block_size) + - ",length1=" + value_to_string(length1 * block_size) + - ",offset2=" + value_to_string(offset2 * block_size) + - ",length2=" + value_to_string(length2 * block_size) + - ",offset3=" + value_to_string(offset3 * block_size) + - ",length3=" + value_to_string(length3 * block_size) + ")"; - case OpType::WRITE: - return "Write (offset=" + value_to_string(offset1 * block_size) + - ",length=" + value_to_string(length1 * block_size) + ")"; - case OpType::WRITE2: - return "Write2 (offset1=" + value_to_string(offset1 * block_size) + - ",length1=" + value_to_string(length1 * block_size) + - ",offset2=" + value_to_string(offset2 * block_size) + - ",length2=" + value_to_string(length2 * block_size) + ")"; - case OpType::WRITE3: - return "Write3 (offset1=" + value_to_string(offset1 * block_size) + - ",length1=" + value_to_string(length1 * block_size) + - ",offset2=" + value_to_string(offset2 * block_size) + - ",length2=" + value_to_string(length2 * block_size) + - ",offset3=" + value_to_string(offset3 * block_size) + - ",length3=" + value_to_string(length3 * block_size) + ")"; - default: - break; - } - return "Unknown"; +TripleReadOp::TripleReadOp(uint64_t offset1, uint64_t length1, uint64_t offset2, + uint64_t length2, uint64_t offset3, uint64_t length3) + : ReadWriteOp({offset1, offset2, offset3}, + {length1, length2, length3}) {} + +std::unique_ptr TripleReadOp::generate( + uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3) { + return std::make_unique(offset1, length1, offset2, length2, + offset3, length3); +} + +SingleWriteOp::SingleWriteOp(uint64_t offset, uint64_t length) + : ReadWriteOp({offset}, {length}) {} + +std::unique_ptr SingleWriteOp::generate(uint64_t offset, + uint64_t length) { + return std::make_unique(offset, length); +} + +DoubleWriteOp::DoubleWriteOp(uint64_t offset1, uint64_t length1, + uint64_t offset2, uint64_t length2) + : ReadWriteOp({offset1, offset2}, {length1, length2}) {} + +std::unique_ptr DoubleWriteOp::generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2) { + return std::make_unique(offset1, length1, offset2, length2); +} + +TripleWriteOp::TripleWriteOp(uint64_t offset1, uint64_t length1, + uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3) + : ReadWriteOp({offset1, offset2, offset3}, + {length1, length2, length3}) {} + +std::unique_ptr TripleWriteOp::generate( + uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3) { + return std::make_unique(offset1, length1, offset2, length2, + offset3, length3); +} + +SingleFailedWriteOp::SingleFailedWriteOp(uint64_t offset, uint64_t length) + : ReadWriteOp({offset}, {length}) {} + +std::unique_ptr SingleFailedWriteOp::generate( + uint64_t offset, uint64_t length) { + return std::make_unique(offset, length); +} + +DoubleFailedWriteOp::DoubleFailedWriteOp(uint64_t offset1, uint64_t length1, + uint64_t offset2, uint64_t length2) + : ReadWriteOp({offset1, offset2}, + {length1, length2}) {} + +std::unique_ptr DoubleFailedWriteOp::generate( + uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2) { + return std::make_unique(offset1, length1, offset2, + length2); +} + +TripleFailedWriteOp::TripleFailedWriteOp(uint64_t offset1, uint64_t length1, + uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3) + : ReadWriteOp({offset1, offset2, offset3}, + {length1, length2, length3}) {} + +std::unique_ptr TripleFailedWriteOp::generate( + uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3) { + return std::make_unique(offset1, length1, offset2, + length2, offset3, length3); +} + +template +ceph::io_exerciser::InjectErrorOp::InjectErrorOp( + int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration) + : TestOp(), + shard(shard), + type(type), + when(when), + duration(duration) {} + +template +std::string ceph::io_exerciser::InjectErrorOp::to_string( + uint64_t blocksize) const { + std::string_view inject_type = get_inject_type_string(); + return fmt::format( + "Inject {} error on shard {} of type {}" + " after {} successful inject(s) lasting {} inject(s)", + inject_type, shard, type.value_or(0), when.value_or(0), + duration.value_or(1)); +} + +ceph::io_exerciser::InjectReadErrorOp::InjectReadErrorOp( + int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration) + : InjectErrorOp(shard, type, when, duration) {} + +std::unique_ptr +ceph::io_exerciser ::InjectReadErrorOp::generate( + int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration) { + return std::make_unique(shard, type, when, duration); +} + +ceph::io_exerciser::InjectWriteErrorOp::InjectWriteErrorOp( + int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration) + : InjectErrorOp(shard, type, when, duration) {} + +std::unique_ptr +ceph::io_exerciser ::InjectWriteErrorOp::generate( + int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration) { + return std::make_unique(shard, type, when, duration); +} + +template +ceph::io_exerciser::ClearErrorInjectOp::ClearErrorInjectOp( + int shard, const std::optional& type) + : TestOp(), shard(shard), type(type) {} + +template +std::string ceph::io_exerciser::ClearErrorInjectOp::to_string( + uint64_t blocksize) const { + std::string_view inject_type = get_inject_type_string(); + return fmt::format("Clear {} injects on shard {} of type {}", inject_type, + shard, type.value_or(0)); +} + +ceph::io_exerciser::ClearReadErrorInjectOp::ClearReadErrorInjectOp( + int shard, const std::optional& type) + : ClearErrorInjectOp(shard, type) {} + +std::unique_ptr +ceph::io_exerciser ::ClearReadErrorInjectOp::generate( + int shard, const std::optional& type) { + return std::make_unique(shard, type); +} + +ceph::io_exerciser::ClearWriteErrorInjectOp::ClearWriteErrorInjectOp( + int shard, const std::optional& type) + : ClearErrorInjectOp(shard, type) {} + +std::unique_ptr +ceph::io_exerciser ::ClearWriteErrorInjectOp::generate( + int shard, const std::optional& type) { + return std::make_unique(shard, type); } \ No newline at end of file diff --git a/src/common/io_exerciser/IoOp.h b/src/common/io_exerciser/IoOp.h index 60c02a93d4e22..1887eafcc1f41 100644 --- a/src/common/io_exerciser/IoOp.h +++ b/src/common/io_exerciser/IoOp.h @@ -1,13 +1,13 @@ #pragma once -#include +#include #include -#include "include/ceph_assert.h" +#include +#include + +#include "OpType.h" /* Overview - * - * enum OpType - * Enumeration of different types of I/O operation * * class IoOp * Stores details for an I/O operation. Generated by IoSequences @@ -15,80 +15,234 @@ */ namespace ceph { - namespace io_exerciser { - - enum class OpType { - Done, // End of I/O sequence - BARRIER, // Barrier - all prior I/Os must complete - CREATE, // Create object and pattern with data - REMOVE, // Remove object - READ, // Read - READ2, // 2 Reads in one op - READ3, // 3 Reads in one op - WRITE, // Write - WRITE2, // 2 Writes in one op - WRITE3 // 3 Writes in one op - }; - - class IoOp { - protected: - std::string value_to_string(uint64_t v) const; - - public: - OpType op; - uint64_t offset1; - uint64_t length1; - uint64_t offset2; - uint64_t length2; - uint64_t offset3; - uint64_t length3; - - IoOp( OpType op, - uint64_t offset1 = 0, uint64_t length1 = 0, - uint64_t offset2 = 0, uint64_t length2 = 0, - uint64_t offset3 = 0, uint64_t length3 = 0 ); - - static std::unique_ptr generate_done(); - - static std::unique_ptr generate_barrier(); - - static std::unique_ptr generate_create(uint64_t size); - - static std::unique_ptr generate_remove(); - - static std::unique_ptr generate_read(uint64_t offset, +namespace io_exerciser { + +class IoOp { + public: + IoOp(); + virtual ~IoOp() = default; + virtual std::string to_string(uint64_t block_size) const = 0; + virtual constexpr OpType getOpType() const = 0; +}; + +template +class TestOp : public IoOp { + public: + TestOp(); + constexpr OpType getOpType() const override { return opType; } +}; + +class DoneOp : public TestOp { + public: + DoneOp(); + static std::unique_ptr generate(); + std::string to_string(uint64_t block_size) const override; +}; + +class BarrierOp : public TestOp { + public: + BarrierOp(); + static std::unique_ptr generate(); + std::string to_string(uint64_t block_size) const override; +}; + +class CreateOp : public TestOp { + public: + CreateOp(uint64_t size); + static std::unique_ptr generate(uint64_t size); + std::string to_string(uint64_t block_size) const override; + uint64_t size; +}; + +class RemoveOp : public TestOp { + public: + RemoveOp(); + static std::unique_ptr generate(); + std::string to_string(uint64_t block_size) const override; +}; + +template +class ReadWriteOp : public TestOp { + public: + std::array offset; + std::array length; + + protected: + ReadWriteOp(std::array&& offset, + std::array&& length); + std::string to_string(uint64_t block_size) const override; +}; + +class SingleReadOp : public ReadWriteOp { + public: + SingleReadOp(uint64_t offset, uint64_t length); + static std::unique_ptr generate(uint64_t offset, + uint64_t length); +}; + +class DoubleReadOp : public ReadWriteOp { + public: + DoubleReadOp(uint64_t offset1, uint64_t length1, uint64_t offset2, + uint64_t length2); + static std::unique_ptr generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2); +}; + +class TripleReadOp : public ReadWriteOp { + public: + TripleReadOp(uint64_t offset1, uint64_t length1, uint64_t offset2, + uint64_t length2, uint64_t offset3, uint64_t length3); + static std::unique_ptr generate( + uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3); +}; + +class SingleWriteOp : public ReadWriteOp { + public: + SingleWriteOp(uint64_t offset, uint64_t length); + static std::unique_ptr generate(uint64_t offset, uint64_t length); +}; + +class DoubleWriteOp : public ReadWriteOp { + public: + DoubleWriteOp(uint64_t offset1, uint64_t length1, uint64_t offset2, + uint64_t length2); + static std::unique_ptr generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2); +}; + +class TripleWriteOp : public ReadWriteOp { + public: + TripleWriteOp(uint64_t offset1, uint64_t length1, uint64_t offset2, + uint64_t length2, uint64_t offset3, uint64_t length3); + static std::unique_ptr generate( + uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3); +}; + +class SingleFailedWriteOp : public ReadWriteOp { + public: + SingleFailedWriteOp(uint64_t offset, uint64_t length); + static std::unique_ptr generate(uint64_t offset, + uint64_t length); +}; + +class DoubleFailedWriteOp : public ReadWriteOp { + public: + DoubleFailedWriteOp(uint64_t offset1, uint64_t length1, uint64_t offset2, + uint64_t length2); + static std::unique_ptr generate(uint64_t offset1, + uint64_t length1, + uint64_t offset2, + uint64_t length2); +}; + +class TripleFailedWriteOp : public ReadWriteOp { + public: + TripleFailedWriteOp(uint64_t offset1, uint64_t length1, uint64_t offset2, + uint64_t length2, uint64_t offset3, uint64_t length3); + static std::unique_ptr generate( + uint64_t offset1, uint64_t length1, uint64_t offset2, uint64_t length2, + uint64_t offset3, uint64_t length3); +}; + +template +class InjectErrorOp : public TestOp { + public: + InjectErrorOp(int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration); + + std::string to_string(uint64_t block_size) const override; + + int shard; + std::optional type; + std::optional when; + std::optional duration; + + protected: + virtual inline constexpr std::string_view get_inject_type_string() const = 0; +}; + +class InjectReadErrorOp : public InjectErrorOp { + public: + InjectReadErrorOp(int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration); + + static std::unique_ptr generate( + int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration); + + protected: + inline constexpr std::string_view get_inject_type_string() const override { + return "read"; + } +}; + +class InjectWriteErrorOp : public InjectErrorOp { + public: + InjectWriteErrorOp(int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration); + + static std::unique_ptr generate( + int shard, const std::optional& type, + const std::optional& when, + const std::optional& duration); + + protected: + inline constexpr std::string_view get_inject_type_string() const override { + return "write"; + } +}; + +template +class ClearErrorInjectOp : public TestOp { + public: + ClearErrorInjectOp(int shard, const std::optional& type); + + std::string to_string(uint64_t block_size) const override; + + int shard; + std::optional type; + + protected: + virtual inline constexpr std::string_view get_inject_type_string() const = 0; +}; + +class ClearReadErrorInjectOp + : public ClearErrorInjectOp { + public: + ClearReadErrorInjectOp(int shard, const std::optional& type); + + static std::unique_ptr generate( + int shard, const std::optional& type); + + protected: + inline constexpr std::string_view get_inject_type_string() const override { + return "read"; + } +}; + +class ClearWriteErrorInjectOp + : public ClearErrorInjectOp { + public: + ClearWriteErrorInjectOp(int shard, const std::optional& type); + + static std::unique_ptr generate( + int shard, const std::optional& type); - static std::unique_ptr generate_read2(uint64_t offset1, - uint64_t length1, - uint64_t offset2, - uint64_t length2); - - static std::unique_ptr generate_read3(uint64_t offset1, - uint64_t length1, - uint64_t offset2, - uint64_t length2, - uint64_t offset3, - uint64_t length3); - - static std::unique_ptr generate_write(uint64_t offset, - uint64_t length); - - static std::unique_ptr generate_write2(uint64_t offset1, - uint64_t length1, - uint64_t offset2, - uint64_t length2); - - static std::unique_ptr generate_write3(uint64_t offset1, - uint64_t length1, - uint64_t offset2, - uint64_t length2, - uint64_t offset3, - uint64_t length3); - - bool done(); - - std::string to_string(uint64_t block_size) const; - }; + protected: + inline constexpr std::string_view get_inject_type_string() const override { + return "write"; } -} \ No newline at end of file +}; +} // namespace io_exerciser +} // namespace ceph \ No newline at end of file diff --git a/src/common/io_exerciser/IoSequence.cc b/src/common/io_exerciser/IoSequence.cc index 4a7ca0593d1d2..83f1cc595a534 100644 --- a/src/common/io_exerciser/IoSequence.cc +++ b/src/common/io_exerciser/IoSequence.cc @@ -1,12 +1,12 @@ #include "IoSequence.h" +using IoOp = ceph::io_exerciser::IoOp; using Sequence = ceph::io_exerciser::Sequence; using IoSequence = ceph::io_exerciser::IoSequence; -std::ostream& ceph::io_exerciser::operator<<(std::ostream& os, const Sequence& seq) -{ - switch (seq) - { +std::ostream& ceph::io_exerciser::operator<<(std::ostream& os, + const Sequence& seq) { + switch (seq) { case Sequence::SEQUENCE_SEQ0: os << "SEQUENCE_SEQ0"; break; @@ -37,6 +37,9 @@ std::ostream& ceph::io_exerciser::operator<<(std::ostream& os, const Sequence& s case Sequence::SEQUENCE_SEQ9: os << "SEQUENCE_SEQ9"; break; + case Sequence::SEQUENCE_SEQ10: + os << "SEQUENCE_SEQ10"; + break; case Sequence::SEQUENCE_END: os << "SEQUENCE_END"; break; @@ -44,19 +47,12 @@ std::ostream& ceph::io_exerciser::operator<<(std::ostream& os, const Sequence& s return os; } -IoSequence::IoSequence(std::pair obj_size_range, - int seed) : - min_obj_size(obj_size_range.first), max_obj_size(obj_size_range.second), - create(true), barrier(false), done(false), remove(false), - obj_size(min_obj_size), step(-1), seed(seed) -{ - rng.seed(seed); +bool IoSequence::is_supported(Sequence sequence) const { + return sequence != Sequence::SEQUENCE_SEQ10; } -std::unique_ptr IoSequence::generate_sequence(Sequence s, - std::pair obj_size_range, - int seed) -{ +std::unique_ptr IoSequence::generate_sequence( + Sequence s, std::pair obj_size_range, int seed) { switch (s) { case Sequence::SEQUENCE_SEQ0: return std::make_unique(obj_size_range, seed); @@ -78,24 +74,39 @@ std::unique_ptr IoSequence::generate_sequence(Sequence s, return std::make_unique(obj_size_range, seed); case Sequence::SEQUENCE_SEQ9: return std::make_unique(obj_size_range, seed); + case Sequence::SEQUENCE_SEQ10: + ceph_abort_msg( + "Sequence 10 only supported for erasure coded pools " + "through the EcIoSequence interface"); + return nullptr; default: break; } return nullptr; } -int IoSequence::get_step() const -{ - return step; +IoSequence::IoSequence(std::pair obj_size_range, int seed) + : min_obj_size(obj_size_range.first), + max_obj_size(obj_size_range.second), + create(true), + barrier(false), + done(false), + remove(false), + obj_size(min_obj_size), + step(-1), + seed(seed) { + rng.seed(seed); } -int IoSequence::get_seed() const -{ - return seed; +std::string ceph::io_exerciser::IoSequence::get_name_with_seqseed() const { + return get_name() + " (seqseed " + std::to_string(get_seed()) + ")"; } -void IoSequence::set_min_object_size(uint64_t size) -{ +int IoSequence::get_step() const { return step; } + +int IoSequence::get_seed() const { return seed; } + +void IoSequence::set_min_object_size(uint64_t size) { min_obj_size = size; if (obj_size < size) { obj_size = size; @@ -105,23 +116,20 @@ void IoSequence::set_min_object_size(uint64_t size) } } -void IoSequence::set_max_object_size(uint64_t size) -{ +void IoSequence::set_max_object_size(uint64_t size) { max_obj_size = size; if (obj_size > size) { done = true; } } -void IoSequence::select_random_object_size() -{ +void IoSequence::select_random_object_size() { if (max_obj_size != min_obj_size) { obj_size = min_obj_size + rng(max_obj_size - min_obj_size); } } -std::unique_ptr IoSequence::increment_object_size() -{ +std::unique_ptr IoSequence::increment_object_size() { obj_size++; if (obj_size > max_obj_size) { done = true; @@ -129,106 +137,118 @@ std::unique_ptr IoSequence::increment_object_size() create = true; barrier = true; remove = true; - return IoOp::generate_barrier(); + return BarrierOp::generate(); } -std::unique_ptr IoSequence::next() -{ +Sequence IoSequence::getNextSupportedSequenceId() const { + Sequence sequence = get_id(); + ++sequence; + for (; sequence < Sequence::SEQUENCE_END; ++sequence) { + if (is_supported(sequence)) { + return sequence; + } + } + + return Sequence::SEQUENCE_END; +} + +std::unique_ptr IoSequence::next() { step++; if (remove) { remove = false; - return IoOp::generate_remove(); + return RemoveOp::generate(); } if (barrier) { barrier = false; - return IoOp::generate_barrier(); + return BarrierOp::generate(); } if (done) { - return IoOp::generate_done(); + return DoneOp::generate(); } if (create) { create = false; barrier = true; - return IoOp::generate_create(obj_size); + return CreateOp::generate(obj_size); } return _next(); } - - -ceph::io_exerciser::Seq0::Seq0(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed), offset(0) -{ +ceph::io_exerciser::Seq0::Seq0(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed), offset(0) { select_random_object_size(); length = 1 + rng(obj_size - 1); } -std::string ceph::io_exerciser::Seq0::get_name() const -{ +Sequence ceph::io_exerciser::Seq0::get_id() const { + return Sequence::SEQUENCE_SEQ0; +} + +std::string ceph::io_exerciser::Seq0::get_name() const { return "Sequential reads of length " + std::to_string(length) + - " with queue depth 1 (seqseed " + std::to_string(get_seed()) + ")"; + " with queue depth 1"; } -std::unique_ptr ceph::io_exerciser::Seq0::_next() -{ +std::unique_ptr ceph::io_exerciser::Seq0::_next() { std::unique_ptr r; if (offset >= obj_size) { done = true; barrier = true; remove = true; - return IoOp::generate_barrier(); + return BarrierOp::generate(); } if (offset + length > obj_size) { - r = IoOp::generate_read(offset, obj_size - offset); + r = SingleReadOp::generate(offset, obj_size - offset); } else { - r = IoOp::generate_read(offset, length); + r = SingleReadOp::generate(offset, length); } offset += length; return r; } - - -ceph::io_exerciser::Seq1::Seq1(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed) -{ +ceph::io_exerciser::Seq1::Seq1(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed) { select_random_object_size(); count = 3 * obj_size; } -std::string ceph::io_exerciser::Seq1::get_name() const -{ - return "Random offset, random length read/write I/O with queue depth 1 (seqseed " - + std::to_string(get_seed()) + ")"; +Sequence ceph::io_exerciser::Seq1::get_id() const { + return Sequence::SEQUENCE_SEQ1; } -std::unique_ptr ceph::io_exerciser::Seq1::_next() -{ +std::string ceph::io_exerciser::Seq1::get_name() const { + return "Random offset, random length read/write I/O with queue depth 1"; +} + +std::unique_ptr ceph::io_exerciser::Seq1::_next() { barrier = true; if (count-- == 0) { done = true; remove = true; - return IoOp::generate_barrier(); + return BarrierOp::generate(); } uint64_t offset = rng(obj_size - 1); uint64_t length = 1 + rng(obj_size - 1 - offset); - return (rng(2) != 0) ? IoOp::generate_write(offset, length) : - IoOp::generate_read(offset, length); -} + if (rng(2) != 0) { + return SingleWriteOp::generate(offset, length); + } else { + return SingleReadOp::generate(offset, length); + } +} +ceph::io_exerciser::Seq2::Seq2(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed), offset(0), length(0) {} -ceph::io_exerciser::Seq2::Seq2(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed), offset(0), length(0) {} +Sequence ceph::io_exerciser::Seq2::get_id() const { + return Sequence::SEQUENCE_SEQ2; +} -std::string ceph::io_exerciser::Seq2::get_name() const -{ +std::string ceph::io_exerciser::Seq2::get_name() const { return "Permutations of offset and length read I/O"; } -std::unique_ptr ceph::io_exerciser::Seq2::_next() -{ +std::unique_ptr ceph::io_exerciser::Seq2::_next() { length++; if (length > obj_size - offset) { length = 1; @@ -239,24 +259,23 @@ std::unique_ptr ceph::io_exerciser::Seq2::_next() return increment_object_size(); } } - return IoOp::generate_read(offset, length); + return SingleReadOp::generate(offset, length); } - - -ceph::io_exerciser::Seq3::Seq3(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed), offset1(0), offset2(0) -{ +ceph::io_exerciser::Seq3::Seq3(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed), offset1(0), offset2(0) { set_min_object_size(2); } -std::string ceph::io_exerciser::Seq3::get_name() const -{ +Sequence ceph::io_exerciser::Seq3::get_id() const { + return Sequence::SEQUENCE_SEQ3; +} + +std::string ceph::io_exerciser::Seq3::get_name() const { return "Permutations of offset 2-region 1-block read I/O"; } -std::unique_ptr ceph::io_exerciser::Seq3::_next() -{ +std::unique_ptr ceph::io_exerciser::Seq3::_next() { offset2++; if (offset2 >= obj_size - offset1) { offset2 = 1; @@ -267,24 +286,23 @@ std::unique_ptr ceph::io_exerciser::Seq3::_next() return increment_object_size(); } } - return IoOp::generate_read2(offset1, 1, offset1 + offset2, 1); + return DoubleReadOp::generate(offset1, 1, offset1 + offset2, 1); } - - -ceph::io_exerciser::Seq4::Seq4(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed), offset1(0), offset2(1) -{ +ceph::io_exerciser::Seq4::Seq4(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed), offset1(0), offset2(1) { set_min_object_size(3); } -std::string ceph::io_exerciser::Seq4::get_name() const -{ +Sequence ceph::io_exerciser::Seq4::get_id() const { + return Sequence::SEQUENCE_SEQ4; +} + +std::string ceph::io_exerciser::Seq4::get_name() const { return "Permutations of offset 3-region 1-block read I/O"; } -std::unique_ptr ceph::io_exerciser::Seq4::_next() -{ +std::unique_ptr ceph::io_exerciser::Seq4::_next() { offset2++; if (offset2 >= obj_size - offset1) { offset2 = 2; @@ -295,33 +313,35 @@ std::unique_ptr ceph::io_exerciser::Seq4::_next() return increment_object_size(); } } - return IoOp::generate_read3(offset1, 1, - offset1 + offset2, 1, - (offset1 * 2 + offset2)/2, 1); + return TripleReadOp::generate(offset1, 1, (offset1 + offset2), 1, + (offset1 * 2 + offset2) / 2, 1); } +ceph::io_exerciser::Seq5::Seq5(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed), + offset(0), + length(1), + doneread(false), + donebarrier(false) {} +Sequence ceph::io_exerciser::Seq5::get_id() const { + return Sequence::SEQUENCE_SEQ5; +} -ceph::io_exerciser::Seq5::Seq5(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed), offset(0), length(1), - doneread(false), donebarrier(false) {} - -std::string ceph::io_exerciser::Seq5::get_name() const -{ +std::string ceph::io_exerciser::Seq5::get_name() const { return "Permutation of length sequential writes"; } -std::unique_ptr ceph::io_exerciser::Seq5::_next() -{ +std::unique_ptr ceph::io_exerciser::Seq5::_next() { if (offset >= obj_size) { if (!doneread) { if (!donebarrier) { donebarrier = true; - return IoOp::generate_barrier(); + return BarrierOp::generate(); } doneread = true; barrier = true; - return IoOp::generate_read(0, obj_size); + return SingleReadOp::generate(0, obj_size); } doneread = false; donebarrier = false; @@ -333,33 +353,36 @@ std::unique_ptr ceph::io_exerciser::Seq5::_next() } } uint64_t io_len = (offset + length > obj_size) ? (obj_size - offset) : length; - std::unique_ptr r = IoOp::generate_write(offset, io_len); + std::unique_ptr r = SingleWriteOp::generate(offset, io_len); offset += io_len; return r; } +ceph::io_exerciser::Seq6::Seq6(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed), + offset(0), + length(1), + doneread(false), + donebarrier(false) {} +Sequence ceph::io_exerciser::Seq6::get_id() const { + return Sequence::SEQUENCE_SEQ6; +} -ceph::io_exerciser::Seq6::Seq6(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed), offset(0), length(1), - doneread(false), donebarrier(false) {} - -std::string ceph::io_exerciser::Seq6::get_name() const -{ +std::string ceph::io_exerciser::Seq6::get_name() const { return "Permutation of length sequential writes, different alignment"; } -std::unique_ptr ceph::io_exerciser::Seq6::_next() -{ +std::unique_ptr ceph::io_exerciser::Seq6::_next() { if (offset >= obj_size) { if (!doneread) { if (!donebarrier) { donebarrier = true; - return IoOp::generate_barrier(); + return BarrierOp::generate(); } doneread = true; barrier = true; - return IoOp::generate_read(0, obj_size); + return SingleReadOp::generate(0, obj_size); } doneread = false; donebarrier = false; @@ -374,74 +397,72 @@ std::unique_ptr ceph::io_exerciser::Seq6::_next() if (io_len == 0) { io_len = length; } - std::unique_ptr r = IoOp::generate_write(offset, io_len); + std::unique_ptr r = SingleWriteOp::generate(offset, io_len); offset += io_len; return r; } - - -ceph::io_exerciser::Seq7::Seq7(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed) -{ +ceph::io_exerciser::Seq7::Seq7(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed) { set_min_object_size(2); offset = obj_size; } -std::string ceph::io_exerciser::Seq7::get_name() const -{ +Sequence ceph::io_exerciser::Seq7::get_id() const { + return Sequence::SEQUENCE_SEQ7; +} + +std::string ceph::io_exerciser::Seq7::get_name() const { return "Permutations of offset 2-region 1-block writes"; } -std::unique_ptr ceph::io_exerciser::Seq7::_next() -{ +std::unique_ptr ceph::io_exerciser::Seq7::_next() { if (!doneread) { if (!donebarrier) { donebarrier = true; - return IoOp::generate_barrier(); + return BarrierOp::generate(); } doneread = true; barrier = true; - return IoOp::generate_read(0, obj_size); + return SingleReadOp::generate(0, obj_size); } if (offset == 0) { doneread = false; donebarrier = false; - offset = obj_size+1; + offset = obj_size + 1; return increment_object_size(); } offset--; - if (offset == obj_size/2) { + if (offset == obj_size / 2) { return _next(); } doneread = false; donebarrier = false; - return IoOp::generate_write2(offset, 1, obj_size/2, 1); + return DoubleReadOp::generate(offset, 1, obj_size / 2, 1); } - - -ceph::io_exerciser::Seq8::Seq8(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed), offset1(0), offset2(1) -{ +ceph::io_exerciser::Seq8::Seq8(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed), offset1(0), offset2(1) { set_min_object_size(3); } -std::string ceph::io_exerciser::Seq8::get_name() const -{ +Sequence ceph::io_exerciser::Seq8::get_id() const { + return Sequence::SEQUENCE_SEQ8; +} + +std::string ceph::io_exerciser::Seq8::get_name() const { return "Permutations of offset 3-region 1-block write I/O"; } -std::unique_ptr ceph::io_exerciser::Seq8::_next() -{ +std::unique_ptr ceph::io_exerciser::Seq8::_next() { if (!doneread) { if (!donebarrier) { donebarrier = true; - return IoOp::generate_barrier(); + return BarrierOp::generate(); } doneread = true; barrier = true; - return IoOp::generate_read(0, obj_size); + return SingleReadOp::generate(0, obj_size); } offset2++; if (offset2 >= obj_size - offset1) { @@ -455,34 +476,30 @@ std::unique_ptr ceph::io_exerciser::Seq8::_next() } doneread = false; donebarrier = false; - return IoOp::generate_write3(offset1, 1, - offset1 + offset2, 1, - (offset1 * 2 + offset2)/2, 1); + return TripleWriteOp::generate(offset1, 1, offset1 + offset2, 1, + (offset1 * 2 + offset2) / 2, 1); } +ceph::io_exerciser::Seq9::Seq9(std::pair obj_size_range, int seed) + : IoSequence(obj_size_range, seed), offset(0), length(0) {} - -ceph::io_exerciser::Seq9::Seq9(std::pair obj_size_range, int seed) : - IoSequence(obj_size_range, seed), offset(0), length(0) -{ - +Sequence ceph::io_exerciser::Seq9::get_id() const { + return Sequence::SEQUENCE_SEQ9; } -std::string ceph::io_exerciser::Seq9::get_name() const -{ +std::string ceph::io_exerciser::Seq9::get_name() const { return "Permutations of offset and length write I/O"; } -std::unique_ptr ceph::io_exerciser::Seq9::_next() -{ +std::unique_ptr ceph::io_exerciser::Seq9::_next() { if (!doneread) { if (!donebarrier) { donebarrier = true; - return IoOp::generate_barrier(); + return BarrierOp::generate(); } doneread = true; barrier = true; - return IoOp::generate_read(0, obj_size); + return SingleReadOp::generate(0, obj_size); } length++; if (length > obj_size - offset) { @@ -496,5 +513,5 @@ std::unique_ptr ceph::io_exerciser::Seq9::_next() } doneread = false; donebarrier = false; - return IoOp::generate_write(offset, length); + return SingleWriteOp::generate(offset, length); } \ No newline at end of file diff --git a/src/common/io_exerciser/IoSequence.h b/src/common/io_exerciser/IoSequence.h index 114ff76303f46..b6c254cf096e0 100644 --- a/src/common/io_exerciser/IoSequence.h +++ b/src/common/io_exerciser/IoSequence.h @@ -3,7 +3,6 @@ #pragma once #include "IoOp.h" - #include "include/random.h" /* Overview @@ -29,195 +28,209 @@ */ namespace ceph { - namespace io_exerciser { - - enum class Sequence { - SEQUENCE_SEQ0, - SEQUENCE_SEQ1, - SEQUENCE_SEQ2, - SEQUENCE_SEQ3, - SEQUENCE_SEQ4, - SEQUENCE_SEQ5, - SEQUENCE_SEQ6, - SEQUENCE_SEQ7, - SEQUENCE_SEQ8, - SEQUENCE_SEQ9, - // - SEQUENCE_END, - SEQUENCE_BEGIN = SEQUENCE_SEQ0 - }; - - inline Sequence operator++( Sequence& s ) - { - return s = (Sequence)(((int)(s) + 1)); - } - - std::ostream& operator<<(std::ostream& os, const Sequence& seq); - - /* I/O Sequences */ - - class IoSequence { - public: - virtual ~IoSequence() = default; - - virtual std::string get_name() const = 0; - int get_step() const; - int get_seed() const; - - std::unique_ptr next(); - - static std::unique_ptr - generate_sequence(Sequence s, std::pair obj_size_range, int seed ); - - protected: - uint64_t min_obj_size; - uint64_t max_obj_size; - bool create; - bool barrier; - bool done; - bool remove; - uint64_t obj_size; - int step; - int seed; - ceph::util::random_number_generator rng = - ceph::util::random_number_generator(); - - IoSequence(std::pair obj_size_range, int seed); - - virtual std::unique_ptr _next() = 0; - - void set_min_object_size(uint64_t size); - void set_max_object_size(uint64_t size); - void select_random_object_size(); - std::unique_ptr increment_object_size(); - - }; - - class Seq0: public IoSequence { - public: - Seq0(std::pair obj_size_range, int seed); - - std::string get_name() const override; - std::unique_ptr _next() override; - - private: - uint64_t offset; - uint64_t length; - }; - - class Seq1: public IoSequence { - public: - Seq1(std::pair obj_size_range, int seed); - - std::string get_name() const override; - std::unique_ptr _next(); - - private: - int count; - }; - - class Seq2: public IoSequence { - public: - Seq2(std::pair obj_size_range, int seed); - - std::string get_name() const override; - std::unique_ptr _next() override; - - private: - uint64_t offset; - uint64_t length; - }; - - class Seq3: public IoSequence { - public: - Seq3(std::pair obj_size_range, int seed); - - std::string get_name() const override; - std::unique_ptr _next() override; - private: - uint64_t offset1; - uint64_t offset2; - }; - - class Seq4: public IoSequence { - public: - Seq4(std::pair obj_size_range, int seed); - - std::string get_name() const override; - std::unique_ptr _next() override; - - private: - uint64_t offset1; - uint64_t offset2; - }; - - class Seq5: public IoSequence { - public: - Seq5(std::pair obj_size_range, int seed); - - std::string get_name() const override; - std::unique_ptr _next() override; - - private: - uint64_t offset; - uint64_t length; - bool doneread; - bool donebarrier; - }; - - class Seq6: public IoSequence { - public: - Seq6(std::pair obj_size_range, int seed); - - std::string get_name() const override; - std::unique_ptr _next() override; - - private: - uint64_t offset; - uint64_t length; - bool doneread; - bool donebarrier; - }; - - class Seq7: public IoSequence { - public: - Seq7(std::pair obj_size_range, int seed); - - std::string get_name() const override; - std::unique_ptr _next() override; - - private: - uint64_t offset; - bool doneread = true; - bool donebarrier = false; - }; - - class Seq8: public IoSequence { - public: - Seq8(std::pair obj_size_range, int seed); - - std::string get_name() const override; - std::unique_ptr _next() override; - private: - uint64_t offset1; - uint64_t offset2; - bool doneread = true; - bool donebarrier = false; - }; - - class Seq9: public IoSequence { - private: - uint64_t offset; - uint64_t length; - bool doneread = true; - bool donebarrier = false; - - public: - Seq9(std::pair obj_size_range, int seed); - - std::string get_name() const override; - - std::unique_ptr _next() override; - }; - } -} \ No newline at end of file +namespace io_exerciser { + +enum class Sequence { + SEQUENCE_SEQ0, + SEQUENCE_SEQ1, + SEQUENCE_SEQ2, + SEQUENCE_SEQ3, + SEQUENCE_SEQ4, + SEQUENCE_SEQ5, + SEQUENCE_SEQ6, + SEQUENCE_SEQ7, + SEQUENCE_SEQ8, + SEQUENCE_SEQ9, + SEQUENCE_SEQ10, + + SEQUENCE_END, + SEQUENCE_BEGIN = SEQUENCE_SEQ0 +}; + +inline Sequence operator++(Sequence& s) { + return s = (Sequence)(((int)(s) + 1)); +} + +std::ostream& operator<<(std::ostream& os, const Sequence& seq); + +/* I/O Sequences */ + +class IoSequence { + public: + virtual ~IoSequence() = default; + + virtual Sequence get_id() const = 0; + virtual std::string get_name_with_seqseed() const; + virtual std::string get_name() const = 0; + int get_step() const; + int get_seed() const; + + virtual Sequence getNextSupportedSequenceId() const; + virtual std::unique_ptr next(); + + virtual bool is_supported(Sequence sequence) const; + static std::unique_ptr generate_sequence( + Sequence s, std::pair obj_size_range, int seed); + + protected: + uint64_t min_obj_size; + uint64_t max_obj_size; + bool create; + bool barrier; + bool done; + bool remove; + uint64_t obj_size; + int step; + int seed; + ceph::util::random_number_generator rng = + ceph::util::random_number_generator(); + + IoSequence(std::pair obj_size_range, int seed); + + virtual std::unique_ptr _next() = 0; + + void set_min_object_size(uint64_t size); + void set_max_object_size(uint64_t size); + void select_random_object_size(); + std::unique_ptr increment_object_size(); +}; + +class Seq0 : public IoSequence { + public: + Seq0(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset; + uint64_t length; +}; + +class Seq1 : public IoSequence { + public: + Seq1(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + int count; +}; + +class Seq2 : public IoSequence { + public: + Seq2(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset; + uint64_t length; +}; + +class Seq3 : public IoSequence { + public: + Seq3(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset1; + uint64_t offset2; +}; + +class Seq4 : public IoSequence { + public: + Seq4(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset1; + uint64_t offset2; +}; + +class Seq5 : public IoSequence { + public: + Seq5(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset; + uint64_t length; + bool doneread; + bool donebarrier; +}; + +class Seq6 : public IoSequence { + public: + Seq6(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset; + uint64_t length; + bool doneread; + bool donebarrier; +}; + +class Seq7 : public IoSequence { + public: + Seq7(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset; + bool doneread = true; + bool donebarrier = false; +}; + +class Seq8 : public IoSequence { + public: + Seq8(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; + + private: + uint64_t offset1; + uint64_t offset2; + bool doneread = true; + bool donebarrier = false; +}; + +class Seq9 : public IoSequence { + private: + uint64_t offset; + uint64_t length; + bool doneread = true; + bool donebarrier = false; + + public: + Seq9(std::pair obj_size_range, int seed); + + Sequence get_id() const override; + std::string get_name() const override; + std::unique_ptr _next() override; +}; +} // namespace io_exerciser +} // namespace ceph \ No newline at end of file diff --git a/src/common/io_exerciser/Model.cc b/src/common/io_exerciser/Model.cc index 50812ecbb155d..6548e1eda7a80 100644 --- a/src/common/io_exerciser/Model.cc +++ b/src/common/io_exerciser/Model.cc @@ -4,25 +4,11 @@ using Model = ceph::io_exerciser::Model; -Model::Model(const std::string& oid, uint64_t block_size) : -num_io(0), -oid(oid), -block_size(block_size) -{ +Model::Model(const std::string& oid, uint64_t block_size) + : num_io(0), oid(oid), block_size(block_size) {} -} +const uint64_t Model::get_block_size() const { return block_size; } -const uint64_t Model::get_block_size() const -{ - return block_size; -} +const std::string Model::get_oid() const { return oid; } -const std::string Model::get_oid() const -{ - return oid; -} - -int Model::get_num_io() const -{ - return num_io; -} \ No newline at end of file +int Model::get_num_io() const { return num_io; } \ No newline at end of file diff --git a/src/common/io_exerciser/Model.h b/src/common/io_exerciser/Model.h index 58d107409a654..9e421e79a78de 100644 --- a/src/common/io_exerciser/Model.h +++ b/src/common/io_exerciser/Model.h @@ -1,15 +1,13 @@ #pragma once -#include "IoOp.h" - #include -#include "librados/librados_asio.h" - -#include "include/interval_set.h" -#include "global/global_init.h" -#include "global/global_context.h" +#include "IoOp.h" #include "common/Thread.h" +#include "global/global_context.h" +#include "global/global_init.h" +#include "include/interval_set.h" +#include "librados/librados_asio.h" /* Overview * @@ -21,29 +19,27 @@ */ namespace ceph { - namespace io_exerciser { - - class Model - { - protected: - int num_io{0}; - std::string oid; - uint64_t block_size; - - public: - Model(const std::string& oid, uint64_t block_size); - virtual ~Model() = default; - - virtual bool readyForIoOp(IoOp& op) = 0; - virtual void applyIoOp(IoOp& op) = 0; - - const std::string get_oid() const; - const uint64_t get_block_size() const; - int get_num_io() const; - }; - - /* Simple RADOS I/O generator */ - - - } -} \ No newline at end of file +namespace io_exerciser { + +class Model { + protected: + int num_io{0}; + std::string oid; + uint64_t block_size; + + public: + Model(const std::string& oid, uint64_t block_size); + virtual ~Model() = default; + + virtual bool readyForIoOp(IoOp& op) = 0; + virtual void applyIoOp(IoOp& op) = 0; + + const std::string get_oid() const; + const uint64_t get_block_size() const; + int get_num_io() const; +}; + +/* Simple RADOS I/O generator */ + +} // namespace io_exerciser +} // namespace ceph \ No newline at end of file diff --git a/src/common/io_exerciser/ObjectModel.cc b/src/common/io_exerciser/ObjectModel.cc index 589f6434282b3..454d7254cf2a8 100644 --- a/src/common/io_exerciser/ObjectModel.cc +++ b/src/common/io_exerciser/ObjectModel.cc @@ -6,25 +6,20 @@ using ObjectModel = ceph::io_exerciser::ObjectModel; -ObjectModel::ObjectModel(const std::string& oid, uint64_t block_size, int seed) : - Model(oid, block_size), created(false) -{ +ObjectModel::ObjectModel(const std::string& oid, uint64_t block_size, int seed) + : Model(oid, block_size), created(false) { rng.seed(seed); } -int ObjectModel::get_seed(uint64_t offset) const -{ +int ObjectModel::get_seed(uint64_t offset) const { ceph_assert(offset < contents.size()); return contents[offset]; } -std::vector ObjectModel::get_seed_offsets(int seed) const -{ +std::vector ObjectModel::get_seed_offsets(int seed) const { std::vector offsets; - for (size_t i = 0; i < contents.size(); i++) - { - if (contents[i] == seed) - { + for (size_t i = 0; i < contents.size(); i++) { + if (contents[i] == seed) { offsets.push_back(i); } } @@ -32,8 +27,7 @@ std::vector ObjectModel::get_seed_offsets(int seed) const return offsets; } -std::string ObjectModel::to_string(int mask) const -{ +std::string ObjectModel::to_string(int mask) const { if (!created) { return "Object does not exist"; } @@ -48,107 +42,127 @@ std::string ObjectModel::to_string(int mask) const return result; } -bool ObjectModel::readyForIoOp(IoOp& op) -{ - return true; -} - -void ObjectModel::applyIoOp(IoOp& op) -{ - auto generate_random = [&rng = rng]() { - return rng(); - }; - - switch (op.op) { - case OpType::BARRIER: - reads.clear(); - writes.clear(); - break; - - case OpType::CREATE: - ceph_assert(!created); - ceph_assert(reads.empty()); - ceph_assert(writes.empty()); - created = true; - contents.resize(op.length1); - std::generate(std::execution::seq, contents.begin(), contents.end(), - generate_random); - break; - - case OpType::REMOVE: - ceph_assert(created); - ceph_assert(reads.empty()); - ceph_assert(writes.empty()); - created = false; - contents.resize(0); - break; - - case OpType::READ3: - ceph_assert(created); - ceph_assert(op.offset3 + op.length3 <= contents.size()); - // Not allowed: read overlapping with parallel write - ceph_assert(!writes.intersects(op.offset3, op.length3)); - reads.union_insert(op.offset3, op.length3); - [[fallthrough]]; - - case OpType::READ2: - ceph_assert(created); - ceph_assert(op.offset2 + op.length2 <= contents.size()); - // Not allowed: read overlapping with parallel write - ceph_assert(!writes.intersects(op.offset2, op.length2)); - reads.union_insert(op.offset2, op.length2); - [[fallthrough]]; - - case OpType::READ: - ceph_assert(created); - ceph_assert(op.offset1 + op.length1 <= contents.size()); - // Not allowed: read overlapping with parallel write - ceph_assert(!writes.intersects(op.offset1, op.length1)); - reads.union_insert(op.offset1, op.length1); - num_io++; - break; - - case OpType::WRITE3: - ceph_assert(created); - // Not allowed: write overlapping with parallel read or write - ceph_assert(!reads.intersects(op.offset3, op.length3)); - ceph_assert(!writes.intersects(op.offset3, op.length3)); - writes.union_insert(op.offset3, op.length3); - ceph_assert(op.offset3 + op.length3 <= contents.size()); - std::generate(std::execution::seq, - std::next(contents.begin(), op.offset3), - std::next(contents.begin(), op.offset3 + op.length3), - generate_random); - [[fallthrough]]; - - case OpType::WRITE2: - ceph_assert(created); - // Not allowed: write overlapping with parallel read or write - ceph_assert(!reads.intersects(op.offset2, op.length2)); - ceph_assert(!writes.intersects(op.offset2, op.length2)); - writes.union_insert(op.offset2, op.length2); - ceph_assert(op.offset2 + op.length2 <= contents.size()); - std::generate(std::execution::seq, - std::next(contents.begin(), op.offset2), - std::next(contents.begin(), op.offset2 + op.length2), - generate_random); - [[fallthrough]]; - - case OpType::WRITE: - ceph_assert(created); - // Not allowed: write overlapping with parallel read or write - ceph_assert(!reads.intersects(op.offset1, op.length1)); - ceph_assert(!writes.intersects(op.offset1, op.length1)); - writes.union_insert(op.offset1, op.length1); - ceph_assert(op.offset1 + op.length1 <= contents.size()); - std::generate(std::execution::seq, - std::next(contents.begin(), op.offset1), - std::next(contents.begin(), op.offset1 + op.length1), - generate_random); - num_io++; - break; - default: - break; +bool ObjectModel::readyForIoOp(IoOp& op) { return true; } + +void ObjectModel::applyIoOp(IoOp& op) { + auto generate_random = [&rng = rng]() { return rng(); }; + + auto verify_and_record_read_op = + [&contents = contents, &created = created, &num_io = num_io, + &reads = reads, + &writes = writes](ReadWriteOp& readOp) { + ceph_assert(created); + for (int i = 0; i < N; i++) { + ceph_assert(readOp.offset[i] + readOp.length[i] <= contents.size()); + // Not allowed: read overlapping with parallel write + ceph_assert(!writes.intersects(readOp.offset[i], readOp.length[i])); + reads.union_insert(readOp.offset[i], readOp.length[i]); + } + num_io++; + }; + + auto verify_write_and_record_and_generate_seed = + [&generate_random, &contents = contents, &created = created, + &num_io = num_io, &reads = reads, + &writes = writes](ReadWriteOp writeOp) { + ceph_assert(created); + for (int i = 0; i < N; i++) { + // Not allowed: write overlapping with parallel read or write + ceph_assert(!reads.intersects(writeOp.offset[i], writeOp.length[i])); + ceph_assert(!writes.intersects(writeOp.offset[i], writeOp.length[i])); + writes.union_insert(writeOp.offset[i], writeOp.length[i]); + ceph_assert(writeOp.offset[i] + writeOp.length[i] <= contents.size()); + std::generate(std::execution::seq, + std::next(contents.begin(), writeOp.offset[i]), + std::next(contents.begin(), + writeOp.offset[i] + writeOp.length[i]), + generate_random); + } + num_io++; + }; + + auto verify_failed_write_and_record = + [&contents = contents, &created = created, &num_io = num_io, + &reads = reads, + &writes = writes](ReadWriteOp writeOp) { + // Ensure write should still be valid, even though we are expecting OSD + // failure + ceph_assert(created); + for (int i = 0; i < N; i++) { + // Not allowed: write overlapping with parallel read or write + ceph_assert(!reads.intersects(writeOp.offset[i], writeOp.length[i])); + ceph_assert(!writes.intersects(writeOp.offset[i], writeOp.length[i])); + writes.union_insert(writeOp.offset[i], writeOp.length[i]); + ceph_assert(writeOp.offset[i] + writeOp.length[i] <= contents.size()); + } + num_io++; + }; + + switch (op.getOpType()) { + case OpType::Barrier: + reads.clear(); + writes.clear(); + break; + + case OpType::Create: + ceph_assert(!created); + ceph_assert(reads.empty()); + ceph_assert(writes.empty()); + created = true; + contents.resize(static_cast(op).size); + std::generate(std::execution::seq, contents.begin(), contents.end(), + generate_random); + break; + + case OpType::Remove: + ceph_assert(created); + ceph_assert(reads.empty()); + ceph_assert(writes.empty()); + created = false; + contents.resize(0); + break; + + case OpType::Read: { + SingleReadOp& readOp = static_cast(op); + verify_and_record_read_op(readOp); + } break; + case OpType::Read2: { + DoubleReadOp& readOp = static_cast(op); + verify_and_record_read_op(readOp); + } break; + case OpType::Read3: { + TripleReadOp& readOp = static_cast(op); + verify_and_record_read_op(readOp); + } break; + + case OpType::Write: { + ceph_assert(created); + SingleWriteOp& writeOp = static_cast(op); + verify_write_and_record_and_generate_seed(writeOp); + } break; + case OpType::Write2: { + DoubleWriteOp& writeOp = static_cast(op); + verify_write_and_record_and_generate_seed(writeOp); + } break; + case OpType::Write3: { + TripleWriteOp& writeOp = static_cast(op); + verify_write_and_record_and_generate_seed(writeOp); + } break; + case OpType::FailedWrite: { + ceph_assert(created); + SingleWriteOp& writeOp = static_cast(op); + verify_failed_write_and_record(writeOp); + } break; + case OpType::FailedWrite2: { + DoubleWriteOp& writeOp = static_cast(op); + verify_failed_write_and_record(writeOp); + } break; + case OpType::FailedWrite3: { + TripleWriteOp& writeOp = static_cast(op); + verify_failed_write_and_record(writeOp); + } break; + default: + break; } } diff --git a/src/common/io_exerciser/ObjectModel.h b/src/common/io_exerciser/ObjectModel.h index 93c70f4142978..cad1307b84e30 100644 --- a/src/common/io_exerciser/ObjectModel.h +++ b/src/common/io_exerciser/ObjectModel.h @@ -14,40 +14,41 @@ */ namespace ceph { - namespace io_exerciser { - /* Model of an object to track its data contents */ - - class ObjectModel : public Model { - private: - bool created; - std::vector contents; - ceph::util::random_number_generator rng = - ceph::util::random_number_generator(); - - // Track read and write I/Os that can be submitted in - // parallel to detect violations: - // - // * Read may not overlap with a parallel write - // * Write may not overlap with a parallel read or write - // * Create / remove may not be in parallel with read or write - // - // Fix broken test cases by adding barrier ops to restrict - // I/O exercisers from issuing conflicting ops in parallel - interval_set reads; - interval_set writes; - public: - ObjectModel(const std::string& oid, uint64_t block_size, int seed); - - int get_seed(uint64_t offset) const; - std::vector get_seed_offsets(int seed) const; - - std::string to_string(int mask = -1) const; - - bool readyForIoOp(IoOp& op); - void applyIoOp(IoOp& op); - - void encode(ceph::buffer::list& bl) const; - void decode(ceph::buffer::list::const_iterator& bl); - }; - } -} \ No newline at end of file +namespace io_exerciser { +/* Model of an object to track its data contents */ + +class ObjectModel : public Model { + private: + bool created; + std::vector contents; + ceph::util::random_number_generator rng = + ceph::util::random_number_generator(); + + // Track read and write I/Os that can be submitted in + // parallel to detect violations: + // + // * Read may not overlap with a parallel write + // * Write may not overlap with a parallel read or write + // * Create / remove may not be in parallel with read or write + // + // Fix broken test cases by adding barrier ops to restrict + // I/O exercisers from issuing conflicting ops in parallel + interval_set reads; + interval_set writes; + + public: + ObjectModel(const std::string& oid, uint64_t block_size, int seed); + + int get_seed(uint64_t offset) const; + std::vector get_seed_offsets(int seed) const; + + std::string to_string(int mask = -1) const; + + bool readyForIoOp(IoOp& op); + void applyIoOp(IoOp& op); + + void encode(ceph::buffer::list& bl) const; + void decode(ceph::buffer::list::const_iterator& bl); +}; +} // namespace io_exerciser +} // namespace ceph \ No newline at end of file diff --git a/src/common/io_exerciser/OpType.h b/src/common/io_exerciser/OpType.h new file mode 100644 index 0000000000000..7cddb805e458c --- /dev/null +++ b/src/common/io_exerciser/OpType.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include + +/* Overview + * + * enum OpType + * Enumeration of different types of I/O operation + * + */ + +namespace ceph { +namespace io_exerciser { +enum class OpType { + Done, // End of I/O sequence + Barrier, // Barrier - all prior I/Os must complete + Create, // Create object and pattern with data + Remove, // Remove object + Read, // Read + Read2, // Two reads in a single op + Read3, // Three reads in a single op + Write, // Write + Write2, // Two writes in a single op + Write3, // Three writes in a single op + FailedWrite, // A write which should fail + FailedWrite2, // Two writes in one op which should fail + FailedWrite3, // Three writes in one op which should fail + InjectReadError, // Op to tell OSD to inject read errors + InjectWriteError, // Op to tell OSD to inject write errors + ClearReadErrorInject, // Op to tell OSD to clear read error injects + ClearWriteErrorInject // Op to tell OSD to clear write error injects +}; + +enum class InjectOpType { + None, + ReadEIO, + ReadMissingShard, + WriteFailAndRollback, + WriteOSDAbort +}; +} // namespace io_exerciser +} // namespace ceph + +template <> +struct fmt::formatter { + constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); } + + auto format(ceph::io_exerciser::OpType opType, + fmt::format_context& ctx) const -> fmt::format_context::iterator { + switch (opType) { + case ceph::io_exerciser::OpType::Done: + return fmt::format_to(ctx.out(), "Done"); + case ceph::io_exerciser::OpType::Barrier: + return fmt::format_to(ctx.out(), "Barrier"); + case ceph::io_exerciser::OpType::Create: + return fmt::format_to(ctx.out(), "Create"); + case ceph::io_exerciser::OpType::Remove: + return fmt::format_to(ctx.out(), "Remove"); + case ceph::io_exerciser::OpType::Read: + return fmt::format_to(ctx.out(), "Read"); + case ceph::io_exerciser::OpType::Read2: + return fmt::format_to(ctx.out(), "Read2"); + case ceph::io_exerciser::OpType::Read3: + return fmt::format_to(ctx.out(), "Read3"); + case ceph::io_exerciser::OpType::Write: + return fmt::format_to(ctx.out(), "Write"); + case ceph::io_exerciser::OpType::Write2: + return fmt::format_to(ctx.out(), "Write2"); + case ceph::io_exerciser::OpType::Write3: + return fmt::format_to(ctx.out(), "Write3"); + case ceph::io_exerciser::OpType::FailedWrite: + return fmt::format_to(ctx.out(), "FailedWrite"); + case ceph::io_exerciser::OpType::FailedWrite2: + return fmt::format_to(ctx.out(), "FailedWrite2"); + case ceph::io_exerciser::OpType::FailedWrite3: + return fmt::format_to(ctx.out(), "FailedWrite3"); + case ceph::io_exerciser::OpType::InjectReadError: + return fmt::format_to(ctx.out(), "InjectReadError"); + case ceph::io_exerciser::OpType::InjectWriteError: + return fmt::format_to(ctx.out(), "InjectWriteError"); + case ceph::io_exerciser::OpType::ClearReadErrorInject: + return fmt::format_to(ctx.out(), "ClearReadErrorInject"); + case ceph::io_exerciser::OpType::ClearWriteErrorInject: + return fmt::format_to(ctx.out(), "ClearWriteErrorInject"); + default: + ceph_abort_msg("Unknown OpType"); + return fmt::format_to(ctx.out(), "Unknown OpType"); + } + } +}; \ No newline at end of file diff --git a/src/common/io_exerciser/RadosIo.cc b/src/common/io_exerciser/RadosIo.cc index 44b82260263a2..4451900b7bb47 100644 --- a/src/common/io_exerciser/RadosIo.cc +++ b/src/common/io_exerciser/RadosIo.cc @@ -1,300 +1,429 @@ #include "RadosIo.h" +#include +#include + +#include + #include "DataGenerator.h" +#include "common/ceph_json.h" +#include "common/json/OSDStructures.h" using RadosIo = ceph::io_exerciser::RadosIo; -RadosIo::RadosIo(librados::Rados& rados, - boost::asio::io_context& asio, - const std::string& pool, - const std::string& oid, - uint64_t block_size, - int seed, - int threads, - ceph::mutex& lock, - ceph::condition_variable& cond) : - Model(oid, block_size), - rados(rados), - asio(asio), - om(std::make_unique(oid, block_size, seed)), - db(data_generation::DataGenerator::create_generator( - data_generation::GenerationType::HeaderedSeededRandom, *om)), - pool(pool), - threads(threads), - lock(lock), - cond(cond), - outstanding_io(0) -{ +RadosIo::RadosIo(librados::Rados& rados, boost::asio::io_context& asio, + const std::string& pool, const std::string& oid, + const std::optional>& cached_shard_order, + uint64_t block_size, int seed, int threads, ceph::mutex& lock, + ceph::condition_variable& cond) + : Model(oid, block_size), + rados(rados), + asio(asio), + om(std::make_unique(oid, block_size, seed)), + db(data_generation::DataGenerator::create_generator( + data_generation::GenerationType::HeaderedSeededRandom, *om)), + pool(pool), + cached_shard_order(cached_shard_order), + threads(threads), + lock(lock), + cond(cond), + outstanding_io(0) { int rc; rc = rados.ioctx_create(pool.c_str(), io); ceph_assert(rc == 0); allow_ec_overwrites(true); } -RadosIo::~RadosIo() -{ -} +RadosIo::~RadosIo() {} -void RadosIo::start_io() -{ +void RadosIo::start_io() { std::lock_guard l(lock); outstanding_io++; } -void RadosIo::finish_io() -{ +void RadosIo::finish_io() { std::lock_guard l(lock); ceph_assert(outstanding_io > 0); outstanding_io--; cond.notify_all(); } -void RadosIo::wait_for_io(int count) -{ +void RadosIo::wait_for_io(int count) { std::unique_lock l(lock); while (outstanding_io > count) { cond.wait(l); } } -void RadosIo::allow_ec_overwrites(bool allow) -{ +void RadosIo::allow_ec_overwrites(bool allow) { int rc; bufferlist inbl, outbl; - std::string cmdstr = - "{\"prefix\": \"osd pool set\", \"pool\": \"" + pool + "\", \ + std::string cmdstr = "{\"prefix\": \"osd pool set\", \"pool\": \"" + pool + + "\", \ \"var\": \"allow_ec_overwrites\", \"val\": \"" + - (allow ? "true" : "false") + "\"}"; + (allow ? "true" : "false") + "\"}"; rc = rados.mon_command(cmdstr, inbl, &outbl, nullptr); ceph_assert(rc == 0); } -RadosIo::AsyncOpInfo::AsyncOpInfo(uint64_t offset1, uint64_t length1, - uint64_t offset2, uint64_t length2, - uint64_t offset3, uint64_t length3 ) : - offset1(offset1), length1(length1), - offset2(offset2), length2(length2), - offset3(offset3), length3(length3) -{ - -} +template +RadosIo::AsyncOpInfo::AsyncOpInfo(const std::array& offset, + const std::array& length) + : offset(offset), length(length) {} -bool RadosIo::readyForIoOp(IoOp &op) -{ - ceph_assert(ceph_mutex_is_locked_by_me(lock)); //Must be called with lock held +bool RadosIo::readyForIoOp(IoOp& op) { + ceph_assert( + ceph_mutex_is_locked_by_me(lock)); // Must be called with lock held if (!om->readyForIoOp(op)) { return false; } - switch (op.op) { - case OpType::Done: - case OpType::BARRIER: - return outstanding_io == 0; - default: - return outstanding_io < threads; + + switch (op.getOpType()) { + case OpType::Done: + case OpType::Barrier: + return outstanding_io == 0; + default: + return outstanding_io < threads; } } -void RadosIo::applyIoOp(IoOp &op) -{ - std::shared_ptr op_info; - +void RadosIo::applyIoOp(IoOp& op) { om->applyIoOp(op); // If there are thread concurrent I/Os in flight then wait for // at least one I/O to complete - wait_for_io(threads-1); - - switch (op.op) { - case OpType::Done: - [[ fallthrough ]]; - case OpType::BARRIER: - // Wait for all outstanding I/O to complete - wait_for_io(0); - break; - - case OpType::CREATE: - { + wait_for_io(threads - 1); + + switch (op.getOpType()) { + case OpType::Done: + [[fallthrough]]; + case OpType::Barrier: + // Wait for all outstanding I/O to complete + wait_for_io(0); + break; + + case OpType::Create: { start_io(); - op_info = std::make_shared(0, op.length1); - op_info->bl1 = db->generate_data(0, op.length1); - op_info->wop.write_full(op_info->bl1); - auto create_cb = [this] (boost::system::error_code ec, - version_t ver) { + uint64_t opSize = static_cast(op).size; + std::shared_ptr> op_info = + std::make_shared>(std::array{0}, + std::array{opSize}); + op_info->bufferlist[0] = db->generate_data(0, opSize); + op_info->wop.write_full(op_info->bufferlist[0]); + auto create_cb = [this](boost::system::error_code ec, version_t ver) { ceph_assert(ec == boost::system::errc::success); finish_io(); }; - librados::async_operate(asio, io, oid, - &op_info->wop, 0, nullptr, create_cb); + librados::async_operate(asio, io, oid, &op_info->wop, 0, nullptr, + create_cb); + break; } - break; - case OpType::REMOVE: - { + case OpType::Remove: { start_io(); - op_info = std::make_shared(); + auto op_info = std::make_shared>(); op_info->wop.remove(); - auto remove_cb = [this] (boost::system::error_code ec, - version_t ver) { + auto remove_cb = [this](boost::system::error_code ec, version_t ver) { ceph_assert(ec == boost::system::errc::success); finish_io(); }; - librados::async_operate(asio, io, oid, - &op_info->wop, 0, nullptr, remove_cb); + librados::async_operate(asio, io, oid, &op_info->wop, 0, nullptr, + remove_cb); + break; } - break; + case OpType::Read: + [[fallthrough]]; + case OpType::Read2: + [[fallthrough]]; + case OpType::Read3: + [[fallthrough]]; + case OpType::Write: + [[fallthrough]]; + case OpType::Write2: + [[fallthrough]]; + case OpType::Write3: + [[fallthrough]]; + case OpType::FailedWrite: + [[fallthrough]]; + case OpType::FailedWrite2: + [[fallthrough]]; + case OpType::FailedWrite3: + applyReadWriteOp(op); + break; + case OpType::InjectReadError: + [[fallthrough]]; + case OpType::InjectWriteError: + [[fallthrough]]; + case OpType::ClearReadErrorInject: + [[fallthrough]]; + case OpType::ClearWriteErrorInject: + applyInjectOp(op); + break; + default: + ceph_abort_msg("Unrecognised Op"); + break; + } +} - case OpType::READ: - { - start_io(); - op_info = std::make_shared(op.offset1, op.length1); - op_info->rop.read(op.offset1 * block_size, - op.length1 * block_size, - &op_info->bl1, nullptr); - auto read_cb = [this, op_info] (boost::system::error_code ec, - version_t ver, - bufferlist bl) { - ceph_assert(ec == boost::system::errc::success); - ceph_assert(db->validate(op_info->bl1, - op_info->offset1, - op_info->length1)); - finish_io(); - }; - librados::async_operate(asio, io, oid, - &op_info->rop, 0, nullptr, read_cb); - num_io++; +void RadosIo::applyReadWriteOp(IoOp& op) { + auto applyReadOp = [this]( + ReadWriteOp readOp) { + auto op_info = + std::make_shared>(readOp.offset, readOp.length); + + for (int i = 0; i < N; i++) { + op_info->rop.read(readOp.offset[i] * block_size, + readOp.length[i] * block_size, &op_info->bufferlist[i], + nullptr); } - break; + auto read_cb = [this, op_info](boost::system::error_code ec, version_t ver, + bufferlist bl) { + ceph_assert(ec == boost::system::errc::success); + for (int i = 0; i < N; i++) { + ceph_assert(db->validate(op_info->bufferlist[i], op_info->offset[i], + op_info->length[i])); + } + finish_io(); + }; + librados::async_operate(asio, io, oid, &op_info->rop, 0, nullptr, read_cb); + num_io++; + }; - case OpType::READ2: - { - start_io(); - op_info = std::make_shared(op.offset1, - op.length1, - op.offset2, - op.length2); - - op_info->rop.read(op.offset1 * block_size, - op.length1 * block_size, - &op_info->bl1, nullptr); - op_info->rop.read(op.offset2 * block_size, - op.length2 * block_size, - &op_info->bl2, nullptr); - auto read2_cb = [this, op_info] (boost::system::error_code ec, - version_t ver, - bufferlist bl) { - ceph_assert(ec == boost::system::errc::success); - ceph_assert(db->validate(op_info->bl1, - op_info->offset1, - op_info->length1)); - ceph_assert(db->validate(op_info->bl2, - op_info->offset2, - op_info->length2)); - finish_io(); - }; - librados::async_operate(asio, io, oid, - &op_info->rop, 0, nullptr, read2_cb); - num_io++; + auto applyWriteOp = [this]( + ReadWriteOp writeOp) { + auto op_info = + std::make_shared>(writeOp.offset, writeOp.length); + for (int i = 0; i < N; i++) { + op_info->bufferlist[i] = + db->generate_data(writeOp.offset[i], writeOp.length[i]); + op_info->wop.write(writeOp.offset[i] * block_size, + op_info->bufferlist[i]); } - break; + auto write_cb = [this](boost::system::error_code ec, version_t ver) { + ceph_assert(ec == boost::system::errc::success); + finish_io(); + }; + librados::async_operate(asio, io, oid, &op_info->wop, 0, nullptr, write_cb); + num_io++; + }; - case OpType::READ3: - { - start_io(); - op_info = std::make_shared(op.offset1, op.length1, - op.offset2, op.length2, - op.offset3, op.length3); - op_info->rop.read(op.offset1 * block_size, - op.length1 * block_size, - &op_info->bl1, nullptr); - op_info->rop.read(op.offset2 * block_size, - op.length2 * block_size, - &op_info->bl2, nullptr); - op_info->rop.read(op.offset3 * block_size, - op.length3 * block_size, - &op_info->bl3, nullptr); - auto read3_cb = [this, op_info] (boost::system::error_code ec, - version_t ver, - bufferlist bl) { - ceph_assert(ec == boost::system::errc::success); - ceph_assert(db->validate(op_info->bl1, - op_info->offset1, - op_info->length1)); - ceph_assert(db->validate(op_info->bl2, - op_info->offset2, - op_info->length2)); - ceph_assert(db->validate(op_info->bl3, - op_info->offset3, - op_info->length3)); - finish_io(); - }; - librados::async_operate(asio, io, oid, - &op_info->rop, 0, nullptr, read3_cb); - num_io++; + auto applyFailedWriteOp = [this]( + ReadWriteOp writeOp) { + auto op_info = + std::make_shared>(writeOp.offset, writeOp.length); + for (int i = 0; i < N; i++) { + op_info->bufferlist[i] = + db->generate_data(writeOp.offset[i], writeOp.length[i]); + op_info->wop.write(writeOp.offset[i] * block_size, + op_info->bufferlist[i]); } - break; + auto write_cb = [this, writeOp](boost::system::error_code ec, + version_t ver) { + ceph_assert(ec != boost::system::errc::success); + finish_io(); + }; + librados::async_operate(asio, io, oid, &op_info->wop, 0, nullptr, write_cb); + num_io++; + }; - case OpType::WRITE: - { + switch (op.getOpType()) { + case OpType::Read: { start_io(); - op_info = std::make_shared(op.offset1, op.length1); - op_info->bl1 = db->generate_data(op.offset1, op.length1); - - op_info->wop.write(op.offset1 * block_size, op_info->bl1); - auto write_cb = [this] (boost::system::error_code ec, - version_t ver) { - ceph_assert(ec == boost::system::errc::success); - finish_io(); - }; - librados::async_operate(asio, io, oid, - &op_info->wop, 0, nullptr, write_cb); - num_io++; + SingleReadOp& readOp = static_cast(op); + applyReadOp(readOp); + break; } - break; - - case OpType::WRITE2: - { + case OpType::Read2: { start_io(); - op_info = std::make_shared(op.offset1, op.length1, - op.offset2, op.length2); - op_info->bl1 = db->generate_data(op.offset1, op.length1); - op_info->bl2 = db->generate_data(op.offset2, op.length2); - op_info->wop.write(op.offset1 * block_size, op_info->bl1); - op_info->wop.write(op.offset2 * block_size, op_info->bl2); - auto write2_cb = [this] (boost::system::error_code ec, - version_t ver) { - ceph_assert(ec == boost::system::errc::success); - finish_io(); - }; - librados::async_operate(asio, io, oid, - &op_info->wop, 0, nullptr, write2_cb); - num_io++; + DoubleReadOp& readOp = static_cast(op); + applyReadOp(readOp); + break; + } + case OpType::Read3: { + start_io(); + TripleReadOp& readOp = static_cast(op); + applyReadOp(readOp); + break; + } + case OpType::Write: { + start_io(); + SingleWriteOp& writeOp = static_cast(op); + applyWriteOp(writeOp); + break; + } + case OpType::Write2: { + start_io(); + DoubleWriteOp& writeOp = static_cast(op); + applyWriteOp(writeOp); + break; + } + case OpType::Write3: { + start_io(); + TripleWriteOp& writeOp = static_cast(op); + applyWriteOp(writeOp); + break; } - break; - case OpType::WRITE3: - { + case OpType::FailedWrite: { start_io(); - op_info = std::make_shared(op.offset1, op.length1, - op.offset2, op.length2, - op.offset3, op.length3); - op_info->bl1 = db->generate_data(op.offset1, op.length1); - op_info->bl2 = db->generate_data(op.offset2, op.length2); - op_info->bl3 = db->generate_data(op.offset3, op.length3); - op_info->wop.write(op.offset1 * block_size, op_info->bl1); - op_info->wop.write(op.offset2 * block_size, op_info->bl2); - op_info->wop.write(op.offset3 * block_size, op_info->bl3); - auto write3_cb = [this] (boost::system::error_code ec, - version_t ver) { - ceph_assert(ec == boost::system::errc::success); - finish_io(); - }; - librados::async_operate(asio, io, oid, - &op_info->wop, 0, nullptr, write3_cb); - num_io++; + SingleFailedWriteOp& writeOp = static_cast(op); + applyFailedWriteOp(writeOp); + break; + } + case OpType::FailedWrite2: { + start_io(); + DoubleFailedWriteOp& writeOp = static_cast(op); + applyFailedWriteOp(writeOp); + break; + } + case OpType::FailedWrite3: { + start_io(); + TripleFailedWriteOp& writeOp = static_cast(op); + applyFailedWriteOp(writeOp); + break; } - break; - default: - break; + default: + ceph_abort_msg( + fmt::format("Unsupported Read/Write operation ({})", op.getOpType())); + break; } } + +void RadosIo::applyInjectOp(IoOp& op) { + bufferlist osdmap_inbl, inject_inbl, osdmap_outbl, inject_outbl; + auto formatter = std::make_unique(false); + std::ostringstream oss; + + int osd = -1; + std::vector shard_order; + + ceph::messaging::osd::OSDMapRequest osdMapRequest{pool, get_oid(), ""}; + encode_json("OSDMapRequest", osdMapRequest, formatter.get()); + formatter->flush(oss); + int rc = rados.mon_command(oss.str(), osdmap_inbl, &osdmap_outbl, nullptr); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(osdmap_outbl.c_str(), osdmap_outbl.length()); + ceph_assert(success); + + ceph::messaging::osd::OSDMapReply reply; + reply.decode_json(&p); + + osd = reply.acting_primary; + shard_order = reply.acting; + + switch (op.getOpType()) { + case OpType::InjectReadError: { + InjectReadErrorOp& errorOp = static_cast(op); + + if (errorOp.type == 0) { + ceph::messaging::osd::InjectECErrorRequest + injectErrorRequest{pool, oid, errorOp.shard, + errorOp.type, errorOp.when, errorOp.duration}; + encode_json("InjectECErrorRequest", injectErrorRequest, + formatter.get()); + } else if (errorOp.type == 1) { + ceph::messaging::osd::InjectECErrorRequest< + InjectOpType::ReadMissingShard> + injectErrorRequest{pool, oid, errorOp.shard, + errorOp.type, errorOp.when, errorOp.duration}; + encode_json("InjectECErrorRequest", injectErrorRequest, + formatter.get()); + } else { + ceph_abort_msg("Unsupported inject type"); + } + formatter->flush(oss); + int rc = rados.osd_command(osd, oss.str(), inject_inbl, &inject_outbl, + nullptr); + ceph_assert(rc == 0); + break; + } + case OpType::InjectWriteError: { + InjectWriteErrorOp& errorOp = static_cast(op); + + if (errorOp.type == 0) { + ceph::messaging::osd::InjectECErrorRequest< + InjectOpType::WriteFailAndRollback> + injectErrorRequest{pool, oid, errorOp.shard, + errorOp.type, errorOp.when, errorOp.duration}; + encode_json("InjectECErrorRequest", injectErrorRequest, + formatter.get()); + } else if (errorOp.type == 3) { + ceph::messaging::osd::InjectECErrorRequest + injectErrorRequest{pool, oid, errorOp.shard, + errorOp.type, errorOp.when, errorOp.duration}; + encode_json("InjectECErrorRequest", injectErrorRequest, + formatter.get()); + + // This inject is sent directly to the shard we want to inject the error + // on + osd = shard_order[errorOp.shard]; + } else { + ceph_abort("Unsupported inject type"); + } + + formatter->flush(oss); + int rc = rados.osd_command(osd, oss.str(), inject_inbl, &inject_outbl, + nullptr); + ceph_assert(rc == 0); + break; + } + case OpType::ClearReadErrorInject: { + ClearReadErrorInjectOp& errorOp = + static_cast(op); + + if (errorOp.type == 0) { + ceph::messaging::osd::InjectECClearErrorRequest + clearErrorInject{pool, oid, errorOp.shard, errorOp.type}; + encode_json("InjectECClearErrorRequest", clearErrorInject, + formatter.get()); + } else if (errorOp.type == 1) { + ceph::messaging::osd::InjectECClearErrorRequest< + InjectOpType::ReadMissingShard> + clearErrorInject{pool, oid, errorOp.shard, errorOp.type}; + encode_json("InjectECClearErrorRequest", clearErrorInject, + formatter.get()); + } else { + ceph_abort("Unsupported inject type"); + } + + formatter->flush(oss); + int rc = rados.osd_command(osd, oss.str(), inject_inbl, &inject_outbl, + nullptr); + ceph_assert(rc == 0); + break; + } + case OpType::ClearWriteErrorInject: { + ClearReadErrorInjectOp& errorOp = + static_cast(op); + + if (errorOp.type == 0) { + ceph::messaging::osd::InjectECClearErrorRequest< + InjectOpType::WriteFailAndRollback> + clearErrorInject{pool, oid, errorOp.shard, errorOp.type}; + encode_json("InjectECClearErrorRequest", clearErrorInject, + formatter.get()); + } else if (errorOp.type == 3) { + ceph::messaging::osd::InjectECClearErrorRequest< + InjectOpType::WriteOSDAbort> + clearErrorInject{pool, oid, errorOp.shard, errorOp.type}; + encode_json("InjectECClearErrorRequest", clearErrorInject, + formatter.get()); + } else { + ceph_abort("Unsupported inject type"); + } + + formatter->flush(oss); + int rc = rados.osd_command(osd, oss.str(), inject_inbl, &inject_outbl, + nullptr); + ceph_assert(rc == 0); + break; + } + default: + ceph_abort_msg( + fmt::format("Unsupported inject operation ({})", op.getOpType())); + break; + } +} \ No newline at end of file diff --git a/src/common/io_exerciser/RadosIo.h b/src/common/io_exerciser/RadosIo.h index 179c5bba3aea0..a5c66ad4768ab 100644 --- a/src/common/io_exerciser/RadosIo.h +++ b/src/common/io_exerciser/RadosIo.h @@ -10,71 +10,65 @@ * in the object. Uses DataBuffer to create and validate * data buffers. When there are not barrier I/Os this may * issue multiple async I/Os in parallel. - * + * */ namespace ceph { - namespace io_exerciser { - namespace data_generation { - class DataGenerator; - } - - class RadosIo: public Model { - protected: - librados::Rados& rados; - boost::asio::io_context& asio; - std::unique_ptr om; - std::unique_ptr db; - std::string pool; - int threads; - ceph::mutex& lock; - ceph::condition_variable& cond; - librados::IoCtx io; - int outstanding_io; +namespace io_exerciser { +namespace data_generation { +class DataGenerator; +} + +class RadosIo : public Model { + protected: + librados::Rados& rados; + boost::asio::io_context& asio; + std::unique_ptr om; + std::unique_ptr db; + std::string pool; + std::optional> cached_shard_order; + int threads; + ceph::mutex& lock; + ceph::condition_variable& cond; + librados::IoCtx io; + int outstanding_io; + + void start_io(); + void finish_io(); + void wait_for_io(int count); + + public: + RadosIo(librados::Rados& rados, boost::asio::io_context& asio, + const std::string& pool, const std::string& oid, + const std::optional>& cached_shard_order, + uint64_t block_size, int seed, int threads, ceph::mutex& lock, + ceph::condition_variable& cond); - void start_io(); - void finish_io(); - void wait_for_io(int count); - - public: - RadosIo(librados::Rados& rados, - boost::asio::io_context& asio, - const std::string& pool, - const std::string& oid, - uint64_t block_size, - int seed, - int threads, - ceph::mutex& lock, - ceph::condition_variable& cond); + ~RadosIo(); - ~RadosIo(); + void allow_ec_overwrites(bool allow); - void allow_ec_overwrites(bool allow); + template + class AsyncOpInfo { + public: + librados::ObjectReadOperation rop; + librados::ObjectWriteOperation wop; + std::array bufferlist; + std::array offset; + std::array length; - class AsyncOpInfo { - public: - librados::ObjectReadOperation rop; - librados::ObjectWriteOperation wop; - ceph::buffer::list bl1; - ceph::buffer::list bl2; - ceph::buffer::list bl3; - uint64_t offset1; - uint64_t length1; - uint64_t offset2; - uint64_t length2; - uint64_t offset3; - uint64_t length3; + AsyncOpInfo(const std::array& offset = {}, + const std::array& length = {}); + ~AsyncOpInfo() = default; + }; - AsyncOpInfo(uint64_t offset1 = 0, uint64_t length1 = 0, - uint64_t offset2 = 0, uint64_t length2 = 0, - uint64_t offset3 = 0, uint64_t length3 = 0 ); - ~AsyncOpInfo() = default; - }; + // Must be called with lock held + bool readyForIoOp(IoOp& op); + void applyIoOp(IoOp& op); - // Must be called with lock held - bool readyForIoOp(IoOp& op); - - void applyIoOp(IoOp& op); - }; - } -} \ No newline at end of file + private: + void applyReadWriteOp(IoOp& op); + void applyInjectOp(IoOp& op); +}; +} // namespace io_exerciser +} // namespace ceph \ No newline at end of file diff --git a/src/common/json/BalancerStructures.cc b/src/common/json/BalancerStructures.cc new file mode 100644 index 0000000000000..48dfb84376113 --- /dev/null +++ b/src/common/json/BalancerStructures.cc @@ -0,0 +1,38 @@ +#include "BalancerStructures.h" + +#include "common/ceph_json.h" + +using namespace ceph::messaging::balancer; + +void BalancerOffRequest::dump(Formatter* f) const { + encode_json("prefix", "balancer off", f); +} + +void BalancerOffRequest::decode_json(JSONObj* obj) {} + +void BalancerStatusRequest::dump(Formatter* f) const { + encode_json("prefix", "balancer status", f); +} + +void BalancerStatusRequest::decode_json(JSONObj* obj) {} + +void BalancerStatusReply::dump(Formatter* f) const { + encode_json("active", active, f); + encode_json("last_optimization_duration", last_optimization_duration, f); + encode_json("last_optimization_started", last_optimization_started, f); + encode_json("mode", mode, f); + encode_json("no_optimization_needed", no_optimization_needed, f); + encode_json("optimize_result", optimize_result, f); +} + +void BalancerStatusReply::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("active", active, obj); + JSONDecoder::decode_json("last_optimization_duration", + last_optimization_duration, obj); + JSONDecoder::decode_json("last_optimization_started", + last_optimization_started, obj); + JSONDecoder::decode_json("mode", mode, obj); + JSONDecoder::decode_json("no_optimization_needed", no_optimization_needed, + obj); + JSONDecoder::decode_json("optimize_result", optimize_result, obj); +} \ No newline at end of file diff --git a/src/common/json/BalancerStructures.h b/src/common/json/BalancerStructures.h new file mode 100644 index 0000000000000..bbf5c748eb312 --- /dev/null +++ b/src/common/json/BalancerStructures.h @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include "include/types.h" + +class JSONObj; + +namespace ceph { +namespace messaging { +namespace balancer { +struct BalancerOffRequest { + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct BalancerStatusRequest { + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct BalancerStatusReply { + bool active; + std::string last_optimization_duration; + std::string last_optimization_started; + std::string mode; + bool no_optimization_needed; + std::string optimize_result; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; +} // namespace balancer +} // namespace messaging +} // namespace ceph \ No newline at end of file diff --git a/src/common/json/CMakeLists.txt b/src/common/json/CMakeLists.txt new file mode 100644 index 0000000000000..1497daf93dbbb --- /dev/null +++ b/src/common/json/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(json_structures STATIC + BalancerStructures.cc ConfigStructures.cc OSDStructures.cc) + + target_link_libraries(json_structures global) \ No newline at end of file diff --git a/src/common/json/ConfigStructures.cc b/src/common/json/ConfigStructures.cc new file mode 100644 index 0000000000000..651278d002a94 --- /dev/null +++ b/src/common/json/ConfigStructures.cc @@ -0,0 +1,20 @@ +#include "ConfigStructures.h" + +#include "common/ceph_json.h" + +using namespace ceph::messaging::config; + +void ConfigSetRequest::dump(Formatter* f) const { + encode_json("prefix", "config set", f); + encode_json("who", who, f); + encode_json("name", name, f); + encode_json("value", value, f); + encode_json("force", force, f); +} + +void ConfigSetRequest::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("who", who, obj); + JSONDecoder::decode_json("name", name, obj); + JSONDecoder::decode_json("value", value, obj); + JSONDecoder::decode_json("force", force, obj); +} \ No newline at end of file diff --git a/src/common/json/ConfigStructures.h b/src/common/json/ConfigStructures.h new file mode 100644 index 0000000000000..554229d75f4f4 --- /dev/null +++ b/src/common/json/ConfigStructures.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "include/types.h" + +class JSONObj; + +namespace ceph { +namespace messaging { +namespace config { +struct ConfigSetRequest { + std::string who; + std::string name; + std::string value; + std::optional force; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; +} // namespace config +} // namespace messaging +} // namespace ceph \ No newline at end of file diff --git a/src/common/json/OSDStructures.cc b/src/common/json/OSDStructures.cc new file mode 100644 index 0000000000000..aaac5f6e16938 --- /dev/null +++ b/src/common/json/OSDStructures.cc @@ -0,0 +1,150 @@ +#include "OSDStructures.h" + +#include "common/ceph_json.h" +#include "common/io_exerciser/OpType.h" + +using namespace ceph::messaging::osd; + +void OSDMapRequest::dump(Formatter* f) const { + encode_json("prefix", "osd map", f); + encode_json("pool", pool, f); + encode_json("object", object, f); + encode_json("nspace", nspace, f); + encode_json("format", format, f); +} + +void OSDMapRequest::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("object", object, obj); + JSONDecoder::decode_json("nspace", nspace, obj); + JSONDecoder::decode_json("format", format, obj); +} + +void OSDMapReply::dump(Formatter* f) const { + encode_json("epoch", epoch, f); + encode_json("pool", pool, f); + encode_json("pool_id", pool_id, f); + encode_json("objname", objname, f); + encode_json("raw_pgid", raw_pgid, f); + encode_json("pgid", pgid, f); + encode_json("up", up, f); + encode_json("up_primary", up_primary, f); + encode_json("acting", acting, f); + encode_json("acting_primary", acting_primary, f); +} + +void OSDMapReply::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("epoch", epoch, obj); + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("pool_id", pool_id, obj); + JSONDecoder::decode_json("objname", objname, obj); + JSONDecoder::decode_json("raw_pgid", raw_pgid, obj); + JSONDecoder::decode_json("pgid", pgid, obj); + JSONDecoder::decode_json("up", up, obj); + JSONDecoder::decode_json("up_primary", up_primary, obj); + JSONDecoder::decode_json("acting", acting, obj); + JSONDecoder::decode_json("acting_primary", acting_primary, obj); +} + +void OSDPoolGetRequest::dump(Formatter* f) const { + encode_json("prefix", "osd pool get", f); + encode_json("pool", pool, f); + encode_json("var", var, f); + encode_json("format", format, f); +} + +void OSDPoolGetRequest::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("var", var, obj); + JSONDecoder::decode_json("format", format, obj); +} + +void OSDPoolGetReply::dump(Formatter* f) const { + encode_json("erasure_code_profile", erasure_code_profile, f); +} + +void OSDPoolGetReply::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("erasure_code_profile", erasure_code_profile, obj); +} + +void OSDECProfileGetRequest::dump(Formatter* f) const { + encode_json("prefix", "osd pool get", f); + encode_json("name", name, f); + encode_json("format", format, f); +} + +void OSDECProfileGetRequest::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("name", name, obj); + JSONDecoder::decode_json("format", format, obj); +} + +void OSDECProfileGetReply::dump(Formatter* f) const { + encode_json("crush-device-class", crush_device_class, f); + encode_json("crush-failure-domain", crush_failure_domain, f); + encode_json("crush-num-failure-domains", crush_num_failure_domains, f); + encode_json("crush-osds-per-failure-domain", crush_osds_per_failure_domain, + f); + encode_json("crush-root", crush_root, f); + encode_json("jerasure-per-chunk-alignment", jerasure_per_chunk_alignment, f); + encode_json("k", k, f); + encode_json("m", m, f); + encode_json("plugin", plugin, f); + encode_json("technique", technique, f); + encode_json("w", w, f); +} + +void OSDECProfileGetReply::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("crush-device-class", crush_device_class, obj); + JSONDecoder::decode_json("crush-failure-domain", crush_failure_domain, obj); + JSONDecoder::decode_json("crush-num-failure-domains", + crush_num_failure_domains, obj); + JSONDecoder::decode_json("crush-osds-per-failure-domain", + crush_osds_per_failure_domain, obj); + JSONDecoder::decode_json("crush-root", crush_root, obj); + JSONDecoder::decode_json("jerasure-per-chunk-alignment", + jerasure_per_chunk_alignment, obj); + JSONDecoder::decode_json("k", k, obj); + JSONDecoder::decode_json("m", m, obj); + JSONDecoder::decode_json("plugin", plugin, obj); + JSONDecoder::decode_json("technique", technique, obj); + JSONDecoder::decode_json("w", w, obj); +} + +void OSDECProfileSetRequest::dump(Formatter* f) const { + encode_json("prefix", "osd erasure-code-profile set", f); + encode_json("name", name, f); + encode_json("profile", profile, f); +} + +void OSDECProfileSetRequest::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("name", name, obj); + JSONDecoder::decode_json("profile", profile, obj); +} + +void OSDECPoolCreateRequest::dump(Formatter* f) const { + encode_json("prefix", "osd pool create", f); + encode_json("pool", pool, f); + encode_json("pool_type", pool_type, f); + encode_json("pg_num", pg_num, f); + encode_json("pgp_num", pgp_num, f); + encode_json("erasure_code_profile", erasure_code_profile, f); +} + +void OSDECPoolCreateRequest::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("pool_type", pool_type, obj); + JSONDecoder::decode_json("pg_num", pg_num, obj); + JSONDecoder::decode_json("pgp_num", pgp_num, obj); + JSONDecoder::decode_json("erasure_code_profile", erasure_code_profile, obj); +} + +void OSDSetRequest::dump(Formatter* f) const { + encode_json("prefix", "osd set", f); + encode_json("key", key, f); + encode_json("yes_i_really_mean_it", yes_i_really_mean_it, f); +} + +void OSDSetRequest::decode_json(JSONObj* obj) { + JSONDecoder::decode_json("key", key, obj); + JSONDecoder::decode_json("yes_i_really_mean_it", yes_i_really_mean_it, obj); +} \ No newline at end of file diff --git a/src/common/json/OSDStructures.h b/src/common/json/OSDStructures.h new file mode 100644 index 0000000000000..3e4528a099fc0 --- /dev/null +++ b/src/common/json/OSDStructures.h @@ -0,0 +1,189 @@ +#pragma once + +#include +#include +#include + +#include "common/ceph_json.h" +#include "common/io_exerciser/OpType.h" +#include "include/types.h" + +class JSONObj; + +namespace ceph { +namespace messaging { +namespace osd { +struct OSDMapRequest { + std::string pool; + std::string object; + std::string nspace; + std::string format = "json"; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct OSDMapReply { + epoch_t epoch; + std::string pool; + uint64_t pool_id; + std::string objname; + std::string raw_pgid; + std::string pgid; + std::vector up; + int up_primary; + std::vector acting; + int acting_primary; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct OSDPoolGetRequest { + std::string pool; + std::string var = "erasure_code_profile"; + std::string format = "json"; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct OSDPoolGetReply { + std::string erasure_code_profile; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct OSDECProfileGetRequest { + std::string name; + std::string format = "json"; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct OSDECProfileGetReply { + std::string crush_device_class; + std::string crush_failure_domain; + int crush_num_failure_domains; + int crush_osds_per_failure_domain; + std::string crush_root; + bool jerasure_per_chunk_alignment; + int k; + int m; + std::string plugin; + std::string technique; + std::string w; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct OSDECProfileSetRequest { + std::string name; + std::vector profile; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct OSDECPoolCreateRequest { + std::string pool; + std::string pool_type; + int pg_num; + int pgp_num; + std::string erasure_code_profile; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +struct OSDSetRequest { + std::string key; + std::optional yes_i_really_mean_it = std::nullopt; + + void dump(Formatter* f) const; + void decode_json(JSONObj* obj); +}; + +// These structures are sent directly to the relevant OSD +// rather than the monitor +template +struct InjectECErrorRequest { + std::string pool; + std::string objname; + int shardid; + std::optional type; + std::optional when; + std::optional duration; + + void dump(Formatter* f) const { + switch (op_type) { + case io_exerciser::InjectOpType::ReadEIO: + [[fallthrough]]; + case io_exerciser::InjectOpType::ReadMissingShard: + ::encode_json("prefix", "injectecreaderr", f); + break; + case io_exerciser::InjectOpType::WriteFailAndRollback: + [[fallthrough]]; + case io_exerciser::InjectOpType::WriteOSDAbort: + ::encode_json("prefix", "injectecwriteerr", f); + break; + default: + ceph_abort_msg("Unsupported Inject Type"); + } + ::encode_json("pool", pool, f); + ::encode_json("objname", objname, f); + ::encode_json("shardid", shardid, f); + ::encode_json("type", type, f); + ::encode_json("when", when, f); + ::encode_json("duration", duration, f); + } + void decode_json(JSONObj* obj) { + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("objname", objname, obj); + JSONDecoder::decode_json("shardid", shardid, obj); + JSONDecoder::decode_json("type", type, obj); + JSONDecoder::decode_json("when", when, obj); + JSONDecoder::decode_json("duration", duration, obj); + } +}; + +template +struct InjectECClearErrorRequest { + std::string pool; + std::string objname; + int shardid; + std::optional type; + + void dump(Formatter* f) const { + switch (op_type) { + case io_exerciser::InjectOpType::ReadEIO: + [[fallthrough]]; + case io_exerciser::InjectOpType::ReadMissingShard: + ::encode_json("prefix", "injectecclearreaderr", f); + break; + case io_exerciser::InjectOpType::WriteFailAndRollback: + [[fallthrough]]; + case io_exerciser::InjectOpType::WriteOSDAbort: + ::encode_json("prefix", "injectecclearwriteerr", f); + break; + default: + ceph_abort_msg("Unsupported Inject Type"); + } + ::encode_json("pool", pool, f); + ::encode_json("objname", objname, f); + ::encode_json("shardid", shardid, f); + ::encode_json("type", type, f); + } + void decode_json(JSONObj* obj) { + JSONDecoder::decode_json("pool", pool, obj); + JSONDecoder::decode_json("objname", objname, obj); + JSONDecoder::decode_json("shardid", shardid, obj); + JSONDecoder::decode_json("type", type, obj); + } +}; +} // namespace osd +} // namespace messaging +} // namespace ceph \ No newline at end of file diff --git a/src/osd/ECBackend.cc b/src/osd/ECBackend.cc index fa2570aba42af..8630b038812ee 100644 --- a/src/osd/ECBackend.cc +++ b/src/osd/ECBackend.cc @@ -945,6 +945,10 @@ void ECBackend::handle_sub_write( } trace.event("handle_sub_write"); + if (cct->_conf->bluestore_debug_inject_read_err && + ec_inject_test_write_error3(op.soid)) { + ceph_abort_msg("Error inject - OSD down"); + } if (!get_parent()->pgb_is_primary()) get_parent()->update_stats(op.stats); ObjectStore::Transaction localt; @@ -1191,6 +1195,15 @@ void ECBackend::handle_sub_write_reply( i->second->on_all_commit = 0; i->second->trace.event("ec write all committed"); } + if (cct->_conf->bluestore_debug_inject_read_err && + (i->second->pending_commit.size() == 1) && + ec_inject_test_write_error2(i->second->hoid)) { + std::string cmd = + "{ \"prefix\": \"osd down\", \"ids\": [\"" + std::to_string( get_parent()->whoami() ) + "\"] }"; + vector vcmd{cmd}; + dout(0) << __func__ << " Error inject - marking OSD down" << dendl; + get_parent()->start_mon_command(vcmd, {}, nullptr, nullptr, nullptr); + } rmw_pipeline.check_ops(); } @@ -1208,6 +1221,19 @@ void ECBackend::handle_sub_read_reply( return; } ReadOp &rop = iter->second; + if (cct->_conf->bluestore_debug_inject_read_err) { + for (auto i = op.buffers_read.begin(); + i != op.buffers_read.end(); + ++i) { + if (ec_inject_test_read_error0(ghobject_t(i->first, ghobject_t::NO_GEN, op.from.shard))) { + dout(0) << __func__ << " Error inject - EIO error for shard " << op.from.shard << dendl; + op.buffers_read.erase(i->first); + op.attrs_read.erase(i->first); + op.errors[i->first] = -EIO; + } + + } + } for (auto i = op.buffers_read.begin(); i != op.buffers_read.end(); ++i) { diff --git a/src/osd/ECCommon.cc b/src/osd/ECCommon.cc index 609ac3141ae30..59077547fcb9e 100644 --- a/src/osd/ECCommon.cc +++ b/src/osd/ECCommon.cc @@ -226,8 +226,14 @@ void ECCommon::ReadPipeline::get_all_avail_shards( ++i) { dout(10) << __func__ << ": checking acting " << *i << dendl; const pg_missing_t &missing = get_parent()->get_shard_missing(*i); - if (error_shards.find(*i) != error_shards.end()) + if (error_shards.contains(*i)) { continue; + } + if (cct->_conf->bluestore_debug_inject_read_err && + ec_inject_test_read_error1(ghobject_t(hoid, ghobject_t::NO_GEN, i->shard))) { + dout(0) << __func__ << " Error inject - Missing shard " << i->shard << dendl; + continue; + } if (!missing.is_missing(hoid)) { ceph_assert(!have.count(i->shard)); have.insert(i->shard); @@ -912,6 +918,11 @@ bool ECCommon::RMWPipeline::try_reads_to_commit() if (*i == get_parent()->whoami_shard()) { should_write_local = true; local_write_op.claim(sop); + } else if (cct->_conf->bluestore_debug_inject_read_err && + ec_inject_test_write_error1(ghobject_t(op->hoid, + ghobject_t::NO_GEN, i->shard))) { + dout(0) << " Error inject - Dropping write message to shard " << + i->shard << dendl; } else { MOSDECSubOpWrite *r = new MOSDECSubOpWrite(sop); r->pgid = spg_t(get_parent()->primary_spg_t().pgid, i->shard); @@ -1090,3 +1101,305 @@ ECUtil::HashInfoRef ECCommon::UnstableHashInfoRegistry::get_hash_info( } return ref; } + +// Error inject interfaces +static ceph::recursive_mutex ec_inject_lock = + ceph::make_recursive_mutex("ECCommon::ec_inject_lock"); +static std::map> ec_inject_read_failures0; +static std::map> ec_inject_read_failures1; +static std::map> ec_inject_write_failures0; +static std::map> ec_inject_write_failures1; +static std::map> ec_inject_write_failures2; +static std::map> ec_inject_write_failures3; +static std::map ec_inject_write_failures0_shard; +static std::set ec_inject_write_failures0_reqid; + +/** + * Configure a read error inject that typically forces additional reads of + * shards in an EC pool to recover data using the redundancy. With multiple + * errors it is possible to force client reads to fail. + * + * Type 0 - Simulate a medium error. Fail a read with -EIO to force + * additional reads and a decode + * + * Type 1 - Simulate a missing OSD. Dont even try to read a shard + * + * @brief Set up a read error inject for an object in an EC pool. + * @param o Target object for the error inject. + * @param when Error inject starts after this many object store reads. + * @param duration Error inject affects this many object store reads. + * @param type Type of error inject 0 = EIO, 1 = missing shard. + * @return string Result of configuring the error inject. + */ +std::string ec_inject_read_error(const ghobject_t& o, + const int64_t type, + const int64_t when, + const int64_t duration) { + std::lock_guard l(ec_inject_lock); + ghobject_t os = o; + if (os.hobj.oid.name == "*") { + os.hobj.set_hash(0); + } + switch (type) { + case 0: + ec_inject_read_failures0[os] = std::pair(when, duration); + return "ok - read returns EIO"; + case 1: + ec_inject_read_failures1[os] = std::pair(when, duration); + return "ok - read pretends shard is missing"; + default: + break; + } + return "unrecognized error inject type"; +} + +/** + * Configure a write error inject that either fails an OSD or causes a + * client write operation to be rolled back. + * + * Type 0 - Tests rollback. Drop a write I/O to a shard, then simulate an OSD + * down to force rollback to occur, lastly fail the retried write from the + * client so the results of the rollback can be inspected. + * + * Type 1 - Drop a write I/O to a shard. Used on its own this will hang a + * write I/O. + * + * Type 2 - Simulate an OSD down (ceph osd down) to force a new epoch. Usually + * used together with type 1 to force a rollback + * + * Type 3 - Abort when an OSD processes a write I/O to a shard. Typically the + * client write will be commited while the OSD is absent which will result in + * recovery or backfill later when the OSD returns. + * + * @brief Set up a write error inject for an object in an EC pool. + * @param o Target object for the error inject. + * @param when Error inject starts after this many object store reads. + * @param duration Error inject affects this many object store reads. + * @param type Type of error inject 0 = EIO, 1 = missing shard. + * @return string Result of configuring the error inect. + */ +std::string ec_inject_write_error(const ghobject_t& o, + const int64_t type, + const int64_t when, + const int64_t duration) { + std::lock_guard l(ec_inject_lock); + std::map> *failures; + ghobject_t os = o; + bool no_shard = true; + std::string result; + switch (type) { + case 0: + failures = &ec_inject_write_failures0; + result = "ok - drop write, sim OSD down and fail client retry with EINVAL"; + break; + case 1: + failures = &ec_inject_write_failures1; + no_shard = false; + result = "ok - drop write to shard"; + break; + case 2: + failures = &ec_inject_write_failures2; + result = "ok - inject OSD down"; + break; + case 3: + if (duration != 1) { + return "duration must be 1"; + } + failures = &ec_inject_write_failures3; + result = "ok - write abort OSDs"; + break; + default: + return "unrecognized error inject type"; + } + if (no_shard) { + os.set_shard(shard_id_t::NO_SHARD); + } + if (os.hobj.oid.name == "*") { + os.hobj.set_hash(0); + } + (*failures)[os] = std::pair(when, duration); + if (type == 0) { + ec_inject_write_failures0_shard[os] = o.shard_id; + } + return result; +} + +/** + * @brief Clear a previously configured read error inject. + * @param o Target object for the error inject. + * @param type Type of error inject 0 = EIO, 1 = missing shard. + * @return string Indication of how many errors were cleared. + */ +std::string ec_inject_clear_read_error(const ghobject_t& o, + const int64_t type) { + std::lock_guard l(ec_inject_lock); + std::map> *failures; + ghobject_t os = o; + int64_t remaining = 0; + switch (type) { + case 0: + failures = &ec_inject_read_failures0; + break; + case 1: + failures = &ec_inject_read_failures1; + break; + default: + return "unrecognized error inject type"; + } + if (os.hobj.oid.name == "*") { + os.hobj.set_hash(0); + } + auto it = failures->find(os); + if (it != failures->end()) { + remaining = it->second.second; + failures->erase(it); + } + if (remaining == 0) { + return "no outstanding error injects"; + } else if (remaining == 1) { + return "ok - 1 inject cleared"; + } + return "ok - " + std::to_string(remaining) + " injects cleared"; +} + +/** + * @brief Clear a previously configured write error inject. + * @param o Target object for the error inject. + * @param type Type of error inject 0 = EIO, 1 = missing shard. + * @return string Indication of how many errors were cleared. + */ +std::string ec_inject_clear_write_error(const ghobject_t& o, + const int64_t type) { + std::lock_guard l(ec_inject_lock); + std::map> *failures; + ghobject_t os = o; + bool no_shard = true; + int64_t remaining = 0; + switch (type) { + case 0: + failures = &ec_inject_write_failures0; + break; + case 1: + failures = &ec_inject_write_failures1; + no_shard = false; + break; + case 2: + failures = &ec_inject_write_failures2; + break; + case 3: + failures = &ec_inject_write_failures3; + break; + default: + return "unrecognized error inject type"; + } + if (no_shard) { + os.set_shard(shard_id_t::NO_SHARD); + } + if (os.hobj.oid.name == "*") { + os.hobj.set_hash(0); + } + auto it = failures->find(os); + if (it != failures->end()) { + remaining = it->second.second; + failures->erase(it); + if (type == 0) { + ec_inject_write_failures0_shard.erase(os); + } + } + if (remaining == 0) { + return "no outstanding error injects"; + } else if (remaining == 1) { + return "ok - 1 inject cleared"; + } + return "ok - " + std::to_string(remaining) + " injects cleared"; +} + +static bool ec_inject_test_error(const ghobject_t& o, + std::map> *failures) +{ + std::lock_guard l(ec_inject_lock); + auto it = failures->find(o); + if (it == failures->end()) { + ghobject_t os = o; + os.hobj.oid.name = "*"; + os.hobj.set_hash(0); + it = failures->find(os); + } + if (it != failures->end()) { + auto && [when,duration] = it->second; + if (when > 0) { + when--; + return false; + } + if (--duration <= 0) { + failures->erase(it); + } + return true; + } + return false; +} + +bool ec_inject_test_read_error0(const ghobject_t& o) +{ + return ec_inject_test_error(o, &ec_inject_read_failures0); +} + +bool ec_inject_test_read_error1(const ghobject_t& o) +{ + return ec_inject_test_error(o, &ec_inject_read_failures1); +} + +bool ec_inject_test_write_error0(const hobject_t& o, + const osd_reqid_t& reqid) { + std::lock_guard l(ec_inject_lock); + ghobject_t os = ghobject_t(o, ghobject_t::NO_GEN, shard_id_t::NO_SHARD); + if (ec_inject_write_failures0_reqid.count(reqid)) { + // Matched reqid of retried write - flag for failure + ec_inject_write_failures0_reqid.erase(reqid); + return true; + } + auto it = ec_inject_write_failures0.find(os); + if (it == ec_inject_write_failures0.end()) { + os.hobj.oid.name = "*"; + os.hobj.set_hash(0); + it = ec_inject_write_failures0.find(os); + } + if (it != ec_inject_write_failures0.end()) { + auto && [when, duration] = it->second; + auto shard = ec_inject_write_failures0_shard.find(os)->second; + if (when > 0) { + when--; + } else { + if (--duration <= 0) { + ec_inject_write_failures0.erase(it); + ec_inject_write_failures0_shard.erase(os); + } + // Error inject triggered - save reqid + ec_inject_write_failures0_reqid.insert(reqid); + // Set up error inject to drop message to primary + ec_inject_write_error(ghobject_t(o, ghobject_t::NO_GEN, shard), 1, 0, 1); + } + } + return false; +} + +bool ec_inject_test_write_error1(const ghobject_t& o) { + bool rc = ec_inject_test_error(o, &ec_inject_write_failures1); + if (rc) { + // Set up error inject to generate OSD down + ec_inject_write_error(o, 2, 0, 1); + } + return rc; +} + +bool ec_inject_test_write_error2(const hobject_t& o) { + return ec_inject_test_error( + ghobject_t(o, ghobject_t::NO_GEN, shard_id_t::NO_SHARD), + &ec_inject_write_failures2); +} + +bool ec_inject_test_write_error3(const hobject_t& o) { + return ec_inject_test_error( + ghobject_t(o, ghobject_t::NO_GEN, shard_id_t::NO_SHARD), + &ec_inject_write_failures3); +} diff --git a/src/osd/ECCommon.h b/src/osd/ECCommon.h index 7ff9cae7646ab..de4c11ad50f49 100644 --- a/src/osd/ECCommon.h +++ b/src/osd/ECCommon.h @@ -493,6 +493,7 @@ struct ECCommon { ); ///< @return error code, 0 on success void schedule_recovery_work(); + }; /** @@ -843,3 +844,15 @@ void ECCommon::ReadPipeline::filter_read_op( on_schedule_recovery(op); } } + +// Error inject interfaces +std::string ec_inject_read_error(const ghobject_t& o, const int64_t type, const int64_t when, const int64_t duration); +std::string ec_inject_write_error(const ghobject_t& o, const int64_t type, const int64_t when, const int64_t duration); +std::string ec_inject_clear_read_error(const ghobject_t& o, const int64_t type); +std::string ec_inject_clear_write_error(const ghobject_t& o, const int64_t type); +bool ec_inject_test_read_error0(const ghobject_t& o); +bool ec_inject_test_read_error1(const ghobject_t& o); +bool ec_inject_test_write_error0(const hobject_t& o,const osd_reqid_t& reqid); +bool ec_inject_test_write_error1(const ghobject_t& o); +bool ec_inject_test_write_error2(const hobject_t& o); +bool ec_inject_test_write_error3(const hobject_t& o); diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc index 5223eb283e937..9c9e540cf6168 100644 --- a/src/osd/OSD.cc +++ b/src/osd/OSD.cc @@ -37,6 +37,7 @@ #include "osd/PG.h" #include "osd/scrubber/scrub_machine.h" #include "osd/scrubber/pg_scrubber.h" +#include "osd/ECCommon.h" #include "include/types.h" #include "include/compat.h" @@ -4347,6 +4348,46 @@ void OSD::final_init() test_ops_hook, "inject metadata error to an object"); ceph_assert(r == 0); + r = admin_socket->register_command( + "injectecreaderr " \ + "name=pool,type=CephString " \ + "name=objname,type=CephObjectname " \ + "name=shardid,type=CephInt,req=true,range=0|255 " \ + "name=type,type=CephInt,req=false " \ + "name=when,type=CephInt,req=false " \ + "name=duration,type=CephInt,req=false", + test_ops_hook, + "inject error for read of object in an EC pool"); + ceph_assert(r == 0); + r = admin_socket->register_command( + "injectecclearreaderr " \ + "name=pool,type=CephString " \ + "name=objname,type=CephObjectname " \ + "name=shardid,type=CephInt,req=true,range=0|255 " \ + "name=type,type=CephInt,req=false", + test_ops_hook, + "clear read error injects for object in an EC pool"); + ceph_assert(r == 0); + r = admin_socket->register_command( + "injectecwriteerr " \ + "name=pool,type=CephString " \ + "name=objname,type=CephObjectname " \ + "name=shardid,type=CephInt,req=true,range=0|255 " \ + "name=type,type=CephInt,req=false " \ + "name=when,type=CephInt,req=false " \ + "name=duration,type=CephInt,req=false", + test_ops_hook, + "inject error for write of object in an EC pool"); + ceph_assert(r == 0); + r = admin_socket->register_command( + "injectecclearwriteerr " \ + "name=pool,type=CephString " \ + "name=objname,type=CephObjectname " \ + "name=shardid,type=CephInt,req=true,range=0|255 " \ + "name=type,type=CephInt,req=false", + test_ops_hook, + "clear write error inject for object in an EC pool"); + ceph_assert(r == 0); r = admin_socket->register_command( "set_recovery_delay " \ "name=utime,type=CephInt,req=false", @@ -6487,8 +6528,10 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, //directly request the osd make a change. if (command == "setomapval" || command == "rmomapkey" || command == "setomapheader" || command == "getomap" || - command == "truncobj" || command == "injectmdataerr" || - command == "injectdataerr" + command == "truncobj" || + command == "injectmdataerr" || command == "injectdataerr" || + command == "injectecreaderr" || command == "injectecclearreaderr" || + command == "injectecwriteerr" || command == "injectecclearwriteerr" ) { pg_t rawpg; int64_t pool; @@ -6527,8 +6570,21 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, ghobject_t gobj(obj, ghobject_t::NO_GEN, shard_id_t(uint8_t(shardid))); spg_t pgid(curmap->raw_pg_to_pg(rawpg), shard_id_t(shardid)); if (curmap->pg_is_ec(rawpg)) { - if ((command != "injectdataerr") && (command != "injectmdataerr")) { - ss << "Must not call on ec pool, except injectdataerr or injectmdataerr"; + if ((command != "injectdataerr") && + (command != "injectmdataerr") && + (command != "injectecreaderr") && + (command != "injectecclearreaderr") && + (command != "injectecwriteerr") && + (command != "injectecclearwriteerr")) { + ss << "Must not call on ec pool"; + return; + } + } else { + if ((command == "injectecreaderr") || + (command == "injecteclearreaderr") || + (command == "injectecwriteerr") || + (command == "injecteclearwriteerr")) { + ss << "Only supported on ec pool"; return; } } @@ -6607,6 +6663,38 @@ void TestOpsSocketHook::test_ops(OSDService *service, ObjectStore *store, } else if (command == "injectmdataerr") { store->inject_mdata_error(gobj); ss << "ok"; + } else if (command == "injectecreaderr") { + if (service->cct->_conf->bluestore_debug_inject_read_err) { + int64_t type = cmd_getval_or(cmdmap, "type", 0); + int64_t when = cmd_getval_or(cmdmap, "when", 0); + int64_t duration = cmd_getval_or(cmdmap, "duration", 1); + ss << ec_inject_read_error(gobj, type, when, duration); + } else { + ss << "bluestore_debug_inject_read_err not enabled"; + } + } else if (command == "injectecclearreaderr") { + if (service->cct->_conf->bluestore_debug_inject_read_err) { + int64_t type = cmd_getval_or(cmdmap, "type", 0); + ss << ec_inject_clear_read_error(gobj, type); + } else { + ss << "bluestore_debug_inject_read_err not enabled"; + } + } else if (command == "injectecwriteerr") { + if (service->cct->_conf->bluestore_debug_inject_read_err) { + int64_t type = cmd_getval_or(cmdmap, "type", 0); + int64_t when = cmd_getval_or(cmdmap, "when", 0); + int64_t duration = cmd_getval_or(cmdmap, "duration", 1); + ss << ec_inject_write_error(gobj, type, when, duration); + } else { + ss << "bluestore_debug_inject_read_err not enabled"; + } + } else if (command == "injectecclearwriteerr") { + if (service->cct->_conf->bluestore_debug_inject_read_err) { + int64_t type = cmd_getval_or(cmdmap, "type", 0); + ss << ec_inject_clear_write_error(gobj, type); + } else { + ss << "bluestore_debug_inject_read_err not enabled"; + } } return; } diff --git a/src/osd/PGBackend.h b/src/osd/PGBackend.h index b87aa1da6771b..f5eb9ea951e99 100644 --- a/src/osd/PGBackend.h +++ b/src/osd/PGBackend.h @@ -290,6 +290,10 @@ typedef std::shared_ptr OSDMapRef; MessageRef, Connection *con) = 0; virtual void send_message_osd_cluster( Message *m, const ConnectionRef& con) = 0; + virtual void start_mon_command( + const std::vector& cmd, const bufferlist& inbl, + bufferlist *outbl, std::string *outs, + Context *onfinish) = 0; virtual ConnectionRef get_con_osd_cluster(int peer, epoch_t from_epoch) = 0; virtual entity_name_t get_cluster_msgr_name() = 0; diff --git a/src/osd/PrimaryLogPG.cc b/src/osd/PrimaryLogPG.cc index 44f8e85b5ef6c..aa0f84783cd96 100644 --- a/src/osd/PrimaryLogPG.cc +++ b/src/osd/PrimaryLogPG.cc @@ -2286,6 +2286,16 @@ void PrimaryLogPG::do_op(OpRequestRef& op) } } + if (cct->_conf->bluestore_debug_inject_read_err && + op->may_write() && + pool.info.is_erasure() && + ec_inject_test_write_error0(m->get_hobj(), m->get_reqid())) { + // Fail retried write with error + dout(0) << __func__ << " Error inject - Fail retried write with EINVAL" << dendl; + osd->reply_op_error(op, -EINVAL); + return; + } + ObjectContextRef obc; bool can_create = op->may_write(); hobject_t missing_oid; diff --git a/src/osd/PrimaryLogPG.h b/src/osd/PrimaryLogPG.h index f66b5c6e16aed..bf55d5398218b 100644 --- a/src/osd/PrimaryLogPG.h +++ b/src/osd/PrimaryLogPG.h @@ -622,6 +622,12 @@ class PrimaryLogPG : public PG, Message *m, const ConnectionRef& con) override { osd->send_message_osd_cluster(m, con); } + void start_mon_command( + const std::vector& cmd, const bufferlist& inbl, + bufferlist *outbl, std::string *outs, + Context *onfinish) override { + osd->monc->start_mon_command(cmd, inbl, outbl, outs, onfinish); + } ConnectionRef get_con_osd_cluster(int peer, epoch_t from_epoch) override; entity_name_t get_cluster_msgr_name() override { return osd->get_cluster_msgr_name(); @@ -1993,6 +1999,7 @@ class PrimaryLogPG : public PG, private: DynamicPerfStats m_dynamic_perf_stats; + }; inline ostream& operator<<(ostream& out, const PrimaryLogPG::RepGather& repop) @@ -2021,5 +2028,4 @@ inline ostream& operator<<(ostream& out, void intrusive_ptr_add_ref(PrimaryLogPG::RepGather *repop); void intrusive_ptr_release(PrimaryLogPG::RepGather *repop); - #endif diff --git a/src/test/osd/CMakeLists.txt b/src/test/osd/CMakeLists.txt index f2d1471e22e3f..798558ebbe06a 100644 --- a/src/test/osd/CMakeLists.txt +++ b/src/test/osd/CMakeLists.txt @@ -22,7 +22,7 @@ install(TARGETS add_executable(ceph_test_rados_io_sequence ${CMAKE_CURRENT_SOURCE_DIR}/ceph_test_rados_io_sequence.cc) target_link_libraries(ceph_test_rados_io_sequence - librados global object_io_exerciser) + librados global object_io_exerciser json_structures) install(TARGETS ceph_test_rados_io_sequence DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/test/osd/ceph_test_rados_io_sequence.cc b/src/test/osd/ceph_test_rados_io_sequence.cc index 4a768a016e28c..96808ea37e56c 100644 --- a/src/test/osd/ceph_test_rados_io_sequence.cc +++ b/src/test/osd/ceph_test_rados_io_sequence.cc @@ -1,83 +1,104 @@ #include "ceph_test_rados_io_sequence.h" +#include #include #include -#include - -#include "include/random.h" - -#include "librados/librados_asio.h" -#include "common/ceph_argparse.h" -#include "include/interval_set.h" -#include "global/global_init.h" -#include "global/global_context.h" +#include "common/Formatter.h" #include "common/Thread.h" +#include "common/ceph_argparse.h" +#include "common/ceph_json.h" #include "common/debug.h" #include "common/dout.h" #include "common/split.h" #include "common/strtol.h" // for strict_iecstrtoll() +#include "common/ceph_json.h" +#include "common/Formatter.h" #include "common/io_exerciser/DataGenerator.h" +#include "common/io_exerciser/EcIoSequence.h" +#include "common/io_exerciser/IoOp.h" +#include "common/io_exerciser/IoSequence.h" #include "common/io_exerciser/Model.h" #include "common/io_exerciser/ObjectModel.h" #include "common/io_exerciser/RadosIo.h" -#include "common/io_exerciser/IoOp.h" -#include "common/io_exerciser/IoSequence.h" +#include "common/json/BalancerStructures.h" +#include "common/json/ConfigStructures.h" +#include "common/json/OSDStructures.h" +#include "fmt/format.h" +#include "global/global_context.h" +#include "global/global_init.h" +#include "include/interval_set.h" +#include "include/random.h" +#include "json_spirit/json_spirit.h" +#include "librados/librados_asio.h" #define dout_subsys ceph_subsys_rados #define dout_context g_ceph_context +using OpType = ceph::io_exerciser::OpType; + +using DoneOp = ceph::io_exerciser::DoneOp; +using BarrierOp = ceph::io_exerciser::BarrierOp; +using CreateOp = ceph::io_exerciser::CreateOp; +using RemoveOp = ceph::io_exerciser::RemoveOp; +using SingleReadOp = ceph::io_exerciser::SingleReadOp; +using DoubleReadOp = ceph::io_exerciser::DoubleReadOp; +using TripleReadOp = ceph::io_exerciser::TripleReadOp; +using SingleWriteOp = ceph::io_exerciser::SingleWriteOp; +using DoubleWriteOp = ceph::io_exerciser::DoubleWriteOp; +using TripleWriteOp = ceph::io_exerciser::TripleWriteOp; +using SingleFailedWriteOp = ceph::io_exerciser::SingleFailedWriteOp; +using DoubleFailedWriteOp = ceph::io_exerciser::DoubleFailedWriteOp; +using TripleFailedWriteOp = ceph::io_exerciser::TripleFailedWriteOp; + namespace { - struct Size {}; - void validate(boost::any& v, const std::vector& values, - Size *target_type, int) { - po::validators::check_first_occurrence(v); - const std::string &s = po::validators::get_single_string(values); - - std::string parse_error; - uint64_t size = strict_iecstrtoll(s, &parse_error); - if (!parse_error.empty()) { - throw po::validation_error(po::validation_error::invalid_option_value); - } - v = boost::any(size); - } - - struct Pair {}; - void validate(boost::any& v, const std::vector& values, - Pair *target_type, int) { - po::validators::check_first_occurrence(v); - const std::string &s = po::validators::get_single_string(values); - auto part = ceph::split(s).begin(); - std::string parse_error; - int first = strict_iecstrtoll(*part++, &parse_error); - int second = strict_iecstrtoll(*part, &parse_error); - if (!parse_error.empty()) { - throw po::validation_error(po::validation_error::invalid_option_value); - } - v = boost::any(std::pair{first,second}); - } - - struct PluginString {}; - void validate(boost::any& v, const std::vector& values, - PluginString *target_type, int) { - po::validators::check_first_occurrence(v); - const std::string &s = po::validators::get_single_string(values); - - const std::string_view* pluginIt = std::find( - ceph::io_sequence::tester::pluginChoices.begin(), - ceph::io_sequence::tester::pluginChoices.end(), - s - ); - if(ceph::io_sequence::tester::pluginChoices.end() == pluginIt) - { - throw po::validation_error(po::validation_error::invalid_option_value); - } +struct Size {}; +void validate(boost::any& v, const std::vector& values, + Size* target_type, int) { + po::validators::check_first_occurrence(v); + const std::string& s = po::validators::get_single_string(values); - v = boost::any(*pluginIt); + std::string parse_error; + uint64_t size = strict_iecstrtoll(s, &parse_error); + if (!parse_error.empty()) { + throw po::validation_error(po::validation_error::invalid_option_value); } + v = boost::any(size); +} + +struct Pair {}; +void validate(boost::any& v, const std::vector& values, + Pair* target_type, int) { + po::validators::check_first_occurrence(v); + const std::string& s = po::validators::get_single_string(values); + auto part = ceph::split(s).begin(); + std::string parse_error; + int first = strict_iecstrtoll(*part++, &parse_error); + int second = strict_iecstrtoll(*part, &parse_error); + if (!parse_error.empty()) { + throw po::validation_error(po::validation_error::invalid_option_value); + } + v = boost::any(std::pair{first, second}); +} + +struct PluginString {}; +void validate(boost::any& v, const std::vector& values, + PluginString* target_type, int) { + po::validators::check_first_occurrence(v); + const std::string& s = po::validators::get_single_string(values); + + const std::string_view* pluginIt = + std::find(ceph::io_sequence::tester::pluginChoices.begin(), + ceph::io_sequence::tester::pluginChoices.end(), s); + if (ceph::io_sequence::tester::pluginChoices.end() == pluginIt) { + throw po::validation_error(po::validation_error::invalid_option_value); + } + + v = boost::any(*pluginIt); +} - constexpr std::string_view usage[] = { +constexpr std::string_view usage[] = { "Basic usage:", "", "ceph_test_rados_io_sequence", @@ -119,103 +140,99 @@ namespace { "\t are specified with unit of blocksize. Supported commands:", "\t\t create ", "\t\t remove", - "\t\t read|write ", - "\t\t read2|write2 ", - "\t\t read3|write3 ", - "\t\t done" - }; - - po::options_description get_options_description() - { - po::options_description desc("ceph_test_rados_io options"); - desc.add_options() - ("help,h", - "show help message") - ("listsequence,l", - "show list of sequences") - ("dryrun,d", - "test sequence, do not issue any I/O") - ("verbose", - "more verbose output during test") - ("sequence,s", po::value(), - "test specified sequence") - ("seed", po::value(), - "seed for whole test") - ("seqseed", po::value(), - "seed for sequence") - ("blocksize,b", po::value(), - "block size (default 2048)") - ("chunksize,c", po::value(), - "chunk size (default 4096)") - ("pool,p", po::value(), - "pool name") - ("object,o", po::value()->default_value("test"), - "object name") - ("km", po::value(), - "k,m EC pool profile (default 2,2)") - ("plugin", po::value(), - "EC plugin (isa or jerasure)") - ("objectsize", po::value(), - "min,max object size in blocks (default 1,32)") - ("threads,t", po::value(), - "number of threads of I/O per object (default 1)") - ("parallel,p", po::value()->default_value(1), - "number of objects to exercise in parallel") - ("interactive", - "interactive mode, execute IO commands from stdin"); - - return desc; - } - - int parse_io_seq_options( - po::variables_map& vm, - int argc, - char** argv) - { - std::vector unrecognized_options; - try { - po::options_description desc = get_options_description(); - - auto parsed = po::command_line_parser(argc, argv) - .options(desc) - .allow_unregistered() - .run(); - po::store(parsed, vm); - po::notify(vm); - unrecognized_options = po::collect_unrecognized(parsed.options, - po::include_positional); - - if (!unrecognized_options.empty()) - { - std::stringstream ss; - ss << "Unrecognised command options supplied: "; - while (unrecognized_options.size() > 1) - { - ss << unrecognized_options.back().c_str() << ", "; - unrecognized_options.pop_back(); - } - ss << unrecognized_options.back(); - dout(0) << ss.str() << dendl; - return 1; + "\t\t read|write|failedwrite ", + "\t\t read2|write2|failedwrite2 ", + "\t\t read3|write3|failedwrite3 ", + "\t\t injecterror ", + "\t\t clearinject ", + "\t\t done"}; + +po::options_description get_options_description() { + po::options_description desc("ceph_test_rados_io options"); + desc.add_options()("help,h", "show help message")("listsequence,l", + "show list of sequences")( + "dryrun,d", "test sequence, do not issue any I/O")( + "verbose", "more verbose output during test")( + "sequence,s", po::value(), "test specified sequence")( + "seed", po::value(), "seed for whole test")( + "seqseed", po::value(), "seed for sequence")( + "blocksize,b", po::value(), "block size (default 2048)")( + "chunksize,c", po::value(), "chunk size (default 4096)")( + "pool,p", po::value(), "pool name")( + "object,o", po::value()->default_value("test"), + "object name")("km", po::value(), + "k,m EC pool profile (default 2,2)")( + "plugin", po::value(), "EC plugin (isa or jerasure)")( + "objectsize", po::value(), + "min,max object size in blocks (default 1,32)")( + "threads,t", po::value(), + "number of threads of I/O per object (default 1)")( + "parallel,p", po::value()->default_value(1), + "number of objects to exercise in parallel")( + "testrecovery", + "Inject errors during sequences to test recovery processes of OSDs")( + "interactive", "interactive mode, execute IO commands from stdin")( + "allow_pool_autoscaling", + "Allows pool autoscaling. Disabled by default.")( + "allow_pool_balancer", "Enables pool balancing. Disabled by default.")( + "allow_pool_deep_scrubbing", + "Enables pool deep scrub. Disabled by default.")( + "allow_pool_scrubbing", "Enables pool scrubbing. Disabled by default."); + + return desc; +} + +int parse_io_seq_options(po::variables_map& vm, int argc, char** argv) { + std::vector unrecognized_options; + try { + po::options_description desc = get_options_description(); + + auto parsed = po::command_line_parser(argc, argv) + .options(desc) + .allow_unregistered() + .run(); + po::store(parsed, vm); + po::notify(vm); + unrecognized_options = + po::collect_unrecognized(parsed.options, po::include_positional); + + if (!unrecognized_options.empty()) { + std::stringstream ss; + ss << "Unrecognised command options supplied: "; + while (unrecognized_options.size() > 1) { + ss << unrecognized_options.back().c_str() << ", "; + unrecognized_options.pop_back(); } - } catch(const po::error& e) { - std::cerr << "error: " << e.what() << std::endl; + ss << unrecognized_options.back(); + dout(0) << ss.str() << dendl; return 1; } - - return 0; + } catch (const po::error& e) { + std::cerr << "error: " << e.what() << std::endl; + return 1; } + + return 0; } +template +int send_mon_command(S& s, librados::Rados& rados, const char* name, + ceph::buffer::list& inbl, ceph::buffer::list* outbl, Formatter* f) { + std::ostringstream oss; + encode_json(name, s, f); + f->flush(oss); + int rc = rados.mon_command(oss.str(), inbl, outbl, nullptr); + return rc; +} + +} // namespace + template & Ts> -ceph::io_sequence::tester::ProgramOptionSelector - ::ProgramOptionSelector(ceph::util::random_number_generator& rng, - po::variables_map vm, - const std::string& option_name, - bool set_forced, - bool select_first) - : rng(rng), - option_name(option_name) { +ceph::io_sequence::tester::ProgramOptionSelector:: + ProgramOptionSelector(ceph::util::random_number_generator& rng, + po::variables_map vm, const std::string& option_name, + bool set_forced, bool select_first) + : rng(rng), option_name(option_name) { if (set_forced && vm.count(option_name)) { force_value = vm[option_name].as(); } @@ -226,76 +243,54 @@ ceph::io_sequence::tester::ProgramOptionSelector } template & Ts> -bool ceph::io_sequence::tester::ProgramOptionSelector::isForced() -{ +bool ceph::io_sequence::tester::ProgramOptionSelector::isForced() { return force_value.has_value(); } template & Ts> -const T ceph::io_sequence::tester::ProgramOptionSelector::choose() -{ +const T ceph::io_sequence::tester::ProgramOptionSelector::choose() { if (force_value.has_value()) { return *force_value; } else if (first_value.has_value()) { return *std::exchange(first_value, std::nullopt); } else { - return choices[rng(N-1)]; + return choices[rng(N - 1)]; } } - - ceph::io_sequence::tester::SelectObjectSize::SelectObjectSize( - ceph::util::random_number_generator& rng, - po::variables_map vm) - : ProgramOptionSelector(rng, vm, "objectsize", true, true) -{ -} - - + ceph::util::random_number_generator& rng, po::variables_map vm) + : ProgramOptionSelector(rng, vm, "objectsize", true, true) {} ceph::io_sequence::tester::SelectBlockSize::SelectBlockSize( - ceph::util::random_number_generator& rng, - po::variables_map vm) - : ProgramOptionSelector(rng, vm, "blocksize", true, true) -{ -} - - + ceph::util::random_number_generator& rng, po::variables_map vm) + : ProgramOptionSelector(rng, vm, "blocksize", true, true) {} ceph::io_sequence::tester::SelectNumThreads::SelectNumThreads( - ceph::util::random_number_generator& rng, - po::variables_map vm) - : ProgramOptionSelector(rng, vm, "threads", true, true) -{ -} - - + ceph::util::random_number_generator& rng, po::variables_map vm) + : ProgramOptionSelector(rng, vm, "threads", true, true) {} ceph::io_sequence::tester::SelectSeqRange::SelectSeqRange( - ceph::util::random_number_generator& rng, - po::variables_map vm) - : ProgramOptionSelector(rng, vm, "sequence", false, false) -{ + ceph::util::random_number_generator& rng, po::variables_map vm) + : ProgramOptionSelector(rng, vm, "sequence", false, false) { if (vm.count(option_name)) { ceph::io_exerciser::Sequence s = - static_cast(vm["sequence"].as()); + static_cast(vm["sequence"].as()); if (s < ceph::io_exerciser::Sequence::SEQUENCE_BEGIN || s >= ceph::io_exerciser::Sequence::SEQUENCE_END) { dout(0) << "Sequence argument out of range" << dendl; throw po::validation_error(po::validation_error::invalid_option_value); } ceph::io_exerciser::Sequence e = s; - force_value = std::make_optional>( - std::make_pair(s, ++e)); + force_value = std::make_optional< + std::pair>( + std::make_pair(s, ++e)); } } -const std::pair - ceph::io_sequence::tester::SelectSeqRange::choose() { - if (force_value.has_value()) - { +const std::pair +ceph::io_sequence::tester::SelectSeqRange::choose() { + if (force_value.has_value()) { return *force_value; } else { return std::make_pair(ceph::io_exerciser::Sequence::SEQUENCE_BEGIN, @@ -303,45 +298,34 @@ const std::pair } } - - ceph::io_sequence::tester::SelectErasureKM::SelectErasureKM( - ceph::util::random_number_generator& rng, - po::variables_map vm) - : ProgramOptionSelector(rng, vm, "km", true, true) -{ -} - - + ceph::util::random_number_generator& rng, po::variables_map vm) + : ProgramOptionSelector(rng, vm, "km", true, true) {} ceph::io_sequence::tester::SelectErasurePlugin::SelectErasurePlugin( - ceph::util::random_number_generator& rng, - po::variables_map vm) - : ProgramOptionSelector(rng, vm, "plugin", true, false) -{ -} - - - -ceph::io_sequence::tester::SelectErasureChunkSize::SelectErasureChunkSize(ceph::util::random_number_generator& rng, po::variables_map vm) - : ProgramOptionSelector(rng, vm, "stripe_unit", true, false) -{ -} - + ceph::util::random_number_generator& rng, po::variables_map vm) + : ProgramOptionSelector(rng, vm, "plugin", true, false) {} +ceph::io_sequence::tester::SelectErasureChunkSize::SelectErasureChunkSize( + ceph::util::random_number_generator& rng, po::variables_map vm) + : ProgramOptionSelector(rng, vm, "chunksize", true, true) {} ceph::io_sequence::tester::SelectECPool::SelectECPool( - ceph::util::random_number_generator& rng, - po::variables_map vm, - librados::Rados& rados, - bool dry_run) - : ProgramOptionSelector(rng, vm, "pool", false, false), - rados(rados), - dry_run(dry_run), - skm(SelectErasureKM(rng, vm)), - spl(SelectErasurePlugin(rng, vm)), - scs(SelectErasureChunkSize(rng, vm)) -{ + ceph::util::random_number_generator& rng, po::variables_map vm, + librados::Rados& rados, bool dry_run, bool allow_pool_autoscaling, + bool allow_pool_balancer, bool allow_pool_deep_scrubbing, + bool allow_pool_scrubbing, bool test_recovery) + : ProgramOptionSelector(rng, vm, "pool", false, false), + rados(rados), + dry_run(dry_run), + allow_pool_autoscaling(allow_pool_autoscaling), + allow_pool_balancer(allow_pool_balancer), + allow_pool_deep_scrubbing(allow_pool_deep_scrubbing), + allow_pool_scrubbing(allow_pool_scrubbing), + test_recovery(test_recovery), + skm(SelectErasureKM(rng, vm)), + spl(SelectErasurePlugin(rng, vm)), + scs(SelectErasureChunkSize(rng, vm)) { if (!skm.isForced()) { if (vm.count("pool")) { force_value = vm["pool"].as(); @@ -349,147 +333,239 @@ ceph::io_sequence::tester::SelectECPool::SelectECPool( } } -const std::string ceph::io_sequence::tester::SelectECPool::choose() -{ - std::pair value; +const std::string ceph::io_sequence::tester::SelectECPool::choose() { + std::pair value; if (!skm.isForced() && force_value.has_value()) { + int rc; + bufferlist inbl, outbl; + auto formatter = std::make_unique(false); + + ceph::messaging::osd::OSDPoolGetRequest osdPoolGetRequest{*force_value}; + rc = send_mon_command(osdPoolGetRequest, rados, "OSDPoolGetRequest", inbl, + &outbl, formatter.get()); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::messaging::osd::OSDPoolGetReply osdPoolGetReply; + osdPoolGetReply.decode_json(&p); + + ceph::messaging::osd::OSDECProfileGetRequest osdECProfileGetRequest{ + osdPoolGetReply.erasure_code_profile}; + rc = send_mon_command(osdECProfileGetRequest, rados, + "OSDECProfileGetRequest", inbl, &outbl, + formatter.get()); + ceph_assert(rc == 0); + + success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::messaging::osd::OSDECProfileGetReply reply; + reply.decode_json(&p); + k = reply.k; + m = reply.m; return *force_value; } else { value = skm.choose(); } - int k = value.first; - int m = value.second; + k = value.first; + m = value.second; const std::string plugin = std::string(spl.choose()); const uint64_t chunk_size = scs.choose(); - std::string pool_name = "ec_" + plugin + - "_cs" + std::to_string(chunk_size) + - "_k" + std::to_string(k) + - "_m" + std::to_string(m); - if (!dry_run) - { + std::string pool_name = "ec_" + plugin + "_cs" + std::to_string(chunk_size) + + "_k" + std::to_string(k) + "_m" + std::to_string(m); + if (!dry_run) { create_pool(rados, pool_name, plugin, chunk_size, k, m); } return pool_name; } void ceph::io_sequence::tester::SelectECPool::create_pool( - librados::Rados& rados, - const std::string& pool_name, - const std::string& plugin, - uint64_t chunk_size, - int k, int m) -{ + librados::Rados& rados, const std::string& pool_name, + const std::string& plugin, uint64_t chunk_size, int k, int m) { int rc; bufferlist inbl, outbl; - std::string profile_create = - "{\"prefix\": \"osd erasure-code-profile set\", \ - \"name\": \"testprofile-" + pool_name + "\", \ - \"profile\": [ \"plugin=" + plugin + "\", \ - \"k=" + std::to_string(k) + "\", \ - \"m=" + std::to_string(m) + "\", \ - \"stripe_unit=" + std::to_string(chunk_size) + "\", \ - \"crush-failure-domain=osd\"]}"; - rc = rados.mon_command(profile_create, inbl, &outbl, nullptr); + auto formatter = std::make_unique(false); + + ceph::messaging::osd::OSDECProfileSetRequest ecProfileSetRequest{ + fmt::format("testprofile-{}", pool_name), + {fmt::format("plugin={}", plugin), fmt::format("k={}", k), + fmt::format("m={}", m), fmt::format("stripe_unit={}", chunk_size), + fmt::format("crush-failure-domain=osd")}}; + rc = send_mon_command(ecProfileSetRequest, rados, "OSDECProfileSetRequest", + inbl, &outbl, formatter.get()); ceph_assert(rc == 0); - std::string cmdstr = - "{\"prefix\": \"osd pool create\", \ - \"pool\": \"" + pool_name + "\", \ - \"pool_type\": \"erasure\", \ - \"pg_num\": 8, \ - \"pgp_num\": 8, \ - \"erasure_code_profile\": \"testprofile-" + pool_name + "\"}"; - rc = rados.mon_command(cmdstr, inbl, &outbl, nullptr); + + ceph::messaging::osd::OSDECPoolCreateRequest poolCreateRequest{ + pool_name, "erasure", 8, 8, fmt::format("testprofile-{}", pool_name)}; + rc = send_mon_command(poolCreateRequest, rados, "OSDECPoolCreateRequest", + inbl, &outbl, formatter.get()); ceph_assert(rc == 0); -} + if (allow_pool_autoscaling) { + ceph::messaging::osd::OSDSetRequest setNoAutoscaleRequest{"noautoscale", + std::nullopt}; + rc = send_mon_command(setNoAutoscaleRequest, rados, "OSDSetRequest", inbl, + &outbl, formatter.get()); + ceph_assert(rc == 0); + } + + if (allow_pool_balancer) { + ceph::messaging::balancer::BalancerOffRequest balancerOffRequest{}; + rc = send_mon_command(balancerOffRequest, rados, "BalancerOffRequest", inbl, + &outbl, formatter.get()); + ceph_assert(rc == 0); + + ceph::messaging::balancer::BalancerStatusRequest balancerStatusRequest{}; + rc = send_mon_command(balancerStatusRequest, rados, "BalancerStatusRequest", + inbl, &outbl, formatter.get()); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::messaging::balancer::BalancerStatusReply reply; + reply.decode_json(&p); + ceph_assert(!reply.active); + } + if (allow_pool_deep_scrubbing) { + ceph::messaging::osd::OSDSetRequest setNoDeepScrubRequest{"nodeep-scrub", + std::nullopt}; + rc = send_mon_command(setNoDeepScrubRequest, rados, "setNoDeepScrubRequest", + inbl, &outbl, formatter.get()); + ceph_assert(rc == 0); + } + + if (allow_pool_scrubbing) { + ceph::messaging::osd::OSDSetRequest setNoScrubRequest{"noscrub", + std::nullopt}; + rc = send_mon_command(setNoScrubRequest, rados, "OSDSetRequest", inbl, + &outbl, formatter.get()); + ceph_assert(rc == 0); + } + + if (test_recovery) { + ceph::messaging::config::ConfigSetRequest configSetBluestoreDebugRequest{ + "global", "bluestore_debug_inject_read_err", "true", std::nullopt}; + rc = send_mon_command(configSetBluestoreDebugRequest, rados, + "ConfigSetRequest", inbl, &outbl, + formatter.get()); + ceph_assert(rc == 0); + + ceph::messaging::config::ConfigSetRequest configSetMaxMarkdownRequest{ + "global", "osd_max_markdown_count", "99999999", std::nullopt}; + rc = + send_mon_command(configSetMaxMarkdownRequest, rados, "ConfigSetRequest", + inbl, &outbl, formatter.get()); + ceph_assert(rc == 0); + } +} -ceph::io_sequence::tester::TestObject::TestObject( const std::string oid, - librados::Rados& rados, - boost::asio::io_context& asio, - SelectBlockSize& sbs, - SelectECPool& spo, - SelectObjectSize& sos, - SelectNumThreads& snt, - SelectSeqRange& ssr, - ceph::util::random_number_generator& rng, - ceph::mutex& lock, - ceph::condition_variable& cond, - bool dryrun, - bool verbose, - std::optional seqseed) : - rng(rng), verbose(verbose), seqseed(seqseed) -{ +ceph::io_sequence::tester::TestObject::TestObject( + const std::string oid, librados::Rados& rados, + boost::asio::io_context& asio, SelectBlockSize& sbs, SelectECPool& spo, + SelectObjectSize& sos, SelectNumThreads& snt, SelectSeqRange& ssr, + ceph::util::random_number_generator& rng, ceph::mutex& lock, + ceph::condition_variable& cond, bool dryrun, bool verbose, + std::optional seqseed, bool testrecovery) + : rng(rng), verbose(verbose), seqseed(seqseed), testrecovery(testrecovery) { if (dryrun) { - verbose = true; - exerciser_model = std::make_unique(oid, - sbs.choose(), - rng()); + exerciser_model = std::make_unique( + oid, sbs.choose(), rng()); } else { const std::string pool = spo.choose(); + poolK = spo.getChosenK(); + poolM = spo.getChosenM(); + int threads = snt.choose(); - exerciser_model = std::make_unique(rados, - asio, - pool, - oid, - sbs.choose(), - rng(), - threads, - lock, - cond); - dout(0) << "= " << oid << " pool=" << pool - << " threads=" << threads - << " blocksize=" << exerciser_model->get_block_size() - << " =" << dendl; + + bufferlist inbl, outbl; + auto formatter = std::make_unique(false); + + std::optional> cached_shard_order = std::nullopt; + + if (!spo.get_allow_pool_autoscaling() && !spo.get_allow_pool_balancer() && + !spo.get_allow_pool_deep_scrubbing() && + !spo.get_allow_pool_scrubbing()) { + ceph::messaging::osd::OSDMapRequest osdMapRequest{pool, oid, ""}; + int rc = send_mon_command(osdMapRequest, rados, "OSDMapRequest", inbl, + &outbl, formatter.get()); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::messaging::osd::OSDMapReply reply{}; + reply.decode_json(&p); + cached_shard_order = reply.acting; + } + + exerciser_model = std::make_unique( + rados, asio, pool, oid, cached_shard_order, sbs.choose(), rng(), + threads, lock, cond); + dout(0) << "= " << oid << " pool=" << pool << " threads=" << threads + << " blocksize=" << exerciser_model->get_block_size() << " =" + << dendl; } obj_size_range = sos.choose(); seq_range = ssr.choose(); curseq = seq_range.first; - seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq, - obj_size_range, - seqseed.value_or(rng())); + + if (testrecovery) { + seq = ceph::io_exerciser::EcIoSequence::generate_sequence( + curseq, obj_size_range, poolK, poolM, seqseed.value_or(rng())); + } else { + seq = ceph::io_exerciser::IoSequence::generate_sequence( + curseq, obj_size_range, seqseed.value_or(rng())); + } + op = seq->next(); done = false; - dout(0) << "== " << exerciser_model->get_oid() << " " - << curseq << " " - << seq->get_name() - << " ==" <get_oid() << " " << curseq << " " + << seq->get_name_with_seqseed() << " ==" << dendl; } -bool ceph::io_sequence::tester::TestObject::readyForIo() -{ +bool ceph::io_sequence::tester::TestObject::readyForIo() { return exerciser_model->readyForIoOp(*op); } -bool ceph::io_sequence::tester::TestObject::next() -{ +bool ceph::io_sequence::tester::TestObject::next() { if (!done) { if (verbose) { - dout(0) << exerciser_model->get_oid() - << " Step " << seq->get_step() << ": " - << op->to_string(exerciser_model->get_block_size()) << dendl; + dout(0) << exerciser_model->get_oid() << " Step " << seq->get_step() + << ": " << op->to_string(exerciser_model->get_block_size()) + << dendl; } else { - dout(5) << exerciser_model->get_oid() - << " Step " << seq->get_step() << ": " - << op->to_string(exerciser_model->get_block_size()) << dendl; + dout(5) << exerciser_model->get_oid() << " Step " << seq->get_step() + << ": " << op->to_string(exerciser_model->get_block_size()) + << dendl; } exerciser_model->applyIoOp(*op); - if (op->done()) { - ++curseq; - if (curseq == seq_range.second) { + if (op->getOpType() == ceph::io_exerciser::OpType::Done) { + curseq = seq->getNextSupportedSequenceId(); + if (curseq >= seq_range.second) { done = true; dout(0) << exerciser_model->get_oid() << " Number of IOs = " << exerciser_model->get_num_io() << dendl; } else { - seq = ceph::io_exerciser::IoSequence::generate_sequence(curseq, - obj_size_range, - seqseed.value_or(rng())); - dout(0) << "== " << exerciser_model->get_oid() << " " - << curseq << " " << seq->get_name() - << " ==" <get_oid() << " " << curseq << " " + << seq->get_name_with_seqseed() << " ==" << dendl; op = seq->next(); } } else { @@ -499,27 +575,30 @@ bool ceph::io_sequence::tester::TestObject::next() return done; } -bool ceph::io_sequence::tester::TestObject::finished() -{ - return done; -} +bool ceph::io_sequence::tester::TestObject::finished() { return done; } -int ceph::io_sequence::tester::TestObject::get_num_io() -{ +int ceph::io_sequence::tester::TestObject::get_num_io() { return exerciser_model->get_num_io(); } ceph::io_sequence::tester::TestRunner::TestRunner(po::variables_map& vm, - librados::Rados& rados) : - rados(rados), - seed(vm.contains("seed") ? vm["seed"].as() : time(nullptr)), - rng(ceph::util::random_number_generator(seed)), - sbs{rng, vm}, - sos{rng, vm}, - spo{rng, vm, rados, vm.contains("dryrun")}, - snt{rng, vm}, - ssr{rng, vm} -{ + librados::Rados& rados) + : rados(rados), + seed(vm.contains("seed") ? vm["seed"].as() : time(nullptr)), + rng(ceph::util::random_number_generator(seed)), + sbs{rng, vm}, + sos{rng, vm}, + spo{rng, + vm, + rados, + vm.contains("dryrun"), + vm.contains("allow_pool_autoscaling"), + vm.contains("allow_pool_balancer"), + vm.contains("allow_pool_deep_scrubbing"), + vm.contains("allow_pool_scrubbing"), + vm.contains("test_recovery")}, + snt{rng, vm}, + ssr{rng, vm} { dout(0) << "Test using seed " << seed << dendl; verbose = vm.contains("verbose"); @@ -532,19 +611,23 @@ ceph::io_sequence::tester::TestRunner::TestRunner(po::variables_map& vm, num_objects = vm["parallel"].as(); object_name = vm["object"].as(); interactive = vm.contains("interactive"); + testrecovery = vm.contains("testrecovery"); + + allow_pool_autoscaling = vm.contains("allow_pool_autoscaling"); + allow_pool_balancer = vm.contains("allow_pool_balancer"); + allow_pool_deep_scrubbing = vm.contains("allow_pool_deep_scrubbing"); + allow_pool_scrubbing = vm.contains("allow_pool_scrubbing"); - if (!dryrun) - { + if (!dryrun) { guard.emplace(boost::asio::make_work_guard(asio)); - thread = make_named_thread("io_thread",[&asio = asio] { asio.run(); }); + thread = make_named_thread("io_thread", [&asio = asio] { asio.run(); }); } show_help = vm.contains("help"); show_sequence = vm.contains("listsequence"); } -ceph::io_sequence::tester::TestRunner::~TestRunner() -{ +ceph::io_sequence::tester::TestRunner::~TestRunner() { if (!dryrun) { guard = std::nullopt; asio.stop(); @@ -553,34 +636,38 @@ ceph::io_sequence::tester::TestRunner::~TestRunner() } } -void ceph::io_sequence::tester::TestRunner::help() -{ +void ceph::io_sequence::tester::TestRunner::help() { std::cout << get_options_description() << std::endl; for (auto line : usage) { std::cout << line << std::endl; } } -void ceph::io_sequence::tester::TestRunner::list_sequence() -{ +void ceph::io_sequence::tester::TestRunner::list_sequence(bool testrecovery) { // List seqeunces - std::pair obj_size_range = sos.choose(); - for (ceph::io_exerciser::Sequence s - = ceph::io_exerciser::Sequence::SEQUENCE_BEGIN; - s < ceph::io_exerciser::Sequence::SEQUENCE_END; ++s) { - std::unique_ptr seq = - ceph::io_exerciser::IoSequence::generate_sequence(s, - obj_size_range, - seqseed.value_or(rng())); - dout(0) << s << " " << seq->get_name() << dendl; + std::pair obj_size_range = sos.choose(); + ceph::io_exerciser::Sequence s = ceph::io_exerciser::Sequence::SEQUENCE_BEGIN; + std::unique_ptr seq; + if (testrecovery) { + seq = ceph::io_exerciser::EcIoSequence::generate_sequence( + s, obj_size_range, spo.getChosenK(), spo.getChosenM(), + seqseed.value_or(rng())); + } else { + seq = ceph::io_exerciser::IoSequence::generate_sequence( + s, obj_size_range, seqseed.value_or(rng())); } + + do { + dout(0) << s << " " << seq->get_name_with_seqseed() << dendl; + s = seq->getNextSupportedSequenceId(); + } while (s != ceph::io_exerciser::Sequence::SEQUENCE_END); } -std::string ceph::io_sequence::tester::TestRunner::get_token() -{ - static std::string line; - static ceph::split split = ceph::split(""); - static ceph::spliterator tokens; +void ceph::io_sequence::tester::TestRunner::clear_tokens() { + tokens = split.end(); +} + +std::string ceph::io_sequence::tester::TestRunner::get_token() { while (line.empty() || tokens == split.end()) { if (!std::getline(std::cin, line)) { throw std::runtime_error("End of input"); @@ -591,127 +678,211 @@ std::string ceph::io_sequence::tester::TestRunner::get_token() return std::string(*tokens++); } -uint64_t ceph::io_sequence::tester::TestRunner::get_numeric_token() -{ +std::optional +ceph::io_sequence::tester::TestRunner ::get_optional_token() { + std::optional ret = std::nullopt; + if (tokens != split.end()) { + ret = std::string(*tokens++); + } + return ret; +} + +uint64_t ceph::io_sequence::tester::TestRunner::get_numeric_token() { std::string parse_error; std::string token = get_token(); uint64_t num = strict_iecstrtoll(token, &parse_error); if (!parse_error.empty()) { - throw std::runtime_error("Invalid number "+token); + throw std::runtime_error("Invalid number " + token); } return num; } -bool ceph::io_sequence::tester::TestRunner::run_test() -{ - if (show_help) - { +std::optional +ceph::io_sequence::tester::TestRunner ::get_optional_numeric_token() { + std::string parse_error; + std::optional token = get_optional_token(); + if (token) { + uint64_t num = strict_iecstrtoll(*token, &parse_error); + if (!parse_error.empty()) { + throw std::runtime_error("Invalid number " + *token); + } + return num; + } + + return std::optional(std::nullopt); +} + +bool ceph::io_sequence::tester::TestRunner::run_test() { + if (show_help) { help(); return true; - } - else if (show_sequence) - { - list_sequence(); + } else if (show_sequence) { + list_sequence(testrecovery); return true; - } - else if (interactive) - { + } else if (interactive) { return run_interactive_test(); - } - else - { + } else { return run_automated_test(); } } -bool ceph::io_sequence::tester::TestRunner::run_interactive_test() -{ +bool ceph::io_sequence::tester::TestRunner::run_interactive_test() { bool done = false; std::unique_ptr ioop; std::unique_ptr model; if (dryrun) { - model = std::make_unique(object_name, - sbs.choose(), - rng()); + model = std::make_unique( + object_name, sbs.choose(), rng()); } else { const std::string pool = spo.choose(); - model = std::make_unique(rados, asio, pool, - object_name, sbs.choose(), - rng(), 1, // 1 thread - lock, cond); + + bufferlist inbl, outbl; + auto formatter = std::make_unique(false); + + ceph::messaging::osd::OSDMapRequest osdMapRequest{pool, object_name, ""}; + int rc = send_mon_command(osdMapRequest, rados, "OSDMapRequest", inbl, + &outbl, formatter.get()); + ceph_assert(rc == 0); + + JSONParser p; + bool success = p.parse(outbl.c_str(), outbl.length()); + ceph_assert(success); + + ceph::messaging::osd::OSDMapReply reply{}; + reply.decode_json(&p); + + model = std::make_unique( + rados, asio, pool, object_name, reply.acting, sbs.choose(), rng(), + 1, // 1 thread + lock, cond); } while (!done) { const std::string op = get_token(); - if (!op.compare("done") || !op.compare("q") || !op.compare("quit")) { - ioop = ceph::io_exerciser::IoOp::generate_done(); - } else if (!op.compare("create")) { - ioop = ceph::io_exerciser::IoOp::generate_create(get_numeric_token()); - } else if (!op.compare("remove") || !op.compare("delete")) { - ioop = ceph::io_exerciser::IoOp::generate_remove(); - } else if (!op.compare("read")) { + if (op == "done" || op == "q" || op == "quit") { + ioop = ceph::io_exerciser::DoneOp::generate(); + } else if (op == "create") { + ioop = ceph::io_exerciser::CreateOp::generate(get_numeric_token()); + } else if (op == "remove" || op == "delete") { + ioop = ceph::io_exerciser::RemoveOp::generate(); + } else if (op == "read") { uint64_t offset = get_numeric_token(); uint64_t length = get_numeric_token(); - ioop = ceph::io_exerciser::IoOp::generate_read(offset, length); - } else if (!op.compare("read2")) { + ioop = ceph::io_exerciser::SingleReadOp::generate(offset, length); + } else if (op == "read2") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); uint64_t offset2 = get_numeric_token(); uint64_t length2 = get_numeric_token(); - ioop = ceph::io_exerciser::IoOp::generate_read2(offset1, length1, - offset2, length2); - } else if (!op.compare("read3")) { + ioop = DoubleReadOp::generate(offset1, length1, offset2, length2); + } else if (op == "read3") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); uint64_t offset2 = get_numeric_token(); uint64_t length2 = get_numeric_token(); uint64_t offset3 = get_numeric_token(); uint64_t length3 = get_numeric_token(); - ioop = ceph::io_exerciser::IoOp::generate_read3(offset1, length1, - offset2, length2, - offset3, length3); - } else if (!op.compare("write")) { + ioop = TripleReadOp::generate(offset1, length1, offset2, length2, offset3, + length3); + } else if (op == "write") { uint64_t offset = get_numeric_token(); uint64_t length = get_numeric_token(); - ioop = ceph::io_exerciser::IoOp::generate_write(offset, length); - } else if (!op.compare("write2")) { + ioop = SingleWriteOp::generate(offset, length); + } else if (op == "write2") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); uint64_t offset2 = get_numeric_token(); uint64_t length2 = get_numeric_token(); - ioop = ceph::io_exerciser::IoOp::generate_write2(offset1, length1, - offset2, length2); - } else if (!op.compare("write3")) { + ioop = DoubleWriteOp::generate(offset1, length1, offset2, length2); + } else if (op == "write3") { uint64_t offset1 = get_numeric_token(); uint64_t length1 = get_numeric_token(); uint64_t offset2 = get_numeric_token(); uint64_t length2 = get_numeric_token(); uint64_t offset3 = get_numeric_token(); uint64_t length3 = get_numeric_token(); - ioop = ceph::io_exerciser::IoOp::generate_write3(offset1, length1, - offset2, length2, - offset3, length3); + ioop = TripleWriteOp::generate(offset1, length1, offset2, length2, + offset3, length3); + } else if (op == "failedwrite") { + uint64_t offset = get_numeric_token(); + uint64_t length = get_numeric_token(); + ioop = SingleFailedWriteOp::generate(offset, length); + } else if (op == "failedwrite2") { + uint64_t offset1 = get_numeric_token(); + uint64_t length1 = get_numeric_token(); + uint64_t offset2 = get_numeric_token(); + uint64_t length2 = get_numeric_token(); + ioop = DoubleFailedWriteOp::generate(offset1, length1, offset2, length2); + } else if (op == "failedwrite3") { + uint64_t offset1 = get_numeric_token(); + uint64_t length1 = get_numeric_token(); + uint64_t offset2 = get_numeric_token(); + uint64_t length2 = get_numeric_token(); + uint64_t offset3 = get_numeric_token(); + uint64_t length3 = get_numeric_token(); + ioop = TripleFailedWriteOp::generate(offset1, length1, offset2, length2, + offset3, length3); + } else if (op == "injecterror") { + std::string inject_type = get_token(); + int shard = get_numeric_token(); + std::optional type = get_optional_numeric_token(); + std::optional when = get_optional_numeric_token(); + std::optional duration = get_optional_numeric_token(); + if (inject_type == "read") { + ioop = ceph::io_exerciser::InjectReadErrorOp::generate(shard, type, + when, duration); + } else if (inject_type == "write") { + ioop = ceph::io_exerciser::InjectWriteErrorOp::generate(shard, type, + when, duration); + } else { + clear_tokens(); + ioop.reset(); + dout(0) << fmt::format("Invalid error inject {}. No action performed.", + inject_type) + << dendl; + } + } else if (op == "clearinject") { + std::string inject_type = get_token(); + int shard = get_numeric_token(); + std::optional type = get_optional_numeric_token(); + if (inject_type == "read") { + ioop = + ceph::io_exerciser::ClearReadErrorInjectOp::generate(shard, type); + } else if (inject_type == "write") { + ioop = + ceph::io_exerciser::ClearWriteErrorInjectOp::generate(shard, type); + } else { + clear_tokens(); + ioop.reset(); + dout(0) << fmt::format("Invalid error inject {}. No action performed.", + inject_type) + << dendl; + } } else { - throw std::runtime_error("Invalid operation "+op); + clear_tokens(); + ioop.reset(); + dout(0) << fmt::format("Invalid op {}. No action performed.", op) + << dendl; } - dout(0) << ioop->to_string(model->get_block_size()) << dendl; - model->applyIoOp(*ioop); - done = ioop->done(); - if (!done) { - ioop = ceph::io_exerciser::IoOp::generate_barrier(); + if (ioop) { + dout(0) << ioop->to_string(model->get_block_size()) << dendl; model->applyIoOp(*ioop); + done = ioop->getOpType() == ceph::io_exerciser::OpType::Done; + if (!done) { + ioop = ceph::io_exerciser::BarrierOp::generate(); + model->applyIoOp(*ioop); + } } } return true; } -bool ceph::io_sequence::tester::TestRunner::run_automated_test() -{ +bool ceph::io_sequence::tester::TestRunner::run_automated_test() { // Create a test for each object - std::vector> test_objects; + std::vector> + test_objects; for (int obj = 0; obj < num_objects; obj++) { std::string name; @@ -721,15 +892,9 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() name = object_name + std::to_string(obj); } test_objects.push_back( - std::make_shared( - name, - rados, asio, - sbs, spo, sos, snt, ssr, - rng, lock, cond, - dryrun, verbose, - seqseed - ) - ); + std::make_shared( + name, rados, asio, sbs, spo, sos, snt, ssr, rng, lock, cond, dryrun, + verbose, seqseed, testrecovery)); } if (!dryrun) { rados.wait_for_latest_osdmap(); @@ -748,16 +913,15 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() for (auto obj = test_objects.begin(); obj != test_objects.end(); ++obj) { std::shared_ptr to = *obj; if (!to->finished()) { - lock.lock(); - bool ready = to->readyForIo(); - lock.unlock(); - if (ready) - { - to->next(); - started_io = true; - } else { - need_wait = true; - } + lock.lock(); + bool ready = to->readyForIo(); + lock.unlock(); + if (ready) { + to->next(); + started_io = true; + } else { + need_wait = true; + } } } if (!started_io && need_wait) { @@ -767,8 +931,7 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() std::shared_ptr to = *obj; if (!to->finished()) { need_wait = !to->readyForIo(); - if (!need_wait) - { + if (!need_wait) { break; } } @@ -788,18 +951,16 @@ bool ceph::io_sequence::tester::TestRunner::run_automated_test() return true; } -int main(int argc, char **argv) -{ +int main(int argc, char** argv) { auto args = argv_to_vec(argc, argv); env_to_vec(args); auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT, - CODE_ENVIRONMENT_UTILITY, 0); + CODE_ENVIRONMENT_UTILITY, 0); common_init_finish(cct.get()); po::variables_map vm; int rc = parse_io_seq_options(vm, argc, argv); - if (rc != 0) - { + if (rc != 0) { return rc; } @@ -814,7 +975,7 @@ int main(int argc, char **argv) std::unique_ptr runner; try { runner = std::make_unique(vm, rados); - } catch(const po::error& e) { + } catch (const po::error& e) { return 1; } runner->run_test(); diff --git a/src/test/osd/ceph_test_rados_io_sequence.h b/src/test/osd/ceph_test_rados_io_sequence.h index 4e21d02570004..9af5f706b2fdf 100644 --- a/src/test/osd/ceph_test_rados_io_sequence.h +++ b/src/test/osd/ceph_test_rados_io_sequence.h @@ -1,34 +1,36 @@ +#include +#include #include -#include "include/random.h" - -#include "global/global_init.h" -#include "global/global_context.h" - #include "common/io_exerciser/IoOp.h" #include "common/io_exerciser/IoSequence.h" #include "common/io_exerciser/Model.h" - +#include "common/split.h" +#include "global/global_context.h" +#include "global/global_init.h" +#include "include/random.h" #include "librados/librados_asio.h" #include #include +#include + /* Overview * * class ProgramOptionSelector - * Base class for selector objects below with common code for + * Base class for selector objects below with common code for * selecting options - * + * * class SelectObjectSize * Selects min and max object sizes for a test * * class SelectErasureKM * Selects an EC k and m value for a test - * + * * class SelectErasurePlugin * Selects an plugin for a test - * + * * class SelectECPool * Selects an EC pool (plugin,k and m) for a test. Also creates the * pool as well. @@ -58,287 +60,279 @@ namespace po = boost::program_options; -namespace ceph -{ - namespace io_sequence::tester - { - // Choices for min and max object size - inline constexpr size_t objectSizeSize = 10; - inline constexpr std::array,objectSizeSize> - objectSizeChoices = {{ - {1,32}, // Default - best for boundary checking - {12,14}, - {28,30}, - {36,38}, - {42,44}, - {52,54}, - {66,68}, - {72,74}, - {83,83}, - {97,97} - }}; - - // Choices for block size - inline constexpr int blockSizeSize = 5; - inline constexpr std::array blockSizeChoices = {{ - 2048, // Default - test boundaries for EC 4K chunk size - 512, - 3767, - 4096, - 32768 - }}; - - // Choices for number of threads - inline constexpr int threadArraySize = 4; - inline constexpr std::array threadCountChoices = {{ - 1, // Default - 2, - 4, - 8 - }}; - - // Choices for EC k+m profile - inline constexpr int kmSize = 6; - inline constexpr std::array, kmSize> kmChoices = {{ - {2,2}, // Default - reasonable coverage - {2,1}, - {2,3}, - {3,2}, - {4,2}, - {5,1} - }}; - - // Choices for EC chunk size - inline constexpr int chunkSizeSize = 3; - inline constexpr std::array chunkSizeChoices = {{ - 4*1024, - 64*1024, - 256*1024 - }}; - - // Choices for plugin - inline constexpr int pluginListSize = 2; - inline constexpr std::array pluginChoices = {{ - "jerasure", - "isa" - }}; - - inline constexpr std::array, - 0> sequencePairs = {{}}; - - inline constexpr std::array poolChoices = {{}}; - - template & Ts> - class ProgramOptionSelector - { - public: - ProgramOptionSelector(ceph::util::random_number_generator& rng, - po::variables_map vm, - const std::string& option_name, - bool set_forced, - bool select_first - ); - virtual ~ProgramOptionSelector() = default; - bool isForced(); - virtual const T choose(); - - protected: - ceph::util::random_number_generator& rng; - static constexpr std::array choices = Ts; - - std::optional force_value; - std::optional first_value; - - std::string option_name; - }; - - class SelectObjectSize - : public ProgramOptionSelector, - io_sequence::tester::objectSizeSize, - io_sequence::tester::objectSizeChoices> - { - public: - SelectObjectSize(ceph::util::random_number_generator& rng, - po::variables_map vm); - }; - - class SelectBlockSize - : public ProgramOptionSelector - { - public: - SelectBlockSize(ceph::util::random_number_generator& rng, - po::variables_map vm); - }; - - class SelectNumThreads - : public ProgramOptionSelector - { - public: - SelectNumThreads(ceph::util::random_number_generator& rng, - po::variables_map vm); - }; - - class SelectSeqRange - : public ProgramOptionSelector, - 0, io_sequence::tester::sequencePairs> - { - public: - SelectSeqRange(ceph::util::random_number_generator& rng, - po::variables_map vm); - - const std::pair choose() override; - }; - - class SelectErasureKM - : public ProgramOptionSelector, - io_sequence::tester::kmSize, - io_sequence::tester::kmChoices> - { - public: - SelectErasureKM(ceph::util::random_number_generator& rng, +namespace ceph { +namespace io_sequence::tester { +// Choices for min and max object size +inline constexpr size_t objectSizeSize = 10; +inline constexpr std::array, objectSizeSize> + objectSizeChoices = {{{1, 32}, // Default - best for boundary checking + {12, 14}, + {28, 30}, + {36, 38}, + {42, 44}, + {52, 54}, + {66, 68}, + {72, 74}, + {83, 83}, + {97, 97}}}; + +// Choices for block size +inline constexpr int blockSizeSize = 5; +inline constexpr std::array blockSizeChoices = { + {2048, // Default - test boundaries for EC 4K chunk size + 512, 3767, 4096, 32768}}; + +// Choices for number of threads +inline constexpr int threadArraySize = 4; +inline constexpr std::array threadCountChoices = { + {1, // Default + 2, 4, 8}}; + +// Choices for EC k+m profile +inline constexpr int kmSize = 6; +inline constexpr std::array, kmSize> kmChoices = { + {{2, 2}, // Default - reasonable coverage + {2, 1}, + {2, 3}, + {3, 2}, + {4, 2}, + {5, 1}}}; + +// Choices for EC chunk size +inline constexpr int chunkSizeSize = 3; +inline constexpr std::array chunkSizeChoices = { + {4 * 1024, 64 * 1024, 256 * 1024}}; + +// Choices for plugin +inline constexpr int pluginListSize = 2; +inline constexpr std::array pluginChoices = { + {"jerasure", "isa"}}; + +inline constexpr std::array< + std::pair, 0> + sequencePairs = {{}}; + +inline constexpr std::array poolChoices = {{}}; + +template & Ts> +class ProgramOptionSelector { + public: + ProgramOptionSelector(ceph::util::random_number_generator& rng, + po::variables_map vm, const std::string& option_name, + bool set_forced, bool select_first); + virtual ~ProgramOptionSelector() = default; + bool isForced(); + virtual const T choose(); + + protected: + ceph::util::random_number_generator& rng; + static constexpr std::array choices = Ts; + + std::optional force_value; + std::optional first_value; + + std::string option_name; +}; + +class SelectObjectSize + : public ProgramOptionSelector, + io_sequence::tester::objectSizeSize, + io_sequence::tester::objectSizeChoices> { + public: + SelectObjectSize(ceph::util::random_number_generator& rng, + po::variables_map vm); +}; + +class SelectBlockSize + : public ProgramOptionSelector { + public: + SelectBlockSize(ceph::util::random_number_generator& rng, + po::variables_map vm); +}; + +class SelectNumThreads + : public ProgramOptionSelector { + public: + SelectNumThreads(ceph::util::random_number_generator& rng, + po::variables_map vm); +}; + +class SelectSeqRange + : public ProgramOptionSelector< + std::pair, + 0, io_sequence::tester::sequencePairs> { + public: + SelectSeqRange(ceph::util::random_number_generator& rng, + po::variables_map vm); + + const std::pair + choose() override; +}; + +class SelectErasureKM + : public ProgramOptionSelector, + io_sequence::tester::kmSize, + io_sequence::tester::kmChoices> { + public: + SelectErasureKM(ceph::util::random_number_generator& rng, + po::variables_map vm); +}; + +class SelectErasurePlugin + : public ProgramOptionSelector { + public: + SelectErasurePlugin(ceph::util::random_number_generator& rng, po::variables_map vm); - }; - - class SelectErasurePlugin - : public ProgramOptionSelector - { - public: - SelectErasurePlugin(ceph::util::random_number_generator& rng, - po::variables_map vm); - }; - - class SelectErasureChunkSize - : public ProgramOptionSelector - { - public: - SelectErasureChunkSize(ceph::util::random_number_generator& rng, po::variables_map vm); - }; - - class SelectECPool - : public ProgramOptionSelector - { - public: - SelectECPool(ceph::util::random_number_generator& rng, - po::variables_map vm, - librados::Rados& rados, - bool dry_run); - const std::string choose() override; - - private: - void create_pool(librados::Rados& rados, - const std::string& pool_name, - const std::string& plugin, - uint64_t chunk_size, - int k, int m); - - protected: - librados::Rados& rados; - bool dry_run; - - SelectErasureKM skm; - SelectErasurePlugin spl; - SelectErasureChunkSize scs; - }; - - class TestObject - { - public: - TestObject( const std::string oid, - librados::Rados& rados, - boost::asio::io_context& asio, - ceph::io_sequence::tester::SelectBlockSize& sbs, - ceph::io_sequence::tester::SelectECPool& spl, - ceph::io_sequence::tester::SelectObjectSize& sos, - ceph::io_sequence::tester::SelectNumThreads& snt, - ceph::io_sequence::tester::SelectSeqRange& ssr, - ceph::util::random_number_generator& rng, - ceph::mutex& lock, - ceph::condition_variable& cond, - bool dryrun, - bool verbose, - std::optional seqseed); - - int get_num_io(); - bool readyForIo(); - bool next(); - bool finished(); - - protected: - std::unique_ptr exerciser_model; - std::pair obj_size_range; - std::pair seq_range; - ceph::io_exerciser::Sequence curseq; - std::unique_ptr seq; - std::unique_ptr op; - bool done; - ceph::util::random_number_generator& rng; - bool verbose; - std::optional seqseed; - }; - - class TestRunner - { - public: - TestRunner(po::variables_map& vm, librados::Rados& rados); - ~TestRunner(); - - bool run_test(); - - private: - librados::Rados& rados; - int seed; - ceph::util::random_number_generator rng; - - ceph::io_sequence::tester::SelectBlockSize sbs; - ceph::io_sequence::tester::SelectObjectSize sos; - ceph::io_sequence::tester::SelectECPool spo; - ceph::io_sequence::tester::SelectNumThreads snt; - ceph::io_sequence::tester::SelectSeqRange ssr; - - boost::asio::io_context asio; - std::thread thread; - std::optional> guard; - ceph::mutex lock = ceph::make_mutex("RadosIo::lock"); - ceph::condition_variable cond; - - bool input_valid; - - bool verbose; - bool dryrun; - std::optional seqseed; - bool interactive; - - bool show_sequence; - bool show_help; - - int num_objects; - std::string object_name; - - std::string get_token(); - uint64_t get_numeric_token(); - - bool run_automated_test(); - - bool run_interactive_test(); - - void help(); - void list_sequence(); - }; - } -} +}; + +class SelectErasureChunkSize + : public ProgramOptionSelector { + public: + SelectErasureChunkSize(ceph::util::random_number_generator& rng, + po::variables_map vm); +}; + +class SelectECPool + : public ProgramOptionSelector { + public: + SelectECPool(ceph::util::random_number_generator& rng, + po::variables_map vm, librados::Rados& rados, bool dry_run, + bool allow_pool_autoscaling, bool allow_pool_balancer, + bool allow_pool_deep_scrubbing, bool allow_pool_scrubbing, + bool test_recovery); + const std::string choose() override; + + bool get_allow_pool_autoscaling() { return allow_pool_autoscaling; } + bool get_allow_pool_balancer() { return allow_pool_balancer; } + bool get_allow_pool_deep_scrubbing() { return allow_pool_deep_scrubbing; } + bool get_allow_pool_scrubbing() { return allow_pool_scrubbing; } + int getChosenK() const { return k; } + int getChosenM() const { return m; } + + private: + void create_pool(librados::Rados& rados, const std::string& pool_name, + const std::string& plugin, uint64_t chunk_size, int k, + int m); + + protected: + librados::Rados& rados; + bool dry_run; + bool allow_pool_autoscaling; + bool allow_pool_balancer; + bool allow_pool_deep_scrubbing; + bool allow_pool_scrubbing; + bool test_recovery; + int k; + int m; + + SelectErasureKM skm; + SelectErasurePlugin spl; + SelectErasureChunkSize scs; +}; + +class TestObject { + public: + TestObject(const std::string oid, librados::Rados& rados, + boost::asio::io_context& asio, + ceph::io_sequence::tester::SelectBlockSize& sbs, + ceph::io_sequence::tester::SelectECPool& spl, + ceph::io_sequence::tester::SelectObjectSize& sos, + ceph::io_sequence::tester::SelectNumThreads& snt, + ceph::io_sequence::tester::SelectSeqRange& ssr, + ceph::util::random_number_generator& rng, ceph::mutex& lock, + ceph::condition_variable& cond, bool dryrun, bool verbose, + std::optional seqseed, bool testRecovery); + + int get_num_io(); + bool readyForIo(); + bool next(); + bool finished(); + + protected: + std::unique_ptr exerciser_model; + std::pair obj_size_range; + std::pair + seq_range; + ceph::io_exerciser::Sequence curseq; + std::unique_ptr seq; + std::unique_ptr op; + bool done; + ceph::util::random_number_generator& rng; + bool verbose; + std::optional seqseed; + int poolK; + int poolM; + bool testrecovery; +}; + +class TestRunner { + public: + TestRunner(po::variables_map& vm, librados::Rados& rados); + ~TestRunner(); + + bool run_test(); + + private: + librados::Rados& rados; + int seed; + ceph::util::random_number_generator rng; + + ceph::io_sequence::tester::SelectBlockSize sbs; + ceph::io_sequence::tester::SelectObjectSize sos; + ceph::io_sequence::tester::SelectECPool spo; + ceph::io_sequence::tester::SelectNumThreads snt; + ceph::io_sequence::tester::SelectSeqRange ssr; + + boost::asio::io_context asio; + std::thread thread; + std::optional< + boost::asio::executor_work_guard> + guard; + ceph::mutex lock = ceph::make_mutex("RadosIo::lock"); + ceph::condition_variable cond; + + bool input_valid; + + bool verbose; + bool dryrun; + std::optional seqseed; + bool interactive; + + bool testrecovery; + + bool allow_pool_autoscaling; + bool allow_pool_balancer; + bool allow_pool_deep_scrubbing; + bool allow_pool_scrubbing; + + bool show_sequence; + bool show_help; + + int num_objects; + std::string object_name; + + std::string line; + ceph::split split = ceph::split(""); + ceph::spliterator tokens; + + void clear_tokens(); + std::string get_token(); + std::optional get_optional_token(); + uint64_t get_numeric_token(); + std::optional get_optional_numeric_token(); + + bool run_automated_test(); + + bool run_interactive_test(); + + void help(); + void list_sequence(bool testrecovery); +}; +} // namespace io_sequence::tester +} // namespace ceph