diff --git a/src/catalog/manager.cpp b/src/catalog/manager.cpp index 7c17c53d7bc..4795673b7c6 100644 --- a/src/catalog/manager.cpp +++ b/src/catalog/manager.cpp @@ -47,4 +47,4 @@ void Manager::DropIndirectionArray(const oid_t oid) { void Manager::ClearIndirectionArray() { indirection_array_locator_.clear(); } } // namespace catalog -} // namespace peloton +} // namespace peloton \ No newline at end of file diff --git a/src/codegen/deleter.cpp b/src/codegen/deleter.cpp index 74591b23137..157e3534159 100644 --- a/src/codegen/deleter.cpp +++ b/src/codegen/deleter.cpp @@ -38,6 +38,7 @@ void Deleter::Delete(uint32_t tile_group_id, uint32_t tuple_offset) { auto *txn = executor_context_->GetTransaction(); auto tile_group = table_->GetTileGroupById(tile_group_id); + PELOTON_ASSERT(tile_group != nullptr); auto *tile_group_header = tile_group->GetHeader(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); diff --git a/src/codegen/inserter.cpp b/src/codegen/inserter.cpp index abe00ccbb07..108fb1fe6d3 100644 --- a/src/codegen/inserter.cpp +++ b/src/codegen/inserter.cpp @@ -34,6 +34,8 @@ void Inserter::Init(storage::DataTable *table, char *Inserter::AllocateTupleStorage() { location_ = table_->GetEmptyTupleSlot(nullptr); + PELOTON_ASSERT(location_.IsNull() == false); + // Get the tile offset assuming that it is a row store auto tile_group = table_->GetTileGroupById(location_.block); auto layout = tile_group->GetLayout(); @@ -54,8 +56,11 @@ void Inserter::Insert() { auto *txn = executor_context_->GetTransaction(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - ContainerTuple tuple( - table_->GetTileGroupById(location_.block).get(), location_.offset); + auto tile_group = table_->GetTileGroupById(location_.block).get(); + PELOTON_ASSERT(tile_group != nullptr); + + ContainerTuple tuple(tile_group, location_.offset); + ItemPointer *index_entry_ptr = nullptr; bool result = table_->InsertTuple(&tuple, location_, txn, &index_entry_ptr); if (result == false) { diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 1782f219c83..a0b808fce34 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -61,6 +61,7 @@ char *Updater::Prepare(uint32_t tile_group_id, uint32_t tuple_offset) { auto *txn = executor_context_->GetTransaction(); auto tile_group = table_->GetTileGroupById(tile_group_id).get(); + PELOTON_ASSERT(tile_group != nullptr); auto *tile_group_header = tile_group->GetHeader(); old_location_.block = tile_group_id; old_location_.offset = tuple_offset; @@ -91,6 +92,7 @@ char *Updater::PreparePK(uint32_t tile_group_id, uint32_t tuple_offset) { auto *txn = executor_context_->GetTransaction(); auto tile_group = table_->GetTileGroupById(tile_group_id).get(); + PELOTON_ASSERT(tile_group != nullptr); auto *tile_group_header = tile_group->GetHeader(); // Check ownership @@ -135,6 +137,7 @@ void Updater::Update() { table_->GetOid()); auto *txn = executor_context_->GetTransaction(); auto tile_group = table_->GetTileGroupById(old_location_.block).get(); + PELOTON_ASSERT(tile_group != nullptr); auto *tile_group_header = tile_group->GetHeader(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); // Either update in-place @@ -147,8 +150,10 @@ void Updater::Update() { } // Or, update with a new version - ContainerTuple new_tuple( - table_->GetTileGroupById(new_location_.block).get(), new_location_.offset); + auto new_tile_group = table_->GetTileGroupById(new_location_.block); + PELOTON_ASSERT(new_tile_group != nullptr); + ContainerTuple new_tuple(new_tile_group.get(), + new_location_.offset); ItemPointer *indirection = tile_group_header->GetIndirection(old_location_.offset); auto result = table_->InstallVersion(&new_tuple, target_list_, txn, @@ -171,6 +176,7 @@ void Updater::UpdatePK() { table_->GetOid()); auto *txn = executor_context_->GetTransaction(); auto tile_group = table_->GetTileGroupById(new_location_.block).get(); + PELOTON_ASSERT(tile_group != nullptr); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); // Insert a new tuple diff --git a/src/common/container/cuckoo_map.cpp b/src/common/container/cuckoo_map.cpp index b7690754c83..72bb1505f7a 100644 --- a/src/common/container/cuckoo_map.cpp +++ b/src/common/container/cuckoo_map.cpp @@ -18,6 +18,9 @@ #include "common/item_pointer.h" #include "common/logger.h" #include "common/macros.h" +#include "common/container/lock_free_queue.h" +#include "gc/recycle_stack.h" +#include "storage/data_table.h" namespace peloton { @@ -125,4 +128,7 @@ template class CuckooMap, std::shared_ptr>; // Used in StatementCacheManager template class CuckooMap; +// Used in TransactionLevelGCManager +template class CuckooMap>; + } // namespace peloton diff --git a/src/common/container/lock_free_array.cpp b/src/common/container/lock_free_array.cpp index bceb0a093ba..7680c5d30d0 100644 --- a/src/common/container/lock_free_array.cpp +++ b/src/common/container/lock_free_array.cpp @@ -53,6 +53,7 @@ template void LOCK_FREE_ARRAY_TYPE::Erase(const std::size_t &offset, const ValueType &invalid_value) { LOG_TRACE("Erase at %lu", offset); + PELOTON_ASSERT(lock_free_array.size() > offset); lock_free_array.at(offset) = invalid_value; } @@ -107,6 +108,20 @@ bool LOCK_FREE_ARRAY_TYPE::Contains(const ValueType &value) const { return exists; } +template +ssize_t LOCK_FREE_ARRAY_TYPE::Lookup(const ValueType &value) { + for (std::size_t array_itr = 0; array_itr < lock_free_array.size(); + array_itr++) { + auto array_value = lock_free_array.at(array_itr); + // Check array value + if (array_value == value) { + return array_itr; + } + } + + return -1; +} + // Explicit template instantiation template class LockFreeArray>; diff --git a/src/common/init.cpp b/src/common/init.cpp index fdc085e6ce3..d7a7d946b51 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -52,7 +52,7 @@ void PelotonInit::Initialize() { threadpool::MonoQueuePool::GetExecutionInstance().Startup(); int parallelism = (CONNECTION_THREAD_COUNT + 3) / 4; - storage::DataTable::SetActiveTileGroupCount(parallelism); + storage::DataTable::SetDefaultActiveTileGroupCount(parallelism); storage::DataTable::SetActiveIndirectionArrayCount(parallelism); // start epoch. diff --git a/src/common/internal_types.cpp b/src/common/internal_types.cpp index 855f7ef2d9b..75531b69946 100644 --- a/src/common/internal_types.cpp +++ b/src/common/internal_types.cpp @@ -2967,8 +2967,8 @@ std::string GCVersionTypeToString(GCVersionType type) { case GCVersionType::ABORT_UPDATE: { return "ABORT_UPDATE"; } - case GCVersionType::ABORT_DELETE: { - return "ABORT_DELETE"; + case GCVersionType::TOMBSTONE: { + return "TOMBSTONE"; } case GCVersionType::ABORT_INSERT: { return "ABORT_INSERT"; @@ -2997,8 +2997,8 @@ GCVersionType StringToGCVersionType(const std::string &str) { return GCVersionType::COMMIT_INS_DEL; } else if (upper_str == "ABORT_UPDATE") { return GCVersionType::ABORT_UPDATE; - } else if (upper_str == "ABORT_DELETE") { - return GCVersionType::ABORT_DELETE; + } else if (upper_str == "TOMBSTONE") { + return GCVersionType::TOMBSTONE; } else if (upper_str == "ABORT_INSERT") { return GCVersionType::ABORT_INSERT; } else if (upper_str == "ABORT_INS_DEL") { diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index 3de2620a275..0d0557eaccd 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -290,9 +290,9 @@ bool TimestampOrderingTransactionManager::PerformRead(TransactionContext *const ////////////////////////////////////////////////////////// else { PELOTON_ASSERT(current_txn->GetIsolationLevel() == - IsolationLevelType::SERIALIZABLE || - current_txn->GetIsolationLevel() == - IsolationLevelType::REPEATABLE_READS); + IsolationLevelType::SERIALIZABLE || + current_txn->GetIsolationLevel() == + IsolationLevelType::REPEATABLE_READS); oid_t tuple_id = location.offset; @@ -580,6 +580,9 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(old_location); } +// Performs Delete on a tuple that was created by the current transaction, and +// never +// installed into the database void TimestampOrderingTransactionManager::PerformDelete( TransactionContext *const current_txn, const ItemPointer &location) { PELOTON_ASSERT(!current_txn->IsReadOnly()); @@ -737,6 +740,9 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( gc_set->operator[](tile_group_id)[tuple_slot] = GCVersionType::COMMIT_DELETE; + gc_set->operator[](new_version.block)[new_version.offset] = + GCVersionType::TOMBSTONE; + log_manager.LogDelete(ItemPointer(tile_group_id, tuple_slot)); } else if (tuple_entry.second == RWType::INSERT) { @@ -855,9 +861,11 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( // before we unlink the aborted version from version list ItemPointer *index_entry_ptr = tile_group_header->GetIndirection(tuple_slot); - UNUSED_ATTRIBUTE auto res = AtomicUpdateItemPointer( - index_entry_ptr, ItemPointer(tile_group_id, tuple_slot)); - PELOTON_ASSERT(res == true); + if (index_entry_ptr) { + UNUSED_ATTRIBUTE auto res = AtomicUpdateItemPointer( + index_entry_ptr, ItemPointer(tile_group_id, tuple_slot)); + PELOTON_ASSERT(res == true); + } ////////////////////////////////////////////////// // we should set the version before releasing the lock. @@ -903,9 +911,11 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( // before we unlink the aborted version from version list ItemPointer *index_entry_ptr = tile_group_header->GetIndirection(tuple_slot); - UNUSED_ATTRIBUTE auto res = AtomicUpdateItemPointer( - index_entry_ptr, ItemPointer(tile_group_id, tuple_slot)); - PELOTON_ASSERT(res == true); + if (index_entry_ptr) { + UNUSED_ATTRIBUTE auto res = AtomicUpdateItemPointer( + index_entry_ptr, ItemPointer(tile_group_id, tuple_slot)); + PELOTON_ASSERT(res == true); + } ////////////////////////////////////////////////// // we should set the version before releasing the lock. @@ -923,7 +933,7 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( // add the version to gc set. gc_set->operator[](new_version.block)[new_version.offset] = - GCVersionType::ABORT_DELETE; + GCVersionType::TOMBSTONE; } else if (tuple_entry.second == RWType::INSERT) { tile_group_header->SetBeginCommitId(tuple_slot, MAX_CID); diff --git a/src/executor/hybrid_scan_executor.cpp b/src/executor/hybrid_scan_executor.cpp index 361358b5e7f..7d91a9b1525 100644 --- a/src/executor/hybrid_scan_executor.cpp +++ b/src/executor/hybrid_scan_executor.cpp @@ -122,6 +122,9 @@ bool HybridScanExecutor::DInit() { tile_group = table_->GetTileGroup(table_tile_group_count_ - 1); } + // TODO: Handle possibility of freed tile_group + PELOTON_ASSERT(tile_group != nullptr); + oid_t tuple_id = 0; ItemPointer location(tile_group->GetTileGroupId(), tuple_id); block_threshold = location.block; @@ -189,6 +192,10 @@ bool HybridScanExecutor::SeqScanUtil() { while (current_tile_group_offset_ < table_tile_group_count_) { LOG_TRACE("Current tile group offset : %u", current_tile_group_offset_); auto tile_group = table_->GetTileGroup(current_tile_group_offset_++); + if (tile_group == nullptr) { + continue; + } + auto tile_group_header = tile_group->GetHeader(); oid_t active_tuple_count = tile_group->GetNextTupleSlot(); @@ -380,6 +387,9 @@ bool HybridScanExecutor::ExecPrimaryIndexLookup() { auto storage_manager = storage::StorageManager::GetInstance(); auto tile_group = storage_manager->GetTileGroup(tuple_location.block); + + // TODO: Handle possibility of freed tile_group + PELOTON_ASSERT(tile_group != nullptr); auto tile_group_header = tile_group.get()->GetHeader(); // perform transaction read @@ -427,6 +437,9 @@ bool HybridScanExecutor::ExecPrimaryIndexLookup() { } tile_group = storage_manager->GetTileGroup(tuple_location.block); + + // TODO: Handle possibility of freed tile_group + PELOTON_ASSERT(tile_group != nullptr); tile_group_header = tile_group.get()->GetHeader(); } } @@ -437,6 +450,9 @@ bool HybridScanExecutor::ExecPrimaryIndexLookup() { auto storage_manager = storage::StorageManager::GetInstance(); auto tile_group = storage_manager->GetTileGroup(tuples.first); + // TODO: Handle possibility of freed tile_group + PELOTON_ASSERT(tile_group != nullptr); + std::unique_ptr logical_tile(LogicalTileFactory::GetTile()); // Add relevant columns to logical tile diff --git a/src/executor/index_scan_executor.cpp b/src/executor/index_scan_executor.cpp index 9184352dc5d..7c9d2fe1ccf 100644 --- a/src/executor/index_scan_executor.cpp +++ b/src/executor/index_scan_executor.cpp @@ -217,6 +217,7 @@ bool IndexScanExecutor::ExecPrimaryIndexLookup() { for (auto tuple_location_ptr : tuple_location_ptrs) { ItemPointer tuple_location = *tuple_location_ptr; auto tile_group = storage_manager->GetTileGroup(tuple_location.block); + PELOTON_ASSERT(tile_group != nullptr); auto tile_group_header = tile_group.get()->GetHeader(); size_t chain_length = 0; @@ -294,6 +295,7 @@ bool IndexScanExecutor::ExecPrimaryIndexLookup() { *(tile_group_header->GetIndirection(tuple_location.offset)); auto storage_manager = storage::StorageManager::GetInstance(); tile_group = storage_manager->GetTileGroup(tuple_location.block); + PELOTON_ASSERT(tile_group != nullptr); tile_group_header = tile_group.get()->GetHeader(); chain_length = 0; continue; @@ -320,6 +322,7 @@ bool IndexScanExecutor::ExecPrimaryIndexLookup() { // search for next version. auto storage_manager = storage::StorageManager::GetInstance(); tile_group = storage_manager->GetTileGroup(tuple_location.block); + PELOTON_ASSERT(tile_group != nullptr); tile_group_header = tile_group.get()->GetHeader(); continue; } @@ -355,6 +358,7 @@ bool IndexScanExecutor::ExecPrimaryIndexLookup() { // into the result vector auto storage_manager = storage::StorageManager::GetInstance(); auto tile_group = storage_manager->GetTileGroup(current_tile_group_oid); + PELOTON_ASSERT(tile_group != nullptr); std::unique_ptr logical_tile(LogicalTileFactory::GetTile()); // Add relevant columns to logical tile logical_tile->AddColumns(tile_group, full_column_ids_); @@ -374,6 +378,7 @@ bool IndexScanExecutor::ExecPrimaryIndexLookup() { // Add the remaining tuples to the result vector if ((current_tile_group_oid != INVALID_OID) && (!tuples.empty())) { auto tile_group = storage_manager->GetTileGroup(current_tile_group_oid); + PELOTON_ASSERT(tile_group != nullptr); std::unique_ptr logical_tile(LogicalTileFactory::GetTile()); // Add relevant columns to logical tile logical_tile->AddColumns(tile_group, full_column_ids_); @@ -464,6 +469,7 @@ bool IndexScanExecutor::ExecSecondaryIndexLookup() { ItemPointer tuple_location = *tuple_location_ptr; if (tuple_location.block != last_block) { tile_group = storage_manager->GetTileGroup(tuple_location.block); + PELOTON_ASSERT(tile_group != nullptr); tile_group_header = tile_group.get()->GetHeader(); } #ifdef LOG_TRACE_ENABLED @@ -565,6 +571,7 @@ bool IndexScanExecutor::ExecSecondaryIndexLookup() { tuple_location = *(tile_group_header->GetIndirection(tuple_location.offset)); tile_group = storage_manager->GetTileGroup(tuple_location.block); + PELOTON_ASSERT(tile_group != nullptr); tile_group_header = tile_group.get()->GetHeader(); chain_length = 0; continue; @@ -594,6 +601,7 @@ bool IndexScanExecutor::ExecSecondaryIndexLookup() { // search for next version. tile_group = storage_manager->GetTileGroup(tuple_location.block); + PELOTON_ASSERT(tile_group != nullptr); tile_group_header = tile_group.get()->GetHeader(); } } @@ -621,6 +629,7 @@ bool IndexScanExecutor::ExecSecondaryIndexLookup() { // Since the tile_group_oids differ, fill in the current tile group // into the result vector auto tile_group = storage_manager->GetTileGroup(current_tile_group_oid); + PELOTON_ASSERT(tile_group != nullptr); std::unique_ptr logical_tile(LogicalTileFactory::GetTile()); // Add relevant columns to logical tile logical_tile->AddColumns(tile_group, full_column_ids_); @@ -640,6 +649,7 @@ bool IndexScanExecutor::ExecSecondaryIndexLookup() { // Add the remaining tuples (if any) to the result vector if ((current_tile_group_oid != INVALID_OID) && (!tuples.empty())) { auto tile_group = storage_manager->GetTileGroup(current_tile_group_oid); + PELOTON_ASSERT(tile_group != nullptr); std::unique_ptr logical_tile(LogicalTileFactory::GetTile()); // Add relevant columns to logical tile logical_tile->AddColumns(tile_group, full_column_ids_); @@ -691,6 +701,7 @@ bool IndexScanExecutor::CheckKeyConditions(const ItemPointer &tuple_location) { auto storage_manager = storage::StorageManager::GetInstance(); auto tile_group = storage_manager->GetTileGroup(tuple_location.block); + PELOTON_ASSERT(tile_group != nullptr); ContainerTuple tuple(tile_group.get(), tuple_location.offset); diff --git a/src/executor/seq_scan_executor.cpp b/src/executor/seq_scan_executor.cpp index 5546635879a..de3413026d0 100644 --- a/src/executor/seq_scan_executor.cpp +++ b/src/executor/seq_scan_executor.cpp @@ -154,6 +154,12 @@ bool SeqScanExecutor::DExecute() { while (current_tile_group_offset_ < table_tile_group_count_) { auto tile_group = target_table_->GetTileGroup(current_tile_group_offset_++); + + if (tile_group == nullptr) { + // tile group was freed so, continue to next tile group + continue; + } + auto tile_group_header = tile_group->GetHeader(); oid_t active_tuple_count = tile_group->GetNextTupleSlot(); diff --git a/src/gc/recycle_stack.cpp b/src/gc/recycle_stack.cpp new file mode 100644 index 00000000000..7e5a5984a31 --- /dev/null +++ b/src/gc/recycle_stack.cpp @@ -0,0 +1,129 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// recycle_stack.cpp +// +// Identification: src/gc/recycle_stack.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "gc/recycle_stack.h" + +#include "common/logger.h" + +namespace peloton { + +namespace gc { + +RecycleStack::~RecycleStack() { + // acquire head lock + while (head_.lock.test_and_set(std::memory_order_acquire)) + ; + + auto curr = head_.next; + + // iterate through entire stack, remove all nodes + while (curr != nullptr) { + // acquire lock on curr + while (curr->lock.test_and_set(std::memory_order_acquire)) + ; + + head_.next = curr->next; // unlink curr + // no need to release lock on curr because no one can be waiting on it + // bceause we have lock on head_ + delete curr; + + curr = head_.next; + } + + head_.lock.clear(std::memory_order_release); +} + +void RecycleStack::Push(const ItemPointer &location) { + // acquire head lock + while (head_.lock.test_and_set(std::memory_order_acquire)) + ; + + auto node = new Node{location, head_.next, ATOMIC_FLAG_INIT}; + head_.next = node; + + head_.lock.clear(std::memory_order_release); +} + +ItemPointer RecycleStack::TryPop() { + ItemPointer location = INVALID_ITEMPOINTER; + + LOG_TRACE("Trying to pop a recycled slot"); + + // try to acquire head lock + if (!head_.lock.test_and_set(std::memory_order_acquire)) { + LOG_TRACE("Acquired head lock"); + auto node = head_.next; + if (node != nullptr) { + // try to acquire first node in list + if (!node->lock.test_and_set(std::memory_order_acquire)) { + LOG_TRACE("Acquired first node lock"); + head_.next = node->next; + location = node->location; + // no need to release lock on node because no one can be waiting on it + // because we have lock on head_ + delete node; + } + } + // release lock + head_.lock.clear(std::memory_order_release); + } + + return location; +} + +uint32_t RecycleStack::RemoveAllWithTileGroup(const oid_t &tile_group_id) { + uint32_t remove_count = 0; + + LOG_TRACE("Removing all recycled slots for TileGroup %u", tile_group_id); + + // acquire head lock + while (head_.lock.test_and_set(std::memory_order_acquire)) + ; + + auto prev = &head_; + auto curr = prev->next; + + // iterate through entire stack, remove any nodes with matching tile_group_id + while (curr != nullptr) { + // acquire lock on curr + while (curr->lock.test_and_set(std::memory_order_acquire)) + ; + + // check if we want to remove this node + if (curr->location.block == tile_group_id) { + prev->next = curr->next; // unlink curr + // no need to release lock on curr because no one can be waiting on it + // bceause we have lock on prev + delete curr; + remove_count++; + + curr = prev->next; + continue; // need to check if null and acquire lock on new curr + } + + // iterate + prev->lock.clear(std::memory_order_release); + prev = curr; + curr = prev->next; + } + + // prev was set to curr, which needs to be freed + prev->lock.clear(std::memory_order_release); + + LOG_TRACE("Removed %u recycled slots for TileGroup %u", remove_count, + tile_group_id); + + return remove_count; +} + +} // namespace gc +} // namespace peloton diff --git a/src/gc/tile_group_compactor.cpp b/src/gc/tile_group_compactor.cpp new file mode 100644 index 00000000000..82abf1a592a --- /dev/null +++ b/src/gc/tile_group_compactor.cpp @@ -0,0 +1,162 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// transaction_level_gc_manager.cpp +// +// Identification: src/gc/transaction_level_gc_manager.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "gc/tile_group_compactor.h" + +namespace peloton { +namespace gc { + +void TileGroupCompactor::CompactTileGroup(const oid_t &tile_group_id) { + LOG_TRACE("Attempting to move tuples out of tile_group %u", tile_group_id); + + size_t attempts = 0; + size_t max_attempts = 100; + + constexpr auto minPauseTime = std::chrono::microseconds(1); + constexpr auto maxPauseTime = std::chrono::microseconds(100000); + + auto pause_time = minPauseTime; + + while (attempts < max_attempts && threadpool::MonoQueuePool::GetInstance().IsRunning()) { + auto tile_group = + storage::StorageManager::GetInstance()->GetTileGroup(tile_group_id); + if (tile_group == nullptr) { + LOG_TRACE("tile_group %u no longer exists", tile_group_id); + return; // this tile group no longer exists + } + + storage::DataTable *table = + dynamic_cast(tile_group->GetAbstractTable()); + + if (table == nullptr) { + return; // this table no longer exists + } + bool success = MoveTuplesOutOfTileGroup(table, tile_group); + + if (success) { + LOG_TRACE("Moved tuples out of tile_group %u", tile_group_id); + return; + } + + LOG_TRACE("Moving tuples out of tile_group %u failed, retrying...", + tile_group_id); + // Otherwise, transaction failed, so we'll retry with exponential backoff + std::this_thread::sleep_for(pause_time); + pause_time = std::min(pause_time * 2, maxPauseTime); + } +} + +bool TileGroupCompactor::MoveTuplesOutOfTileGroup( + storage::DataTable *table, std::shared_ptr tile_group) { + auto tile_group_id = tile_group->GetTileGroupId(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto *txn = txn_manager.BeginTransaction(); + + std::unique_ptr executor_context( + new executor::ExecutorContext(txn)); + + auto tile_group_header = tile_group->GetHeader(); + PELOTON_ASSERT(tile_group_header != nullptr); + + // Construct Project Info (outside loop so only done once) + TargetList target_list; + DirectMapList direct_map_list; + size_t num_columns = table->GetSchema()->GetColumnCount(); + + for (size_t i = 0; i < num_columns; i++) { + DirectMap direct_map = std::make_pair(i, std::make_pair(0, i)); + direct_map_list.push_back(direct_map); + } + + std::unique_ptr project_info( + new planner::ProjectInfo(std::move(target_list), + std::move(direct_map_list))); + + // Update tuples in the given tile group + for (oid_t physical_tuple_id = 0; + physical_tuple_id < tile_group->GetAllocatedTupleCount(); + physical_tuple_id++) { + ItemPointer old_location(tile_group_id, physical_tuple_id); + + auto visibility = txn_manager.IsVisible( + txn, tile_group_header, physical_tuple_id, VisibilityIdType::COMMIT_ID); + if (visibility != VisibilityType::OK) { + // ignore garbage tuples because they don't prevent tile group freeing + continue; + } + + LOG_TRACE("Moving Physical Tuple id : %u ", physical_tuple_id); + + bool is_ownable = + txn_manager.IsOwnable(txn, tile_group_header, physical_tuple_id); + if (!is_ownable) { + LOG_TRACE("Failed to move tuple. Not ownable."); + txn_manager.SetTransactionResult(txn, ResultType::FAILURE); + txn_manager.AbortTransaction(txn); + return false; + } + + // if the tuple is not owned by any transaction and is visible to + // current transaction, we'll try to move it to a new tile group + bool acquired_ownership = + txn_manager.AcquireOwnership(txn, tile_group_header, physical_tuple_id); + if (!acquired_ownership) { + LOG_TRACE("Failed to move tuple. Could not acquire ownership of tuple."); + txn_manager.SetTransactionResult(txn, ResultType::FAILURE); + txn_manager.AbortTransaction(txn); + return false; + } + + txn->RecordReadOwn(old_location); + + // check again now that we have ownsership + // to ensure that this is stil the latest version + bool is_latest_version = + tile_group_header->GetPrevItemPointer(physical_tuple_id).IsNull(); + if (is_latest_version == false) { + // if a tuple is not the latest version, then there's no point in moving + // it + // this also does not conflict with our compaction operation, so don't + // abort + LOG_TRACE("Skipping tuple, not latest version."); + txn_manager.YieldOwnership(txn, tile_group_header, physical_tuple_id); + continue; + } + + ItemPointer new_location = table->AcquireVersion(); + PELOTON_ASSERT(new_location.IsNull() == false); + + auto manager = storage::StorageManager::GetInstance(); + auto new_tile_group = manager->GetTileGroup(new_location.block); + + ContainerTuple new_tuple(new_tile_group.get(), + new_location.offset); + + ContainerTuple old_tuple(tile_group.get(), + physical_tuple_id); + + project_info->Evaluate(&new_tuple, &old_tuple, nullptr, + executor_context.get()); + + LOG_TRACE("perform move old location: %u, %u", old_location.block, + old_location.offset); + LOG_TRACE("perform move new location: %u, %u", new_location.block, + new_location.offset); + txn_manager.PerformUpdate(txn, old_location, new_location); + } + + txn_manager.CommitTransaction(txn); + return true; +} + +} // namespace gc +} // namespace peloton \ No newline at end of file diff --git a/src/gc/transaction_level_gc_manager.cpp b/src/gc/transaction_level_gc_manager.cpp index 804ae21f05b..af6806c7c1d 100644 --- a/src/gc/transaction_level_gc_manager.cpp +++ b/src/gc/transaction_level_gc_manager.cpp @@ -14,10 +14,15 @@ #include "brain/query_logger.h" #include "catalog/manager.h" +#include "catalog/catalog.h" #include "common/container_tuple.h" #include "concurrency/epoch_manager_factory.h" #include "concurrency/transaction_manager_factory.h" +#include "executor/executor_context.h" +#include "executor/logical_tile.h" +#include "executor/logical_tile_factory.h" #include "index/index.h" +#include "gc/tile_group_compactor.h" #include "settings/settings_manager.h" #include "storage/database.h" #include "storage/storage_manager.h" @@ -28,10 +33,113 @@ namespace peloton { namespace gc { +TransactionLevelGCManager::TransactionLevelGCManager( + const uint32_t &thread_count) + : gc_thread_count_(thread_count), + local_unlink_queues_(thread_count), + reclaim_maps_(thread_count) { + unlink_queues_.reserve(thread_count); + + for (uint32_t i = 0; i < gc_thread_count_; ++i) { + unlink_queues_.emplace_back( + std::make_shared>( + INITIAL_UNLINK_QUEUE_LENGTH)); + } + + immutable_queue_ = + std::make_shared>(INITIAL_TG_QUEUE_LENGTH); + compaction_queue_ = + std::make_shared>(INITIAL_TG_QUEUE_LENGTH); + recycle_stacks_ = std::make_shared< + peloton::CuckooMap>>( + INITIAL_MAP_SIZE); +} + +void TransactionLevelGCManager::TransactionLevelGCManager::Reset() { + local_unlink_queues_.clear(); + local_unlink_queues_.resize(gc_thread_count_); + + reclaim_maps_.clear(); + reclaim_maps_.resize(gc_thread_count_); + + unlink_queues_.clear(); + unlink_queues_.reserve(gc_thread_count_); + + for (uint32_t i = 0; i < gc_thread_count_; ++i) { + unlink_queues_.emplace_back( + std::make_shared>( + INITIAL_UNLINK_QUEUE_LENGTH)); + } + + immutable_queue_ = + std::make_shared>(INITIAL_TG_QUEUE_LENGTH); + compaction_queue_ = + std::make_shared>(INITIAL_TG_QUEUE_LENGTH); + recycle_stacks_ = std::make_shared< + peloton::CuckooMap>>( + INITIAL_MAP_SIZE); + + is_running_ = false; + tile_group_recycling_threshold_ = settings::SettingsManager::GetDouble( + settings::SettingId::tile_group_recycling_threshold); + tile_group_freeing_ = settings::SettingsManager::GetBool( + settings::SettingId::tile_group_freeing); + tile_group_compaction_ = settings::SettingsManager::GetBool( + settings::SettingId::tile_group_compaction); +} + +TransactionLevelGCManager &TransactionLevelGCManager::GetInstance( + const uint32_t &thread_count) { + static TransactionLevelGCManager gc_manager(thread_count); + return gc_manager; +} + +void TransactionLevelGCManager::StartGC( + std::vector> &gc_threads) { + LOG_TRACE("Starting GC"); + + is_running_ = true; + gc_threads.resize(gc_thread_count_); + + for (uint32_t i = 0; i < gc_thread_count_; ++i) { + gc_threads[i].reset( + new std::thread(&TransactionLevelGCManager::Running, this, i)); + } +} + +void TransactionLevelGCManager::StartGC() { + LOG_TRACE("Starting GC"); + is_running_ = true; + + for (uint32_t i = 0; i < gc_thread_count_; ++i) { + thread_pool.SubmitDedicatedTask(&TransactionLevelGCManager::Running, this, + std::move(i)); + } +}; + +void TransactionLevelGCManager::RegisterTable(const oid_t &table_id) { + // if table already registered, ignore it + if (recycle_stacks_->Contains(table_id)) { + return; + } + // Insert a new entry for the table + auto recycle_stack = std::make_shared(); + recycle_stacks_->Insert(table_id, recycle_stack); +} + +void TransactionLevelGCManager::DeregisterTable(const oid_t &table_id) { + recycle_stacks_->Erase(table_id); +} + +// Assumes that location is a valid ItemPointer bool TransactionLevelGCManager::ResetTuple(const ItemPointer &location) { auto storage_manager = storage::StorageManager::GetInstance(); auto tile_group = storage_manager->GetTileGroup(location.block).get(); + if (tile_group == nullptr) { + return false; + } + auto tile_group_header = tile_group->GetHeader(); // Reset the header @@ -51,8 +159,8 @@ bool TransactionLevelGCManager::ResetTuple(const ItemPointer &location) { return true; } -void TransactionLevelGCManager::Running(const int &thread_id) { - PELOTON_ASSERT(is_running_ == true); +void TransactionLevelGCManager::Running(const uint32_t &thread_id) { + PELOTON_ASSERT(is_running_); uint32_t backoff_shifts = 0; while (true) { auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); @@ -65,13 +173,17 @@ void TransactionLevelGCManager::Running(const int &thread_id) { continue; } - int reclaimed_count = Reclaim(thread_id, expired_eid); + int immutable_count = ProcessImmutableQueue(); + int compaction_count = ProcessCompactionQueue(); int unlinked_count = Unlink(thread_id, expired_eid); + int reclaimed_count = Reclaim(thread_id, expired_eid); - if (is_running_ == false) { + if (!is_running_) { return; } - if (reclaimed_count == 0 && unlinked_count == 0) { + + if (immutable_count == 0 && reclaimed_count == 0 && unlinked_count == 0 && + compaction_count == 0) { // sleep at most 0.8192 s if (backoff_shifts < 13) { ++backoff_shifts; @@ -91,8 +203,8 @@ void TransactionLevelGCManager::RecycleTransaction( epoch_manager.ExitEpoch(txn->GetThreadId(), txn->GetEpochId()); - if (!txn->IsReadOnly() && \ - txn->GetResult() != ResultType::SUCCESS && txn->IsGCSetEmpty() != true) { + if (!txn->IsReadOnly() && txn->GetResult() != ResultType::SUCCESS && + !txn->IsGCSetEmpty()) { txn->SetEpochId(epoch_manager.GetNextEpochId()); } @@ -100,43 +212,43 @@ void TransactionLevelGCManager::RecycleTransaction( unlink_queues_[HashToThread(txn->GetThreadId())]->Enqueue(txn); } -int TransactionLevelGCManager::Unlink(const int &thread_id, - const eid_t &expired_eid) { - int tuple_counter = 0; +uint32_t TransactionLevelGCManager::Unlink(const uint32_t &thread_id, + const eid_t &expired_eid) { + uint32_t tuple_counter = 0; // check if any garbage can be unlinked from indexes. - // every time we garbage collect at most MAX_ATTEMPT_COUNT tuples. + // every time we garbage collect at most MAX_PROCESSED_COUNT tuples. std::vector garbages; // First iterate the local unlink queue local_unlink_queues_[thread_id].remove_if( - [&garbages, &tuple_counter, expired_eid, - this](concurrency::TransactionContext *txn_ctx) -> bool { - bool res = txn_ctx->GetEpochId() <= expired_eid; - if (res == true) { + [&garbages, &tuple_counter, expired_eid, this]( + concurrency::TransactionContext *txn_ctx) -> bool { + bool result = txn_ctx->GetEpochId() <= expired_eid; + if (result) { // unlink versions from version chain and indexes - UnlinkVersions(txn_ctx); + RemoveVersionsFromIndexes(txn_ctx); // Add to the garbage map garbages.push_back(txn_ctx); tuple_counter++; } - return res; + return result; }); - for (size_t i = 0; i < MAX_ATTEMPT_COUNT; ++i) { + for (size_t i = 0; i < MAX_PROCESSED_COUNT; ++i) { concurrency::TransactionContext *txn_ctx; // if there's no more tuples in the queue, then break. - if (unlink_queues_[thread_id]->Dequeue(txn_ctx) == false) { + if (!unlink_queues_[thread_id]->Dequeue(txn_ctx)) { break; } // Log the query into query_history_catalog if (settings::SettingsManager::GetBool(settings::SettingId::brain)) { std::vector query_strings = txn_ctx->GetQueryStrings(); - if (query_strings.size() != 0) { + if (!query_strings.empty()) { uint64_t timestamp = txn_ctx->GetTimestamp(); auto &pool = threadpool::MonoQueuePool::GetBrainInstance(); - for (auto query_string : query_strings) { + for (const auto &query_string : query_strings) { pool.SubmitTask([query_string, timestamp] { brain::QueryLogger::LogQuery(query_string, timestamp); }); @@ -153,13 +265,12 @@ int TransactionLevelGCManager::Unlink(const int &thread_id, } if (txn_ctx->GetEpochId() <= expired_eid) { - // as the global expired epoch id is no less than the garbage version's - // epoch id, it means that no active transactions can read the version. As - // a result, we can delete all the tuples from the indexes to which it - // belongs. + // since this txn's epochId is <= the global expired epoch id + // no active transactions can read the version. Asa result, + // we can delete remove all of its garbage tuples from the indexes // unlink versions from version chain and indexes - UnlinkVersions(txn_ctx); + RemoveVersionsFromIndexes(txn_ctx); // Add to the garbage map garbages.push_back(txn_ctx); tuple_counter++; @@ -170,7 +281,7 @@ int TransactionLevelGCManager::Unlink(const int &thread_id, } } // end for - // once the current epoch id is expired, then we know all the transactions + // once the current epoch is expired, we know all the transactions // that are active at this time point will be committed/aborted. // at that time point, it is safe to recycle the version. eid_t safe_expired_eid = @@ -184,9 +295,9 @@ int TransactionLevelGCManager::Unlink(const int &thread_id, } // executed by a single thread. so no synchronization is required. -int TransactionLevelGCManager::Reclaim(const int &thread_id, - const eid_t &expired_eid) { - int gc_counter = 0; +uint32_t TransactionLevelGCManager::Reclaim(const uint32_t &thread_id, + const eid_t &expired_eid) { + uint32_t gc_counter = 0; // we delete garbage in the free list auto garbage_ctx_entry = reclaim_maps_[thread_id].begin(); @@ -194,10 +305,11 @@ int TransactionLevelGCManager::Reclaim(const int &thread_id, const eid_t garbage_eid = garbage_ctx_entry->first; auto txn_ctx = garbage_ctx_entry->second; - // if the global expired epoch id is no less than the garbage version's - // epoch id, then recycle the garbage version + // if the the garbage version's epoch id is expired + // then recycle the garbage version if (garbage_eid <= expired_eid) { - AddToRecycleMap(txn_ctx); + RecycleTupleSlots(txn_ctx); + RemoveObjectLevelGarbage(txn_ctx); // Remove from the original map garbage_ctx_entry = reclaim_maps_[thread_id].erase(garbage_ctx_entry); @@ -211,48 +323,104 @@ int TransactionLevelGCManager::Reclaim(const int &thread_id, return gc_counter; } -// Multiple GC thread share the same recycle map -void TransactionLevelGCManager::AddToRecycleMap( +// Multiple GC threads share the same recycle map +void TransactionLevelGCManager::RecycleTupleSlots( concurrency::TransactionContext *txn_ctx) { + // for each tile group that this txn created garbage tuples in for (auto &entry : *(txn_ctx->GetGCSetPtr().get())) { - auto storage_manager = storage::StorageManager::GetInstance(); - auto tile_group = storage_manager->GetTileGroup(entry.first); + auto tile_group_id = entry.first; - // During the resetting, a table may be deconstructed because of the DROP - // TABLE request - if (tile_group == nullptr) { - delete txn_ctx; - return; + // recycle each garbage tuple in the tile group + for (auto &element : entry.second) { + auto offset = element.first; + RecycleTupleSlot(ItemPointer(tile_group_id, offset)); } + } +} - PELOTON_ASSERT(tile_group != nullptr); +void TransactionLevelGCManager::RecycleTupleSlot(const ItemPointer &location) { + auto tile_group_id = location.block; + auto tile_group = storage::StorageManager::GetInstance()->GetTileGroup(tile_group_id); - storage::DataTable *table = - dynamic_cast(tile_group->GetAbstractTable()); - PELOTON_ASSERT(table != nullptr); + // During the resetting, + // a table may be deconstructed because of a DROP TABLE request + if (tile_group == nullptr) { + return; + } - oid_t table_id = table->GetOid(); - auto tile_group_header = tile_group->GetHeader(); - PELOTON_ASSERT(tile_group_header != nullptr); - bool immutable = tile_group_header->GetImmutability(); + oid_t table_id = tile_group->GetTableId(); + auto table = storage::StorageManager::GetInstance()->GetTableWithOid( + tile_group->GetDatabaseId(), table_id); + if (table == nullptr) { + // Guard against the table being dropped out from under us + return; + } - for (auto &element : entry.second) { - // as this transaction has been committed, we should reclaim older - // versions. - ItemPointer location(entry.first, element.first); + auto recycle_stack = GetTableRecycleStack(table_id); + if (recycle_stack == nullptr) { + return; + } - // If the tuple being reset no longer exists, just skip it - if (ResetTuple(location) == false) { - continue; - } - // if immutable is false and the entry for table_id exists. - if ((!immutable) && - recycle_queue_map_.find(table_id) != recycle_queue_map_.end()) { - recycle_queue_map_[table_id]->Enqueue(location); - } + // If the tuple being reset no longer exists, just skip it + if (!ResetTuple(location)) { + return; + } + + auto tile_group_header = tile_group->GetHeader(); + PELOTON_ASSERT(tile_group_header != nullptr); + if (tile_group_header == nullptr) { + return; + } + + tile_group_header->IncrementGCReaders(); + auto num_recycled = tile_group_header->IncrementRecycled() + 1; + auto tuples_per_tile_group = table->GetTuplesPerTileGroup(); + bool immutable = tile_group_header->GetImmutability(); + auto max_recycled = static_cast(tuples_per_tile_group * + GetTileGroupRecyclingThreshold()); + + // check if tile group should no longer be recycled from, and potentially + // compacted + if (!immutable && num_recycled >= max_recycled && + !table->IsActiveTileGroup(tile_group_id) && GetTileGroupFreeing()) { + LOG_TRACE("Setting tile_group %u to immutable", tile_group_id); + tile_group_header->SetImmutabilityWithoutNotifyingGC(); + LOG_TRACE("Purging tile_group %u recycled slots", tile_group_id); + recycle_stack->RemoveAllWithTileGroup(tile_group_id); + + immutable = true; + + // create task to compact this tile group + // add to the worker queue + if (GetTileGroupCompaction()) { + LOG_TRACE("Adding tile_group %u to compaction queue", tile_group_id); + AddToCompactionQueue(tile_group_id); } } + if (!immutable) { + // this slot should be recycled, add it to the recycle stack + recycle_stack->Push(location); + } + + // if this is the last remaining tuple recycled, free tile group + if (num_recycled == tuples_per_tile_group && GetTileGroupFreeing()) { + // Spin here until the other GC threads stop operating on this TileGroup + while (tile_group_header->GetGCReaders() > 1) + ; + LOG_TRACE("Dropping tile_group %u", tile_group_id); + table->DropTileGroup(tile_group_id); + } + + tile_group_header->DecrementGCReaders(); + + LOG_TRACE("Recycled tuple slot count for tile_group %u is %zu", tile_group_id, + tile_group_header->GetNumRecycled()); +} + +void TransactionLevelGCManager::RemoveObjectLevelGarbage( + concurrency::TransactionContext *txn_ctx) { + // Perform object-level GC (e.g. dropped tables, indexes, databases) auto storage_manager = storage::StorageManager::GetInstance(); for (auto &entry : *(txn_ctx->GetGCObjectSetPtr().get())) { oid_t database_oid = std::get<0>(entry); @@ -263,77 +431,112 @@ void TransactionLevelGCManager::AddToRecycleMap( PELOTON_ASSERT(database != nullptr); if (table_oid == INVALID_OID) { storage_manager->RemoveDatabaseFromStorageManager(database_oid); + LOG_TRACE("GCing database %u", database_oid); continue; } auto table = database->GetTableWithOid(table_oid); PELOTON_ASSERT(table != nullptr); if (index_oid == INVALID_OID) { database->DropTableWithOid(table_oid); - LOG_DEBUG("GCing table %u", table_oid); + LOG_TRACE("GCing table %u", table_oid); continue; } auto index = table->GetIndexWithOid(index_oid); PELOTON_ASSERT(index != nullptr); table->DropIndexWithOid(index_oid); - LOG_DEBUG("GCing index %u", index_oid); + LOG_TRACE("GCing index %u", index_oid); } delete txn_ctx; } -// this function returns a free tuple slot, if one exists -// called by data_table. -ItemPointer TransactionLevelGCManager::ReturnFreeSlot(const oid_t &table_id) { - // for catalog tables, we directly return invalid item pointer. - if (recycle_queue_map_.find(table_id) == recycle_queue_map_.end()) { +// looks for a free tuple slot that can now be reused +// called by data_table, which passes in a pointer to itself +ItemPointer TransactionLevelGCManager::GetRecycledTupleSlot( + storage::DataTable *table) { + if (table == nullptr) { return INVALID_ITEMPOINTER; } - ItemPointer location; - PELOTON_ASSERT(recycle_queue_map_.find(table_id) != recycle_queue_map_.end()); - auto recycle_queue = recycle_queue_map_[table_id]; - if (recycle_queue->Dequeue(location) == true) { - LOG_TRACE("Reuse tuple(%u, %u) in table %u", location.block, - location.offset, table_id); - return location; + auto table_id = table->GetOid(); + auto recycle_stack = GetTableRecycleStack(table_id); + if (recycle_stack == nullptr) { + // Table does not have a recycle stack, likely a catalog table + LOG_TRACE("No recycle queue for table %u", table_id); + return INVALID_ITEMPOINTER; } - return INVALID_ITEMPOINTER; + + // Try to get a slot that can be recycled + ItemPointer location = recycle_stack->TryPop(); + if (location.IsNull()) { + LOG_TRACE("Failed to reuse tuple slot for table %u", table_id); + return INVALID_ITEMPOINTER; + } + + LOG_TRACE("Reuse tuple(%u, %u) for table %u", location.block, location.offset, + table_id); + + auto tile_group_id = location.block; + + auto tile_group = table->GetTileGroupById(tile_group_id); + PELOTON_ASSERT(tile_group != nullptr); + + auto tile_group_header = tile_group->GetHeader(); + PELOTON_ASSERT(tile_group_header != nullptr); + + tile_group_header->DecrementRecycled(); + return location; } -void TransactionLevelGCManager::ClearGarbage(int thread_id) { +void TransactionLevelGCManager::ClearGarbage(const uint32_t &thread_id) { + // order matters + + while (!immutable_queue_->IsEmpty()) { + ProcessImmutableQueue(); + } + + while (!compaction_queue_->IsEmpty()) { + ProcessCompactionQueue(); + } + while (!unlink_queues_[thread_id]->IsEmpty() || !local_unlink_queues_[thread_id].empty()) { Unlink(thread_id, MAX_CID); } - while (reclaim_maps_[thread_id].size() != 0) { + while (!reclaim_maps_[thread_id].empty()) { Reclaim(thread_id, MAX_CID); } - - return; } void TransactionLevelGCManager::StopGC() { LOG_TRACE("Stopping GC"); this->is_running_ = false; // clear the garbage in each GC thread - for (int thread_id = 0; thread_id < gc_thread_count_; ++thread_id) { + for (uint32_t thread_id = 0; thread_id < gc_thread_count_; ++thread_id) { ClearGarbage(thread_id); } } -void TransactionLevelGCManager::UnlinkVersions( +void TransactionLevelGCManager::RemoveVersionsFromIndexes( concurrency::TransactionContext *txn_ctx) { + // for each tile group that this txn created garbage tuples in for (auto entry : *(txn_ctx->GetGCSetPtr().get())) { - for (auto &element : entry.second) { - UnlinkVersion(ItemPointer(entry.first, element.first), element.second); + auto tile_group_id = entry.first; + auto garbage_tuples = entry.second; + + // for each garbage tuple in the tile group + for (auto &element : garbage_tuples) { + auto offset = element.first; + auto gc_type = element.second; + RemoveVersionFromIndexes(ItemPointer(tile_group_id, offset), gc_type); } } } -// delete a tuple from all its indexes it belongs to. -void TransactionLevelGCManager::UnlinkVersion(const ItemPointer location, - GCVersionType type) { +// unlink garbage tuples and update indexes appropriately (according to gc type) +void TransactionLevelGCManager::RemoveVersionFromIndexes( + const ItemPointer &location, const GCVersionType &type) { // get indirection from the indirection array. auto tile_group = storage::StorageManager::GetInstance()->GetTileGroup(location.block); @@ -344,9 +547,7 @@ void TransactionLevelGCManager::UnlinkVersion(const ItemPointer location, return; } - auto tile_group_header = - storage::StorageManager::GetInstance()->GetTileGroup(location.block)->GetHeader(); - + auto tile_group_header = tile_group->GetHeader(); ItemPointer *indirection = tile_group_header->GetIndirection(location.offset); // do nothing if indirection is null @@ -357,40 +558,107 @@ void TransactionLevelGCManager::UnlinkVersion(const ItemPointer location, ContainerTuple current_tuple(tile_group.get(), location.offset); - storage::DataTable *table = + auto table = dynamic_cast(tile_group->GetAbstractTable()); - PELOTON_ASSERT(table != nullptr); + if (table == nullptr) { + // guard against table being GC'd by another GC thread + return; + } - // NOTE: for now, we only consider unlinking tuple versions from primary - // indexes. if (type == GCVersionType::COMMIT_UPDATE) { // the gc'd version is an old version. - // this version needs to be reclaimed by the GC. - // if the version differs from the previous one in some columns where - // secondary indexes are built on, then we need to unlink the previous - // version from the secondary index. - } else if (type == GCVersionType::COMMIT_DELETE) { - // the gc'd version is an old version. - // need to recycle this version as well as its newer (empty) version. - // we also need to delete the tuple from the primary and secondary - // indexes. + // this old version needs to be reclaimed by the GC. + // if this old version differs from the newest version in some columns that + // secondary indexes are built on, then we need to delete this old version + // from those secondary indexes + ContainerTuple older_tuple(tile_group.get(), + location.offset); + + ItemPointer newer_location = + tile_group_header->GetPrevItemPointer(location.offset); + + if (newer_location == INVALID_ITEMPOINTER) { + return; + } + + auto newer_tile_group = + storage::StorageManager::GetInstance()->GetTileGroup(newer_location.block); + ContainerTuple newer_tuple(newer_tile_group.get(), + newer_location.offset); + // remove the older version from all the indexes + // where it no longer matches the newer version + for (uint32_t idx = 0; idx < table->GetIndexCount(); ++idx) { + auto index = table->GetIndex(idx); + if (index == nullptr) continue; + auto index_schema = index->GetKeySchema(); + auto indexed_columns = index_schema->GetIndexedColumns(); + + // build keys + std::unique_ptr older_key( + new storage::Tuple(index_schema, true)); + older_key->SetFromTuple(&older_tuple, indexed_columns, index->GetPool()); + std::unique_ptr newer_key( + new storage::Tuple(index_schema, true)); + newer_key->SetFromTuple(&newer_tuple, indexed_columns, index->GetPool()); + + // if older_key is different, delete it from index + if (newer_key->Compare(*older_key) != 0) { + index->DeleteEntry(older_key.get(), indirection); + } + } } else if (type == GCVersionType::ABORT_UPDATE) { // the gc'd version is a newly created version. // if the version differs from the previous one in some columns where // secondary indexes are built on, then we need to unlink this version // from the secondary index. + ContainerTuple newer_tuple(tile_group.get(), + location.offset); + + ItemPointer older_location = + tile_group_header->GetNextItemPointer(location.offset); + + if (older_location == INVALID_ITEMPOINTER) { + return; + } + + auto older_tile_group = + storage::StorageManager::GetInstance()->GetTileGroup(older_location.block); + ContainerTuple older_tuple(older_tile_group.get(), + older_location.offset); + // remove the newer version from all the indexes + // where it no longer matches the older version + for (uint32_t idx = 0; idx < table->GetIndexCount(); ++idx) { + auto index = table->GetIndex(idx); + if (index == nullptr) continue; + auto index_schema = index->GetKeySchema(); + auto indexed_columns = index_schema->GetIndexedColumns(); - } else if (type == GCVersionType::ABORT_DELETE) { + // build keys + std::unique_ptr older_key( + new storage::Tuple(index_schema, true)); + older_key->SetFromTuple(&older_tuple, indexed_columns, index->GetPool()); + std::unique_ptr newer_key( + new storage::Tuple(index_schema, true)); + newer_key->SetFromTuple(&newer_tuple, indexed_columns, index->GetPool()); + + // if newer_key is different, delete it from index + if (newer_key->Compare(*older_key) != 0) { + index->DeleteEntry(newer_key.get(), indirection); + } + } + + } else if (type == GCVersionType::TOMBSTONE) { // the gc'd version is a newly created empty version. // need to recycle this version. // no index manipulation needs to be made. } else { PELOTON_ASSERT(type == GCVersionType::ABORT_INSERT || type == GCVersionType::COMMIT_INS_DEL || - type == GCVersionType::ABORT_INS_DEL); + type == GCVersionType::ABORT_INS_DEL || + type == GCVersionType::COMMIT_DELETE); // attempt to unlink the version from all the indexes. - for (size_t idx = 0; idx < table->GetIndexCount(); ++idx) { + for (uint32_t idx = 0; idx < table->GetIndexCount(); ++idx) { auto index = table->GetIndex(idx); if (index == nullptr) continue; auto index_schema = index->GetKeySchema(); @@ -407,5 +675,80 @@ void TransactionLevelGCManager::UnlinkVersion(const ItemPointer location, } } +inline unsigned int TransactionLevelGCManager::HashToThread( + const size_t &thread_id) { + return (unsigned int)thread_id % gc_thread_count_; +} + +std::shared_ptr TransactionLevelGCManager::GetTableRecycleStack( + const oid_t &table_id) const { + std::shared_ptr recycle_stack; + if (recycle_stacks_->Find(table_id, recycle_stack)) { + return recycle_stack; + } else { + return nullptr; + } +} + +uint32_t TransactionLevelGCManager::ProcessImmutableQueue() { + uint32_t num_processed = 0; + oid_t tile_group_id; + + for (size_t i = 0; i < MAX_PROCESSED_COUNT; ++i) { + // if there are no more tile_groups in the queue, then break. + if (!immutable_queue_->Dequeue(tile_group_id)) { + break; + } + + auto tile_group = + storage::StorageManager::GetInstance()->GetTileGroup(tile_group_id); + if (tile_group == nullptr) { + continue; + } + + oid_t table_id = tile_group->GetTableId(); + auto recycle_stack = GetTableRecycleStack(table_id); + if (recycle_stack == nullptr) { + continue; + } + + recycle_stack->RemoveAllWithTileGroup(tile_group_id); + num_processed++; + } + return num_processed; +} + +void TransactionLevelGCManager::AddToImmutableQueue( + const oid_t &tile_group_id) { + immutable_queue_->Enqueue(tile_group_id); +} + +void TransactionLevelGCManager::AddToCompactionQueue( + const oid_t &tile_group_id) { + compaction_queue_->Enqueue(tile_group_id); +} + +uint32_t TransactionLevelGCManager::ProcessCompactionQueue() { + uint32_t num_processed = 0; + oid_t tile_group_id; + + for (size_t i = 0; i < MAX_PROCESSED_COUNT; ++i) { + // if there are no more tile_groups in the queue, then break. + if (!compaction_queue_->Dequeue(tile_group_id)) { + break; + } + + // Submit task to compact this tile group asynchronously + // Task is responsible for ensuring the tile group still exists + // when it runs + auto &pool = threadpool::MonoQueuePool::GetInstance(); + pool.SubmitTask([tile_group_id] { + TileGroupCompactor::CompactTileGroup(tile_group_id); + }); + num_processed++; + } + return num_processed; +} + } // namespace gc -} // namespace peloton +} // namespace peloton \ No newline at end of file diff --git a/src/include/catalog/manager.h b/src/include/catalog/manager.h index 87aee3fcc2e..db39dca80d5 100644 --- a/src/include/catalog/manager.h +++ b/src/include/catalog/manager.h @@ -75,4 +75,4 @@ class Manager { }; } // namespace catalog -} // namespace peloton +} // namespace peloton \ No newline at end of file diff --git a/src/include/common/container/lock_free_array.h b/src/include/common/container/lock_free_array.h index a0d79ad8be7..ba715c0e851 100644 --- a/src/include/common/container/lock_free_array.h +++ b/src/include/common/container/lock_free_array.h @@ -97,6 +97,13 @@ class LockFreeArray { */ bool Contains(const ValueType &value) const; + /** + * Finds the offset of an element given its value + * @param value Element to search the array for + * @return -1 if element not found, offset of element otherwise + */ + ssize_t Lookup(const ValueType &value); + private: // lock free array tbb::concurrent_vector> diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 22598226407..f0f04613873 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -1235,9 +1235,9 @@ enum class GCVersionType { COMMIT_DELETE, // a version that is deleted during txn commit. COMMIT_INS_DEL, // a version that is inserted and deleted during txn commit. ABORT_UPDATE, // a version that is updated during txn abort. - ABORT_DELETE, // a version that is deleted during txn abort. ABORT_INSERT, // a version that is inserted during txn abort. - ABORT_INS_DEL, // a version that is inserted and deleted during txn commit. + ABORT_INS_DEL, // a version that is inserted and deleted during txn abort. + TOMBSTONE, // a version that signifies that the tuple has been deleted. }; std::string GCVersionTypeToString(GCVersionType type); GCVersionType StringToGCVersionType(const std::string &str); diff --git a/src/include/gc/gc_manager.h b/src/include/gc/gc_manager.h index 1adce28a944..d373df7e159 100644 --- a/src/include/gc/gc_manager.h +++ b/src/include/gc/gc_manager.h @@ -20,6 +20,8 @@ #include "common/logger.h" #include "common/macros.h" #include "common/internal_types.h" +#include "settings/settings_manager.h" +#include "storage/data_table.h" namespace peloton { @@ -44,7 +46,14 @@ class GCManager { GCManager(GCManager &&) = delete; GCManager &operator=(GCManager &&) = delete; - GCManager() : is_running_(false) {} + GCManager() + : is_running_(false), + tile_group_recycling_threshold_(settings::SettingsManager::GetDouble( + settings::SettingId::tile_group_recycling_threshold)), + tile_group_freeing_(settings::SettingsManager::GetBool( + settings::SettingId::tile_group_freeing)), + tile_group_compaction_(settings::SettingsManager::GetBool( + settings::SettingId::tile_group_compaction)) {} virtual ~GCManager() {} @@ -53,10 +62,18 @@ class GCManager { return gc_manager; } - virtual void Reset() { is_running_ = false; } + virtual void Reset() { + is_running_ = false; + tile_group_recycling_threshold_ = settings::SettingsManager::GetDouble( + settings::SettingId::tile_group_recycling_threshold); + tile_group_freeing_ = settings::SettingsManager::GetBool( + settings::SettingId::tile_group_freeing); + tile_group_compaction_ = settings::SettingsManager::GetBool( + settings::SettingId::tile_group_compaction); + } // Get status of whether GC thread is running or not - bool GetStatus() { return this->is_running_; } + bool GetStatus() { return is_running_; } virtual void StartGC( std::vector> &UNUSED_ATTRIBUTE) {} @@ -65,10 +82,13 @@ class GCManager { virtual void StopGC() {} - virtual ItemPointer ReturnFreeSlot(const oid_t &table_id UNUSED_ATTRIBUTE) { + virtual ItemPointer GetRecycledTupleSlot(storage::DataTable *table + UNUSED_ATTRIBUTE) { return INVALID_ITEMPOINTER; } + virtual void RecycleTupleSlot(const ItemPointer &location UNUSED_ATTRIBUTE) {} + virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} @@ -78,12 +98,33 @@ class GCManager { virtual void RecycleTransaction( concurrency::TransactionContext *txn UNUSED_ATTRIBUTE) {} + virtual void AddToImmutableQueue(const oid_t &tile_group_id + UNUSED_ATTRIBUTE) {} + + virtual void SetTileGroupRecyclingThreshold(const double &threshold + UNUSED_ATTRIBUTE) {} + + virtual double GetTileGroupRecyclingThreshold() const { + return tile_group_recycling_threshold_; + } + + virtual void SetTileGroupFreeing(const bool &free UNUSED_ATTRIBUTE) {} + + virtual bool GetTileGroupFreeing() const { return tile_group_freeing_; } + + virtual void SetTileGroupCompaction(const bool &compact UNUSED_ATTRIBUTE) {} + + virtual bool GetTileGroupCompaction() const { return tile_group_compaction_; } + protected: void CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, oid_t tuple_id); protected: volatile bool is_running_; + volatile double tile_group_recycling_threshold_; + volatile bool tile_group_freeing_; + volatile bool tile_group_compaction_; }; } // namespace gc diff --git a/src/include/gc/recycle_stack.h b/src/include/gc/recycle_stack.h new file mode 100644 index 00000000000..eba06f281c1 --- /dev/null +++ b/src/include/gc/recycle_stack.h @@ -0,0 +1,95 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// recycle_stack.h +// +// Identification: src/include/gc/recycle_stack.h +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include + +#include "common/item_pointer.h" + +namespace peloton { + +namespace gc { + +/** + * @brief Concurrent stack for the Garbage Collector to store recycled tuples + * + * The goals of this structure are: + * -# Provide fast, best-effort removal of recycled ItemPointers on the + * critical path for worker threads + * -# Provide possibly-slow, guaranteed insertion of recycled ItemPointers + * for background GC threads + * -# Provide possibly-slow, guaranteed correct removal of recycled + * ItemPointers for background GC threads + * -# Thread-safe for multiple GC threads and multiple worker threads + */ +class RecycleStack { + public: + /** + * @return Empty RecycleStack + */ + RecycleStack(){}; + + /** + * @brief Removes all elements from the stack + */ + ~RecycleStack(); + + /** + * @brief Adds the provided ItemPointer to the top of the stack + * + * Intended for the Garbage Collector to use in a background thread so + * performance is not critical. Will spin on the head lock until acquired + * + * @param location[in] ItemPointer to be added to the top of the stack + */ + void Push(const ItemPointer &location); + + /** + * @brief Attempts to remove an ItemPointer from the top of the stack + * + * Intended for the critical path during Insert by a worker thread so + * performance is critical. Will not spin on the head lock until acquired + * + * @return ItemPointer from the top of the stack. Can be an + * INVALID_ITEMPOINTER if stack is empty or failed to acquire head lock + */ + ItemPointer TryPop(); + + /** + * @brief Removes all ItemPointers from the RecycleStack belonging to + * the provded TileGroup oid. + * + * Intended for the Garbage Collector to use in a background thread + * so performance is not critical. Will spin on the head lock until + * acquired, and then iterate through the Recycle Stack using hand + * over hand locking + * + * @param[in] tile_group_id The global oid of the TileGroup that + * should have all of its ItemPointers removed from the RecycleStack + * + * @return Number of elements removed from the stack. Useful for + * debugging + */ + uint32_t RemoveAllWithTileGroup(const oid_t &tile_group_id); + + private: + struct Node { + ItemPointer location; + Node *next; + std::atomic_flag lock; + }; + + Node head_{INVALID_ITEMPOINTER, nullptr, ATOMIC_FLAG_INIT}; +}; // class RecycleStack +} // namespace gc +} // namespace peloton diff --git a/src/include/gc/tile_group_compactor.h b/src/include/gc/tile_group_compactor.h new file mode 100644 index 00000000000..3d21a124a0c --- /dev/null +++ b/src/include/gc/tile_group_compactor.h @@ -0,0 +1,77 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// tile_group_compactor.h +// +// Identification: src/include/gc/transaction_level_gc_manager.h +// +// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include +#include +#include +#include +#include + +#include "brain/query_logger.h" +#include "catalog/manager.h" +#include "catalog/catalog.h" +#include "common/container_tuple.h" +#include "common/logger.h" +#include "common/thread_pool.h" +#include "common/internal_types.h" +#include "concurrency/transaction_context.h" +#include "concurrency/transaction_manager_factory.h" +#include "executor/executor_context.h" +#include "index/index.h" +#include "settings/settings_manager.h" +#include "storage/database.h" +#include "storage/storage_manager.h" +#include "storage/tile_group.h" +#include "storage/tuple.h" +#include "threadpool/mono_queue_pool.h" + +namespace peloton { + +namespace gc { + +class TileGroupCompactor { + public: + /** + * @brief Repeatedly tries to move all of the tuples out of a TileGroup + * in a transactional manner. + * + * This function is intended to be added to the MonoQueuePool as a task. + * + * @param[in] tile_group_id Global oid of the TileGroup to be compacted. + * TileGroup should be marked as immutable first otherwise tuples may + * be reinserted into the same TileGroup. + */ + static void CompactTileGroup(const oid_t &tile_group_id); + + /** + * @brief Creates a transaction and performs Updates on each visible + * tuple with the same tuple contents. + * + * The net effect is that all visible tuples are reinserted into the + * table in other TileGroups. + * Intended to be used by CompactTileGroup(), but can also be modified + * to handle online schema changes. + * + * @param[in] table Pointer to the table for this request + * @param[in] tile_group Smart pointer to the TileGroup for this request. + * TileGroup should be marked as immutable first otherwise tuples may + * be reinserted into the same TileGroup. + * @return True if transaction succeeds, false otherwise + */ + static bool MoveTuplesOutOfTileGroup( + storage::DataTable *table, + std::shared_ptr tile_group); +}; +} +} // namespace peloton diff --git a/src/include/gc/transaction_level_gc_manager.h b/src/include/gc/transaction_level_gc_manager.h index 89704685d39..c78e68218be 100644 --- a/src/include/gc/transaction_level_gc_manager.h +++ b/src/include/gc/transaction_level_gc_manager.h @@ -6,7 +6,7 @@ // // Identification: src/include/gc/transaction_level_gc_manager.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-18, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -18,150 +18,284 @@ #include #include +#include "common/container/cuckoo_map.h" +#include "common/container/lock_free_queue.h" #include "common/init.h" #include "common/logger.h" #include "common/thread_pool.h" +#include "common/internal_types.h" #include "concurrency/transaction_context.h" #include "gc/gc_manager.h" -#include "common/internal_types.h" - -#include "common/container/lock_free_queue.h" +#include "gc/recycle_stack.h" namespace peloton { + namespace gc { -#define MAX_QUEUE_LENGTH 100000 -#define MAX_ATTEMPT_COUNT 100000 +static constexpr size_t INITIAL_UNLINK_QUEUE_LENGTH = 100000; +static constexpr size_t INITIAL_TG_QUEUE_LENGTH = 1000; +static constexpr size_t INITIAL_MAP_SIZE = 256; +static constexpr size_t MAX_PROCESSED_COUNT = 100000; class TransactionLevelGCManager : public GCManager { public: - TransactionLevelGCManager(const int thread_count) - : gc_thread_count_(thread_count), reclaim_maps_(thread_count) { - unlink_queues_.reserve(thread_count); - for (int i = 0; i < gc_thread_count_; ++i) { - std::shared_ptr> - unlink_queue(new LockFreeQueue( - MAX_QUEUE_LENGTH)); - unlink_queues_.push_back(unlink_queue); - local_unlink_queues_.emplace_back(); - } - } - - virtual ~TransactionLevelGCManager() {} - - // this function cleans up all the member variables in the class object. - virtual void Reset() override { - unlink_queues_.clear(); - local_unlink_queues_.clear(); - - unlink_queues_.reserve(gc_thread_count_); - for (int i = 0; i < gc_thread_count_; ++i) { - std::shared_ptr> - unlink_queue(new LockFreeQueue( - MAX_QUEUE_LENGTH)); - unlink_queues_.push_back(unlink_queue); - local_unlink_queues_.emplace_back(); - } - - reclaim_maps_.clear(); - reclaim_maps_.resize(gc_thread_count_); - recycle_queue_map_.clear(); + /** + * @brief TransactionLevelGCManager should be created with GetInstance() + */ + TransactionLevelGCManager() = delete; - is_running_ = false; - } + /** + * @brief Resets member variables and data structures to defaults. + * + * Intended for testing purposes only. + * + * @warning This leaks tuple slots, txns, etc. if StopGC() not called first! + */ + virtual void Reset() override; - static TransactionLevelGCManager &GetInstance(const int thread_count = 1) { - static TransactionLevelGCManager gc_manager(thread_count); - return gc_manager; - } + /** + * + * @param[in] thread_count Number of Garbage Collector threads + * @return Singleton instance of the TransactionLevelGCManager + */ + static TransactionLevelGCManager &GetInstance( + const uint32_t &thread_count = 1); virtual void StartGC( - std::vector> &gc_threads) override { - LOG_TRACE("Starting GC"); - this->is_running_ = true; - gc_threads.resize(gc_thread_count_); - for (int i = 0; i < gc_thread_count_; ++i) { - gc_threads[i].reset( - new std::thread(&TransactionLevelGCManager::Running, this, i)); - } - } + std::vector> &gc_threads) override; - virtual void StartGC() override { - LOG_TRACE("Starting GC"); - this->is_running_ = true; - for (int i = 0; i < gc_thread_count_; ++i) { - thread_pool.SubmitDedicatedTask(&TransactionLevelGCManager::Running, this, - std::move(i)); - } - }; + /** + * @brief Launches GC threads + */ + virtual void StartGC() override; /** - * @brief This stops the Garbage Collector when Peloton shuts down - * - * @return No return value. + * @brief Clears garbage for each GC thread and then ends the threads */ virtual void StopGC() override; + /** + * @brief Registers the provided table with the GC to recycle its + * tuple slots + * @param[in] table_id Global oid of the table to start recycling + * slots for + */ + virtual void RegisterTable(const oid_t &table_id) override; + + /** + * @brief Deregisters the provided table with the GC to recycle its + * tuple slots + * @param[in] table_id Global oid of the table to stop recycling + * slots for + */ + virtual void DeregisterTable(const oid_t &table_id) override; + + /** + * @brief Passes a transaction's context to the GC for freeing and + * possible recycling + * @param[id] txn TransactionContext pointer for the GC to process. + * @warning txn will be freed by the GC, so do not dereference it + * after calling this function with txn + */ virtual void RecycleTransaction( concurrency::TransactionContext *txn) override; - virtual ItemPointer ReturnFreeSlot(const oid_t &table_id) override; + /** + * @brief Attempt to get a recycled ItemPointer for this table from the GC + * @param[in] table Pointer of the table to return a recycled ItemPointer for + * @return ItemPointer to a recycled tuple slot on success, + * INVALID_ITEMPOINTER + * otherwise + */ + virtual ItemPointer GetRecycledTupleSlot(storage::DataTable *table) override; - virtual void RegisterTable(const oid_t &table_id) override { - // Insert a new entry for the table - if (recycle_queue_map_.find(table_id) == recycle_queue_map_.end()) { - std::shared_ptr> recycle_queue( - new LockFreeQueue(MAX_QUEUE_LENGTH)); - recycle_queue_map_[table_id] = recycle_queue; - } + /** + * @brief Recycle the provided tuple slot. May trigger TileGroup compaction or + * TileGroup freeing if enabled + * @param[id] location ItemPointer of the tuple slot to be recycled + */ + virtual void RecycleTupleSlot(const ItemPointer &location) override; + + /** + * + * @return Number of tables currently registered with the GC for recycling + */ + virtual size_t GetTableCount() override { return recycle_stacks_->GetSize(); } + + /** + * @brief Process unlink queue for provided thread + * @param[id] thread_id Zero-indexed thread id to access unlink queue + * @param[id] expired_eid Expired epoch from the EpochManager + * @return Number of processed tuples + */ + uint32_t Unlink(const uint32_t &thread_id, const eid_t &expired_eid); + + /** + * @brief Process reclaim queue for provided thread + * @param[id] thread_id Zero-indexed thread id to access reclaim queue + * @param[id] expired_eid Expired epoch from the EpochManager + * @return Number of processed objects + */ + uint32_t Reclaim(const uint32_t &thread_id, const eid_t &expired_eid); + + /** + * @brief Adds the provided TileGroup oid to a queue to be marked + * immutable the next time a GC thread wakes up + * @param[in] tile_group_id Global oid of the TileGroup + */ + virtual void AddToImmutableQueue(const oid_t &tile_group_id) override; + + /** + * @brief Adds the provided TileGroup oid to a queue to be marked + * for compaction the next time a GC thread wakes up + * @param[in] tile_group_id Global oid of the TileGroup + */ + virtual void AddToCompactionQueue(const oid_t &tile_group_id); + + /** + * @brief Override the GC recycling threshold from settings.h + * @param[in] threshold The ratio of recycled tuples in a TileGroup before + * stopping recycling + */ + virtual void SetTileGroupRecyclingThreshold( + const double &threshold) override { + tile_group_recycling_threshold_ = threshold; } - virtual void DeregisterTable(const oid_t &table_id) override { - // Remove dropped tables - if (recycle_queue_map_.find(table_id) != recycle_queue_map_.end()) { - recycle_queue_map_.erase(table_id); - } + /** + * @brief The current GC recycling threshold + * @return The ratio of recycled tuples in a TileGroup before stopping + * recycling + */ + virtual double GetTileGroupRecyclingThreshold() const override { + return tile_group_recycling_threshold_; } - virtual size_t GetTableCount() override { return recycle_queue_map_.size(); } + /** + * @brief Override the GC TileGroup freeing setting from settings.h + * @param[in] free True to set GC to free TileGroups, false otherwise + */ + virtual void SetTileGroupFreeing(const bool &free) override { + tile_group_freeing_ = free; + } - int Unlink(const int &thread_id, const eid_t &expired_eid); + /** + * @brief The current GC TileGroup freeing setting + * @return True if the GC is set to free TileGroups + */ + virtual bool GetTileGroupFreeing() const override { + return tile_group_freeing_; + } - int Reclaim(const int &thread_id, const eid_t &expired_eid); + /** + * @brief Override the GC TileGroup compaction setting from settings.h + * @param[in] compact True to set GC to compact TileGroups, false otherwise + * @warning Setting to true expects TileGroupFreeing to be set to true first + */ + virtual void SetTileGroupCompaction(const bool &compact) override { + tile_group_compaction_ = compact; + if (tile_group_compaction_) { + PELOTON_ASSERT(tile_group_freeing_); + } + } - private: - inline unsigned int HashToThread(const size_t &thread_id) { - return (unsigned int)thread_id % gc_thread_count_; + /** + * @brief The current GC TileGroup compaction setting + * @return True if the GC is set to compact TileGroups + */ + virtual bool GetTileGroupCompaction() const override { + return tile_group_compaction_; } /** - * @brief Unlink and reclaim the tuples remained in a garbage collection - * thread when the Garbage Collector stops. + * @brief Unlink and reclaim the objects currently in queues * - * @return No return value. + * Meant to be used primarily internally by GC and in tests, not + * by outside classes + * + * @param[in] thread_id */ - void ClearGarbage(int thread_id); + void ClearGarbage(const uint32_t &thread_id); - void Running(const int &thread_id); + // iterates through immutable tile group queue and purges all tile groups + // from the recycles queues + /** + * @brief Empties the immutable queue and for each TileGroup removes + * its ItemPointers from its table's RecycleStack + * @return Number of TileGroups processed + */ + uint32_t ProcessImmutableQueue(); - void AddToRecycleMap(concurrency::TransactionContext *txn_ctx); + /** + * @brief Empties the compaction queue and for each TileGroup enqueues + * it in the MonoQueuePool for compaction + * @return Number of TileGroups processed + */ + uint32_t ProcessCompactionQueue(); + private: + TransactionLevelGCManager(const uint32_t &thread_count); + + virtual ~TransactionLevelGCManager() {} + + /** + * @brief Helper function to easily look up a table's RecycleStack + * @param[id] table_id Global oid of the table + * @return Smart pointer to the RecycleStack for the provided table. + * May be nullptr if the table is not registered with the GC + */ + std::shared_ptr GetTableRecycleStack( + const oid_t &table_id) const; + + inline unsigned int HashToThread(const size_t &thread_id); + + /** + * @brief Primary function for GC threads: wakes up, runs GC, has + * exponential backoff if queues are empty + * @param[id] thread_id Zero-indexed thread id for queue access + */ + void Running(const uint32_t &thread_id); + + /** + * @brief Recycles all of the tuple slots in transaction context's GCSet + * @param[in] txn_ctx TransactionConext pointer containing GCSet to be + * processed + */ + void RecycleTupleSlots(concurrency::TransactionContext *txn_ctx); + + /** + * @brief Recycles all of the objects in transaction context's GCObjectSet + * @param[in] txn_ctx TransactionConext pointer containing GCObjectSet + * to be processed + */ + void RemoveObjectLevelGarbage(concurrency::TransactionContext *txn_ctx); + + /** + * @brief Resets a tuple slot's version chain info and varlen pool + * @return True on success, false if TileGroup no longer exists + */ bool ResetTuple(const ItemPointer &); - // this function iterates the gc context and unlinks every version - // from the indexes. - // this function will call the UnlinkVersion() function. - void UnlinkVersions(concurrency::TransactionContext *txn_ctx); + /** + * @brief Unlinks all tuples in GCSet from indexes. + * @param[in] txn_ctx TransactionConext pointer containing GCSet to be + * processed + */ + void RemoveVersionsFromIndexes(concurrency::TransactionContext *txn_ctx); // this function unlinks a specified version from the index. - void UnlinkVersion(const ItemPointer location, const GCVersionType type); + /** + * @brief Unlinks provided tuple from indexes + * @param[in] location ItemPointer to garbage tuple to be processed + * @param[in] type GCVersionType for the provided garbage tuple + */ + void RemoveVersionFromIndexes(const ItemPointer &location, + const GCVersionType &type); - private: //===--------------------------------------------------------------------===// // Data members //===--------------------------------------------------------------------===// - - int gc_thread_count_; + uint32_t gc_thread_count_; // queues for to-be-unlinked tuples. // # unlink_queues == # gc_threads @@ -181,11 +315,18 @@ class TransactionLevelGCManager : public GCManager { std::vector> reclaim_maps_; + // queues of tile groups to be purged from recycle_stacks + // oid_t here is global TileGroup oid + std::shared_ptr> immutable_queue_; + + // queues of tile groups to be compacted + // oid_t here is global TileGroup oid + std::shared_ptr> compaction_queue_; + // queues for to-be-reused tuples. - // # recycle_queue_maps == # tables - std::unordered_map>> - recycle_queue_map_; + // oid_t here is global DataTable oid + std::shared_ptr>> + recycle_stacks_; }; } } // namespace peloton diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index 757cc9043e6..851fb92a825 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -127,6 +127,25 @@ SETTING_int(min_parallel_table_scan_size, 1, std::numeric_limits::max(), true, true) +//===----------------------------------------------------------------------===// +// Garbage Collection and TileGroup Compaction +//===----------------------------------------------------------------------===// + +SETTING_double(tile_group_recycling_threshold, + "Fraction of recycled slots in a TileGroup before recycling is stopped and TileGroup is enqueued for compaction (if enabled) (default: 0.9)", + 0.9, + 0.5, + 1.0, + false, false) +SETTING_bool(tile_group_freeing, + "Enable TileGroup freeing by the garbage collector (default: false)", + false, + true, false) +SETTING_bool(tile_group_compaction, + "Enable TileGroup compaction by the garbage collector (default: false)", + false, + true, false) + //===----------------------------------------------------------------------===// // WRITE AHEAD LOG //===----------------------------------------------------------------------===// diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 01d14db6be1..4b8dcaad889 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -146,6 +146,8 @@ class DataTable : public AbstractTable { void AddTileGroup(const std::shared_ptr &tile_group); + void DropTileGroup(const oid_t &tile_group_id); + // Offset is a 0-based number local to the table std::shared_ptr GetTileGroup( const std::size_t &tile_group_offset) const; @@ -303,14 +305,23 @@ class DataTable : public AbstractTable { concurrency::TransactionContext *transaction, ItemPointer **index_entry_ptr); - inline static size_t GetActiveTileGroupCount() { + inline static size_t GetDefaultActiveTileGroupCount() { return default_active_tilegroup_count_; } - static void SetActiveTileGroupCount(const size_t active_tile_group_count) { + static void SetDefaultActiveTileGroupCount( + const size_t active_tile_group_count) { default_active_tilegroup_count_ = active_tile_group_count; } + inline size_t GetActiveTileGroupCount() const { + return active_tilegroup_count_; + } + + inline size_t GetTuplesPerTileGroup() const { return tuples_per_tilegroup_; } + + bool IsActiveTileGroup(const oid_t &tile_group_id) const; + inline static size_t GetActiveIndirectionArrayCount() { return default_active_indirection_array_count_; } diff --git a/src/include/storage/storage_manager.h b/src/include/storage/storage_manager.h index a5903524781..cf123d69ada 100644 --- a/src/include/storage/storage_manager.h +++ b/src/include/storage/storage_manager.h @@ -102,15 +102,30 @@ class StorageManager { oid_t GetNextTileGroupId() { return ++tile_group_oid_; } - oid_t GetCurrentTileGroupId() { return tile_group_oid_; } + oid_t GetCurrentTileGroupId() const { return tile_group_oid_; } - void SetNextTileGroupId(oid_t next_oid) { tile_group_oid_ = next_oid; } + oid_t GetNumLiveTileGroups() const { return num_live_tile_groups_.load(); } + void SetNextTileGroupId(const oid_t next_oid) { tile_group_oid_ = next_oid; } + + /** + * @brief Adds/updates the TileGroup in Manager's oid->TileGroup* map + * @param oid[in] Global oid of the TileGroup to be added/updated + * @param location[in] Smart pointer to the TileGroup to be registered + */ void AddTileGroup(const oid_t oid, std::shared_ptr location); - + /** + * @brief Removes the TileGroup from Manager's oid->TileGroup* map + * @param oid[in] Global oid of the TileGroup to be removed + */ void DropTileGroup(const oid_t oid); - + /** + * @brief Gets a smart pointer to a TileGroup based on its global oid + * @param oid[in] Global oid of the TileGroup to be accessed + * @return Smart pointer to the TileGroup. Can be nullptr if TileGroup + * does not exist in the Manager's map (for example: TileGroup dropped) + */ std::shared_ptr GetTileGroup(const oid_t oid); void ClearTileGroup(void); @@ -131,6 +146,7 @@ class StorageManager { // Data members for tile group allocation //===--------------------------------------------------------------------===// std::atomic tile_group_oid_ = ATOMIC_VAR_INIT(START_OID); + std::atomic num_live_tile_groups_ = ATOMIC_VAR_INIT(0); CuckooMap> tile_group_locator_; static std::shared_ptr empty_tile_group_; diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index ea9218f06a8..7ca48008894 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -117,7 +117,6 @@ class TileGroup : public Printable { // this function is called only when building tile groups for aggregation // operations. - // FIXME: GC has recycled some of the tuples, so this count is not accurate uint32_t GetActiveTupleCount() const; uint32_t GetAllocatedTupleCount() const { return num_tuple_slots_; } diff --git a/src/include/storage/tile_group_header.h b/src/include/storage/tile_group_header.h index c7e8c010530..eca4720adbb 100644 --- a/src/include/storage/tile_group_header.h +++ b/src/include/storage/tile_group_header.h @@ -19,8 +19,9 @@ #include "common/macros.h" #include "common/synchronization/spin_latch.h" #include "common/printable.h" -#include "storage/tuple.h" #include "common/internal_types.h" +#include "gc/gc_manager_factory.h" +#include "storage/tuple.h" #include "type/value.h" namespace peloton { @@ -227,23 +228,61 @@ class TileGroupHeader : public Printable { transaction_id); } - /* - * @brief The following method use Compare and Swap to set the tilegroup's - immutable flag to be true. - */ - inline bool SetImmutability() { - return __sync_bool_compare_and_swap(&immutable, false, true); + /** + * @brief Uses Compare and Swap to set the TileGroup's + * immutable flag to be true + * + * Notifies the GC that TileGroup is now immutable to no longer + * hand out recycled slots. This is not guaranteed to be instantaneous + * so recycled slots may still be handed out immediately after + * immutability is set. + * + * @return Result of CAS + */ + bool SetImmutability(); + + /** + * @brief Uses Compare and Swap to set the TileGroup's + * immutable flag to be true + * + * Does not notify the GC. Should only be used by GC when it + * initiates a TileGroup's immutability + * + * @return Result of CAS + */ + inline bool SetImmutabilityWithoutNotifyingGC() { + bool expected = false; + return immutable_.compare_exchange_strong(expected, true); } - - /* - * @brief The following method use Compare and Swap to set the tilegroup's - immutable flag to be false. - */ + + /** + * @brief Uses Compare and Swap to set the TileGroup's + * immutable flag to be false + * + * @warning This should only be used for testing purposes because it violates + * a constraint of Zone Maps and the Garbage Collector that a TileGroup's + * immutability will never change after being set to true + * + * @return Result of CAS + */ inline bool ResetImmutability() { - return __sync_bool_compare_and_swap(&immutable, true, false); + bool expected = true; + return immutable_.compare_exchange_strong(expected, false); } - inline bool GetImmutability() const { return immutable; } + inline bool GetImmutability() const { return immutable_.load(); } + + inline size_t IncrementRecycled() { return num_recycled_.fetch_add(1); } + + inline size_t DecrementRecycled() { return num_recycled_.fetch_sub(1); } + + inline size_t GetNumRecycled() const { return num_recycled_.load(); } + + inline size_t IncrementGCReaders() { return num_gc_readers_.fetch_add(1); } + + inline size_t DecrementGCReaders() { return num_gc_readers_.fetch_sub(1); } + + inline size_t GetGCReaders() { return num_gc_readers_.load(); } void PrintVisibility(txn_id_t txn_id, cid_t at_cid); @@ -304,9 +343,15 @@ class TileGroupHeader : public Printable { common::synchronization::SpinLatch tile_header_lock; - // Immmutable Flag. Should be set by the indextuner to be true. + // Immmutable Flag. Should only be set to true when a TileGroup has used up + // all of its initial slots // By default it will be set to false. - bool immutable; + std::atomic immutable_; + + // metadata used by the garbage collector to recycle tuples + std::atomic + num_recycled_; // num empty tuple slots available for reuse + std::atomic num_gc_readers_; // used as a semaphor by GC }; } // namespace storage diff --git a/src/include/threadpool/mono_queue_pool.h b/src/include/threadpool/mono_queue_pool.h index fbee1985f22..c4514ed9aab 100644 --- a/src/include/threadpool/mono_queue_pool.h +++ b/src/include/threadpool/mono_queue_pool.h @@ -32,6 +32,8 @@ class MonoQueuePool { void Shutdown(); + bool IsRunning() const { return is_running_; } + template void SubmitTask(const F &func); diff --git a/src/index/art_index.cpp b/src/index/art_index.cpp index 18de5d02393..b74c0b19148 100644 --- a/src/index/art_index.cpp +++ b/src/index/art_index.cpp @@ -39,6 +39,7 @@ void LoadKey(void *ctx, TID tid, art::Key &key) { // Get physical tile group auto *item_pointer = reinterpret_cast(tid); auto tile_group = table->GetTileGroupById(item_pointer->block); + PELOTON_ASSERT(tile_group != nullptr); // Construct tuple, project only indexed columns const auto &indexed_cols = index_meta->GetKeyAttrs(); diff --git a/src/optimizer/stats/table_stats_collector.cpp b/src/optimizer/stats/table_stats_collector.cpp index b5b63d17b5f..b54775d7efd 100644 --- a/src/optimizer/stats/table_stats_collector.cpp +++ b/src/optimizer/stats/table_stats_collector.cpp @@ -47,6 +47,9 @@ void TableStatsCollector::CollectColumnStats() { for (size_t offset = 0; offset < tile_group_count; offset++) { std::shared_ptr tile_group = table_->GetTileGroup(offset); + if (tile_group == nullptr) { + continue; + } storage::TileGroupHeader *tile_group_header = tile_group->GetHeader(); oid_t tuple_count = tile_group->GetAllocatedTupleCount(); active_tuple_count_ += tile_group_header->GetActiveTupleCount(); diff --git a/src/optimizer/stats/tuple_sampler.cpp b/src/optimizer/stats/tuple_sampler.cpp index 26831079add..744d0191766 100644 --- a/src/optimizer/stats/tuple_sampler.cpp +++ b/src/optimizer/stats/tuple_sampler.cpp @@ -1,193 +1,198 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// tuple_sampler.cpp -// -// Identification: src/optimizer/tuple_sampler.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "optimizer/stats/tuple_sampler.h" -#include - -#include "storage/data_table.h" -#include "storage/tile.h" -#include "storage/tile_group.h" -#include "storage/tile_group_header.h" -#include "storage/tuple.h" - -namespace peloton { -namespace optimizer { - -/** - * AcquireSampleTuples - Sample a certain number of tuples from a given table. - * This function performs random sampling by generating random tile_group_offset - * and random tuple_offset. - */ -size_t TupleSampler::AcquireSampleTuples(size_t target_sample_count) { - size_t tuple_count = table->GetTupleCount(); - size_t tile_group_count = table->GetTileGroupCount(); - LOG_TRACE("tuple_count = %lu, tile_group_count = %lu", tuple_count, - tile_group_count); - - if (tuple_count < target_sample_count) { - target_sample_count = tuple_count; - } - - size_t rand_tilegroup_offset, rand_tuple_offset; - srand(time(NULL)); - catalog::Schema *tuple_schema = table->GetSchema(); - - while (sampled_tuples.size() < target_sample_count) { - // Generate a random tilegroup offset - rand_tilegroup_offset = rand() % tile_group_count; - storage::TileGroup *tile_group = - table->GetTileGroup(rand_tilegroup_offset).get(); - oid_t tuple_per_group = tile_group->GetActiveTupleCount(); - LOG_TRACE("tile_group: offset: %lu, addr: %p, tuple_per_group: %u", - rand_tilegroup_offset, tile_group, tuple_per_group); - if (tuple_per_group == 0) { - continue; - } - - rand_tuple_offset = rand() % tuple_per_group; - - std::unique_ptr tuple( - new storage::Tuple(tuple_schema, true)); - - LOG_TRACE("tuple_group_offset = %lu, tuple_offset = %lu", - rand_tilegroup_offset, rand_tuple_offset); - if (!GetTupleInTileGroup(tile_group, rand_tuple_offset, tuple)) { - continue; - } - LOG_TRACE("Add sampled tuple: %s", tuple->GetInfo().c_str()); - sampled_tuples.push_back(std::move(tuple)); - } - LOG_TRACE("%lu Sample added - size: %lu", sampled_tuples.size(), - sampled_tuples.size() * tuple_schema->GetLength()); - return sampled_tuples.size(); -} - -/** - * GetTupleInTileGroup - This function is a helper function to get a tuple in - * a tile group. - */ -bool TupleSampler::GetTupleInTileGroup(storage::TileGroup *tile_group, - size_t tuple_offset, - std::unique_ptr &tuple) { - // Tile Group Header - storage::TileGroupHeader *tile_group_header = tile_group->GetHeader(); - - // Check whether tuple is valid at given offset in the tile_group - // Reference: TileGroupHeader::GetActiveTupleCount() - // Check whether the transaction ID is invalid. - txn_id_t tuple_txn_id = tile_group_header->GetTransactionId(tuple_offset); - LOG_TRACE("transaction ID: %" PRId64, tuple_txn_id); - if (tuple_txn_id == INVALID_TXN_ID) { - return false; - } - - size_t tuple_column_itr = 0; - size_t tile_count = tile_group->GetTileCount(); - - LOG_TRACE("tile_count: %lu", tile_count); - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - - storage::Tile *tile = tile_group->GetTile(tile_itr); - const catalog::Schema &schema = *(tile->GetSchema()); - uint32_t tile_column_count = schema.GetColumnCount(); - - char *tile_tuple_location = tile->GetTupleLocation(tuple_offset); - storage::Tuple tile_tuple(&schema, tile_tuple_location); - - for (oid_t tile_column_itr = 0; tile_column_itr < tile_column_count; - tile_column_itr++) { - type::Value val = (tile_tuple.GetValue(tile_column_itr)); - tuple->SetValue(tuple_column_itr, val, pool_.get()); - tuple_column_itr++; - } - } - LOG_TRACE("offset %lu, Tuple info: %s", tuple_offset, - tuple->GetInfo().c_str()); - - return true; -} - -size_t TupleSampler::AcquireSampleTuplesForIndexJoin( - std::vector> &sample_tuples, - std::vector> &matched_tuples, size_t count) { - size_t target = std::min(count, sample_tuples.size()); - std::vector sid; - for (size_t i = 1; i <= target; i++) { - sid.push_back(i); - } - srand(time(NULL)); - for (size_t i = target + 1; i <= count; i++) { - if (rand() % i < target) { - size_t pos = rand() % target; - sid[pos] = i; - } - } - for (auto id : sid) { - size_t chosen = 0; - size_t cnt = 0; - while (cnt < id) { - cnt += matched_tuples.at(chosen).size(); - if (cnt >= id) { - break; - } - chosen++; - } - - size_t offset = rand() % matched_tuples.at(chosen).size(); - auto item = matched_tuples.at(chosen).at(offset); - storage::TileGroup *tile_group = table->GetTileGroupById(item->block).get(); - - std::unique_ptr tuple( - new storage::Tuple(table->GetSchema(), true)); - GetTupleInTileGroup(tile_group, item->offset, tuple); - LOG_TRACE("tuple info %s", tuple->GetInfo().c_str()); - AddJoinTuple(sample_tuples.at(chosen), tuple); - } - LOG_TRACE("join schema info %s", - sampled_tuples[0]->GetSchema()->GetInfo().c_str()); - return sampled_tuples.size(); -} - -void TupleSampler::AddJoinTuple(std::unique_ptr &left_tuple, - std::unique_ptr &right_tuple) { - if (join_schema == nullptr) { - std::unique_ptr left_schema( - catalog::Schema::CopySchema(left_tuple->GetSchema())); - std::unique_ptr right_schema( - catalog::Schema::CopySchema(right_tuple->GetSchema())); - join_schema.reset( - catalog::Schema::AppendSchema(left_schema.get(), right_schema.get())); - } - std::unique_ptr tuple( - new storage::Tuple(join_schema.get(), true)); - for (oid_t i = 0; i < left_tuple->GetColumnCount(); i++) { - tuple->SetValue(i, left_tuple->GetValue(i), pool_.get()); - } - - oid_t column_offset = left_tuple->GetColumnCount(); - for (oid_t i = 0; i < right_tuple->GetColumnCount(); i++) { - tuple->SetValue(i + column_offset, right_tuple->GetValue(i), pool_.get()); - } - LOG_TRACE("join tuple info %s", tuple->GetInfo().c_str()); - - sampled_tuples.push_back(std::move(tuple)); -} - -/** - * GetSampledTuples - This function returns the sampled tuples. - */ -std::vector> &TupleSampler::GetSampledTuples() { - return sampled_tuples; -} - -} // namespace optimizer -} // namespace peloton +//===----------------------------------------------------------------------===// +// +// Peloton +// +// tuple_sampler.cpp +// +// Identification: src/optimizer/tuple_sampler.cpp +// +// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "optimizer/stats/tuple_sampler.h" +#include + +#include "storage/data_table.h" +#include "storage/tile.h" +#include "storage/tile_group.h" +#include "storage/tile_group_header.h" +#include "storage/tuple.h" + +namespace peloton { +namespace optimizer { + +/** + * AcquireSampleTuples - Sample a certain number of tuples from a given table. + * This function performs random sampling by generating random tile_group_offset + * and random tuple_offset. + */ +size_t TupleSampler::AcquireSampleTuples(size_t target_sample_count) { + size_t tuple_count = table->GetTupleCount(); + size_t tile_group_count = table->GetTileGroupCount(); + LOG_TRACE("tuple_count = %lu, tile_group_count = %lu", tuple_count, + tile_group_count); + + if (tuple_count < target_sample_count) { + target_sample_count = tuple_count; + } + + size_t rand_tilegroup_offset, rand_tuple_offset; + srand(time(NULL)); + catalog::Schema *tuple_schema = table->GetSchema(); + + while (sampled_tuples.size() < target_sample_count) { + // Generate a random tilegroup offset + rand_tilegroup_offset = rand() % tile_group_count; + storage::TileGroup *tile_group = + table->GetTileGroup(rand_tilegroup_offset).get(); + + if (tile_group == nullptr) { + continue; + } + + oid_t tuple_per_group = tile_group->GetActiveTupleCount(); + LOG_TRACE("tile_group: offset: %lu, addr: %p, tuple_per_group: %u", + rand_tilegroup_offset, tile_group, tuple_per_group); + if (tuple_per_group == 0) { + continue; + } + + rand_tuple_offset = rand() % tuple_per_group; + + std::unique_ptr tuple( + new storage::Tuple(tuple_schema, true)); + + LOG_TRACE("tuple_group_offset = %lu, tuple_offset = %lu", + rand_tilegroup_offset, rand_tuple_offset); + if (!GetTupleInTileGroup(tile_group, rand_tuple_offset, tuple)) { + continue; + } + LOG_TRACE("Add sampled tuple: %s", tuple->GetInfo().c_str()); + sampled_tuples.push_back(std::move(tuple)); + } + LOG_TRACE("%lu Sample added - size: %lu", sampled_tuples.size(), + sampled_tuples.size() * tuple_schema->GetLength()); + return sampled_tuples.size(); +} + +/** + * GetTupleInTileGroup - This function is a helper function to get a tuple in + * a tile group. + */ +bool TupleSampler::GetTupleInTileGroup(storage::TileGroup *tile_group, + size_t tuple_offset, + std::unique_ptr &tuple) { + // Tile Group Header + storage::TileGroupHeader *tile_group_header = tile_group->GetHeader(); + + // Check whether tuple is valid at given offset in the tile_group + // Reference: TileGroupHeader::GetActiveTupleCount() + // Check whether the transaction ID is invalid. + txn_id_t tuple_txn_id = tile_group_header->GetTransactionId(tuple_offset); + LOG_TRACE("transaction ID: %" PRId64, tuple_txn_id); + if (tuple_txn_id == INVALID_TXN_ID) { + return false; + } + + size_t tuple_column_itr = 0; + size_t tile_count = tile_group->GetTileCount(); + + LOG_TRACE("tile_count: %lu", tile_count); + for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + storage::Tile *tile = tile_group->GetTile(tile_itr); + const catalog::Schema &schema = *(tile->GetSchema()); + uint32_t tile_column_count = schema.GetColumnCount(); + + char *tile_tuple_location = tile->GetTupleLocation(tuple_offset); + storage::Tuple tile_tuple(&schema, tile_tuple_location); + + for (oid_t tile_column_itr = 0; tile_column_itr < tile_column_count; + tile_column_itr++) { + type::Value val = (tile_tuple.GetValue(tile_column_itr)); + tuple->SetValue(tuple_column_itr, val, pool_.get()); + tuple_column_itr++; + } + } + LOG_TRACE("offset %lu, Tuple info: %s", tuple_offset, + tuple->GetInfo().c_str()); + + return true; +} + +size_t TupleSampler::AcquireSampleTuplesForIndexJoin( + std::vector> &sample_tuples, + std::vector> &matched_tuples, size_t count) { + size_t target = std::min(count, sample_tuples.size()); + std::vector sid; + for (size_t i = 1; i <= target; i++) { + sid.push_back(i); + } + srand(time(NULL)); + for (size_t i = target + 1; i <= count; i++) { + if (rand() % i < target) { + size_t pos = rand() % target; + sid[pos] = i; + } + } + for (auto id : sid) { + size_t chosen = 0; + size_t cnt = 0; + while (cnt < id) { + cnt += matched_tuples.at(chosen).size(); + if (cnt >= id) { + break; + } + chosen++; + } + + size_t offset = rand() % matched_tuples.at(chosen).size(); + auto item = matched_tuples.at(chosen).at(offset); + storage::TileGroup *tile_group = table->GetTileGroupById(item->block).get(); + PELOTON_ASSERT(tile_group != nullptr); + + std::unique_ptr tuple( + new storage::Tuple(table->GetSchema(), true)); + GetTupleInTileGroup(tile_group, item->offset, tuple); + LOG_TRACE("tuple info %s", tuple->GetInfo().c_str()); + AddJoinTuple(sample_tuples.at(chosen), tuple); + } + LOG_TRACE("join schema info %s", + sampled_tuples[0]->GetSchema()->GetInfo().c_str()); + return sampled_tuples.size(); +} + +void TupleSampler::AddJoinTuple(std::unique_ptr &left_tuple, + std::unique_ptr &right_tuple) { + if (join_schema == nullptr) { + std::unique_ptr left_schema( + catalog::Schema::CopySchema(left_tuple->GetSchema())); + std::unique_ptr right_schema( + catalog::Schema::CopySchema(right_tuple->GetSchema())); + join_schema.reset( + catalog::Schema::AppendSchema(left_schema.get(), right_schema.get())); + } + std::unique_ptr tuple( + new storage::Tuple(join_schema.get(), true)); + for (oid_t i = 0; i < left_tuple->GetColumnCount(); i++) { + tuple->SetValue(i, left_tuple->GetValue(i), pool_.get()); + } + + oid_t column_offset = left_tuple->GetColumnCount(); + for (oid_t i = 0; i < right_tuple->GetColumnCount(); i++) { + tuple->SetValue(i + column_offset, right_tuple->GetValue(i), pool_.get()); + } + LOG_TRACE("join tuple info %s", tuple->GetInfo().c_str()); + + sampled_tuples.push_back(std::move(tuple)); +} + +/** + * GetSampledTuples - This function returns the sampled tuples. + */ +std::vector> &TupleSampler::GetSampledTuples() { + return sampled_tuples; +} + +} // namespace optimizer +} // namespace peloton diff --git a/src/storage/abstract_table.cpp b/src/storage/abstract_table.cpp index 893da6e5cd9..2885576bc64 100644 --- a/src/storage/abstract_table.cpp +++ b/src/storage/abstract_table.cpp @@ -60,6 +60,10 @@ const std::string AbstractTable::GetInfo() const { if (tile_group_itr > 0) inner << std::endl; auto tile_group = this->GetTileGroup(tile_group_itr); + if (tile_group == nullptr) { + continue; + } + auto tile_tuple_count = tile_group->GetNextTupleSlot(); std::string tileData = tile_group->GetInfo(); diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 9615691d54f..42d5fe5194a 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -81,6 +81,13 @@ DataTable::DataTable(catalog::Schema *schema, const std::string &table_name, active_tile_groups_.resize(active_tilegroup_count_); active_indirection_arrays_.resize(active_indirection_array_count_); + + // Register non-catalog tables for GC + if (is_catalog == false) { + auto &gc_manager = gc::GCManagerFactory::GetInstance(); + gc_manager.RegisterTable(table_oid); + } + // Create tile groups. for (size_t i = 0; i < active_tilegroup_count_; ++i) { AddDefaultTileGroup(i); @@ -232,7 +239,7 @@ ItemPointer DataTable::GetEmptyTupleSlot(const storage::Tuple *tuple) { //=============== garbage collection================== // check if there are recycled tuple slots auto &gc_manager = gc::GCManagerFactory::GetInstance(); - auto free_item_pointer = gc_manager.ReturnFreeSlot(this->table_oid); + auto free_item_pointer = gc_manager.GetRecycledTupleSlot(this); if (free_item_pointer.IsNull() == false) { // when inserting a tuple if (tuple != nullptr) { @@ -338,17 +345,22 @@ ItemPointer DataTable::InsertTuple(const storage::Tuple *tuple, return INVALID_ITEMPOINTER; } - auto result = - InsertTuple(tuple, location, transaction, index_entry_ptr, check_fk); + auto result = InsertTuple(tuple, location, transaction, index_entry_ptr, check_fk); if (result == false) { + // Insertion failed due to some constraint (indexes, etc.) but tuple + // is in the table already, need to give the ItemPointer back to the + // GCManager + auto &gc_manager = gc::GCManagerFactory::GetInstance(); + gc_manager.RecycleTupleSlot(location); + return INVALID_ITEMPOINTER; } return location; } -bool DataTable::InsertTuple(const AbstractTuple *tuple, ItemPointer location, - concurrency::TransactionContext *transaction, - ItemPointer **index_entry_ptr, bool check_fk) { +bool DataTable::InsertTuple(const AbstractTuple *tuple, + ItemPointer location, concurrency::TransactionContext *transaction, + ItemPointer **index_entry_ptr, bool check_fk) { if (CheckConstraints(tuple) == false) { LOG_TRACE("InsertTuple(): Constraint violated"); return false; @@ -373,11 +385,6 @@ bool DataTable::InsertTuple(const AbstractTuple *tuple, ItemPointer location, IncreaseTupleCount(1); return true; } - // Index checks and updates - if (InsertInIndexes(tuple, location, transaction, index_entry_ptr) == false) { - LOG_TRACE("Index constraint violated"); - return false; - } // ForeignKey checks if (check_fk && CheckForeignKeyConstraints(tuple, transaction) == false) { @@ -385,8 +392,14 @@ bool DataTable::InsertTuple(const AbstractTuple *tuple, ItemPointer location, return false; } + // Index checks and updates + if (InsertInIndexes(tuple, location, transaction, index_entry_ptr) == false) { + LOG_TRACE("Index constraint violated"); + return false; + } + PELOTON_ASSERT((*index_entry_ptr)->block == location.block && - (*index_entry_ptr)->offset == location.offset); + (*index_entry_ptr)->offset == location.offset); // Increase the table's number of tuples by 1 IncreaseTupleCount(1); @@ -485,9 +498,19 @@ bool DataTable::InsertInIndexes(const AbstractTuple *tuple, // Handle failure if (res == false) { - // If some of the indexes have been inserted, - // the pointer has a chance to be dereferenced by readers and it cannot be - // deleted + // if an index insert fails, undo all prior inserts on this index + for (index_itr = index_itr + 1; index_itr < index_count; ++index_itr) { + index = GetIndex(index_itr); + if (index == nullptr) continue; + index_schema = index->GetKeySchema(); + indexed_columns = index_schema->GetIndexedColumns(); + std::unique_ptr delete_key( + new storage::Tuple(index_schema, true)); + delete_key->SetFromTuple(tuple, indexed_columns, index->GetPool()); + UNUSED_ATTRIBUTE bool delete_res = + index->DeleteEntry(delete_key.get(), *index_entry_ptr); + PELOTON_ASSERT(delete_res == true); + } *index_entry_ptr = nullptr; return false; } else { @@ -499,10 +522,10 @@ bool DataTable::InsertInIndexes(const AbstractTuple *tuple, return true; } -bool DataTable::InsertInSecondaryIndexes( - const AbstractTuple *tuple, const TargetList *targets_ptr, - concurrency::TransactionContext *transaction, - ItemPointer *index_entry_ptr) { +bool DataTable::InsertInSecondaryIndexes(const AbstractTuple *tuple, + const TargetList *targets_ptr, + concurrency::TransactionContext *transaction, + ItemPointer *index_entry_ptr) { int index_count = GetIndexCount(); // Transform the target list into a hash set // when attempting to perform insertion to a secondary index, @@ -569,8 +592,7 @@ bool DataTable::InsertInSecondaryIndexes( } /** - * @brief This function checks any other table which has a foreign key - *constraint + * @brief This function checks any other table which has a foreign key constraint * referencing the current table, where a tuple is updated/deleted. The final * result depends on the type of cascade action. * @@ -582,15 +604,16 @@ bool DataTable::InsertInSecondaryIndexes( * @param context: The executor context passed from upper level * @param is_update: whether this is a update action (false means delete) * - * @return True if the check is successful (nothing happens) or the cascade - *operation + * @return True if the check is successful (nothing happens) or the cascade operation * is done properly. Otherwise returns false. Note that the transaction result * is not set in this function. */ -bool DataTable::CheckForeignKeySrcAndCascade( - storage::Tuple *prev_tuple, storage::Tuple *new_tuple, - concurrency::TransactionContext *current_txn, - executor::ExecutorContext *context, bool is_update) { +bool DataTable::CheckForeignKeySrcAndCascade(storage::Tuple *prev_tuple, + storage::Tuple *new_tuple, + concurrency::TransactionContext *current_txn, + executor::ExecutorContext *context, + bool is_update) +{ size_t fk_count = GetForeignKeySrcCount(); if (fk_count == 0) return true; @@ -600,7 +623,7 @@ bool DataTable::CheckForeignKeySrcAndCascade( for (size_t iter = 0; iter < fk_count; iter++) { catalog::ForeignKey *fk = GetForeignKeySrc(iter); - + // Check if any row in the source table references the current tuple oid_t source_table_id = fk->GetSourceTableOid(); storage::DataTable *src_table = nullptr; @@ -619,7 +642,8 @@ bool DataTable::CheckForeignKeySrcAndCascade( // Make sure this is the right index to search in if (index->GetMetadata()->GetName().find("_FK_") != std::string::npos && - index->GetMetadata()->GetKeyAttrs() == fk->GetSourceColumnIds()) { + index->GetMetadata()->GetKeyAttrs() == fk->GetSourceColumnIds()) + { LOG_DEBUG("Searching in source tables's fk index...\n"); std::vector key_attrs = fk->GetSourceColumnIds(); @@ -638,6 +662,7 @@ bool DataTable::CheckForeignKeySrcAndCascade( for (ItemPointer *ptr : location_ptrs) { auto src_tile_group = src_table->GetTileGroupById(ptr->block); + PELOTON_ASSERT(src_tile_group != nullptr); auto src_tile_group_header = src_tile_group->GetHeader(); auto visibility = transaction_manager.IsVisible( @@ -796,6 +821,7 @@ bool DataTable::CheckForeignKeyConstraints( // Check the visibility of the result auto tile_group = ref_table->GetTileGroupById(location_ptrs[0]->block); + PELOTON_ASSERT(tile_group != nullptr); auto tile_group_header = tile_group->GetHeader(); auto &transaction_manager = @@ -995,6 +1021,24 @@ void DataTable::AddTileGroup(const std::shared_ptr &tile_group) { LOG_TRACE("Recording tile group : %u ", tile_group_id); } +void DataTable::DropTileGroup(const oid_t &tile_group_id) { + ssize_t tile_group_offset = tile_groups_.Lookup(tile_group_id); + if (tile_group_offset != -1) { + tile_groups_.Erase(tile_group_offset, invalid_tile_group_id); + } + auto storage_manager = storage::StorageManager::GetInstance(); + storage_manager->DropTileGroup(tile_group_id); +} + +bool DataTable::IsActiveTileGroup(const oid_t &tile_group_id) const { + for (auto tile_group : active_tile_groups_) { + if (tile_group_id == tile_group->GetTileGroupId()) { + return true; + } + } + return false; +} + size_t DataTable::GetTileGroupCount() const { return tile_group_count_; } std::shared_ptr DataTable::GetTileGroup( @@ -1290,6 +1334,10 @@ storage::TileGroup *DataTable::TransformTileGroup( auto tile_group_id = tile_groups_.FindValid(tile_group_offset, invalid_tile_group_id); + if (tile_group_id == invalid_tile_group_id) { + LOG_ERROR("Tile group offset not found in table : %u ", tile_group_offset); + return nullptr; + } // Get orig tile group from catalog auto storage_tilegroup = storage::StorageManager::GetInstance(); diff --git a/src/storage/database.cpp b/src/storage/database.cpp index 8a7506805c8..eb2aa1e9d4e 100644 --- a/src/storage/database.cpp +++ b/src/storage/database.cpp @@ -40,18 +40,10 @@ Database::~Database() { // TABLE //===----------------------------------------------------------------------===// -void Database::AddTable(storage::DataTable *table, bool is_catalog) { - { - std::lock_guard lock(database_mutex); - tables.push_back(table); - - if (is_catalog == false) { - // Register table to GC manager. - auto *gc_manager = &gc::GCManagerFactory::GetInstance(); - assert(gc_manager != nullptr); - gc_manager->RegisterTable(table->GetOid()); - } - } +void Database::AddTable(storage::DataTable *table, + bool is_catalog UNUSED_ATTRIBUTE) { + std::lock_guard lock(database_mutex); + tables.push_back(table); } storage::DataTable *Database::GetTableWithOid(const oid_t table_oid) const { diff --git a/src/storage/storage_manager.cpp b/src/storage/storage_manager.cpp index 0cb67d06bc1..f36cccde3b0 100644 --- a/src/storage/storage_manager.cpp +++ b/src/storage/storage_manager.cpp @@ -123,13 +123,19 @@ bool StorageManager::RemoveDatabaseFromStorageManager(oid_t database_oid) { void StorageManager::AddTileGroup(const oid_t oid, std::shared_ptr location) { - // add/update the catalog reference to the tile group + // do this check first, so that count is not updated yet + if (!tile_group_locator_.Contains(oid)) { + // only increment if new tile group + num_live_tile_groups_.fetch_add(1); + } tile_group_locator_.Upsert(oid, location); } void StorageManager::DropTileGroup(const oid_t oid) { // drop the catalog reference to the tile group - tile_group_locator_.Erase(oid); + if (tile_group_locator_.Erase(oid)) { + num_live_tile_groups_.fetch_sub(1); + } } std::shared_ptr StorageManager::GetTileGroup(const oid_t oid) { @@ -141,7 +147,10 @@ std::shared_ptr StorageManager::GetTileGroup(const oid_t oid } // used for logging test -void StorageManager::ClearTileGroup() { tile_group_locator_.Clear(); } +void StorageManager::ClearTileGroup() { + num_live_tile_groups_.store(0); + tile_group_locator_.Clear(); +} } // namespace storage } // namespace peloton diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 8458d167dd0..17c4cbae432 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -56,7 +56,7 @@ TileGroup::TileGroup(BackendType backend_type, } TileGroup::~TileGroup() { - // Drop references on all tiles + LOG_TRACE("TileGroup %d destructed!", tile_group_id); // clean up tile group header delete tile_group_header; diff --git a/src/storage/tile_group_header.cpp b/src/storage/tile_group_header.cpp index 1e0b450144e..14c94a07dca 100644 --- a/src/storage/tile_group_header.cpp +++ b/src/storage/tile_group_header.cpp @@ -22,10 +22,11 @@ #include "common/printable.h" #include "concurrency/transaction_manager_factory.h" #include "gc/gc_manager.h" +#include "gc/gc_manager_factory.h" #include "logging/log_manager.h" #include "storage/backend_manager.h" -#include "type/value.h" #include "storage/tuple.h" +#include "type/value.h" namespace peloton { namespace storage { @@ -60,8 +61,9 @@ TileGroupHeader::TileGroupHeader(const BackendType &backend_type, SetPrevItemPointer(tuple_slot_id, INVALID_ITEMPOINTER); } - // Initially immutabile flag to false initially. - immutable = false; + immutable_ = false; + num_recycled_ = 0; + num_gc_readers_ = 0; } TileGroupHeader::~TileGroupHeader() { @@ -83,7 +85,9 @@ const std::string TileGroupHeader::GetInfo() const { os << "Address:" << this << ", "; os << "NumActiveTuples:"; os << GetActiveTupleCount() << ", "; - os << "Immutable: " << GetImmutability(); + os << "NumRecycled:"; + os << GetNumRecycled() << ", "; + os << "Immutable:" << GetImmutability(); os << ")"; os << std::endl; @@ -247,5 +251,15 @@ oid_t TileGroupHeader::GetActiveTupleCount() const { return active_tuple_slots; } +bool TileGroupHeader::SetImmutability() { + bool expected = false; + bool result = immutable_.compare_exchange_strong(expected, true); + if (result == true) { + auto &gc_manager = gc::GCManagerFactory::GetInstance(); + gc_manager.AddToImmutableQueue(tile_group->GetTileGroupId()); + } + return result; +} + } // namespace storage } // namespace peloton diff --git a/src/storage/zone_map_manager.cpp b/src/storage/zone_map_manager.cpp index 2e6269bf170..7320a1eb343 100644 --- a/src/storage/zone_map_manager.cpp +++ b/src/storage/zone_map_manager.cpp @@ -63,6 +63,9 @@ void ZoneMapManager::CreateZoneMapsForTable( size_t num_tile_groups = table->GetTileGroupCount(); for (size_t i = 0; i < num_tile_groups; i++) { auto tile_group = table->GetTileGroup(i); + if (tile_group == nullptr) { + continue; + } auto tile_group_ptr = tile_group.get(); PELOTON_ASSERT(tile_group_ptr != nullptr); auto tile_group_header = tile_group_ptr->GetHeader(); @@ -92,6 +95,9 @@ void ZoneMapManager::CreateOrUpdateZoneMapForTileGroup( auto schema = table->GetSchema(); size_t num_columns = schema->GetColumnCount(); auto tile_group = table->GetTileGroup(tile_group_idx); + if (tile_group == nullptr) { + return; + } for (oid_t col_itr = 0; col_itr < num_columns; col_itr++) { // Set temp min and temp max as the first value. diff --git a/src/tuning/index_tuner.cpp b/src/tuning/index_tuner.cpp index cbd1bd57926..57a247fb752 100644 --- a/src/tuning/index_tuner.cpp +++ b/src/tuning/index_tuner.cpp @@ -108,6 +108,7 @@ void IndexTuner::BuildIndex(storage::DataTable *table, new storage::Tuple(table_schema, true)); auto tile_group = table->GetTileGroup(index_tile_group_offset); + PELOTON_ASSERT(tile_group != nullptr); auto tile_group_id = tile_group->GetTileGroupId(); oid_t active_tuple_count = tile_group->GetNextTupleSlot(); diff --git a/test/common/container_tuple_test.cpp b/test/common/container_tuple_test.cpp index 54794a5ddd2..17bdfb88346 100644 --- a/test/common/container_tuple_test.cpp +++ b/test/common/container_tuple_test.cpp @@ -63,6 +63,7 @@ TEST_F(ContainerTupleTests, GetInfo) { auto pos = temp_table->InsertTuple(&tuple1); auto tile_group = temp_table->GetTileGroupById(pos.block); + PELOTON_ASSERT(tile_group != nullptr); auto tuple_id = pos.offset; // Now test diff --git a/test/common/internal_types_test.cpp b/test/common/internal_types_test.cpp index 7a616315e20..98635da91eb 100644 --- a/test/common/internal_types_test.cpp +++ b/test/common/internal_types_test.cpp @@ -512,7 +512,7 @@ TEST_F(InternalTypesTests, GarbageCollectionTypeTest) { TEST_F(InternalTypesTests, ProtocolTypeTest) { std::vector list = { - ProtocolType::INVALID, + ProtocolType::INVALID, ProtocolType::TIMESTAMP_ORDERING }; @@ -538,7 +538,7 @@ TEST_F(InternalTypesTests, ProtocolTypeTest) { TEST_F(InternalTypesTests, EpochTypeTest) { std::vector list = { - EpochType::INVALID, + EpochType::INVALID, EpochType::DECENTRALIZED_EPOCH }; @@ -616,8 +616,8 @@ TEST_F(InternalTypesTests, VisibilityTypeTest) { TEST_F(InternalTypesTests, VisibilityIdTypeTest) { std::vector list = { - VisibilityIdType::INVALID, - VisibilityIdType::READ_ID, + VisibilityIdType::INVALID, + VisibilityIdType::READ_ID, VisibilityIdType::COMMIT_ID }; @@ -1073,7 +1073,7 @@ TEST_F(InternalTypesTests, GCVersionTypeTest) { std::vector list = { GCVersionType::INVALID, GCVersionType::COMMIT_UPDATE, GCVersionType::COMMIT_DELETE, GCVersionType::COMMIT_INS_DEL, - GCVersionType::ABORT_UPDATE, GCVersionType::ABORT_DELETE, + GCVersionType::ABORT_UPDATE, GCVersionType::TOMBSTONE, GCVersionType::ABORT_INSERT, GCVersionType::ABORT_INS_DEL, }; diff --git a/test/concurrency/testing_transaction_util.cpp b/test/concurrency/testing_transaction_util.cpp index 7f61cc0b765..d984fd4edbf 100644 --- a/test/concurrency/testing_transaction_util.cpp +++ b/test/concurrency/testing_transaction_util.cpp @@ -212,6 +212,24 @@ storage::DataTable *TestingTransactionUtil::CreateTable( return table; } +void TestingTransactionUtil::AddSecondaryIndex(storage::DataTable *table) { + // Create unique index on the value column + std::vector key_attrs = {1}; + auto tuple_schema = table->GetSchema(); + bool unique = false; + auto key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); + key_schema->SetIndexedColumns(key_attrs); + auto index_metadata2 = new index::IndexMetadata( + "unique_btree_index", 1235, TEST_TABLE_OID, CATALOG_DATABASE_OID, + IndexType::BWTREE, IndexConstraintType::UNIQUE, tuple_schema, key_schema, + key_attrs, unique); + + std::shared_ptr secondary_key_index( + index::IndexFactory::GetIndex(index_metadata2)); + + table->AddIndex(secondary_key_index); +} + std::unique_ptr TestingTransactionUtil::MakeProjectInfoFromTuple(const storage::Tuple *tuple) { TargetList target_list; @@ -456,5 +474,83 @@ bool TestingTransactionUtil::ExecuteScan( } return true; } + +ResultType TestingTransactionUtil::UpdateTuple(storage::DataTable *table, + const int key) { + srand(15721); + + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table, &txn_manager); + scheduler.Txn(0).Update(key, rand() % 15721); + scheduler.Txn(0).Commit(); + scheduler.Run(); + + return scheduler.schedules[0].txn_result; +} + +ResultType TestingTransactionUtil::InsertTuple(storage::DataTable *table, + const int key) { + srand(15721); + + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table, &txn_manager); + scheduler.Txn(0).Insert(key, rand() % 15721); + scheduler.Txn(0).Commit(); + scheduler.Run(); + + return scheduler.schedules[0].txn_result; +} + +ResultType TestingTransactionUtil::BulkInsertTuples(storage::DataTable *table, + const size_t num_tuples) { + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table, &txn_manager); + for (size_t i = 0; i < num_tuples; i++) { + scheduler.Txn(0).Insert(i, i); + } + scheduler.Txn(0).Commit(); + scheduler.Run(); + + return scheduler.schedules[0].txn_result; +} + +ResultType TestingTransactionUtil::BulkDeleteTuples(storage::DataTable *table, + const size_t num_tuples) { + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table, &txn_manager); + for (size_t i = 0; i < num_tuples; i++) { + scheduler.Txn(0).Delete(i, false); + } + scheduler.Txn(0).Commit(); + scheduler.Run(); + + return scheduler.schedules[0].txn_result; +} + +ResultType TestingTransactionUtil::DeleteTuple(storage::DataTable *table, + const int key) { + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table, &txn_manager); + scheduler.Txn(0).Delete(key); + scheduler.Txn(0).Commit(); + scheduler.Run(); + + return scheduler.schedules[0].txn_result; } + +ResultType TestingTransactionUtil::SelectTuple(storage::DataTable *table, + const int key, + std::vector &results) { + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table, &txn_manager); + scheduler.Txn(0).Read(key); + scheduler.Txn(0).Commit(); + scheduler.Run(); + + results = scheduler.schedules[0].results; + + return scheduler.schedules[0].txn_result; } + +} // namespace test +} // namespace peloton diff --git a/test/executor/loader_test.cpp b/test/executor/loader_test.cpp index 14b88882ff4..09c7a8183c7 100644 --- a/test/executor/loader_test.cpp +++ b/test/executor/loader_test.cpp @@ -133,29 +133,34 @@ TEST_F(LoaderTests, LoadingTest) { int total_tuple_count = loader_threads_count * tilegroup_count_per_loader * TEST_TUPLES_PER_TILEGROUP; int max_cached_tuple_count = - TEST_TUPLES_PER_TILEGROUP * storage::DataTable::GetActiveTileGroupCount(); + TEST_TUPLES_PER_TILEGROUP * + storage::DataTable::GetDefaultActiveTileGroupCount(); int max_unfill_cached_tuple_count = (TEST_TUPLES_PER_TILEGROUP - 1) * - storage::DataTable::GetActiveTileGroupCount(); + storage::DataTable::GetDefaultActiveTileGroupCount(); if (total_tuple_count - max_cached_tuple_count <= 0) { if (total_tuple_count <= max_unfill_cached_tuple_count) { - expected_tile_group_count = storage::DataTable::GetActiveTileGroupCount(); + expected_tile_group_count = + storage::DataTable::GetDefaultActiveTileGroupCount(); } else { expected_tile_group_count = - storage::DataTable::GetActiveTileGroupCount() + total_tuple_count - - max_unfill_cached_tuple_count; + storage::DataTable::GetDefaultActiveTileGroupCount() + + total_tuple_count - max_unfill_cached_tuple_count; } } else { - int filled_tile_group_count = total_tuple_count / max_cached_tuple_count * storage::DataTable::GetActiveTileGroupCount(); - + int filled_tile_group_count = + total_tuple_count / max_cached_tuple_count * + storage::DataTable::GetDefaultActiveTileGroupCount(); + if (total_tuple_count - filled_tile_group_count * TEST_TUPLES_PER_TILEGROUP - max_unfill_cached_tuple_count <= 0) { - expected_tile_group_count = filled_tile_group_count + - storage::DataTable::GetActiveTileGroupCount(); + expected_tile_group_count = + filled_tile_group_count + + storage::DataTable::GetDefaultActiveTileGroupCount(); } else { expected_tile_group_count = filled_tile_group_count + - storage::DataTable::GetActiveTileGroupCount() + + storage::DataTable::GetDefaultActiveTileGroupCount() + (total_tuple_count - filled_tile_group_count - max_unfill_cached_tuple_count); } diff --git a/test/gc/garbage_collection_test.cpp b/test/gc/garbage_collection_test.cpp index d3b24b878fc..80d90e3aa53 100644 --- a/test/gc/garbage_collection_test.cpp +++ b/test/gc/garbage_collection_test.cpp @@ -87,6 +87,10 @@ int GarbageNum(storage::DataTable *table) { while (current_tile_group_offset_ < table_tile_group_count_) { auto tile_group = table->GetTileGroup(current_tile_group_offset_++); + if (tile_group == nullptr) { + continue; + } + auto tile_group_header = tile_group->GetHeader(); oid_t active_tuple_count = tile_group->GetNextTupleSlot(); @@ -106,8 +110,8 @@ int GarbageNum(storage::DataTable *table) { // get tuple recycled by GC int RecycledNum(storage::DataTable *table) { int count = 0; - auto table_id = table->GetOid(); - while (!gc::GCManagerFactory::GetInstance().ReturnFreeSlot(table_id).IsNull()) + while ( + !gc::GCManagerFactory::GetInstance().GetRecycledTupleSlot(table).IsNull()) count++; LOG_INFO("recycled version num = %d", count); @@ -134,7 +138,7 @@ TEST_F(GarbageCollectionTests, UpdateTest) { std::unique_ptr table(TestingTransactionUtil::CreateTable( num_key, "UPDATE_TABLE", db_id, INVALID_OID, 1234, true)); - EXPECT_TRUE(gc_manager.GetTableCount() == 1); + EXPECT_EQ(1, gc_manager.GetTableCount()); gc_manager.StartGC(gc_threads); @@ -283,11 +287,7 @@ TEST_F(GarbageCollectionTests, DeleteTest) { // there should be two versions to be recycled by the GC: // the deleted version and the empty version. - // however, the txn will explicitly pass one version (the deleted - // version) to the GC manager. - // The GC itself should be responsible for recycling the - // empty version. - EXPECT_EQ(1, recycle_num); + EXPECT_EQ(2, recycle_num); gc_manager.StopGC(); gc::GCManagerFactory::Configure(0); diff --git a/test/gc/tile_group_compactor_test.cpp b/test/gc/tile_group_compactor_test.cpp new file mode 100644 index 00000000000..7d427160252 --- /dev/null +++ b/test/gc/tile_group_compactor_test.cpp @@ -0,0 +1,504 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// transaction_level_gc_manager_test.cpp +// +// Identification: test/gc/tile_group_compactor_test.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "catalog/catalog.h" +#include "common/harness.h" +#include "concurrency/epoch_manager.h" +#include "concurrency/testing_transaction_util.h" +#include "executor/testing_executor_util.h" +#include "gc/transaction_level_gc_manager.h" +#include "gc/tile_group_compactor.h" +#include "sql/testing_sql_util.h" +#include "storage/data_table.h" +#include "storage/database.h" +#include "storage/storage_manager.h" +#include "storage/tile_group.h" +#include "threadpool/mono_queue_pool.h" + +namespace peloton { + +namespace test { + +//===--------------------------------------------------------------------===// +// TransactionContext-Level GC Manager Tests +//===--------------------------------------------------------------------===// + +class TileGroupCompactorTests : public PelotonTest {}; + +oid_t test_index_oid = 1234; +double recycling_threshold = 0.8; + +// Test that GCManager triggers compaction for sparse tile groups +// And test it doesn't trigger compaction for dense tile groups +// Runs MonoQueuePool to do compaction in separate threads +TEST_F(TileGroupCompactorTests, GCIntegrationTestSparse) { + std::string test_name = "gc_integration_test_sparse"; + + // start worker pool + threadpool::MonoQueuePool::GetInstance().Startup(); + + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(1); + + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + gc_manager.SetTileGroupRecyclingThreshold(recycling_threshold); + gc_manager.SetTileGroupFreeing(true); + gc_manager.SetTileGroupCompaction(true); + + // create database + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + // create table + const int num_key = 0; + size_t tuples_per_tilegroup = 10; + std::unique_ptr table(TestingTransactionUtil::CreateTable( + num_key, "table0", db_id, INVALID_OID, test_index_oid++, true, + tuples_per_tilegroup)); + + auto manager = storage::StorageManager::GetInstance(); + size_t tile_group_count_after_init = manager->GetNumLiveTileGroups(); + LOG_DEBUG("tile_group_count_after_init: %zu\n", tile_group_count_after_init); + + auto current_eid = epoch_manager.GetCurrentEpochId(); + + epoch_manager.SetCurrentEpochId(++current_eid); + + // insert tuples here, this will allocate another tile group + size_t num_inserts = tuples_per_tilegroup; + auto insert_result = + TestingTransactionUtil::BulkInsertTuples(table.get(), num_inserts); + EXPECT_EQ(ResultType::SUCCESS, insert_result); + + // capture num tile groups occupied + size_t tile_group_count_after_insert = manager->GetNumLiveTileGroups(); + LOG_DEBUG("tile_group_count_after_insert: %zu", + tile_group_count_after_insert); + EXPECT_GT(tile_group_count_after_insert, tile_group_count_after_init); + + epoch_manager.SetCurrentEpochId(++current_eid); + + // delete all but 1 of the tuples + // this will create 9 tombstones, so won't fill another tile group + auto delete_result = + TestingTransactionUtil::BulkDeleteTuples(table.get(), num_inserts - 1); + EXPECT_EQ(ResultType::SUCCESS, delete_result); + + size_t tile_group_count_after_delete = manager->GetNumLiveTileGroups(); + LOG_DEBUG("tile_group_count_after_delete: %zu", + tile_group_count_after_delete); + EXPECT_EQ(tile_group_count_after_delete, tile_group_count_after_insert); + + // first clear garbage from outdated versions and tombstones + // should also add tile groups to compaction queue + epoch_manager.SetCurrentEpochId(++current_eid); + gc_manager.ClearGarbage(0); + // submit compaction tasks to worker pool + gc_manager.ProcessCompactionQueue(); + + // sleep to allow tile group compaction to happen + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + + LOG_DEBUG("tile_group_count_after_compact: %u", + manager->GetNumLiveTileGroups()); + + // Run GC to free compacted tile groups + epoch_manager.SetCurrentEpochId(++current_eid); + gc_manager.ClearGarbage(0); + + size_t tile_group_count_after_gc = manager->GetNumLiveTileGroups(); + LOG_DEBUG("tile_group_count_after_gc: %zu", tile_group_count_after_gc); + EXPECT_EQ(tile_group_count_after_gc, tile_group_count_after_init); + + threadpool::MonoQueuePool::GetInstance().Shutdown(); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); +} + +// Test that GCManager doesn't trigger compaction for dense tile groups +// Runs MonoQueuePool to do compaction in separate threads +TEST_F(TileGroupCompactorTests, GCIntegrationTestDense) { + std::string test_name = "gc_integration_test_dense"; + // start worker pool + threadpool::MonoQueuePool::GetInstance().Startup(); + + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(1); + + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + gc_manager.SetTileGroupRecyclingThreshold(recycling_threshold); + gc_manager.SetTileGroupFreeing(true); + gc_manager.SetTileGroupCompaction(true); + + // create database + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + // create table + const int num_key = 0; + size_t tuples_per_tilegroup = 10; + std::unique_ptr table(TestingTransactionUtil::CreateTable( + num_key, "table0", db_id, INVALID_OID, test_index_oid++, true, + tuples_per_tilegroup)); + + auto manager = storage::StorageManager::GetInstance(); + size_t tile_group_count_after_init = manager->GetNumLiveTileGroups(); + LOG_DEBUG("tile_group_count_after_init: %zu\n", tile_group_count_after_init); + + auto current_eid = epoch_manager.GetCurrentEpochId(); + + epoch_manager.SetCurrentEpochId(++current_eid); + + // insert tuples here, this will allocate another tile group + size_t num_inserts = tuples_per_tilegroup; + auto insert_result = + TestingTransactionUtil::BulkInsertTuples(table.get(), num_inserts); + EXPECT_EQ(ResultType::SUCCESS, insert_result); + + // capture num tile groups occupied + size_t tile_group_count_after_insert = manager->GetNumLiveTileGroups(); + LOG_DEBUG("tile_group_count_after_insert: %zu", + tile_group_count_after_insert); + EXPECT_GT(tile_group_count_after_insert, tile_group_count_after_init); + + epoch_manager.SetCurrentEpochId(++current_eid); + + // delete 3/10 of the tuples + // this will create 3 tombstones, so won't fill another tile group + auto delete_result = TestingTransactionUtil::BulkDeleteTuples(table.get(), 3); + EXPECT_EQ(ResultType::SUCCESS, delete_result); + + size_t tile_group_count_after_delete = manager->GetNumLiveTileGroups(); + LOG_DEBUG("tile_group_count_after_delete: %zu", + tile_group_count_after_delete); + EXPECT_EQ(tile_group_count_after_init + 1, tile_group_count_after_delete); + + // first clear garbage from outdated versions and tombstones + // should also add tile groups to compaction queue + epoch_manager.SetCurrentEpochId(++current_eid); + gc_manager.ClearGarbage(0); + // submit compaction tasks to worker pool + gc_manager.ProcessCompactionQueue(); + + // sleep to allow tile group compaction to happen + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + + LOG_DEBUG("tile_group_count_after_compact: %u", + manager->GetNumLiveTileGroups()); + + // Run GC to free compacted tile groups + epoch_manager.SetCurrentEpochId(++current_eid); + gc_manager.ClearGarbage(0); + + size_t tile_group_count_after_gc = manager->GetNumLiveTileGroups(); + LOG_DEBUG("tile_group_count_after_gc: %zu", tile_group_count_after_gc); + EXPECT_EQ(tile_group_count_after_delete, tile_group_count_after_gc); + + threadpool::MonoQueuePool::GetInstance().Shutdown(); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); +} + +// Test compaction during a concurrent update txn +TEST_F(TileGroupCompactorTests, ConcurrentUpdateTest) { + std::string test_name = "concurrentupdatetest"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + gc_manager.SetTileGroupRecyclingThreshold(recycling_threshold); + gc_manager.SetTileGroupFreeing(true); + gc_manager.SetTileGroupCompaction(true); + + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, test_index_oid++, true, 10)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + size_t starting_num_live_tile_groups = storage_manager->GetNumLiveTileGroups(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + + // Fill a tile group with tuples + size_t num_inserts = 10; + auto insert_result = + TestingTransactionUtil::BulkInsertTuples(table.get(), num_inserts); + EXPECT_EQ(ResultType::SUCCESS, insert_result); + + // Delete compaction_threshold tuples from tile_group + size_t num_deletes = 8; + auto delete_result = + TestingTransactionUtil::BulkDeleteTuples(table.get(), num_deletes); + EXPECT_EQ(ResultType::SUCCESS, delete_result); + + // Start txn that updates one of the remaining tuples + // Don't commit yet + auto txn = txn_manager.BeginTransaction(); + auto update_result = + TestingTransactionUtil::ExecuteUpdate(txn, table.get(), 9, 100, true); + EXPECT_TRUE(update_result); + + // Then try to compact the table's first tile_group while update txn is in + // progress + auto starting_tg = table->GetTileGroup(0); + bool compact_result = gc::TileGroupCompactor::MoveTuplesOutOfTileGroup( + table.get(), starting_tg); + EXPECT_FALSE(compact_result); + + // Commit the update txn so that the compaction is able to proceed + txn_manager.CommitTransaction(txn); + + // clear garbage + // Marks first & second tilegroups for compaction + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + auto num_tg_before_compaction = storage_manager->GetNumLiveTileGroups(); + + // Try to compact the tile again. This time it should succeed + compact_result = gc::TileGroupCompactor::MoveTuplesOutOfTileGroup( + table.get(), starting_tg); + EXPECT_TRUE(compact_result); + + // Clear garbage, trigger freeing of compacted tile group + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + // assert num live tile groups decreased + auto num_tg_now = storage_manager->GetNumLiveTileGroups(); + EXPECT_LT(num_tg_now, num_tg_before_compaction); + + // Compact all tile groups + for (size_t i = 0; i < table->GetTileGroupCount(); i++) { + auto tg = table->GetTileGroup(i); + if (tg != nullptr) { + gc::TileGroupCompactor::MoveTuplesOutOfTileGroup(table.get(), tg); + } + } + // Clear garbage from compaction + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + // Assert that num live tile groups is back to starting value + num_tg_now = storage_manager->GetNumLiveTileGroups(); + EXPECT_EQ(starting_num_live_tile_groups, num_tg_now); + + // assert that we have the moved tuple and updated tuple with expected values + // 8 was moved, 9 was updated + std::vector results; + auto ret = TestingTransactionUtil::SelectTuple(table.get(), 8, results); + EXPECT_EQ(ResultType::SUCCESS, ret); + EXPECT_EQ(8, results[0]); + + results.clear(); + ret = TestingTransactionUtil::SelectTuple(table.get(), 9, results); + EXPECT_EQ(ResultType::SUCCESS, ret); + EXPECT_EQ(100, results[0]); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + threadpool::MonoQueuePool::GetInstance().Shutdown(); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Test that TileGroupCompactor can handle: +// - tile groups that are entirely filled with garbage +// - tile groups that no longer exist (already freed) +// - tile groups that belong to dropped tables +TEST_F(TileGroupCompactorTests, EdgeCasesTest) { + std::string test_name = "edgecasestest"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + gc_manager.SetTileGroupRecyclingThreshold(recycling_threshold); + gc_manager.SetTileGroupFreeing(true); + gc_manager.SetTileGroupCompaction(true); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, test_index_oid++, true, 10)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + size_t starting_num_live_tile_groups = storage_manager->GetNumLiveTileGroups(); + + oid_t starting_tgid = table->GetTileGroup(0)->GetTileGroupId(); + + // First insert 1 tile group full of tuples + size_t num_inserts = 10; + auto insert_result = + TestingTransactionUtil::BulkInsertTuples(table.get(), num_inserts); + EXPECT_EQ(ResultType::SUCCESS, insert_result); + + size_t num_deletes = num_inserts; + auto delete_result = + TestingTransactionUtil::BulkDeleteTuples(table.get(), num_deletes); + EXPECT_EQ(ResultType::SUCCESS, delete_result); + + auto post_delete_num_live_tile_groups = + storage_manager->GetNumLiveTileGroups(); + EXPECT_EQ(starting_num_live_tile_groups + 2, + post_delete_num_live_tile_groups); + + // Compact tile group that is all garbage. It should ignore all slots + gc::TileGroupCompactor::CompactTileGroup(starting_tgid); + + // assert num live tile groups did not change + auto current_num_live_tile_groups = storage_manager->GetNumLiveTileGroups(); + EXPECT_EQ(post_delete_num_live_tile_groups, current_num_live_tile_groups); + + // clear garbage, triggers freeing of starting tile group + // also clears tombstones from second tile group + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + // assert num live tile groups decreased by 1 + current_num_live_tile_groups = storage_manager->GetNumLiveTileGroups(); + EXPECT_EQ(starting_num_live_tile_groups, current_num_live_tile_groups); + + // Compact tile group that no longer exists + // it should ignore the tile group (& not crash) + EXPECT_EQ(nullptr, table->GetTileGroupById(starting_tgid)); + gc::TileGroupCompactor::CompactTileGroup(starting_tgid); + + // assert num live tile groups is what it was before started + current_num_live_tile_groups = storage_manager->GetNumLiveTileGroups(); + EXPECT_EQ(starting_num_live_tile_groups, current_num_live_tile_groups); + + table.release(); + + // Compact tile group on a table that was dropped. Shouldn't crash + gc::TileGroupCompactor::CompactTileGroup(starting_tgid); + + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + threadpool::MonoQueuePool::GetInstance().Shutdown(); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Test retry mechanism +TEST_F(TileGroupCompactorTests, RetryTest) { + std::string test_name = "retrytest"; + + // start worker pool + threadpool::MonoQueuePool::GetInstance().Startup(); + + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + gc_manager.SetTileGroupRecyclingThreshold(recycling_threshold); + gc_manager.SetTileGroupFreeing(true); + gc_manager.SetTileGroupCompaction(true); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, test_index_oid++, true, 10)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + + // Fill a tile group with tuples + size_t num_inserts = 10; + auto insert_result = + TestingTransactionUtil::BulkInsertTuples(table.get(), num_inserts); + EXPECT_EQ(ResultType::SUCCESS, insert_result); + + // Delete compaction_threshold tuples from tile_group + size_t num_deletes = 8; + auto delete_result = + TestingTransactionUtil::BulkDeleteTuples(table.get(), num_deletes); + EXPECT_EQ(ResultType::SUCCESS, delete_result); + + // Start txn that updates one of the remaining tuples + // Don't commit yet + auto txn = txn_manager.BeginTransaction(); + auto update_result = + TestingTransactionUtil::ExecuteUpdate(txn, table.get(), 9, 100, true); + EXPECT_TRUE(update_result); + + auto num_tg_before_compaction = storage_manager->GetNumLiveTileGroups(); + + // Now trigger GC, which should add this TG to compaction queue + // Marks first tilegroup for compaction + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + // try to compact the table's first tile_group while update txn is in progress + gc_manager.ProcessCompactionQueue(); + + // sleep to give it time to try and fail compaction + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + + // assert num live tile groups stays the same since compaction is blocked + auto num_tg_now = storage_manager->GetNumLiveTileGroups(); + EXPECT_LE(num_tg_before_compaction, num_tg_now); + + // Commit the update txn so that the compaction is able to proceed + txn_manager.CommitTransaction(txn); + + // assert num live tile groups decreased + num_tg_now = storage_manager->GetNumLiveTileGroups(); + EXPECT_LE(num_tg_before_compaction, num_tg_now); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + threadpool::MonoQueuePool::GetInstance().Shutdown(); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +} // namespace test +} // namespace peloton diff --git a/test/gc/transaction_level_gc_manager_test.cpp b/test/gc/transaction_level_gc_manager_test.cpp index cef62e0cf73..e501c5bc4bb 100644 --- a/test/gc/transaction_level_gc_manager_test.cpp +++ b/test/gc/transaction_level_gc_manager_test.cpp @@ -17,11 +17,14 @@ #include "concurrency/epoch_manager.h" #include "catalog/catalog.h" +#include "sql/testing_sql_util.h" #include "storage/data_table.h" #include "storage/tile_group.h" #include "storage/database.h" #include "storage/storage_manager.h" +#include "gc/tile_group_compactor.h" + namespace peloton { namespace test { @@ -32,55 +35,1021 @@ namespace test { class TransactionLevelGCManagerTests : public PelotonTest {}; -ResultType UpdateTuple(storage::DataTable *table, const int key) { - srand(15721); +int GetNumRecycledTuples(storage::DataTable *table) { + int count = 0; + // auto table_id = table->GetOid(); + while ( + !gc::GCManagerFactory::GetInstance().GetRecycledTupleSlot(table).IsNull()) + count++; + + LOG_INFO("recycled version num = %d", count); + return count; +} + +size_t CountOccurrencesInAllIndexes(storage::DataTable *table, int first_val, + int second_val) { + size_t num_occurrences = 0; + std::unique_ptr tuple( + new storage::Tuple(table->GetSchema(), true)); + auto primary_key = type::ValueFactory::GetIntegerValue(first_val); + auto value = type::ValueFactory::GetIntegerValue(second_val); + + tuple->SetValue(0, primary_key, nullptr); + tuple->SetValue(1, value, nullptr); + + for (size_t idx = 0; idx < table->GetIndexCount(); ++idx) { + auto index = table->GetIndex(idx); + if (index == nullptr) continue; + auto index_schema = index->GetKeySchema(); + auto indexed_columns = index_schema->GetIndexedColumns(); + + // build key. + std::unique_ptr current_key( + new storage::Tuple(index_schema, true)); + current_key->SetFromTuple(tuple.get(), indexed_columns, index->GetPool()); + + std::vector index_entries; + index->ScanKey(current_key.get(), index_entries); + num_occurrences += index_entries.size(); + } + return num_occurrences; +} + +size_t CountOccurrencesInIndex(storage::DataTable *table, int idx, + int first_val, int second_val) { + std::unique_ptr tuple( + new storage::Tuple(table->GetSchema(), true)); + auto primary_key = type::ValueFactory::GetIntegerValue(first_val); + auto value = type::ValueFactory::GetIntegerValue(second_val); + + tuple->SetValue(0, primary_key, nullptr); + tuple->SetValue(1, value, nullptr); + + auto index = table->GetIndex(idx); + if (index == nullptr) return 0; + auto index_schema = index->GetKeySchema(); + auto indexed_columns = index_schema->GetIndexedColumns(); + + // build key. + std::unique_ptr current_key( + new storage::Tuple(index_schema, true)); + current_key->SetFromTuple(tuple.get(), indexed_columns, index->GetPool()); + + std::vector index_entries; + index->ScanKey(current_key.get(), index_entries); + + return index_entries.size(); +} + +//////////////////////////////////////////// +// NEW TESTS +//////////////////////////////////////////// + +// Scenario: Abort Insert (due to other operation) +// Insert tuple +// Some other operation fails +// Abort +// Assert RQ size = 1 +// Assert not present in indexes +TEST_F(TransactionLevelGCManagerTests, AbortInsertTest) { + std::string test_name = "abortinsert"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // insert, then abort + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); + scheduler.Txn(0).Abort(); + scheduler.Run(); + + EXPECT_EQ(ResultType::ABORTED, scheduler.schedules[0].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table.get(), 2, 1)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Fail to insert a tuple +// Scenario: Failed Insert (due to insert failure (e.g. index rejects insert or +// FK constraints) violated) +// Abort +// Assert RQ size = 1 +// Assert old copy in 2 indexes +// Assert new copy in 0 indexes +TEST_F(TransactionLevelGCManagerTests, FailedInsertPrimaryKeyTest) { + std::string test_name = "failedinsertprimarykey"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // insert duplicate key (failure), try to commit + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(2, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 0); + scheduler.Txn(0).Commit(); + scheduler.Txn(1).Insert(0, 1); // primary key already exists in table + scheduler.Txn(1).Commit(); + scheduler.Run(); + + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + EXPECT_EQ(ResultType::ABORTED, scheduler.schedules[1].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + // EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 0, 0, 0)); + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 1, 0, 0)); + + EXPECT_EQ(0, CountOccurrencesInIndex(table.get(), 1, 0, 1)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +//// Scenario: Failed Insert (due to insert failure (e.g. index rejects insert +/// or FK constraints) violated) +//// Fail to insert a tuple +//// Abort +//// Assert RQ size = 1 +//// Assert old tuple in 2 indexes +//// Assert new tuple in 0 indexes +TEST_F(TransactionLevelGCManagerTests, FailedInsertSecondaryKeyTest) { + std::string test_name = "failedinsertsecondarykey"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // insert duplicate value (secondary index requires uniqueness, so fails) + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(2, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); // succeeds + scheduler.Txn(0).Commit(); + scheduler.Txn(1).Insert(1, 1); // fails, dup value + scheduler.Txn(1).Commit(); + scheduler.Run(); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + EXPECT_EQ(ResultType::ABORTED, scheduler.schedules[1].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 0, 0, 1)); + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 1, 0, 1)); + + EXPECT_EQ(0, CountOccurrencesInIndex(table.get(), 0, 1, 1)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +//// Scenario: COMMIT_UPDATE +//// Insert tuple +//// Commit +//// Update tuple +//// Commit +//// Assert RQ size = 1 +//// Assert old version in 1 index (primary key) +//// Assert new version in 2 indexes +TEST_F(TransactionLevelGCManagerTests, CommitUpdateSecondaryKeyTest) { + std::string test_name = "commitupdatesecondarykey"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // insert, commit. update, commit. + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(2, table.get(), &txn_manager); + scheduler.Txn(0).Insert(5, 1); + scheduler.Txn(0).Commit(); + scheduler.Txn(1).Update(5, 2); + scheduler.Txn(1).Commit(); + scheduler.Run(); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[1].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + + EXPECT_EQ(0, CountOccurrencesInIndex(table.get(), 1, 5, 1)); + + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 0, 5, 2)); + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 1, 5, 2)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Scenario: ABORT_UPDATE +// Insert tuple +// Commit +// Update tuple +// Abort +// Assert RQ size = 1 +// Assert old version is in 2 indexes +// Assert new version is in 1 index (primary key) +TEST_F(TransactionLevelGCManagerTests, AbortUpdateSecondaryKeyTest) { + std::string test_name = "abortupdatesecondarykey"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // update, abort auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - TransactionScheduler scheduler(1, table, &txn_manager); - scheduler.Txn(0).Update(key, rand() % 15721); + TransactionScheduler scheduler(2, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); // succeeds scheduler.Txn(0).Commit(); + scheduler.Txn(1).Update(0, 2); + scheduler.Txn(1).Abort(); scheduler.Run(); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + EXPECT_EQ(ResultType::ABORTED, scheduler.schedules[1].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 0, 0, 1)); + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 1, 0, 1)); - return scheduler.schedules[0].txn_result; + EXPECT_EQ(0, CountOccurrencesInIndex(table.get(), 1, 0, 2)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); } -ResultType InsertTuple(storage::DataTable *table, const int key) { - srand(15721); +// Scenario: COMMIT_INS_UPDATE (not a GC type) +// Insert tuple +// Update tuple +// Commit +// Assert RQ.size = 0 +// Assert old tuple in 1 index (primary key) +// Assert new tuple in 2 indexes +// Test is disabled until the reuse of owned tuple slots optimization is removed. +TEST_F(TransactionLevelGCManagerTests, DISABLED_CommitInsertUpdateTest) { + std::string test_name = "commitinsertupdate"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // insert, update, commit auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - TransactionScheduler scheduler(1, table, &txn_manager); - scheduler.Txn(0).Insert(key, rand() % 15721); + TransactionScheduler scheduler(1, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); + scheduler.Txn(0).Update(0, 2); scheduler.Txn(0).Commit(); scheduler.Run(); - return scheduler.schedules[0].txn_result; + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + EXPECT_EQ(0, CountOccurrencesInIndex(table.get(), 1, 0, 1)); + + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 0, 0, 2)); + EXPECT_EQ(1, CountOccurrencesInIndex(table.get(), 1, 0, 2)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); } -ResultType DeleteTuple(storage::DataTable *table, const int key) { - srand(15721); +// Scenario: ABORT_INS_UPDATE +// Insert tuple +// Update tuple +// Abort +// Assert RQ.size = 1 or 2? +// Assert inserted tuple in 0 indexes +// Assert updated tuple in 0 indexes +// Test is disabled until the reuse of owned tuple slots optimization is removed. +TEST_F(TransactionLevelGCManagerTests, DISABLED_AbortInsertUpdateTest) { + std::string test_name = "abortinsertupdate"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // insert, update, abort + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); + scheduler.Txn(0).Update(0, 2); + scheduler.Txn(0).Abort(); + scheduler.Run(); + EXPECT_EQ(ResultType::ABORTED, scheduler.schedules[0].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table.get(), 0, 1)); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table.get(), 0, 2)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Scenario: COMMIT_DELETE +// Insert tuple +// Commit +// Delete tuple +// Commit +// Assert RQ size = 2 +// Assert deleted tuple appears in 0 indexes +TEST_F(TransactionLevelGCManagerTests, CommitDeleteTest) { + std::string test_name = "commitdelete"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // insert, commit, delete, commit auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - TransactionScheduler scheduler(1, table, &txn_manager); - scheduler.Txn(0).Delete(key); + TransactionScheduler scheduler(2, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); scheduler.Txn(0).Commit(); + scheduler.Txn(1).Delete(0); + scheduler.Txn(1).Commit(); scheduler.Run(); - return scheduler.schedules[0].txn_result; + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[1].txn_result); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(2, GetNumRecycledTuples(table.get())); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table.get(), 0, 1)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); } -ResultType SelectTuple(storage::DataTable *table, const int key, - std::vector &results) { - srand(15721); +// Scenario: ABORT_DELETE +// Insert tuple +// Commit +// Delete tuple +// Abort +// Assert RQ size = 1 +// Assert tuple found in 2 indexes +TEST_F(TransactionLevelGCManagerTests, AbortDeleteTest) { + std::string test_name = "abortdelete"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + epoch_manager.SetCurrentEpochId(++current_epoch); + + // delete, abort auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - TransactionScheduler scheduler(1, table, &txn_manager); - scheduler.Txn(0).Read(key); + TransactionScheduler scheduler(2, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); scheduler.Txn(0).Commit(); + scheduler.Txn(1).Delete(0); + scheduler.Txn(1).Abort(); scheduler.Run(); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + EXPECT_EQ(ResultType::ABORTED, scheduler.schedules[1].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); - results = scheduler.schedules[0].results; + EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + EXPECT_EQ(2, CountOccurrencesInAllIndexes(table.get(), 0, 1)); - return scheduler.schedules[0].txn_result; + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Scenario: COMMIT_INS_DEL +// Insert tuple +// Delete tuple +// Commit +// Assert RQ.size = 1 +// Assert tuple found in 0 indexes +TEST_F(TransactionLevelGCManagerTests, CommitInsertDeleteTest) { + std::string test_name = "commitinsertdelete"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // insert, delete, commit + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); + scheduler.Txn(0).Delete(0); + scheduler.Txn(0).Commit(); + scheduler.Run(); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table.get(), 0, 1)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Scenario: ABORT_INS_DEL +// Insert tuple +// Delete tuple +// Abort +// Assert RQ size = 1 +// Assert tuple found in 0 indexes +TEST_F(TransactionLevelGCManagerTests, AbortInsertDeleteTest) { + std::string test_name = "abortinsertdelete"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // insert, delete, abort + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(1, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); + scheduler.Txn(0).Delete(0); + scheduler.Txn(0).Abort(); + scheduler.Run(); + EXPECT_EQ(ResultType::ABORTED, scheduler.schedules[0].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table.get(), 0, 1)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Scenario: COMMIT_UPDATE_DEL +// Insert tuple +// Commit +// Update tuple +// Delete tuple +// Commit +// Assert RQ.size = 2 +// Assert old tuple in 0 indexes +// Assert new tuple in 0 indexes +// Test is disabled until the reuse of owned tuple slots optimization is removed. +TEST_F(TransactionLevelGCManagerTests, DISABLED_CommitUpdateDeleteTest) { + std::string test_name = "commitupdatedelete"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // update, delete, commit + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(2, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); + scheduler.Txn(0).Commit(); + scheduler.Txn(1).Update(0, 2); + scheduler.Txn(1).Delete(0); + scheduler.Txn(1).Commit(); + scheduler.Run(); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(2, GetNumRecycledTuples(table.get())); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table.get(), 0, 1)); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table.get(), 0, 2)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Scenario: ABORT_UPDATE_DEL +// Insert tuple +// Commit +// Update tuple +// Delete tuple +// Abort +// Assert RQ size = 2 +// Assert old tuple in 2 indexes +// Assert new tuple in 1 index (primary key) +// Test is disabled until the reuse of owned tuple slots optimization is removed. +TEST_F(TransactionLevelGCManagerTests, DISABLED_AbortUpdateDeleteTest) { + std::string test_name = "abortupdatedelete"; + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = TestingExecutorUtil::InitializeDatabase(test_name + "db"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + 0, test_name + "table", db_id, INVALID_OID, 1234, true)); + TestingTransactionUtil::AddSecondaryIndex(table.get()); + + EXPECT_EQ(0, GetNumRecycledTuples(table.get())); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + // update, delete, then abort + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + TransactionScheduler scheduler(2, table.get(), &txn_manager); + scheduler.Txn(0).Insert(0, 1); + scheduler.Txn(0).Commit(); + scheduler.Txn(1).Update(0, 2); + scheduler.Txn(1).Delete(0); + scheduler.Txn(1).Abort(); + scheduler.Run(); + EXPECT_EQ(ResultType::SUCCESS, scheduler.schedules[0].txn_result); + EXPECT_EQ(ResultType::ABORTED, scheduler.schedules[1].txn_result); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + EXPECT_EQ(1, GetNumRecycledTuples(table.get())); + + EXPECT_EQ(2, CountOccurrencesInAllIndexes(table.get(), 0, 1)); + EXPECT_EQ(0, CountOccurrencesInIndex(table.get(), 1, 0, 2)); + + table.release(); + TestingExecutorUtil::DeleteDatabase(test_name + "db"); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + EXPECT_FALSE(storage_manager->HasDatabase(db_id)); +} + +// Scenario: Update Primary Key Test +// Insert tuple +// Commit +// Update primary key and value +// Commit +// Assert RQ.size = 2 (primary key update causes delete and insert) +// Assert old tuple in 0 indexes +// Assert new tuple in 2 indexes +TEST_F(TransactionLevelGCManagerTests, CommitUpdatePrimaryKeyTest) { + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + auto catalog = catalog::Catalog::GetInstance(); + catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + txn_manager.CommitTransaction(txn); + auto database = catalog->GetDatabaseWithName(DEFAULT_DB_NAME, txn); + + TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE test(a INT PRIMARY KEY, b INT);"); + auto table = database->GetTable(database->GetTableCount() - 1); + TestingTransactionUtil::AddSecondaryIndex(table); + + EXPECT_EQ(0, GetNumRecycledTuples(table)); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + TestingSQLUtil::ExecuteSQLQuery("INSERT INTO test VALUES (3, 30);"); + + std::vector result; + std::vector tuple_descriptor; + std::string error_message; + int rows_affected; + + // confirm setup + TestingSQLUtil::ExecuteSQLQuery("SELECT * from test WHERE b=30", result, + tuple_descriptor, rows_affected, + error_message); + EXPECT_EQ('3', result[0][0]); + EXPECT_EQ(2, CountOccurrencesInAllIndexes(table, 3, 30)); + + // Perform primary key and value update + TestingSQLUtil::ExecuteSQLQuery("UPDATE test SET a=5, b=40", result, + tuple_descriptor, rows_affected, + error_message); + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + // confirm update + TestingSQLUtil::ExecuteSQLQuery("SELECT * from test WHERE b=40", result, + tuple_descriptor, rows_affected, + error_message); + EXPECT_EQ('5', result[0][0]); + + EXPECT_EQ(2, GetNumRecycledTuples(table)); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table, 3, 30)); + EXPECT_EQ(2, CountOccurrencesInAllIndexes(table, 5, 40)); + + txn = txn_manager.BeginTransaction(); + catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + txn_manager.CommitTransaction(txn); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); +} + +// Scenario: Insert then Update Primary Key Test +// Insert tuple +// Update primary key and value +// Commit +// Assert RQ.size = 2 (primary key update causes delete and insert) +// Assert old tuple in 0 indexes +// Assert new tuple in 2 indexes +TEST_F(TransactionLevelGCManagerTests, DISABLED_CommitInsertUpdatePrimaryKeyTest) { + uint64_t current_epoch = 0; + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(++current_epoch); + std::vector> gc_threads; + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + auto catalog = catalog::Catalog::GetInstance(); + catalog->CreateDatabase(DEFAULT_DB_NAME, txn); + txn_manager.CommitTransaction(txn); + auto database = catalog->GetDatabaseWithName(DEFAULT_DB_NAME, txn); + + TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE test(a INT PRIMARY KEY, b INT);"); + auto table = database->GetTable(database->GetTableCount() - 1); + TestingTransactionUtil::AddSecondaryIndex(table); + + EXPECT_EQ(0, GetNumRecycledTuples(table)); + + epoch_manager.SetCurrentEpochId(++current_epoch); + + TestingSQLUtil::ExecuteSQLQuery("BEGIN;"); + TestingSQLUtil::ExecuteSQLQuery("INSERT INTO test VALUES (3, 30);"); + TestingSQLUtil::ExecuteSQLQuery("UPDATE test SET a=5, b=40;"); + TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + std::vector result; + std::vector tuple_descriptor; + std::string error_message; + int rows_affected; + + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.ClearGarbage(0); + + // confirm update + TestingSQLUtil::ExecuteSQLQuery("SELECT * from test WHERE b=40", result, + tuple_descriptor, rows_affected, + error_message); + EXPECT_EQ('5', result[0][0]); + + EXPECT_EQ(2, GetNumRecycledTuples(table)); + EXPECT_EQ(0, CountOccurrencesInAllIndexes(table, 3, 30)); + EXPECT_EQ(2, CountOccurrencesInAllIndexes(table, 5, 40)); + + txn = txn_manager.BeginTransaction(); + catalog::Catalog::GetInstance()->DropDatabaseWithName(DEFAULT_DB_NAME, txn); + txn_manager.CommitTransaction(txn); + epoch_manager.SetCurrentEpochId(++current_epoch); + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); +} + +// check mem -> insert 100k -> check mem -> delete all -> check mem +TEST_F(TransactionLevelGCManagerTests, FreeTileGroupsTest) { + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(1); + + std::vector> gc_threads; + + gc::GCManagerFactory::Configure(1); + auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); + gc_manager.Reset(); + gc_manager.SetTileGroupFreeing(true); + + auto storage_manager = storage::StorageManager::GetInstance(); + // create database + auto database = TestingExecutorUtil::InitializeDatabase("freetilegroupsdb"); + oid_t db_id = database->GetOid(); + EXPECT_TRUE(storage_manager->HasDatabase(db_id)); + + // create a table with only one key + const int num_key = 0; + size_t tuples_per_tilegroup = 2; + + std::unique_ptr table(TestingTransactionUtil::CreateTable( + num_key, "table1", db_id, INVALID_OID, 1234, true, tuples_per_tilegroup)); + + auto manager = storage::StorageManager::GetInstance(); + size_t tile_group_count_after_init = manager->GetNumLiveTileGroups(); + LOG_DEBUG("tile_group_count_after_init: %zu\n", tile_group_count_after_init); + + auto current_eid = epoch_manager.GetCurrentEpochId(); + + // int round = 1; + for (int round = 1; round <= 3; round++) { + LOG_DEBUG("Round: %d\n", round); + + epoch_manager.SetCurrentEpochId(++current_eid); + //=========================== + // insert tuples here. + //=========================== + size_t num_inserts = 100; + auto insert_result = + TestingTransactionUtil::BulkInsertTuples(table.get(), num_inserts); + EXPECT_EQ(ResultType::SUCCESS, insert_result); + + // capture memory usage + LOG_DEBUG("Round %d: tile_group_count_after_insert: %u", round, + manager->GetNumLiveTileGroups()); + + epoch_manager.SetCurrentEpochId(++current_eid); + //=========================== + // delete the tuples. + //=========================== + auto delete_result = + TestingTransactionUtil::BulkDeleteTuples(table.get(), num_inserts); + EXPECT_EQ(ResultType::SUCCESS, delete_result); + + LOG_DEBUG("Round %d: tile_group_count_after_delete: %u", round, + manager->GetNumLiveTileGroups()); + + epoch_manager.SetCurrentEpochId(++current_eid); + + gc_manager.ClearGarbage(0); + + size_t tile_group_count_after_gc = manager->GetNumLiveTileGroups(); + LOG_DEBUG("Round %d: tile_group_count_after_gc: %zu", round, + tile_group_count_after_gc); + EXPECT_LT(tile_group_count_after_gc, tile_group_count_after_init + 1); + } + + gc_manager.StopGC(); + gc::GCManagerFactory::Configure(0); + + table.release(); + + // DROP! + TestingExecutorUtil::DeleteDatabase("freetilegroupsdb"); + + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + EXPECT_THROW(catalog::Catalog::GetInstance()->GetDatabaseObject( + "freetilegroupsdb", txn), + CatalogException); + txn_manager.CommitTransaction(txn); } // update -> delete @@ -93,21 +1062,21 @@ TEST_F(TransactionLevelGCManagerTests, UpdateDeleteTest) { auto &gc_manager = gc::TransactionLevelGCManager::GetInstance(); auto storage_manager = storage::StorageManager::GetInstance(); // create database - auto database = TestingExecutorUtil::InitializeDatabase("database0"); + auto database = TestingExecutorUtil::InitializeDatabase("updatedeletedb"); oid_t db_id = database->GetOid(); EXPECT_TRUE(storage_manager->HasDatabase(db_id)); // create a table with only one key const int num_key = 1; std::unique_ptr table(TestingTransactionUtil::CreateTable( - num_key, "TABLE0", db_id, INVALID_OID, 1234, true)); + num_key, "updatedeletetable", db_id, INVALID_OID, 1234, true)); EXPECT_TRUE(gc_manager.GetTableCount() == 1); //=========================== // update a version here. //=========================== - auto ret = UpdateTuple(table.get(), 0); + auto ret = TestingTransactionUtil::UpdateTuple(table.get(), 0); EXPECT_TRUE(ret == ResultType::SUCCESS); epoch_manager.SetCurrentEpochId(2); @@ -152,7 +1121,7 @@ TEST_F(TransactionLevelGCManagerTests, UpdateDeleteTest) { //=========================== // delete a version here. //=========================== - ret = DeleteTuple(table.get(), 0); + ret = TestingTransactionUtil::DeleteTuple(table.get(), 0); EXPECT_TRUE(ret == ResultType::SUCCESS); epoch_manager.SetCurrentEpochId(4); @@ -200,12 +1169,12 @@ TEST_F(TransactionLevelGCManagerTests, UpdateDeleteTest) { table.release(); // DROP! - TestingExecutorUtil::DeleteDatabase("database0"); + TestingExecutorUtil::DeleteDatabase("updatedeletedb"); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); EXPECT_THROW( - catalog::Catalog::GetInstance()->GetDatabaseObject("database0", txn), + catalog::Catalog::GetInstance()->GetDatabaseObject("updatedeletedb", txn), CatalogException); txn_manager.CommitTransaction(txn); // EXPECT_FALSE(storage_manager->HasDatabase(db_id)); @@ -224,21 +1193,21 @@ TEST_F(TransactionLevelGCManagerTests, ReInsertTest) { auto storage_manager = storage::StorageManager::GetInstance(); // create database - auto database = TestingExecutorUtil::InitializeDatabase("database1"); + auto database = TestingExecutorUtil::InitializeDatabase("reinsertdb"); oid_t db_id = database->GetOid(); EXPECT_TRUE(storage_manager->HasDatabase(db_id)); // create a table with only one key const int num_key = 1; std::unique_ptr table(TestingTransactionUtil::CreateTable( - num_key, "TABLE1", db_id, INVALID_OID, 1234, true)); + num_key, "reinserttable", db_id, INVALID_OID, 1234, true)); EXPECT_TRUE(gc_manager.GetTableCount() == 1); //=========================== // insert a tuple here. //=========================== - auto ret = InsertTuple(table.get(), 100); + auto ret = TestingTransactionUtil::InsertTuple(table.get(), 100); EXPECT_TRUE(ret == ResultType::SUCCESS); epoch_manager.SetCurrentEpochId(2); @@ -286,14 +1255,14 @@ TEST_F(TransactionLevelGCManagerTests, ReInsertTest) { std::vector results; results.clear(); - ret = SelectTuple(table.get(), 100, results); + ret = TestingTransactionUtil::SelectTuple(table.get(), 100, results); EXPECT_TRUE(ret == ResultType::SUCCESS); EXPECT_TRUE(results[0] != -1); //=========================== // delete the tuple. //=========================== - ret = DeleteTuple(table.get(), 100); + ret = TestingTransactionUtil::DeleteTuple(table.get(), 100); EXPECT_TRUE(ret == ResultType::SUCCESS); epoch_manager.SetCurrentEpochId(4); @@ -339,21 +1308,21 @@ TEST_F(TransactionLevelGCManagerTests, ReInsertTest) { // select the tuple. //=========================== results.clear(); - ret = SelectTuple(table.get(), 100, results); + ret = TestingTransactionUtil::SelectTuple(table.get(), 100, results); EXPECT_TRUE(ret == ResultType::SUCCESS); EXPECT_TRUE(results[0] == -1); //=========================== // insert the tuple again. //=========================== - ret = InsertTuple(table.get(), 100); + ret = TestingTransactionUtil::InsertTuple(table.get(), 100); EXPECT_TRUE(ret == ResultType::SUCCESS); //=========================== // select the tuple. //=========================== results.clear(); - ret = SelectTuple(table.get(), 100, results); + ret = TestingTransactionUtil::SelectTuple(table.get(), 100, results); EXPECT_TRUE(ret == ResultType::SUCCESS); EXPECT_TRUE(results[0] != -1); @@ -363,12 +1332,12 @@ TEST_F(TransactionLevelGCManagerTests, ReInsertTest) { table.release(); // DROP! - TestingExecutorUtil::DeleteDatabase("database1"); + TestingExecutorUtil::DeleteDatabase("reinsertdb"); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); EXPECT_THROW( - catalog::Catalog::GetInstance()->GetDatabaseObject("database0", txn), + catalog::Catalog::GetInstance()->GetDatabaseObject("reinsertdb", txn), CatalogException); txn_manager.CommitTransaction(txn); // EXPECT_FALSE(storage_manager->HasDatabase(db_id)); @@ -401,7 +1370,8 @@ TEST_F(TransactionLevelGCManagerTests, ImmutabilityTest) { const int num_key = 25; const size_t tuples_per_tilegroup = 5; std::unique_ptr table(TestingTransactionUtil::CreateTable( - num_key, "TABLE1", db_id, INVALID_OID, 1234, true, tuples_per_tilegroup)); + num_key, "immutabilitytable", db_id, INVALID_OID, 1234, true, + tuples_per_tilegroup)); EXPECT_TRUE(gc_manager.GetTableCount() == 1); @@ -415,60 +1385,25 @@ TEST_F(TransactionLevelGCManagerTests, ImmutabilityTest) { tile_group_header->SetImmutability(); // Deleting a tuple from the 1st tilegroup - auto ret = DeleteTuple(table.get(), 2); - EXPECT_TRUE(ret == ResultType::SUCCESS); - epoch_manager.SetCurrentEpochId(2); - auto expired_eid = epoch_manager.GetExpiredEpochId(); - EXPECT_EQ(1, expired_eid); - auto current_eid = epoch_manager.GetCurrentEpochId(); - EXPECT_EQ(2, current_eid); - auto reclaimed_count = gc_manager.Reclaim(0, expired_eid); - auto unlinked_count = gc_manager.Unlink(0, expired_eid); - EXPECT_EQ(0, reclaimed_count); - EXPECT_EQ(1, unlinked_count); - - epoch_manager.SetCurrentEpochId(3); - expired_eid = epoch_manager.GetExpiredEpochId(); - EXPECT_EQ(2, expired_eid); - current_eid = epoch_manager.GetCurrentEpochId(); - EXPECT_EQ(3, current_eid); - reclaimed_count = gc_manager.Reclaim(0, expired_eid); - unlinked_count = gc_manager.Unlink(0, expired_eid); - EXPECT_EQ(1, reclaimed_count); - EXPECT_EQ(0, unlinked_count); + auto ret = TestingTransactionUtil::DeleteTuple(table.get(), 2); + gc_manager.ClearGarbage(0); - // ReturnFreeSlot() should return null because deleted tuple was from - // immutable tilegroup. - auto location = gc_manager.ReturnFreeSlot((table.get())->GetOid()); - EXPECT_EQ(location.IsNull(), true); + // ReturnFreeSlot() should not return a tuple slot from the immutable tile + // group + // should be from where ever the tombstone was inserted + auto location = gc_manager.GetRecycledTupleSlot(table.get()); + EXPECT_NE(tile_group->GetTileGroupId(), location.block); // Deleting a tuple from the 2nd tilegroup which is mutable. - ret = DeleteTuple(table.get(), 6); + ret = TestingTransactionUtil::DeleteTuple(table.get(), 6); EXPECT_TRUE(ret == ResultType::SUCCESS); epoch_manager.SetCurrentEpochId(4); - expired_eid = epoch_manager.GetExpiredEpochId(); - EXPECT_EQ(3, expired_eid); - current_eid = epoch_manager.GetCurrentEpochId(); - EXPECT_EQ(4, current_eid); - reclaimed_count = gc_manager.Reclaim(0, expired_eid); - unlinked_count = gc_manager.Unlink(0, expired_eid); - EXPECT_EQ(0, reclaimed_count); - EXPECT_EQ(1, unlinked_count); - - epoch_manager.SetCurrentEpochId(5); - expired_eid = epoch_manager.GetExpiredEpochId(); - EXPECT_EQ(4, expired_eid); - current_eid = epoch_manager.GetCurrentEpochId(); - EXPECT_EQ(5, current_eid); - reclaimed_count = gc_manager.Reclaim(0, expired_eid); - unlinked_count = gc_manager.Unlink(0, expired_eid); - EXPECT_EQ(1, reclaimed_count); - EXPECT_EQ(0, unlinked_count); + gc_manager.ClearGarbage(0); // ReturnFreeSlot() should not return null because deleted tuple was from - // mutable tilegroup. - location = gc_manager.ReturnFreeSlot((table.get())->GetOid()); + // mutable tilegroup + location = gc_manager.GetRecycledTupleSlot(table.get()); EXPECT_EQ(location.IsNull(), false); gc_manager.StopGC(); diff --git a/test/include/concurrency/testing_transaction_util.h b/test/include/concurrency/testing_transaction_util.h index d7bb919a0f3..1cd8ee43ce0 100644 --- a/test/include/concurrency/testing_transaction_util.h +++ b/test/include/concurrency/testing_transaction_util.h @@ -153,6 +153,23 @@ class TestingTransactionUtil { static std::unique_ptr MakeProjectInfoFromTuple( const storage::Tuple *tuple); static expression::ComparisonExpression *MakePredicate(int id); + + static void AddSecondaryIndex(storage::DataTable *table); + + static ResultType UpdateTuple(storage::DataTable *table, const int key); + + static ResultType InsertTuple(storage::DataTable *table, const int key); + + static ResultType BulkInsertTuples(storage::DataTable *table, + const size_t num_tuples); + + static ResultType BulkDeleteTuples(storage::DataTable *table, + const size_t num_tuples); + + static ResultType DeleteTuple(storage::DataTable *table, const int key); + + static ResultType SelectTuple(storage::DataTable *table, const int key, + std::vector &results); }; struct TransactionOperation { diff --git a/test/performance/insert_performance_test.cpp b/test/performance/insert_performance_test.cpp index a65be8e7ead..ece22621337 100644 --- a/test/performance/insert_performance_test.cpp +++ b/test/performance/insert_performance_test.cpp @@ -120,30 +120,34 @@ TEST_F(InsertPerformanceTests, LoadingTest) { int total_tuple_count = loader_threads_count * tilegroup_count_per_loader * TEST_TUPLES_PER_TILEGROUP; int max_cached_tuple_count = - TEST_TUPLES_PER_TILEGROUP * storage::DataTable::GetActiveTileGroupCount(); + TEST_TUPLES_PER_TILEGROUP * + storage::DataTable::GetDefaultActiveTileGroupCount(); int max_unfill_cached_tuple_count = (TEST_TUPLES_PER_TILEGROUP - 1) * - storage::DataTable::GetActiveTileGroupCount(); + storage::DataTable::GetDefaultActiveTileGroupCount(); if (total_tuple_count - max_cached_tuple_count <= 0) { if (total_tuple_count <= max_unfill_cached_tuple_count) { - expected_tile_group_count = storage::DataTable::GetActiveTileGroupCount(); + expected_tile_group_count = + storage::DataTable::GetDefaultActiveTileGroupCount(); } else { expected_tile_group_count = - storage::DataTable::GetActiveTileGroupCount() + total_tuple_count - - max_unfill_cached_tuple_count; + storage::DataTable::GetDefaultActiveTileGroupCount() + + total_tuple_count - max_unfill_cached_tuple_count; } } else { - int filled_tile_group_count = total_tuple_count / max_cached_tuple_count * - storage::DataTable::GetActiveTileGroupCount(); + int filled_tile_group_count = + total_tuple_count / max_cached_tuple_count * + storage::DataTable::GetDefaultActiveTileGroupCount(); if (total_tuple_count - filled_tile_group_count * TEST_TUPLES_PER_TILEGROUP - max_unfill_cached_tuple_count <= 0) { - expected_tile_group_count = filled_tile_group_count + - storage::DataTable::GetActiveTileGroupCount(); + expected_tile_group_count = + filled_tile_group_count + + storage::DataTable::GetDefaultActiveTileGroupCount(); } else { expected_tile_group_count = filled_tile_group_count + - storage::DataTable::GetActiveTileGroupCount() + + storage::DataTable::GetDefaultActiveTileGroupCount() + (total_tuple_count - filled_tile_group_count - max_unfill_cached_tuple_count); } diff --git a/test/sql/update_sql_test.cpp b/test/sql/update_sql_test.cpp index f3584843918..c02b62ff9bd 100644 --- a/test/sql/update_sql_test.cpp +++ b/test/sql/update_sql_test.cpp @@ -299,10 +299,10 @@ TEST_F(UpdateSQLTests, HalloweenProblemTest) { // it would have caused a second update on an already updated Tuple. size_t active_tilegroup_count = 3; - storage::DataTable::SetActiveTileGroupCount(active_tilegroup_count); + storage::DataTable::SetDefaultActiveTileGroupCount(active_tilegroup_count); LOG_DEBUG("Active tile group count = %zu", - storage::DataTable::GetActiveTileGroupCount()); + storage::DataTable::GetDefaultActiveTileGroupCount()); // Create a table first LOG_DEBUG("Creating a table..."); LOG_DEBUG("Query: CREATE TABLE test(a INT, b INT)"); @@ -372,10 +372,10 @@ TEST_F(UpdateSQLTests, HalloweenProblemTestWithPK) { // active_tilegroup_count set to 3, [Reason: Refer to HalloweenProblemTest] size_t active_tilegroup_count = 3; - storage::DataTable::SetActiveTileGroupCount(active_tilegroup_count); + storage::DataTable::SetDefaultActiveTileGroupCount(active_tilegroup_count); LOG_DEBUG("Active tile group count = %zu", - storage::DataTable::GetActiveTileGroupCount()); + storage::DataTable::GetDefaultActiveTileGroupCount()); // Create a table first LOG_DEBUG("Creating a table..."); LOG_DEBUG("Query: CREATE TABLE test(a INT PRIMARY KEY, b INT)"); @@ -469,10 +469,10 @@ TEST_F(UpdateSQLTests, MultiTileGroupUpdateSQLTest) { // active_tilegroup_count set to 3, [Reason: Refer to HalloweenProblemTest] size_t active_tilegroup_count = 3; - storage::DataTable::SetActiveTileGroupCount(active_tilegroup_count); + storage::DataTable::SetDefaultActiveTileGroupCount(active_tilegroup_count); LOG_DEBUG("Active tile group count = %zu", - storage::DataTable::GetActiveTileGroupCount()); + storage::DataTable::GetDefaultActiveTileGroupCount()); // Create a table first LOG_DEBUG("Creating a table..."); LOG_DEBUG("Query: CREATE TABLE test(a INT PRIMARY KEY, b INT)");