From 5686479c5f33087031e1b68b7832245c7886b712 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 14 May 2018 19:45:41 -0400 Subject: [PATCH] Schema - Layout decoupling (#1327) * Added layout.h. Modified Runtime functions to handle row stores. * Modify TileGroup api * Changed Updater & Inserter to use Layout object * Remove calls from optimizer and executor * Removed all calls to schemas vector from TileGroup * Removed Calls to LocateTileAndColumn for TileGroup * Moved definition of column_map_type. Removed column_map from tile_group * Added GetInfo function * make check pases. Need to change a few more function calls * Convert PL_ASSERT -> PELOTON_ASSERT * Modified DataTable and LayoutTunerTest * Added Layout Test for codegen * Get the tests to actually use different layouts * Modified storage layer, builds successfully. Need to fix tests * Moved GetColumnMapStats -> GetColumnLayoutStats * Minor change in TableScanTranslator * Modify TransformTileGroup * Change layout.cpp to better handle HYBRID layouts * use std::make_shared * Modified tests to change calls to GetTileGroup() * make check passes. Need to add test with hybrid layout. * Modify print functions * Add TODOs for catalog * Added LayoutCatalog. Yet to modify TableCatalog * Added catalog functions * Added DeleteLayouts to delete all layouts of a table_oid * LayoutTunerTest fixed * Fix build failures * Access LayoutCatalog via the global Catalog object * Added Multi layout scan test * Fixed tests after catalog refactor * Failing LayoutCatalog test because of deserialization * Fix all tests * Added documentation * Style Fix * Address review comments + minor clean-up in the layout.h API * Change valid_layout_objects_ * Revert unused changes * Addressed Prashanth's initial comments * Modify CreateTable for tests + modify LayoutTuner * Address Pervaze's review comments. * Changed after Prashanth's review --- src/catalog/catalog.cpp | 81 +++- src/catalog/layout_catalog.cpp | 229 ++++++++++++ src/catalog/schema.cpp | 18 +- src/catalog/system_catalogs.cpp | 6 +- src/catalog/table_catalog.cpp | 82 ++++ src/codegen/inserter.cpp | 9 +- src/codegen/runtime_functions.cpp | 47 ++- src/codegen/updater.cpp | 9 +- src/executor/logical_tile.cpp | 6 +- src/executor/logical_tile_factory.cpp | 8 +- src/gc/gc_manager.cpp | 65 ++-- src/include/catalog/catalog.h | 42 ++- src/include/catalog/catalog_defaults.h | 12 +- src/include/catalog/layout_catalog.h | 75 ++++ src/include/catalog/schema.h | 13 +- src/include/catalog/system_catalogs.h | 9 + src/include/catalog/table_catalog.h | 26 ++ src/include/codegen/tile_group.h | 2 +- src/include/common/exception.h | 13 +- src/include/common/internal_types.h | 10 + src/include/gc/gc_manager.h | 3 +- src/include/parser/sql_statement.h | 2 +- src/include/parser/table_ref.h | 2 +- src/include/storage/abstract_table.h | 20 +- src/include/storage/data_table.h | 46 ++- src/include/storage/layout.h | 148 ++++++++ src/include/storage/tile_group.h | 53 +-- src/include/storage/tile_group_factory.h | 2 +- src/include/tuning/clusterer.h | 5 - src/include/tuning/layout_tuner.h | 11 +- src/main/sdbench/sdbench_workload.cpp | 5 +- src/optimizer/stats/tuple_sampler.cpp | 5 +- src/optimizer/stats/tuple_samples_storage.cpp | 10 +- src/storage/abstract_table.cpp | 66 +--- src/storage/data_table.cpp | 130 +++---- src/storage/layout.cpp | 352 ++++++++++++++++++ src/storage/temp_table.cpp | 9 +- src/storage/tile_group.cpp | 95 ++--- src/storage/tile_group_factory.cpp | 8 +- src/tuning/index_tuner.cpp | 2 +- src/tuning/layout_tuner.cpp | 74 ++-- test/binder/binder_test.cpp | 6 +- test/catalog/catalog_test.cpp | 148 +++++++- test/catalog/constraints_test.cpp | 4 +- test/catalog/manager_test.cpp | 13 +- test/codegen/bloom_filter_test.cpp | 6 +- test/codegen/table_scan_translator_test.cpp | 251 ++++++++++++- test/codegen/testing_codegen_util.cpp | 95 ++++- test/executor/create_index_test.cpp | 2 +- test/executor/create_test.cpp | 16 +- test/executor/delete_test.cpp | 4 +- test/executor/drop_test.cpp | 44 ++- test/executor/insert_test.cpp | 4 +- test/executor/logical_tile_test.cpp | 17 +- test/executor/seq_scan_test.cpp | 8 +- test/executor/testing_executor_util.cpp | 42 ++- test/executor/update_test.cpp | 16 +- .../catalog/testing_constraints_util.h | 4 +- test/include/codegen/testing_codegen_util.h | 19 +- test/include/executor/testing_executor_util.h | 8 + test/optimizer/old_optimizer_test.cpp | 2 +- test/optimizer/optimizer_test.cpp | 10 +- test/optimizer/selectivity_test.cpp | 4 +- test/optimizer/table_stats_collector_test.cpp | 4 +- test/optimizer/tuple_samples_storage_test.cpp | 2 +- test/planner/plan_util_test.cpp | 18 +- test/planner/planner_test.cpp | 16 +- test/sql/drop_sql_test.cpp | 8 +- test/sql/optimizer_sql_test.cpp | 4 +- test/statistics/stats_test.cpp | 8 +- test/statistics/testing_stats_util.cpp | 2 +- test/storage/data_table_test.cpp | 2 +- test/storage/database_test.cpp | 9 +- test/storage/tile_group_test.cpp | 34 +- test/trigger/trigger_test.cpp | 12 +- test/tuning/layout_tuner_test.cpp | 49 +-- 76 files changed, 2104 insertions(+), 597 deletions(-) create mode 100644 src/catalog/layout_catalog.cpp create mode 100644 src/include/catalog/layout_catalog.h create mode 100644 src/include/storage/layout.h create mode 100644 src/storage/layout.cpp diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 90b9d13a62a..a36eb71c4bd 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -18,6 +18,7 @@ #include "catalog/index_catalog.h" #include "catalog/index_metrics_catalog.h" #include "catalog/language_catalog.h" +#include "catalog/layout_catalog.h" #include "catalog/proc_catalog.h" #include "catalog/query_history_catalog.h" #include "catalog/query_metrics_catalog.h" @@ -161,7 +162,7 @@ void Catalog::BootstrapSystemCatalogs(storage::Database *database, system_catalogs->GetSchemaCatalog()->InsertSchema( CATALOG_SCHEMA_OID, CATALOG_SCHEMA_NAME, pool_.get(), txn); system_catalogs->GetSchemaCatalog()->InsertSchema( - DEFUALT_SCHEMA_OID, DEFUALT_SCHEMA_NAME, pool_.get(), txn); + DEFAULT_SCHEMA_OID, DEFAULT_SCHEMA_NAME, pool_.get(), txn); // Insert catalog tables into pg_table // pg_database record is shared across different databases @@ -180,6 +181,9 @@ void Catalog::BootstrapSystemCatalogs(storage::Database *database, system_catalogs->GetTableCatalog()->InsertTable( COLUMN_CATALOG_OID, COLUMN_CATALOG_NAME, CATALOG_SCHEMA_NAME, database_oid, pool_.get(), txn); + system_catalogs->GetTableCatalog()->InsertTable( + LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, CATALOG_SCHEMA_NAME, + database_oid, pool_.get(), txn); } void Catalog::Bootstrap() { @@ -295,7 +299,8 @@ ResultType Catalog::CreateTable(const std::string &database_name, const std::string &table_name, std::unique_ptr schema, concurrency::TransactionContext *txn, - bool is_catalog, oid_t tuples_per_tilegroup) { + bool is_catalog, uint32_t tuples_per_tilegroup, + peloton::LayoutType layout_type) { if (txn == nullptr) throw CatalogException("Do not have transaction to create table " + table_name); @@ -348,7 +353,8 @@ ResultType Catalog::CreateTable(const std::string &database_name, bool adapt_table = false; auto table = storage::TableFactory::GetDataTable( database_object->GetDatabaseOid(), table_oid, schema.release(), - table_name, tuples_per_tilegroup, own_schema, adapt_table, is_catalog); + table_name, tuples_per_tilegroup, own_schema, adapt_table, is_catalog, + layout_type); database->AddTable(table, is_catalog); // put data table object into rw_object_set txn->RecordCreate(database_object->GetDatabaseOid(), table_oid, INVALID_OID); @@ -555,6 +561,44 @@ ResultType Catalog::CreateIndex( return ResultType::SUCCESS; } +std::shared_ptr Catalog::CreateLayout( + oid_t database_oid, oid_t table_oid, const column_map_type &column_map, + concurrency::TransactionContext *txn) { + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = storage_manager->GetDatabaseWithOid(database_oid); + auto table = database->GetTableWithOid(table_oid); + + oid_t layout_oid = table->GetNextLayoutOid(); + // Ensure that the new layout + PELOTON_ASSERT(layout_oid < INVALID_OID); + auto new_layout = std::shared_ptr( + new const storage::Layout(column_map, layout_oid)); + + // Add the layout the pg_layout table + auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog(); + bool result = + pg_layout->InsertLayout(table_oid, new_layout, pool_.get(), txn); + if (!result) { + LOG_ERROR("Failed to create a new layout for table %u", table_oid); + return nullptr; + } + return new_layout; +} + +std::shared_ptr Catalog::CreateDefaultLayout( + oid_t database_oid, oid_t table_oid, const column_map_type &column_map, + concurrency::TransactionContext *txn) { + auto new_layout = CreateLayout(database_oid, table_oid, column_map, txn); + // If the layout creation was successful, set it as the default + if (new_layout != nullptr) { + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = storage_manager->GetDatabaseWithOid(database_oid); + auto table = database->GetTableWithOid(table_oid); + table->SetDefaultLayout(new_layout); + } + return new_layout; +} + //===----------------------------------------------------------------------===// // DROP FUNCTIONS //===----------------------------------------------------------------------===// @@ -700,7 +744,6 @@ ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid, auto table_object = database_object->GetTableObject(table_oid); auto index_objects = table_object->GetIndexObjects(); LOG_TRACE("dropping #%d indexes", (int)index_objects.size()); - // delete trigger and records in pg_trigger auto pg_trigger = catalog_map_[database_object->GetDatabaseOid()]->GetTriggerCatalog(); @@ -719,11 +762,15 @@ ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid, catalog_map_[database_object->GetDatabaseOid()]->GetColumnCatalog(); pg_attribute->DeleteColumns(table_oid, txn); + // delete record in pg_layout + auto pg_layout = + catalog_map_[database_object->GetDatabaseOid()]->GetLayoutCatalog(); + pg_layout->DeleteLayouts(table_oid, txn); + // delete record in pg_table auto pg_table = catalog_map_[database_object->GetDatabaseOid()]->GetTableCatalog(); pg_table->DeleteTable(table_oid, txn); - database->GetTableWithOid(table_oid); txn->RecordDrop(database_oid, table_oid, INVALID_OID); @@ -764,6 +811,30 @@ ResultType Catalog::DropIndex(oid_t database_oid, oid_t index_oid, return ResultType::SUCCESS; } +ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, + oid_t layout_oid, + concurrency::TransactionContext *txn) { + // Check if the default_layout of the table is the same. + // If true reset it to a row store. + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = storage_manager->GetDatabaseWithOid(database_oid); + auto table = database->GetTableWithOid(table_oid); + auto default_layout = table->GetDefaultLayout(); + + if (default_layout.GetOid() == layout_oid) { + table->ResetDefaultLayout(); + } + + auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog(); + if (!pg_layout->DeleteLayout(table_oid, layout_oid, txn)) { + auto layout = table->GetDefaultLayout(); + LOG_DEBUG("Layout delete failed. Default layout id: %u", layout.GetOid()); + return ResultType::FAILURE; + } + + return ResultType::SUCCESS; +} + //===--------------------------------------------------------------------===// // GET WITH NAME - CHECK FROM CATALOG TABLES, USING TRANSACTION //===--------------------------------------------------------------------===// diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp new file mode 100644 index 00000000000..bd72854455d --- /dev/null +++ b/src/catalog/layout_catalog.cpp @@ -0,0 +1,229 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// layout_catalog.cpp +// +// Identification: src/catalog/layout_catalog.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "catalog/layout_catalog.h" + +#include "catalog/catalog.h" +#include "catalog/system_catalogs.h" +#include "concurrency/transaction_context.h" +#include "executor/logical_tile.h" +#include "storage/data_table.h" +#include "storage/layout.h" + +namespace peloton { +namespace catalog { + +/** @brief Constructor invoked by the SystemsCatalog constructor. + * @param pg_catalog The database to which this pg_layout belongs. + */ +LayoutCatalog::LayoutCatalog( + storage::Database *pg_catalog, UNUSED_ATTRIBUTE type::AbstractPool *pool, + UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) + : AbstractCatalog(LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, + InitializeSchema().release(), pg_catalog) { + // Add indexes for pg_attribute + AddIndex({ColumnId::TABLE_OID, ColumnId::LAYOUT_OID}, LAYOUT_CATALOG_PKEY_OID, + LAYOUT_CATALOG_NAME "_pkey", IndexConstraintType::PRIMARY_KEY); + AddIndex({ColumnId::TABLE_OID}, LAYOUT_CATALOG_SKEY0_OID, + LAYOUT_CATALOG_NAME "_skey0", IndexConstraintType::DEFAULT); +} +/** @brief Destructor. Do nothing. Layouts will be dropped by DropTable. */ +LayoutCatalog::~LayoutCatalog() {} + +/** @brief Initilailizes the schema for the pg_layout table. + * @return unique_ptr of the schema for pg_layout. + */ +std::unique_ptr LayoutCatalog::InitializeSchema() { + const std::string primary_key_constraint_name = "primary_key"; + const std::string not_null_constraint_name = "not_null"; + + auto table_id_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "table_oid", true); + table_id_column.AddConstraint(catalog::Constraint( + ConstraintType::PRIMARY, primary_key_constraint_name)); + table_id_column.AddConstraint( + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + + auto layout_oid_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "layout_oid", true); + layout_oid_column.AddConstraint(catalog::Constraint( + ConstraintType::PRIMARY, primary_key_constraint_name)); + layout_oid_column.AddConstraint( + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + + auto num_columns_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "num_columns", true); + num_columns_column.AddConstraint( + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + + auto column_map_column = catalog::Column( + type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR), + "column_map", false); + column_map_column.AddConstraint( + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + + std::unique_ptr column_catalog_schema( + new catalog::Schema({table_id_column, layout_oid_column, + num_columns_column, column_map_column})); + + return column_catalog_schema; +} + +/** @brief Insert a layout into the pg_layout table. + * @param table_oid oid of the table to which the new layout belongs. + * @param layout layout to be added to the pg_layout table. + * @param pool to allocate memory for the column_map column. + * @param txn TransactionContext for adding the layout. + * @return true on success. + */ +bool LayoutCatalog::InsertLayout(oid_t table_oid, + std::shared_ptr layout, + type::AbstractPool *pool, + concurrency::TransactionContext *txn) { + // Create the tuple first + std::unique_ptr tuple( + new storage::Tuple(catalog_table_->GetSchema(), true)); + + auto val0 = type::ValueFactory::GetIntegerValue(table_oid); + auto val1 = type::ValueFactory::GetIntegerValue(layout->GetOid()); + auto val2 = type::ValueFactory::GetIntegerValue(layout->GetColumnCount()); + auto val3 = type::ValueFactory::GetVarcharValue(layout->SerializeColumnMap(), + nullptr); + + tuple->SetValue(LayoutCatalog::ColumnId::TABLE_OID, val0, pool); + tuple->SetValue(LayoutCatalog::ColumnId::LAYOUT_OID, val1, pool); + tuple->SetValue(LayoutCatalog::ColumnId::NUM_COLUMNS, val2, pool); + tuple->SetValue(LayoutCatalog::ColumnId::COLUMN_MAP, val3, pool); + + // Insert the tuple + return InsertTuple(std::move(tuple), txn); +} + +/** @brief Delete a layout from the pg_layout table. + * @param table_oid oid of the table to which the old layout belongs. + * @param layout_oid oid of the layout to be deleted. + * @param txn TransactionContext for deleting the layout. + * @return true on success. + */ +bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_oid, + concurrency::TransactionContext *txn) { + oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid & layout_oid + + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + values.push_back(type::ValueFactory::GetIntegerValue(layout_oid).Copy()); + + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid) + ->GetTableCatalog(); + + // delete column from cache + auto table_object = pg_table->GetTableObject(table_oid, txn); + table_object->EvictLayout(layout_oid); + + return DeleteWithIndexScan(index_offset, values, txn); +} + +/** @brief Delete all layouts correponding to a table from the pg_layout. + * @param table_oid oid of the table to delete all layouts. + * @param txn TransactionContext for deleting the layouts. + * @return true on success. + */ +bool LayoutCatalog::DeleteLayouts(oid_t table_oid, + concurrency::TransactionContext *txn) { + oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + + // delete layouts from cache + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableObject(table_oid, txn); + table_object->EvictAllLayouts(); + + return DeleteWithIndexScan(index_offset, values, txn); +} + +/** @brief Get all layouts correponding to a table from the pg_layout. + * @param table_oid oid of the table to fetch all layouts. + * @param txn TransactionContext for getting the layouts. + * @return unordered_map containing a layout_oid -> layout mapping. + */ +const std::unordered_map> +LayoutCatalog::GetLayouts(oid_t table_oid, + concurrency::TransactionContext *txn) { + // Try to find the layouts in the cache + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableObject(table_oid, txn); + PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); + auto layout_objects = table_object->GetLayouts(true); + if (layout_objects.size() != 0) { + return layout_objects; + } + + // Cache miss, get from pg_catalog + std::vector column_ids(all_column_ids); + oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + + auto result_tiles = + GetResultWithIndexScan(column_ids, index_offset, values, txn); + + for (auto &tile : (*result_tiles)) { // Iterate through the result_tiles + for (auto tuple_id : *tile) { + oid_t layout_oid = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::LAYOUT_OID) + .GetAs(); + oid_t num_columns = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::NUM_COLUMNS) + .GetAs(); + std::string column_map_str = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::COLUMN_MAP) + .ToString(); + auto column_map = + storage::Layout::DeserializeColumnMap(num_columns, column_map_str); + auto layout_object = + std::make_shared(column_map, layout_oid); + table_object->InsertLayout(layout_object); + } + } + + return table_object->GetLayouts(); +} + +/** @brief Get the layout by layout_oid from the pg_layout. + * @param table_oid oid of the table to fetch the layout. + * @param layout_oid oid of the layout being queried. + * @param txn TransactionContext for getting the layout. + * @return shared_ptr corresponding to the layout_oid if found. + * nullptr otherwise. + */ +std::shared_ptr LayoutCatalog::GetLayoutWithOid( + oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn) { + auto table_layouts = GetLayouts(table_oid, txn); + for (const auto &layout_entry : table_layouts) { + if (layout_entry.second->GetOid() == layout_oid) { + return layout_entry.second; + } + } + return nullptr; +} + +} // namespace catalog +} // namespace peloton \ No newline at end of file diff --git a/src/catalog/schema.cpp b/src/catalog/schema.cpp index e3e088e607a..ab38f8c8501 100644 --- a/src/catalog/schema.cpp +++ b/src/catalog/schema.cpp @@ -205,14 +205,15 @@ Schema *Schema::FilterSchema(const Schema *schema, } // Append two schema objects -Schema *Schema::AppendSchema(Schema *first, Schema *second) { +Schema *Schema::AppendSchema(const Schema *first, const Schema *second) { return AppendSchemaPtrList({first, second}); } // Append subset of columns in the two given schemas -Schema *Schema::AppendSchema(Schema *first, std::vector &first_set, - Schema *second, std::vector &second_set) { - const std::vector schema_list({first, second}); +Schema *Schema::AppendSchema(const Schema *first, std::vector &first_set, + const Schema *second, + std::vector &second_set) { + const std::vector schema_list({first, second}); const std::vector> subsets({first_set, second_set}); return AppendSchemaPtrList(schema_list, subsets); } @@ -221,7 +222,7 @@ Schema *Schema::AppendSchema(Schema *first, std::vector &first_set, Schema *Schema::AppendSchemaList(std::vector &schema_list) { // All we do here is convert vector to vector. // This is a convenience function. - std::vector schema_ptr_list; + std::vector schema_ptr_list; for (unsigned int i = 0; i < schema_list.size(); i++) { schema_ptr_list.push_back(&schema_list[i]); } @@ -229,7 +230,8 @@ Schema *Schema::AppendSchemaList(std::vector &schema_list) { } // Append given schemas. -Schema *Schema::AppendSchemaPtrList(const std::vector &schema_list) { +Schema *Schema::AppendSchemaPtrList( + const std::vector &schema_list) { std::vector> subsets; for (unsigned int i = 0; i < schema_list.size(); i++) { @@ -246,13 +248,13 @@ Schema *Schema::AppendSchemaPtrList(const std::vector &schema_list) { // Append subsets of columns in the given schemas. Schema *Schema::AppendSchemaPtrList( - const std::vector &schema_list, + const std::vector &schema_list, const std::vector> &subsets) { PELOTON_ASSERT(schema_list.size() == subsets.size()); std::vector columns; for (unsigned int i = 0; i < schema_list.size(); i++) { - Schema *schema = schema_list[i]; + const Schema *schema = schema_list[i]; const std::vector &subset = subsets[i]; unsigned int column_count = schema->GetColumnCount(); diff --git a/src/catalog/system_catalogs.cpp b/src/catalog/system_catalogs.cpp index b1371ddd379..3900c165f74 100644 --- a/src/catalog/system_catalogs.cpp +++ b/src/catalog/system_catalogs.cpp @@ -13,6 +13,7 @@ #include "catalog/system_catalogs.h" #include "catalog/column_catalog.h" #include "catalog/index_catalog.h" +#include "catalog/layout_catalog.h" #include "catalog/table_catalog.h" #include "storage/data_table.h" #include "storage/database.h" @@ -38,6 +39,7 @@ SystemCatalogs::SystemCatalogs(storage::Database *database, pg_namespace_ = new SchemaCatalog(database, pool, txn); pg_table_ = new TableCatalog(database, pool, txn); pg_index_ = new IndexCatalog(database, pool, txn); + pg_layout_ = new LayoutCatalog(database, pool, txn); // TODO: can we move this to BootstrapSystemCatalogs()? // insert column information into pg_attribute @@ -45,7 +47,8 @@ SystemCatalogs::SystemCatalogs(storage::Database *database, {CATALOG_DATABASE_OID, DATABASE_CATALOG_OID}, {database_oid, TABLE_CATALOG_OID}, {database_oid, SCHEMA_CATALOG_OID}, - {database_oid, INDEX_CATALOG_OID}}; + {database_oid, INDEX_CATALOG_OID}, + {database_oid, LAYOUT_CATALOG_OID}}; for (int i = 0; i < (int)shared_tables.size(); i++) { oid_t column_id = 0; @@ -65,6 +68,7 @@ SystemCatalogs::SystemCatalogs(storage::Database *database, SystemCatalogs::~SystemCatalogs() { delete pg_index_; + delete pg_layout_; delete pg_table_; delete pg_attribute_; delete pg_namespace_; diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 34ef723e366..9f181d4c0dc 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -18,6 +18,7 @@ #include "catalog/column_catalog.h" #include "catalog/database_catalog.h" #include "catalog/index_catalog.h" +#include "catalog/layout_catalog.h" #include "catalog/system_catalogs.h" #include "concurrency/transaction_context.h" #include "storage/data_table.h" @@ -330,6 +331,87 @@ TableCatalog::TableCatalog( IndexConstraintType::DEFAULT); } +/** @brief Insert layout object into the cache. + * @param layout Layout object to be inserted + * @return false if layout already exists in cache + */ +bool TableCatalogObject::InsertLayout( + std::shared_ptr layout) { + // Invalid object + if (!layout || (layout->GetOid() == INVALID_OID)) { + return false; + } + + oid_t layout_id = layout->GetOid(); + // layout is already present in the cache. + if (layout_objects_.find(layout_id) != layout_objects_.end()) { + LOG_DEBUG("Layout %u already exists in cache!", layout_id); + return false; + } + + layout_objects_.insert(std::make_pair(layout_id, layout)); + return true; +} + +/** @brief evict all layout objects from cache. */ +void TableCatalogObject::EvictAllLayouts() { + layout_objects_.clear(); + valid_layout_objects_ = false; +} + +/** @brief Get all layout objects of this table. + * Add it to the cache if necessary. + * @param cached_only If set to true, don't fetch the layout objects. + * @return Map from layout_oid to cached layout object. + */ +std::unordered_map> +TableCatalogObject::GetLayouts(bool cached_only) { + if (!valid_layout_objects_ && !cached_only) { + // get layout catalog objects from pg_layout + auto pg_layout = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid) + ->GetLayoutCatalog(); + pg_layout->GetLayouts(table_oid, txn); + valid_column_objects = true; + } + return layout_objects_; +} + +/** @brief Get the layout object of the given layout_id. + * @param layout_id The id of the layout to be fetched. + * @param cached_only If set to true, don't fetch the layout objects. + * @return Layout object of corresponding to the layout_id if present. + */ +std::shared_ptr TableCatalogObject::GetLayout( + oid_t layout_id, bool cached_entry) { + // fetch layout objects in case we have not + GetLayouts(cached_entry); + auto it = layout_objects_.find(layout_id); + if (it != layout_objects_.end()) { + return it->second; + } + return nullptr; +} + +/** @brief Evict layout from the cache. + * @param layout_id Id of the layout to be deleted. + * @return true if layout_id is found and evicted; false if not found. + */ +bool TableCatalogObject::EvictLayout(oid_t layout_id) { + if (!valid_layout_objects_) return false; + + // find layout from the cache + auto it = layout_objects_.find(layout_id); + if (it == layout_objects_.end()) { + return false; // layout_id not found in cache + } + + auto layout = it->second; + PELOTON_ASSERT(layout); + layout_objects_.erase(it); + return true; +} + TableCatalog::~TableCatalog() {} /*@brief private function for initialize schema of pg_table diff --git a/src/codegen/inserter.cpp b/src/codegen/inserter.cpp index ce652cf746e..abe00ccbb07 100644 --- a/src/codegen/inserter.cpp +++ b/src/codegen/inserter.cpp @@ -34,11 +34,12 @@ void Inserter::Init(storage::DataTable *table, char *Inserter::AllocateTupleStorage() { location_ = table_->GetEmptyTupleSlot(nullptr); - // Get the tile offset assuming that it is in a tuple format + // Get the tile offset assuming that it is a row store auto tile_group = table_->GetTileGroupById(location_.block); - oid_t tile_offset, tile_column_offset; - tile_group->LocateTileAndColumn(0, tile_offset, tile_column_offset); - tile_ = tile_group->GetTileReference(tile_offset); + auto layout = tile_group->GetLayout(); + PELOTON_ASSERT(layout.IsRowStore()); + // layout is still a row store. Hence tile offset it 0 + tile_ = tile_group->GetTileReference(0); return tile_->GetTupleLocation(location_.offset); } diff --git a/src/codegen/runtime_functions.cpp b/src/codegen/runtime_functions.cpp index 466478fc55d..40562516f75 100644 --- a/src/codegen/runtime_functions.cpp +++ b/src/codegen/runtime_functions.cpp @@ -19,8 +19,9 @@ #include "expression/abstract_expression.h" #include "expression/expression_util.h" #include "storage/data_table.h" -#include "storage/tile_group.h" +#include "storage/layout.h" #include "storage/tile.h" +#include "storage/tile_group.h" #include "storage/zone_map_manager.h" #include "type/value_factory.h" @@ -102,23 +103,37 @@ void RuntimeFunctions::FillPredicateArray( //===----------------------------------------------------------------------===// void RuntimeFunctions::GetTileGroupLayout(const storage::TileGroup *tile_group, ColumnLayoutInfo *infos, - uint32_t num_cols) { - for (uint32_t col_idx = 0; col_idx < num_cols; col_idx++) { - // Map the current column to a tile and a column offset in the tile - oid_t tile_offset, tile_column_offset; - tile_group->LocateTileAndColumn(col_idx, tile_offset, tile_column_offset); + UNUSED_ATTRIBUTE uint32_t num_cols) { + const auto &layout = tile_group->GetLayout(); + UNUSED_ATTRIBUTE oid_t last_col_idx = INVALID_OID; - // Now grab the column information - auto *tile = tile_group->GetTile(tile_offset); - auto *tile_schema = tile->GetSchema(); - infos[col_idx].column = - tile->GetTupleLocation(0) + tile_schema->GetOffset(tile_column_offset); - infos[col_idx].stride = tile_schema->GetLength(); - infos[col_idx].is_columnar = tile_schema->GetColumnCount() == 1; - LOG_TRACE("Col [%u] start: %p, stride: %u, columnar: %s", col_idx, - infos[col_idx].column, infos[col_idx].stride, - infos[col_idx].is_columnar ? "true" : "false"); + auto tile_map = layout.GetTileMap(); + // Find the mapping for each tile in the layout. + for (auto tile_entry : tile_map) { + // Get the tile schema. + auto tile_idx = tile_entry.first; + auto *tile = tile_group->GetTile(tile_idx); + auto tile_schema = tile->GetSchema(); + // Map the current column to a tile and a column offset in the tile. + for (auto column_entry : tile_entry.second) { + // Now grab the column information + oid_t col_idx = column_entry.first; + oid_t tile_col_offset = column_entry.second; + // Ensure that the col_idx is within the num_cols range + PELOTON_ASSERT(col_idx < num_cols); + infos[col_idx].column = + tile->GetTupleLocation(0) + tile_schema->GetOffset(tile_col_offset); + infos[col_idx].stride = tile_schema->GetLength(); + infos[col_idx].is_columnar = tile_schema->GetColumnCount() == 1; + last_col_idx = col_idx; + LOG_TRACE("Col [%u] start: %p, stride: %u, columnar: %s", col_idx, + infos[col_idx].column, infos[col_idx].stride, + infos[col_idx].is_columnar ? "true" : "false"); + } } + // Ensure that ColumnLayoutInfo for each column has been populated. + PELOTON_ASSERT((last_col_idx != INVALID_OID) && + (last_col_idx == (num_cols - 1))); } void RuntimeFunctions::ThrowDivideByZeroException() { diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 88ef489aed1..1782f219c83 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -44,10 +44,11 @@ void Updater::Init(storage::DataTable *table, char *Updater::GetDataPtr(uint32_t tile_group_id, uint32_t tuple_offset) { auto tile_group = table_->GetTileGroupById(tile_group_id); - // Get the tile offset assuming that it is still in a tuple format - oid_t tile_offset, tile_column_offset; - tile_group->LocateTileAndColumn(0, tile_offset, tile_column_offset); - tile_ = tile_group->GetTileReference(tile_offset); + // Get the tile offset assuming that it is still a row store + // Hence the Tile offset is 0. + UNUSED_ATTRIBUTE const auto &layout = tile_group->GetLayout(); + PELOTON_ASSERT(layout.IsRowStore()); + tile_ = tile_group->GetTileReference(0); return tile_->GetTupleLocation(tuple_offset); } diff --git a/src/executor/logical_tile.cpp b/src/executor/logical_tile.cpp index 3bdeec9b129..f3857748ecf 100644 --- a/src/executor/logical_tile.cpp +++ b/src/executor/logical_tile.cpp @@ -16,6 +16,7 @@ #include "catalog/schema.h" #include "common/macros.h" #include "storage/data_table.h" +#include "storage/layout.h" #include "storage/tile.h" #include "storage/tile_group.h" #include "type/value.h" @@ -396,11 +397,12 @@ void LogicalTile::AddColumns( const std::shared_ptr &tile_group, const std::vector &column_ids) { const int position_list_idx = 0; + auto tile_group_layout = tile_group->GetLayout(); for (oid_t origin_column_id : column_ids) { oid_t base_tile_offset, tile_column_id; - tile_group->LocateTileAndColumn(origin_column_id, base_tile_offset, - tile_column_id); + tile_group_layout.LocateTileAndColumn(origin_column_id, base_tile_offset, + tile_column_id); AddColumn(tile_group->GetTileReference(base_tile_offset), tile_column_id, position_list_idx); diff --git a/src/executor/logical_tile_factory.cpp b/src/executor/logical_tile_factory.cpp index b1278a2b274..793a22ee784 100644 --- a/src/executor/logical_tile_factory.cpp +++ b/src/executor/logical_tile_factory.cpp @@ -89,11 +89,11 @@ LogicalTile *LogicalTileFactory::WrapTileGroup( CreateIdentityPositionList(tile_group->GetActiveTupleCount())); // Construct schema. - std::vector &schemas = tile_group->GetTileSchemas(); - PELOTON_ASSERT(schemas.size() == tile_group->NumTiles()); - for (unsigned int i = 0; i < schemas.size(); i++) { + unsigned int num_tiles = tile_group->NumTiles(); + for (unsigned int i = 0; i < num_tiles; i++) { auto base_tile_ref = tile_group->GetTileReference(i); - for (oid_t col_id = 0; col_id < schemas[i].GetColumnCount(); col_id++) { + auto schema = base_tile_ref->GetSchema(); + for (oid_t col_id = 0; col_id < schema->GetColumnCount(); col_id++) { new_tile->AddColumn(base_tile_ref, col_id, position_list_idx); } } diff --git a/src/gc/gc_manager.cpp b/src/gc/gc_manager.cpp index 7db811f197d..9357ff1496e 100644 --- a/src/gc/gc_manager.cpp +++ b/src/gc/gc_manager.cpp @@ -23,39 +23,40 @@ namespace peloton { namespace gc { // Check a tuple and reclaim all varlen field -void GCManager::CheckAndReclaimVarlenColumns(storage::TileGroup *tg, oid_t tuple_id) { - oid_t tile_count = tg->tile_count; - oid_t tile_col_count; - type::TypeId type_id; - char *tuple_location; - char *field_location; - char *varlen_ptr; - - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tg->tile_schemas[tile_itr]; - - tile_col_count = schema.GetColumnCount(); - - storage::Tile *tile = tg->GetTile(tile_itr); - PELOTON_ASSERT(tile); - for (oid_t tile_col_itr = 0; tile_col_itr < tile_col_count; ++tile_col_itr) { - type_id = schema.GetType(tile_col_itr); - - if ((type_id != type::TypeId::VARCHAR && type_id != type::TypeId::VARBINARY) - || (schema.IsInlined(tile_col_itr) == true)) { - // Not of varlen type, or is inlined, skip - continue; - } - // Get the raw varlen pointer - tuple_location = tile->GetTupleLocation(tuple_id); - field_location = tuple_location + schema.GetOffset(tile_col_itr); - varlen_ptr = type::Value::GetDataFromStorage(type_id, field_location); - // Call the corresponding varlen pool free - if (varlen_ptr != nullptr) { - tile->pool->Free(varlen_ptr); - } - } +void GCManager::CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, + oid_t tuple_id) { + uint32_t tile_count = tile_group->tile_count_; + uint32_t tile_col_count; + type::TypeId type_id; + char *tuple_location; + char *field_location; + char *varlen_ptr; + + for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + storage::Tile *tile = tile_group->GetTile(tile_itr); + PELOTON_ASSERT(tile); + const catalog::Schema *schema = tile->GetSchema(); + tile_col_count = schema->GetColumnCount(); + for (oid_t tile_col_itr = 0; tile_col_itr < tile_col_count; + ++tile_col_itr) { + type_id = schema->GetType(tile_col_itr); + + if ((type_id != type::TypeId::VARCHAR && + type_id != type::TypeId::VARBINARY) || + (schema->IsInlined(tile_col_itr) == true)) { + // Not of varlen type, or is inlined, skip + continue; + } + // Get the raw varlen pointer + tuple_location = tile->GetTupleLocation(tuple_id); + field_location = tuple_location + schema->GetOffset(tile_col_itr); + varlen_ptr = type::Value::GetDataFromStorage(type_id, field_location); + // Call the corresponding varlen pool free + if (varlen_ptr != nullptr) { + tile->pool->Free(varlen_ptr); } + } + } } } diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index cd19d7fe72b..febcb82be4e 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -38,6 +38,7 @@ class Index; namespace storage { class Database; class DataTable; +class Layout; class TableFactory; class Tuple; } // namespace storage @@ -99,7 +100,8 @@ class Catalog { const std::string &database_name, const std::string &schema_name, const std::string &table_name, std::unique_ptr, concurrency::TransactionContext *txn, bool is_catalog = false, - oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP); + uint32_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP, + peloton::LayoutType layout_type = LayoutType::ROW); // Create index for a table ResultType CreateIndex(const std::string &database_name, @@ -118,6 +120,33 @@ class Catalog { concurrency::TransactionContext *txn, bool is_catalog = false); + /** + * @brief create a new layout for a table + * @param database_oid database to which the table belongs to + * @param table_oid table to which the layout has to be added + * @param column_map column_map of the new layout to be created + * @param txn TransactionContext + * @return shared_ptr shared_ptr to the newly created layout in case of + * success. nullptr in case of failure. + */ + std::shared_ptr CreateLayout( + oid_t database_oid, oid_t table_oid, const column_map_type &column_map, + concurrency::TransactionContext *txn); + + /** + * @brief create a new layout for a table and make it the default if + * if the creating is successsful. + * @param database_oid database to which the table belongs to + * @param table_oid table to which the layout has to be added + * @param column_map column_map of the new layout to be created + * @param txn TransactionContext + * @return shared_ptr shared_ptr to the newly created layout in case of + * success. nullptr in case of failure. + */ + std::shared_ptr CreateDefaultLayout( + oid_t database_oid, oid_t table_oid, const column_map_type &column_map, + concurrency::TransactionContext *txn); + //===--------------------------------------------------------------------===// // DROP FUNCTIONS //===--------------------------------------------------------------------===// @@ -145,6 +174,17 @@ class Catalog { // Drop an index, using its index_oid ResultType DropIndex(oid_t database_oid, oid_t index_oid, concurrency::TransactionContext *txn); + + /** @brief Drop layout + * tile_groups + * @param database_oid the database to which the table belongs + * @param table_oid the table to which the layout belongs + * @param layout_oid the layout to be dropped + * @param txn TransactionContext + * @return ResultType(SUCCESS or FAILURE) + */ + ResultType DropLayout(oid_t database_oid, oid_t table_oid, oid_t layout_oid, + concurrency::TransactionContext *txn); //===--------------------------------------------------------------------===// // GET WITH NAME - CHECK FROM CATALOG TABLES, USING TRANSACTION //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/catalog_defaults.h b/src/include/catalog/catalog_defaults.h index 2cfcacbda70..117a7e8ab6d 100644 --- a/src/include/catalog/catalog_defaults.h +++ b/src/include/catalog/catalog_defaults.h @@ -32,10 +32,11 @@ namespace catalog { #define TABLE_CATALOG_NAME "pg_table" #define INDEX_CATALOG_NAME "pg_index" #define COLUMN_CATALOG_NAME "pg_attribute" +#define LAYOUT_CATALOG_NAME "pg_layout" // Local oids from START_OID = 0 to START_OID + OID_OFFSET are reserved #define OID_OFFSET 100 -#define CATALOG_TABLES_COUNT 8 +#define CATALOG_TABLES_COUNT 9 // Oid mask for each type #define DATABASE_OID_MASK (static_cast(catalog::CatalogType::DATABASE)) @@ -52,9 +53,9 @@ namespace catalog { // Reserved schema oid // "public" for default schema, and "pg_catalog" schema for catalog tables #define CATALOG_SCHEMA_OID (0 | SCHEMA_OID_MASK) -#define DEFUALT_SCHEMA_OID (1 | SCHEMA_OID_MASK) +#define DEFAULT_SCHEMA_OID (1 | SCHEMA_OID_MASK) #define CATALOG_SCHEMA_NAME "pg_catalog" -#define DEFUALT_SCHEMA_NAME "public" +#define DEFAULT_SCHEMA_NAME "public" // Reserved pg_xxx table oid #define DATABASE_CATALOG_OID (0 | TABLE_OID_MASK) @@ -62,6 +63,7 @@ namespace catalog { #define TABLE_CATALOG_OID (2 | TABLE_OID_MASK) #define INDEX_CATALOG_OID (3 | TABLE_OID_MASK) #define COLUMN_CATALOG_OID (4 | TABLE_OID_MASK) +#define LAYOUT_CATALOG_OID (5 | TABLE_OID_MASK) // Reserved pg_column index oid #define COLUMN_CATALOG_PKEY_OID (0 | INDEX_OID_MASK) @@ -86,6 +88,10 @@ namespace catalog { #define TABLE_CATALOG_SKEY0_OID (11 | INDEX_OID_MASK) #define TABLE_CATALOG_SKEY1_OID (12 | INDEX_OID_MASK) +// Reserve pg_layout index oid +#define LAYOUT_CATALOG_PKEY_OID (13 | INDEX_OID_MASK) +#define LAYOUT_CATALOG_SKEY0_OID (14 | INDEX_OID_MASK) + // Use upper 8 bits indicating catalog type #define CATALOG_TYPE_OFFSET 24 diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h new file mode 100644 index 00000000000..0fc224f094d --- /dev/null +++ b/src/include/catalog/layout_catalog.h @@ -0,0 +1,75 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// layout_catalog.h +// +// Identification: src/include/catalog/layout_catalog.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "catalog/abstract_catalog.h" + +namespace peloton { + +namespace storage { +class Layout; +} // namespace storage + +namespace catalog { + +class LayoutCatalog : public AbstractCatalog { + public: + + LayoutCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, + concurrency::TransactionContext *txn); + + ~LayoutCatalog(); + + //===--------------------------------------------------------------------===// + // write Related API + //===--------------------------------------------------------------------===// + bool InsertLayout(oid_t table_oid, + std::shared_ptr layout, + type::AbstractPool *pool, + concurrency::TransactionContext *txn); + + bool DeleteLayout(oid_t table_oid, oid_t layout_oid, + concurrency::TransactionContext *txn); + + bool DeleteLayouts(oid_t table_oid, concurrency::TransactionContext *txn); + + //===--------------------------------------------------------------------===// + // Read Related API + //===--------------------------------------------------------------------===// + const std::unordered_map> + GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn); + + std::shared_ptr GetLayoutWithOid( + oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn); + + private: + std::unique_ptr InitializeSchema(); + + enum ColumnId { + TABLE_OID = 0, + LAYOUT_OID = 1, + NUM_COLUMNS = 2, + COLUMN_MAP = 3, + // Add new columns here in creation order + }; + std::vector all_column_ids = {0, 1, 2, 3}; + + enum IndexId { + PRIMARY_KEY = 0, + SKEY_TABLE_OID = 1, + // Add new indexes here in creation order + }; +}; + +} // namespace catalog +} // namespace peloton \ No newline at end of file diff --git a/src/include/catalog/schema.h b/src/include/catalog/schema.h index 8a2feec5dcd..43a62d6444f 100644 --- a/src/include/catalog/schema.h +++ b/src/include/catalog/schema.h @@ -62,21 +62,24 @@ class Schema : public Printable { const std::vector &set); // Append two schema objects - static Schema *AppendSchema(Schema *first, Schema *second); + static Schema *AppendSchema(const Schema *first, const Schema *second); // Append subset of columns in the two given schemas - static Schema *AppendSchema(Schema *first, std::vector &first_set, - Schema *second, std::vector &second_set); + static Schema *AppendSchema(const Schema *first, + std::vector &first_set, + const Schema *second, + std::vector &second_set); // Append given schemas. static Schema *AppendSchemaList(std::vector &schema_list); // Append given schemas. - static Schema *AppendSchemaPtrList(const std::vector &schema_list); + static Schema *AppendSchemaPtrList( + const std::vector &schema_list); // Append subsets of columns in the given schemas. static Schema *AppendSchemaPtrList( - const std::vector &schema_list, + const std::vector &schema_list, const std::vector> &subsets); // Compare two schemas diff --git a/src/include/catalog/system_catalogs.h b/src/include/catalog/system_catalogs.h index 7791c019097..9792d180f9d 100644 --- a/src/include/catalog/system_catalogs.h +++ b/src/include/catalog/system_catalogs.h @@ -34,6 +34,7 @@ class SchemaCatalog; class TableCatalog; class IndexCatalog; class ColumnCatalog; +class LayoutCatalog; class SystemCatalogs { public: @@ -79,6 +80,13 @@ class SystemCatalogs { return pg_index_; } + LayoutCatalog *GetLayoutCatalog() { + if (!pg_layout_) { + throw CatalogException("Layout catalog has not been initialized"); + } + return pg_layout_; + } + TriggerCatalog *GetTriggerCatalog() { if (!pg_trigger_) { throw CatalogException("Trigger catalog has not been initialized"); @@ -112,6 +120,7 @@ class SystemCatalogs { SchemaCatalog *pg_namespace_; TableCatalog *pg_table_; IndexCatalog *pg_index_; + LayoutCatalog *pg_layout_; TriggerCatalog *pg_trigger_; // ProcCatalog *pg_proc; diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index 0dfc3f51fa9..9a01ee6e07f 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -36,6 +36,11 @@ #include "executor/logical_tile.h" namespace peloton { + +namespace storage { +class Layout; +} // namespace storage + namespace catalog { class IndexCatalogObject; @@ -45,6 +50,7 @@ class TableCatalogObject { friend class TableCatalog; friend class IndexCatalog; friend class ColumnCatalog; + friend class LayoutCatalog; public: TableCatalogObject(executor::LogicalTile *tile, @@ -73,6 +79,15 @@ class TableCatalogObject { std::shared_ptr GetColumnObject( const std::string &column_name, bool cached_only = false); + // Evict all layouts from the cache + void EvictAllLayouts(); + + // Get layouts + std::unordered_map> GetLayouts( + bool cached_only = false); + std::shared_ptr GetLayout(oid_t layout_id, + bool cached_entry = false); + inline oid_t GetTableOid() { return table_oid; } inline const std::string &GetTableName() { return table_name; } inline const std::string &GetSchemaName() { return schema_name; } @@ -97,6 +112,11 @@ class TableCatalogObject { bool EvictColumnObject(oid_t column_id); bool EvictColumnObject(const std::string &column_name); + // Insert layout into table object + bool InsertLayout(std::shared_ptr layout); + // Evict layout_id from the table object + bool EvictLayout(oid_t layout_id); + // cache for *all* index catalog objects in this table std::unordered_map> index_objects; std::unordered_map> @@ -110,6 +130,11 @@ class TableCatalogObject { column_names; bool valid_column_objects; + // cache for *all* layout objects in the table + std::unordered_map> + layout_objects_; + bool valid_layout_objects_; + // Pointer to its corresponding transaction concurrency::TransactionContext *txn; }; @@ -119,6 +144,7 @@ class TableCatalog : public AbstractCatalog { friend class DatabaseCatalogObject; friend class ColumnCatalog; friend class IndexCatalog; + friend class LayoutCatalog; friend class Catalog; public: diff --git a/src/include/codegen/tile_group.h b/src/include/codegen/tile_group.h index 38d7192f62c..dcb8888fcde 100644 --- a/src/include/codegen/tile_group.h +++ b/src/include/codegen/tile_group.h @@ -137,7 +137,7 @@ class TileGroup { inline const TileGroup &GetTileGroup() const { return tile_group_; } - inline const ColumnLayout &GetLayout(uint32_t col_idx) const { + const ColumnLayout &GetLayout(uint32_t col_idx) const { return layout_[col_idx]; } diff --git a/src/include/common/exception.h b/src/include/common/exception.h index 4c201891751..7f25ec20e9f 100644 --- a/src/include/common/exception.h +++ b/src/include/common/exception.h @@ -59,7 +59,8 @@ enum class ExceptionType { SETTINGS = 23, // settings related BINDER = 24, // binder related NETWORK = 25, // network related - OPTIMIZER = 26 // optimizer related + OPTIMIZER = 26, // optimizer related + NULL_POINTER = 27 // nullptr exception }; class Exception : public std::runtime_error { @@ -132,6 +133,8 @@ class Exception : public std::runtime_error { return "Settings"; case ExceptionType::OPTIMIZER: return "Optimizer"; + case ExceptionType::NULL_POINTER: + return "NullPointer"; default: return "Unknown"; } @@ -467,4 +470,12 @@ class OptimizerException : public Exception { : Exception(ExceptionType::OPTIMIZER, msg) {} }; +class NullPointerException : public Exception { + NullPointerException() = delete; + + public: + NullPointerException(std::string msg) + : Exception(ExceptionType::NULL_POINTER, msg) {} +}; + } // namespace peloton diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 17020512944..0f12d14a49b 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -1396,6 +1397,15 @@ typedef std::unordered_set ExprSet; +//===--------------------------------------------------------------------===// +// Storage +//===--------------------------------------------------------------------===// + +/* column_map_type used to store the layout of a tile_group + * to + */ +typedef std::map> column_map_type; + //===--------------------------------------------------------------------===// // Wire protocol typedefs //===--------------------------------------------------------------------===// diff --git a/src/include/gc/gc_manager.h b/src/include/gc/gc_manager.h index 2697629f3e2..1adce28a944 100644 --- a/src/include/gc/gc_manager.h +++ b/src/include/gc/gc_manager.h @@ -79,7 +79,8 @@ class GCManager { concurrency::TransactionContext *txn UNUSED_ATTRIBUTE) {} protected: - void CheckAndReclaimVarlenColumns(storage::TileGroup *tg, oid_t tuple_id); + void CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, + oid_t tuple_id); protected: volatile bool is_running_; diff --git a/src/include/parser/sql_statement.h b/src/include/parser/sql_statement.h index 012f4090cb9..8b6343f7ea1 100644 --- a/src/include/parser/sql_statement.h +++ b/src/include/parser/sql_statement.h @@ -73,7 +73,7 @@ class TableRefStatement : public SQLStatement { table_info_->database_name = default_database_name; // if schema name is not specified, then it's default value is "public" if (table_info_->schema_name.empty()) - table_info_->schema_name = DEFUALT_SCHEMA_NAME; + table_info_->schema_name = DEFAULT_SCHEMA_NAME; } virtual inline std::string GetTableName() const { diff --git a/src/include/parser/table_ref.h b/src/include/parser/table_ref.h index e3b4fab31d6..5788f382d52 100644 --- a/src/include/parser/table_ref.h +++ b/src/include/parser/table_ref.h @@ -76,7 +76,7 @@ struct TableRef { } if (table_info_->schema_name.empty()) { - table_info_->schema_name = DEFUALT_SCHEMA_NAME; + table_info_->schema_name = DEFAULT_SCHEMA_NAME; } } diff --git a/src/include/storage/abstract_table.h b/src/include/storage/abstract_table.h index 0d41173a9b6..6b49cf9e431 100644 --- a/src/include/storage/abstract_table.h +++ b/src/include/storage/abstract_table.h @@ -19,9 +19,10 @@ #include #include +#include "common/internal_types.h" #include "common/item_pointer.h" #include "common/printable.h" -#include "common/internal_types.h" +#include "storage/layout.h" //===--------------------------------------------------------------------===// // GUC Variables @@ -29,8 +30,6 @@ namespace peloton { -typedef std::map> column_map_type; - namespace concurrency { class TransactionContext; } @@ -77,13 +76,11 @@ class AbstractTable : public Printable { // LAYOUT TYPE //===--------------------------------------------------------------------===// - void SetLayoutType(peloton::LayoutType layout) { - layout_type = layout; + void SetDefaultLayout(std::shared_ptr layout) { + default_layout_ = layout; } - peloton::LayoutType GetLayoutType() { - return layout_type; - } + std::shared_ptr GetDefaultLayout() { return default_layout_; } //===--------------------------------------------------------------------===// // TILE GROUP //===--------------------------------------------------------------------===// @@ -142,11 +139,9 @@ class AbstractTable : public Printable { //===--------------------------------------------------------------------===// TileGroup *GetTileGroupWithLayout(oid_t database_id, oid_t tile_group_id, - const column_map_type &partitioning, + std::shared_ptr layout, const size_t num_tuples); - column_map_type GetTileGroupLayout() const; - //===--------------------------------------------------------------------===// // MEMBERS //===--------------------------------------------------------------------===// @@ -164,7 +159,8 @@ class AbstractTable : public Printable { */ bool own_schema_; - peloton::LayoutType layout_type; + // Default layout of the table + std::shared_ptr default_layout_; }; } // namespace storage diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index c8348485115..cfabec8f78e 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -18,11 +18,12 @@ #include #include +#include "common/container/lock_free_array.h" #include "common/item_pointer.h" #include "common/platform.h" -#include "common/container/lock_free_array.h" #include "storage/abstract_table.h" #include "storage/indirection_array.h" +#include "storage/layout.h" #include "trigger/trigger.h" //===--------------------------------------------------------------------===// @@ -39,6 +40,7 @@ class Sample; namespace catalog { class ForeignKey; +class Catalog; } // namespace catalog namespace index { @@ -76,6 +78,7 @@ class DataTable : public AbstractTable { friend class TileGroup; friend class TileGroupFactory; friend class TableFactory; + friend class catalog::Catalog; friend class logging::LogManager; DataTable() = delete; @@ -148,7 +151,7 @@ class DataTable : public AbstractTable { size_t GetTileGroupCount() const; // Get a tile group with given layout - TileGroup *GetTileGroupWithLayout(const column_map_type &partitioning); + TileGroup *GetTileGroupWithLayout(std::shared_ptr layout); //===--------------------------------------------------------------------===// // TRIGGER @@ -189,7 +192,6 @@ class DataTable : public AbstractTable { const std::vector> &GetIndexColumns() const { return indexes_columns_; } - //===--------------------------------------------------------------------===// // FOREIGN KEYS //===--------------------------------------------------------------------===// @@ -247,9 +249,17 @@ class DataTable : public AbstractTable { void ClearLayoutSamples(); - void SetDefaultLayout(const column_map_type &layout); + void SetDefaultLayout(std::shared_ptr new_layout) { + PELOTON_ASSERT(new_layout->GetColumnCount() == schema->GetColumnCount()); + default_layout_ = new_layout; + } - column_map_type GetDefaultLayout() const; + void ResetDefaultLayout(LayoutType type = LayoutType::ROW) { + PELOTON_ASSERT((type == LayoutType::ROW) || (type == LayoutType::COLUMN)); + default_layout_ = std::shared_ptr( + new const Layout(schema->GetColumnCount(), type)); + } + const Layout &GetDefaultLayout() const; //===--------------------------------------------------------------------===// // INDEX TUNER @@ -277,8 +287,6 @@ class DataTable : public AbstractTable { bool HasForeignKeys() const { return (foreign_keys_.empty() == false); } - std::map GetColumnMapStats(); - // try to insert into all indexes. // the last argument is the index entry in primary index holding the new // tuple. @@ -353,11 +361,26 @@ class DataTable : public AbstractTable { bool CheckForeignKeyConstraints(const AbstractTuple *tuple, concurrency::TransactionContext *transaction); + //===--------------------------------------------------------------------===// + // LAYOUT HELPERS + //===--------------------------------------------------------------------===// + + // Set the current_layout_oid_ to the given value if the current value + // is less than new_layout_oid. Return true on success. + // To be used for recovery. + bool SetCurrentLayoutOid(oid_t new_layout_oid); + + // Performs an atomic increment on the current_layout_oid_ + // and returns the incremented value. + oid_t GetNextLayoutOid() { return ++current_layout_oid_; } + private: + //===--------------------------------------------------------------------===// + // STATIC MEMBERS + //===--------------------------------------------------------------------===// static size_t default_active_tilegroup_count_; static size_t default_active_indirection_array_count_; - private: //===--------------------------------------------------------------------===// // MEMBERS //===--------------------------------------------------------------------===// @@ -414,6 +437,10 @@ class DataTable : public AbstractTable { // dirty flag. for detecting whether the tile group has been used. bool dirty_ = false; + // Last used layout_oid. Used while creating new layouts + // Initialized to COLUMN_STORE_OID since its the highest predefined value. + std::atomic current_layout_oid_; + //===--------------------------------------------------------------------===// // TUNING MEMBERS //===--------------------------------------------------------------------===// @@ -421,9 +448,6 @@ class DataTable : public AbstractTable { // adapt table bool adapt_table_ = true; - // default partition map for table - column_map_type default_partition_; - // samples for layout tuning std::vector layout_samples_; diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h new file mode 100644 index 00000000000..ce726d5d729 --- /dev/null +++ b/src/include/storage/layout.h @@ -0,0 +1,148 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// layout.h +// +// Identification: src/include/storage/layout.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include + +#include "common/internal_types.h" +#include "common/printable.h" + +//@{ +/** @brief Pre-defined OIDs for ROW and COLUMN store. */ +#define ROW_STORE_OID 0 +#define COLUMN_STORE_OID 1 +//@} + +namespace peloton { + +namespace catalog { +class Schema; +} // namespace catalog + +namespace storage { + +/** @brief used to store the mapping between a tile and its columns + * to vector{} + */ +typedef std::map>> TileToColumnMap; + +/** + * @brief Class to store the physical layout of a TileGroup. + */ +class Layout : public Printable { + public: + /** + * @brief Constructor for predefined layout types. + * @param num_columns Number of columns in the layouts. + * @param layout_type Has to be LayoutType::ROW or LayoutType::COLUMN. + */ + Layout(const oid_t num_columns, LayoutType layout_type = LayoutType::ROW); + + /** + * @brief Constructor for arbitrary column_maps. + * If the column_map is of type LayoutType::HYBRID, + * the layout_oid_ is set to INVALID_OID. For all other + * layouts, the pre-defined OIDs are used. Should be used + * only for testing & TempTables, the layout isn't persistent. + * @param column_map Column map of the layout to be constructed + * + */ + Layout(const column_map_type &column_map); + + /** + * @brief Constructor for arbitrary column_maps. + * @param column_map Column map of the layout to be constructed. + * @param layout_oid Per-table unique OID. Generted by DataTable. + */ + Layout(const column_map_type &column_map, oid_t layout_oid); + + /** @brief Check whether this layout is a row store. */ + bool IsRowStore() const { return (layout_type_ == LayoutType::ROW); } + + /** @brief Check whether this layout is a column store. */ + bool IsColumnStore() const { return (layout_type_ == LayoutType::COLUMN); } + + /** @brief Return the layout_oid_ of this object. */ + oid_t GetOid() const { return layout_oid_; } + + /** @brief Locates the tile and column to which the specified + * specified tile group column_id. + * It updates the tile_id and tile_column_id references. + * @param column_id The Id of the column to be located. + * @param tile_id A reference to update the tile_id. + * @param tile_column_id A reference to update the tile_column_id. + */ + void LocateTileAndColumn(oid_t column_id, oid_t &tile_id, + oid_t &tile_column_id) const; + + /** @brief Returns the layout difference w.r.t. the other Layout. + * @double The delta between the layouts. Used by LayoutTuner class. + */ + double GetLayoutDifference(const storage::Layout &other) const; + + /** @brief Returns the tile_id of the given column_id. */ + oid_t GetTileIdFromColumnId(oid_t column_id) const; + + /** @brief Returns the column offset of the column_id. */ + oid_t GetTileColumnOffset(oid_t column_id) const; + + /** @brief Returns the number of columns in the layout. */ + uint32_t GetColumnCount() const { return num_columns_; } + + /** @brief Returns the tile-columns map for each tile in the TileGroup. */ + TileToColumnMap GetTileMap() const; + + /** @brief Constructs the schema for the given layout. This function + * is used only in TempTables and LogicalTiles. + */ + std::vector GetLayoutSchemas( + catalog::Schema *const schema) const; + + /** @brief Returns the layout statistics used by the LayoutTuner. */ + std::map GetLayoutStats() const; + + /** @brief Serialzies the column_map to be added to pg_layout. */ + std::string SerializeColumnMap() const; + + /** @brief Deserializes the string that has been read from pg_layout. */ + static column_map_type DeserializeColumnMap(oid_t num_columns, + std::string column_map_str); + + /** @brief Returns a string containing the column_map of this layout. */ + std::string GetColumnMapInfo() const; + + /** @brief Returns a string representation for debugging function. */ + const std::string GetInfo() const; + + //@{ + /** @brief Operators for checking equality of two layouts. */ + friend bool operator==(const Layout &lhs, const Layout &rhs); + friend bool operator!=(const Layout &lhs, const Layout &rhs); + //@} + + private: + /** @brief Layout Oid of the layout object. */ + oid_t layout_oid_; + + /** @brief Number of columns in the layout. */ + oid_t num_columns_; + + /** @brief column_map of the layout. */ + column_map_type column_layout_; + + /** @brief Layout type is always (ROW, COLUMN or HYBRID). */ + LayoutType layout_type_; +}; + +} // namespace storage +} // namespace peloton diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index 946c65f5f8c..ea9218f06a8 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -18,11 +18,12 @@ #include #include +#include "common/internal_types.h" #include "common/item_pointer.h" #include "common/printable.h" #include "planner/project_info.h" +#include "storage/layout.h" #include "type/abstract_pool.h" -#include "common/internal_types.h" #include "type/value.h" namespace peloton { @@ -53,8 +54,6 @@ class AbstractTable; class TileGroupIterator; class RollbackSegment; -typedef std::map> column_map_type; - /** * Represents a group of tiles logically horizontally contiguous. * @@ -76,7 +75,7 @@ class TileGroup : public Printable { // Tile group constructor TileGroup(BackendType backend_type, TileGroupHeader *tile_group_header, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, int tuple_count); + std::shared_ptr layout, int tuple_count); ~TileGroup(); @@ -119,9 +118,9 @@ class TileGroup : public Printable { // this function is called only when building tile groups for aggregation // operations. // FIXME: GC has recycled some of the tuples, so this count is not accurate - oid_t GetActiveTupleCount() const; + uint32_t GetActiveTupleCount() const; - oid_t GetAllocatedTupleCount() const { return num_tuple_slots; } + uint32_t GetAllocatedTupleCount() const { return num_tuple_slots_; } TileGroupHeader *GetHeader() const { return tile_group_header; } @@ -131,7 +130,7 @@ class TileGroup : public Printable { // Get the tile at given offset in the tile group inline Tile *GetTile(const oid_t tile_offset) const { - PELOTON_ASSERT(tile_offset < tile_count); + PELOTON_ASSERT(tile_offset < tile_count_); Tile *tile = tiles[tile_offset].get(); return tile; } @@ -143,10 +142,6 @@ class TileGroup : public Printable { peloton::type::AbstractPool *GetTilePool(const oid_t tile_id) const; - const std::map> &GetColumnMap() const { - return column_map; - } - oid_t GetTileGroupId() const; oid_t GetDatabaseId() const { return database_id; } @@ -157,34 +152,18 @@ class TileGroup : public Printable { void SetTileGroupId(oid_t tile_group_id_) { tile_group_id = tile_group_id_; } - std::vector &GetTileSchemas() { return tile_schemas; } - - size_t GetTileCount() const { return tile_count; } - - // Sets the tile id and column id w.r.t that tile corresponding to - // the specified tile group column id. - inline void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, - oid_t &tile_column_offset) const { - PELOTON_ASSERT(column_map.count(column_offset) != 0); - // get the entry in the column map - auto entry = column_map.at(column_offset); - tile_offset = entry.first; - tile_column_offset = entry.second; - } - - oid_t GetTileIdFromColumnId(oid_t column_id); - - oid_t GetTileColumnId(oid_t column_id); + size_t GetTileCount() const { return tile_count_; } type::Value GetValue(oid_t tuple_id, oid_t column_id); void SetValue(type::Value &value, oid_t tuple_id, oid_t column_id); - double GetSchemaDifference(const storage::column_map_type &new_column_map); - // Sync the contents void Sync(); + // Get the layout of the TileGroup. Used to locate columns. + const storage::Layout &GetLayout() const { return *tile_group_layout_; } + protected: //===--------------------------------------------------------------------===// // Data members @@ -198,9 +177,6 @@ class TileGroup : public Printable { // Backend type BackendType backend_type; - // mapping to tile schemas - std::vector tile_schemas; - // set of tiles std::vector> tiles; @@ -211,16 +187,15 @@ class TileGroup : public Printable { AbstractTable *table; // this design is fantastic!!! // number of tuple slots allocated - oid_t num_tuple_slots; + uint32_t num_tuple_slots_; // number of tiles - oid_t tile_count; + uint32_t tile_count_; std::mutex tile_group_mutex; - // column to tile mapping : - // to - column_map_type column_map; + // Refernce to the layout of the TileGroup + std::shared_ptr tile_group_layout_; }; } // namespace storage diff --git a/src/include/storage/tile_group_factory.h b/src/include/storage/tile_group_factory.h index e6d99dd907d..3e615aba479 100644 --- a/src/include/storage/tile_group_factory.h +++ b/src/include/storage/tile_group_factory.h @@ -32,7 +32,7 @@ class TileGroupFactory { static TileGroup *GetTileGroup(oid_t database_id, oid_t table_id, oid_t tile_group_id, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, + std::shared_ptr layout, int tuple_count); }; diff --git a/src/include/tuning/clusterer.h b/src/include/tuning/clusterer.h index 1bdc8d1ee15..85b57696055 100644 --- a/src/include/tuning/clusterer.h +++ b/src/include/tuning/clusterer.h @@ -30,11 +30,6 @@ namespace tuning { // Clusterer //===--------------------------------------------------------------------===// -/** - * Column Id to < Tile Id, Tile Column Id > - */ -typedef std::map> column_map_type; - /** * Sequential k-Means Clustering */ diff --git a/src/include/tuning/layout_tuner.h b/src/include/tuning/layout_tuner.h index bf17e9cd933..14a6fe74e8d 100644 --- a/src/include/tuning/layout_tuner.h +++ b/src/include/tuning/layout_tuner.h @@ -81,22 +81,15 @@ class LayoutTuner { */ void ClearTables(); - /** - * @brief Gets the column map information. - * - * @param[in] column_map The column map - * - * @return The column map information. - */ - std::string GetColumnMapInfo(const column_map_type &column_map); protected: /** * Update layout of table * * @param table The table + * @return true if the update succeeds, false otherwise */ - void UpdateDefaultPartition(storage::DataTable *table); + bool UpdateDefaultPartition(storage::DataTable *table); private: /** diff --git a/src/main/sdbench/sdbench_workload.cpp b/src/main/sdbench/sdbench_workload.cpp index 8468d88ffe3..3d50b037e8c 100644 --- a/src/main/sdbench/sdbench_workload.cpp +++ b/src/main/sdbench/sdbench_workload.cpp @@ -561,8 +561,9 @@ static void CopyColumn(oid_t col_itr) { // Begin copy oid_t orig_tile_offset, orig_tile_column_offset; auto orig_tile_group = sdbench_table->GetTileGroup(tile_group_itr); - orig_tile_group->LocateTileAndColumn(col_itr, orig_tile_offset, - orig_tile_column_offset); + auto orig_layout = orig_tile_group->GetLayout(); + orig_layout.LocateTileAndColumn(col_itr, orig_tile_offset, + orig_tile_column_offset); auto orig_tile = orig_tile_group->GetTile(orig_tile_offset); oid_t tuple_count = state.tuples_per_tilegroup; for (oid_t tuple_itr = 0; tuple_itr < tuple_count; tuple_itr++) { diff --git a/src/optimizer/stats/tuple_sampler.cpp b/src/optimizer/stats/tuple_sampler.cpp index 470a5e4392d..26831079add 100644 --- a/src/optimizer/stats/tuple_sampler.cpp +++ b/src/optimizer/stats/tuple_sampler.cpp @@ -91,15 +91,14 @@ bool TupleSampler::GetTupleInTileGroup(storage::TileGroup *tile_group, } size_t tuple_column_itr = 0; - auto tile_schemas = tile_group->GetTileSchemas(); size_t tile_count = tile_group->GetTileCount(); LOG_TRACE("tile_count: %lu", tile_count); for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tile_schemas[tile_itr]; - oid_t tile_column_count = schema.GetColumnCount(); storage::Tile *tile = tile_group->GetTile(tile_itr); + const catalog::Schema &schema = *(tile->GetSchema()); + uint32_t tile_column_count = schema.GetColumnCount(); char *tile_tuple_location = tile->GetTupleLocation(tuple_offset); storage::Tuple tile_tuple(&schema, tile_tuple_location); diff --git a/src/optimizer/stats/tuple_samples_storage.cpp b/src/optimizer/stats/tuple_samples_storage.cpp index 9ae85256758..edb41c2f820 100644 --- a/src/optimizer/stats/tuple_samples_storage.cpp +++ b/src/optimizer/stats/tuple_samples_storage.cpp @@ -66,11 +66,11 @@ void TupleSamplesStorage::AddSamplesTable( auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); catalog->CreateTable(std::string(SAMPLES_DB_NAME), - std::string(DEFUALT_SCHEMA_NAME), samples_table_name, + std::string(DEFAULT_SCHEMA_NAME), samples_table_name, std::move(schema_ptr), txn, is_catalog); auto samples_table = catalog->GetTableWithName( - std::string(SAMPLES_DB_NAME), std::string(DEFUALT_SCHEMA_NAME), + std::string(SAMPLES_DB_NAME), std::string(DEFAULT_SCHEMA_NAME), samples_table_name, txn); for (auto &tuple : sampled_tuples) { @@ -94,7 +94,7 @@ ResultType TupleSamplesStorage::DeleteSamplesTable( ResultType result = ResultType::FAILURE; try { result = catalog->DropTable(std::string(SAMPLES_DB_NAME), - std::string(DEFUALT_SCHEMA_NAME), + std::string(DEFAULT_SCHEMA_NAME), samples_table_name, txn); } catch (CatalogException &e) { // Samples table does not exist, no need to drop @@ -187,7 +187,7 @@ TupleSamplesStorage::GetTupleSamples(oid_t database_id, oid_t table_id) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto data_table = catalog->GetTableWithName(std::string(SAMPLES_DB_NAME), - std::string(DEFUALT_SCHEMA_NAME), + std::string(DEFAULT_SCHEMA_NAME), samples_table_name, txn); auto col_count = data_table->GetSchema()->GetColumnCount(); @@ -214,7 +214,7 @@ void TupleSamplesStorage::GetColumnSamples( auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto data_table = catalog->GetTableWithName(std::string(SAMPLES_DB_NAME), - std::string(DEFUALT_SCHEMA_NAME), + std::string(DEFAULT_SCHEMA_NAME), samples_table_name, txn); std::vector column_ids({column_id}); diff --git a/src/storage/abstract_table.cpp b/src/storage/abstract_table.cpp index ee0607252d7..893da6e5cd9 100644 --- a/src/storage/abstract_table.cpp +++ b/src/storage/abstract_table.cpp @@ -26,67 +26,27 @@ namespace storage { AbstractTable::AbstractTable(oid_t table_oid, catalog::Schema *schema, bool own_schema, peloton::LayoutType layout_type) - : table_oid(table_oid), schema(schema), own_schema_(own_schema), layout_type(layout_type) {} + : table_oid(table_oid), schema(schema), own_schema_(own_schema) { + // The default Layout should always be ROW or COLUMN + PELOTON_ASSERT((layout_type == LayoutType::ROW) || + (layout_type == LayoutType::COLUMN)); + default_layout_ = std::shared_ptr( + new Layout(schema->GetColumnCount(), layout_type)); +} AbstractTable::~AbstractTable() { // clean up schema if (own_schema_) delete schema; } -column_map_type AbstractTable::GetTileGroupLayout() const { - column_map_type column_map; - - auto col_count = schema->GetColumnCount(); - - // pure row layout map - if (layout_type == LayoutType::ROW) { - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(0, col_itr); - } - } - // pure column layout map - else if (layout_type == LayoutType::COLUMN) { - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(col_itr, 0); - } - } - // hybrid layout map - else if (layout_type == LayoutType::HYBRID) { - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(0, col_itr); - } - } else { - throw Exception("Unknown tilegroup layout option : " + - LayoutTypeToString(layout_type)); - } - - return column_map; -} - TileGroup *AbstractTable::GetTileGroupWithLayout( - oid_t database_id, oid_t tile_group_id, const column_map_type &partitioning, - const size_t num_tuples) { - std::vector schemas; - - // Figure out the columns in each tile in new layout - std::map, oid_t> tile_column_map; - for (auto entry : partitioning) { - tile_column_map[entry.second] = entry.first; - } - - // Build the schema tile at a time - std::map> tile_schemas; - for (auto entry : tile_column_map) { - tile_schemas[entry.first.first].push_back(schema->GetColumn(entry.second)); - } - for (auto entry : tile_schemas) { - catalog::Schema tile_schema(entry.second); - schemas.push_back(tile_schema); - } + oid_t database_id, oid_t tile_group_id, + std::shared_ptr layout, const size_t num_tuples) { + // Populate the schema for each tile + std::vector schemas = layout->GetLayoutSchemas(schema); - TileGroup *tile_group = - TileGroupFactory::GetTileGroup(database_id, GetOid(), tile_group_id, this, - schemas, partitioning, num_tuples); + TileGroup *tile_group = TileGroupFactory::GetTileGroup( + database_id, GetOid(), tile_group_id, this, schemas, layout, num_tuples); return tile_group; } diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 1f3d9195038..2cc158eb837 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -15,7 +15,9 @@ #include "catalog/catalog.h" #include "catalog/foreign_key.h" +#include "catalog/layout_catalog.h" #include "catalog/system_catalogs.h" +#include "catalog/table_catalog.h" #include "common/container_tuple.h" #include "common/exception.h" #include "common/logger.h" @@ -65,14 +67,9 @@ DataTable::DataTable(catalog::Schema *schema, const std::string &table_name, database_oid(database_oid), table_name(table_name), tuples_per_tilegroup_(tuples_per_tilegroup), + current_layout_oid_(ATOMIC_VAR_INIT(COLUMN_STORE_OID)), adapt_table_(adapt_table), trigger_list_(new trigger::TriggerList()) { - // Init default partition - auto col_count = schema->GetColumnCount(); - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - default_partition_[col_itr] = std::make_pair(0, col_itr); - } - if (is_catalog == true) { active_tilegroup_count_ = 1; active_indirection_array_count_ = 1; @@ -873,10 +870,10 @@ void DataTable::ResetDirty() { dirty_ = false; } //===--------------------------------------------------------------------===// TileGroup *DataTable::GetTileGroupWithLayout( - const column_map_type &partitioning) { + std::shared_ptr layout) { oid_t tile_group_id = catalog::Manager::GetInstance().GetNextTileGroupId(); - return (AbstractTable::GetTileGroupWithLayout( - database_oid, tile_group_id, partitioning, tuples_per_tilegroup_)); + return (AbstractTable::GetTileGroupWithLayout(database_oid, tile_group_id, + layout, tuples_per_tilegroup_)); } oid_t DataTable::AddDefaultIndirectionArray( @@ -901,14 +898,11 @@ oid_t DataTable::AddDefaultTileGroup() { } oid_t DataTable::AddDefaultTileGroup(const size_t &active_tile_group_id) { - column_map_type column_map; oid_t tile_group_id = INVALID_OID; - // Figure out the partitioning for given tilegroup layout - column_map = GetTileGroupLayout(); - // Create a tile group with that partitioning - std::shared_ptr tile_group(GetTileGroupWithLayout(column_map)); + std::shared_ptr tile_group( + GetTileGroupWithLayout(default_layout_)); PELOTON_ASSERT(tile_group.get()); tile_group_id = tile_group->GetTileGroupId(); @@ -939,16 +933,20 @@ void DataTable::AddTileGroupWithOidForRecovery(const oid_t &tile_group_id) { std::vector schemas; schemas.push_back(*schema); + std::shared_ptr layout = nullptr; - column_map_type column_map; - // default column map - auto col_count = schema->GetColumnCount(); - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(0, col_itr); + // The TileGroup for recovery is always added in ROW layout, + // This was a part of the previous design. If you are planning + // to change this, make sure the layout is added to the catalog + if (default_layout_->IsRowStore()) { + layout = default_layout_; + } else { + layout = std::shared_ptr( + new const Layout(schema->GetColumnCount())); } std::shared_ptr tile_group(TileGroupFactory::GetTileGroup( - database_oid, table_oid, tile_group_id, this, schemas, column_map, + database_oid, table_oid, tile_group_id, this, schemas, layout, tuples_per_tilegroup_)); auto tile_groups_exists = tile_groups_.Contains(tile_group_id); @@ -1196,25 +1194,28 @@ catalog::ForeignKey *DataTable::GetForeignKeySrc(const size_t offset) const { // Get the schema for the new transformed tile group std::vector TransformTileGroupSchema( - storage::TileGroup *tile_group, const column_map_type &column_map) { + storage::TileGroup *tile_group, const Layout &layout) { std::vector new_schema; oid_t orig_tile_offset, orig_tile_column_offset; oid_t new_tile_offset, new_tile_column_offset; + auto tile_group_layout = tile_group->GetLayout(); // First, get info from the original tile group's schema std::map> schemas; - auto orig_schemas = tile_group->GetTileSchemas(); - for (auto column_map_entry : column_map) { - new_tile_offset = column_map_entry.second.first; - new_tile_column_offset = column_map_entry.second.second; - oid_t column_offset = column_map_entry.first; - - tile_group->LocateTileAndColumn(column_offset, orig_tile_offset, - orig_tile_column_offset); - // Get the column info from original schema - auto orig_schema = orig_schemas[orig_tile_offset]; - auto column_info = orig_schema.GetColumn(orig_tile_column_offset); + uint32_t column_count = layout.GetColumnCount(); + for (oid_t col_id = 0; col_id < column_count; col_id++) { + // Get TileGroup layout's tile and offset for col_id. + tile_group_layout.LocateTileAndColumn(col_id, orig_tile_offset, + orig_tile_column_offset); + // Get new layout's tile and offset for col_id. + layout.LocateTileAndColumn(col_id, new_tile_offset, new_tile_column_offset); + + // Get the column info from original tile + auto tile = tile_group->GetTile(orig_tile_offset); + PELOTON_ASSERT(tile != nullptr); + auto orig_schema = tile->GetSchema(); + auto column_info = orig_schema->GetColumn(orig_tile_column_offset); schemas[new_tile_offset][new_tile_column_offset] = column_info; } @@ -1234,24 +1235,29 @@ std::vector TransformTileGroupSchema( // Set the transformed tile group column-at-a-time void SetTransformedTileGroup(storage::TileGroup *orig_tile_group, storage::TileGroup *new_tile_group) { - // Check the schema of the two tile groups - auto new_column_map = new_tile_group->GetColumnMap(); - auto orig_column_map = orig_tile_group->GetColumnMap(); - PELOTON_ASSERT(new_column_map.size() == orig_column_map.size()); + auto new_layout = new_tile_group->GetLayout(); + auto orig_layout = orig_tile_group->GetLayout(); + + // Check that both tile groups have the same schema + // Currently done by checking that the number of columns are equal + // TODO Pooja: Handle schema equality for multiple schema versions. + UNUSED_ATTRIBUTE auto new_column_count = new_layout.GetColumnCount(); + UNUSED_ATTRIBUTE auto orig_column_count = orig_layout.GetColumnCount(); + PELOTON_ASSERT(new_column_count == orig_column_count); oid_t orig_tile_offset, orig_tile_column_offset; oid_t new_tile_offset, new_tile_column_offset; - auto column_count = new_column_map.size(); + auto column_count = new_column_count; auto tuple_count = orig_tile_group->GetAllocatedTupleCount(); // Go over each column copying onto the new tile group for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { // Locate the original base tile and tile column offset - orig_tile_group->LocateTileAndColumn(column_itr, orig_tile_offset, - orig_tile_column_offset); + orig_layout.LocateTileAndColumn(column_itr, orig_tile_offset, + orig_tile_column_offset); - new_tile_group->LocateTileAndColumn(column_itr, new_tile_offset, - new_tile_column_offset); + new_layout.LocateTileAndColumn(column_itr, new_tile_offset, + new_tile_column_offset); auto orig_tile = orig_tile_group->GetTile(orig_tile_offset); auto new_tile = new_tile_group->GetTile(new_tile_offset); @@ -1284,7 +1290,7 @@ storage::TileGroup *DataTable::TransformTileGroup( // Get orig tile group from catalog auto &catalog_manager = catalog::Manager::GetInstance(); auto tile_group = catalog_manager.GetTileGroup(tile_group_id); - auto diff = tile_group->GetSchemaDifference(default_partition_); + auto diff = tile_group->GetLayout().GetLayoutDifference(*default_layout_); // Check threshold for transformation if (diff < theta) { @@ -1295,15 +1301,14 @@ storage::TileGroup *DataTable::TransformTileGroup( // Get the schema for the new transformed tile group auto new_schema = - TransformTileGroupSchema(tile_group.get(), default_partition_); + TransformTileGroupSchema(tile_group.get(), *default_layout_); // Allocate space for the transformed tile group std::shared_ptr new_tile_group( TileGroupFactory::GetTileGroup( tile_group->GetDatabaseId(), tile_group->GetTableId(), tile_group->GetTileGroupId(), tile_group->GetAbstractTable(), - new_schema, default_partition_, - tile_group->GetAllocatedTupleCount())); + new_schema, default_layout_, tile_group->GetAllocatedTupleCount())); // Set the transformed tile group column-at-a-time SetTransformedTileGroup(tile_group.get(), new_tile_group.get()); @@ -1361,29 +1366,7 @@ void DataTable::ClearIndexSamples() { } } -std::map DataTable::GetColumnMapStats() { - std::map column_map_stats; - - // Cluster per-tile column count - for (auto entry : default_partition_) { - auto tile_id = entry.second.first; - auto column_map_itr = column_map_stats.find(tile_id); - if (column_map_itr == column_map_stats.end()) - column_map_stats[tile_id] = 1; - else - column_map_stats[tile_id]++; - } - - return column_map_stats; -} - -void DataTable::SetDefaultLayout(const column_map_type &layout) { - default_partition_ = layout; -} - -column_map_type DataTable::GetDefaultLayout() const { - return default_partition_; -} +const Layout &DataTable::GetDefaultLayout() const { return *default_layout_; } void DataTable::AddTrigger(trigger::Trigger new_trigger) { trigger_list_->AddTrigger(new_trigger); @@ -1432,5 +1415,16 @@ bool DataTable::operator==(const DataTable &rhs) const { return true; } +bool DataTable::SetCurrentLayoutOid(oid_t new_layout_oid) { + oid_t old_oid = current_layout_oid_; + while (old_oid <= new_layout_oid) { + if (current_layout_oid_.compare_exchange_strong(old_oid, new_layout_oid)) { + return true; + } + old_oid = current_layout_oid_; + } + return false; +} + } // namespace storage } // namespace peloton diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp new file mode 100644 index 00000000000..c03b721087f --- /dev/null +++ b/src/storage/layout.cpp @@ -0,0 +1,352 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// layout.cpp +// +// Identification: src/storage/layout.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include +#include + +#include "catalog/column.h" +#include "catalog/schema.h" +#include "storage/layout.h" +#include "util/stringbox_util.h" + +namespace peloton { +namespace storage { + +// Constructor for the layout class with column_count +// The default layout is always a ROW_STORE +Layout::Layout(const oid_t num_columns, LayoutType layout_type) + : num_columns_(num_columns), layout_type_(layout_type) {} + +// Constructor for the Layout class with column_map +Layout::Layout(const column_map_type &column_map) + : num_columns_(column_map.size()), column_layout_(column_map) { + // Figure out the layout type if not present + bool row_layout = true, column_layout = true; + for (oid_t column_id = 0; column_id < num_columns_; column_id++) { + if (column_layout_[column_id].first != 0) { + row_layout = false; + } + if (column_layout_[column_id].first != column_id) { + column_layout = false; + } + } + + // Assign the layout_type_ + // If a table has one column, it would be LayoutType::Row + if (row_layout) { + layout_type_ = LayoutType::ROW; + layout_oid_ = ROW_STORE_OID; + } else if (column_layout) { + layout_type_ = LayoutType::COLUMN; + layout_oid_ = COLUMN_STORE_OID; + } else { + // layout_oid_ is set to INVALID_OID, indicating that this + // layout is not stored in the catalog and thus not persistent. + // To be used only in TempTable or Tests. + layout_type_ = LayoutType::HYBRID; + layout_oid_ = INVALID_OID; + } + + if (layout_type_ != LayoutType::HYBRID) { + // Since this is a predefined layout, we don't need the map + column_layout_.clear(); + } +} + +// Constructor for Layout class with predefined layout_oid +Layout::Layout(const column_map_type &column_map, oid_t layout_id) + : layout_oid_(layout_id), + num_columns_(column_map.size()), + column_layout_(column_map) { + if (layout_oid_ == ROW_STORE_OID) { + layout_type_ = LayoutType::ROW; + } else if (layout_oid_ == COLUMN_STORE_OID) { + layout_type_ = LayoutType::COLUMN; + } else { + layout_type_ = LayoutType::HYBRID; + } + + if (layout_type_ != LayoutType::HYBRID) { + // Since this is a predefined layout, we don't need the map + column_layout_.clear(); + } +} + +void Layout::LocateTileAndColumn(oid_t column_id, oid_t &tile_id, + oid_t &tile_column_id) const { + // Ensure that the column_offset is not out of bound + PELOTON_ASSERT(num_columns_ > column_id); + + // For row store layout, tile_id is always 0 and the + // column_id and tile column_id are the same. + if (layout_type_ == LayoutType::ROW) { + tile_id = 0; + tile_column_id = column_id; + return; + } + + // For column store layout, tile_id is same as column_id + // and the tile_column_id is always 0. + if (layout_type_ == LayoutType::COLUMN) { + tile_id = column_id; + tile_column_id = 0; + return; + } + + // For other layouts, fetch the layout and + // get the entry in the column map + auto entry = column_layout_.at(column_id); + tile_id = entry.first; + tile_column_id = entry.second; +} + +double Layout::GetLayoutDifference(const storage::Layout &other) const { + double theta = 0; + double diff = 0; + + // Calculate theta only for TileGroups with the same schema. + // Check the schema before invoking this function. + PELOTON_ASSERT(this->num_columns_ == other.num_columns_); + + if ((this->layout_oid_ != other.layout_oid_)) { + for (oid_t col_itr = 0; col_itr < num_columns_; col_itr++) { + if (this->GetTileIdFromColumnId(col_itr) != + other.GetTileIdFromColumnId(col_itr)) { + diff++; + } + } + // compute theta + theta = diff / num_columns_; + } + + return theta; +} + +oid_t Layout::GetTileIdFromColumnId(oid_t column_id) const { + oid_t tile_column_id, tile_offset; + LocateTileAndColumn(column_id, tile_offset, tile_column_id); + return tile_offset; +} + +TileToColumnMap Layout::GetTileMap() const { + TileToColumnMap tile_map; + + if (layout_type_ == LayoutType::ROW) { + // Row store layout, hence all columns are contained in tile 0. + // The column_offset is always the same as column_id. + tile_map[0] = {}; + for (oid_t column_id = 0; column_id < num_columns_; column_id++) { + tile_map[0].emplace_back(column_id, column_id); + } + } else if (layout_type_ == LayoutType::COLUMN) { + // Column store layout, hence all columns are contained in separate tiles. + // The column_offset within the tile is always 0. + for (oid_t column_id = 0; column_id < num_columns_; column_id++) { + tile_map[column_id] = {std::make_pair(column_id, 0)}; + } + } else { + // Hybrid layout, hence figure out the mapping from the column_layout_. + for (auto entry : column_layout_) { + auto column_id = entry.first; + auto tile_id = entry.second.first; + auto column_offset = entry.second.second; + if (tile_map.find(tile_id) == tile_map.end()) { + tile_map[tile_id] = {}; + } + tile_map[tile_id].emplace_back(column_id, column_offset); + } + } + return tile_map; +} + +oid_t Layout::GetTileColumnOffset(oid_t column_id) const { + oid_t tile_column_id, tile_offset; + LocateTileAndColumn(column_id, tile_offset, tile_column_id); + return tile_column_id; +} + +std::vector Layout::GetLayoutSchemas( + catalog::Schema *const schema) const { + std::vector schemas; + + // Build the schema tile at a time + std::map> tile_schemas; + + // Handle schemas population based on the layout type + if (layout_type_ == LayoutType::ROW) { + schemas.push_back(*schema); + } else if (layout_type_ == LayoutType::COLUMN) { + for (oid_t col_id = 0; col_id < num_columns_; col_id++) { + std::vector tile_schema({schema->GetColumn(col_id)}); + schemas.push_back(tile_schema); + } + } else { + // For LayoutType::HYBRID, use the old technique. + // This only works if the order of columns in the tile are the same + // as the schema. This snipet was initially in abstract_table.cpp. + for (auto column_info : column_layout_) { + tile_schemas[column_info.second.first].push_back( + schema->GetColumn(column_info.first)); + } + + for (auto entry : tile_schemas) { + catalog::Schema tile_schema(entry.second); + schemas.push_back(tile_schema); + } + } + + return schemas; +} + +std::map Layout::GetLayoutStats() const { + std::map column_map_stats; + + // Cluster per-tile column count + for (auto entry : column_layout_) { + auto tile_id = entry.second.first; + auto column_map_itr = column_map_stats.find(tile_id); + if (column_map_itr == column_map_stats.end()) + column_map_stats[tile_id] = 1; + else + column_map_stats[tile_id]++; + } + + return column_map_stats; +} + +std::string Layout::SerializeColumnMap() const { + std::stringstream ss; + + for (auto column_info : column_layout_) { + ss << column_info.first << ":"; + ss << column_info.second.first << ":"; + ss << column_info.second.second << ","; + } + + return ss.str(); +} + +column_map_type Layout::DeserializeColumnMap(oid_t num_columns, + std::string column_map_str) { + column_map_type column_map; + std::stringstream ss(column_map_str); + + for (oid_t col_id = 0; col_id < num_columns; col_id++) { + oid_t str_col_id, tile_id, tile_col_id; + // Read col_id from column_map_str + ss >> str_col_id; + PELOTON_ASSERT(str_col_id == col_id); + + PELOTON_ASSERT(ss.peek() == ':'); + ss.ignore(); + // Read tile_id from column_map_str + ss >> tile_id; + + PELOTON_ASSERT(ss.peek() == ':'); + ss.ignore(); + // Read tile_col_id from column_map_str + ss >> tile_col_id; + + // Insert the column info into column_map + column_map[col_id] = std::make_pair(tile_id, tile_col_id); + + if (ss.peek() == ',') { + ss.ignore(); + } + } + + return column_map; +} + +std::string Layout::GetColumnMapInfo() const { + std::stringstream ss; + std::map> tile_column_map; + + if (layout_type_ == LayoutType::ROW) { + // Row store always contains only 1 tile. The tile_id is always 0. + oid_t tile_id = 0; + tile_column_map[tile_id] = {}; + for (oid_t col_id = 0; col_id < num_columns_; col_id++) { + tile_column_map[tile_id].push_back(col_id); + } + } else if (layout_type_ == LayoutType::COLUMN) { + // Column store always contains 1 column per tile. + oid_t tile_col_id = 0; + for (oid_t col_id = 0; col_id < num_columns_; col_id++) { + tile_column_map[col_id].push_back(tile_col_id); + } + } else { + for (auto column_info : column_layout_) { + oid_t col_id = column_info.first; + oid_t tile_id = column_info.second.first; + if (tile_column_map.find(tile_id) == tile_column_map.end()) { + tile_column_map[tile_id] = {}; + } + tile_column_map[tile_id].push_back(col_id); + } + } + + // Construct a string from tile_col_map + for (auto tile_info : tile_column_map) { + ss << tile_info.first << " : "; + for (auto col_id : tile_info.second) { + ss << col_id << " "; + } + ss << " :: "; + } + + return ss.str(); +} + +const std::string Layout::GetInfo() const { + std::ostringstream os; + + os << peloton::GETINFO_DOUBLE_STAR << " Layout[#" << layout_oid_ << "] " + << peloton::GETINFO_DOUBLE_STAR << std::endl; + os << "Number of columns[" << num_columns_ << "] " << std::endl; + os << "LayoutType[" << LayoutTypeToString(layout_type_) << std::endl; + os << "ColumnMap[ " << GetColumnMapInfo() << "] " << std::endl; + + return peloton::StringUtil::Prefix(peloton::StringBoxUtil::Box(os.str()), + GETINFO_SPACER); +} + +bool operator==(const Layout &lhs, const Layout &rhs) { + // Check the equality of layout_type_ + if (lhs.layout_type_ != rhs.layout_type_) { + return false; + } + + // Check the equality of layout_oid_ + if (lhs.GetOid() != rhs.GetOid()) { + return false; + } + // Check the equality of column_count_ + if (lhs.GetColumnCount() != rhs.GetColumnCount()) { + return false; + } + + // Check for the equality of the column_layout_ for + // LayoutType::HYBRID + if (lhs.layout_type_ == LayoutType::HYBRID) { + return (lhs.column_layout_ == rhs.column_layout_); + } + // In case of LayoutType::ROW or LayoutType::COLUMN, + // there is no need to check for column_layout_ equality + + return true; +} + +bool operator!=(const Layout &lhs, const Layout &rhs) { return !(lhs == rhs); } + +} // namespace storage +} // namespace peloton \ No newline at end of file diff --git a/src/storage/temp_table.cpp b/src/storage/temp_table.cpp index b0739b912ce..d9db400e3b1 100644 --- a/src/storage/temp_table.cpp +++ b/src/storage/temp_table.cpp @@ -104,8 +104,6 @@ std::shared_ptr TempTable::GetTileGroupById( } oid_t TempTable::AddDefaultTileGroup() { - column_map_type column_map; - // Well, a TempTable doesn't really care about TileGroupIds // And nobody else in the system should be referencing our boys directly, // so we're just going to use a simple counter for these ids @@ -114,14 +112,11 @@ oid_t TempTable::AddDefaultTileGroup() { oid_t tile_group_id = TEMPTABLE_TILEGROUP_ID + static_cast(tile_groups_.size()); - // Figure out the partitioning for given tilegroup layout - column_map = - AbstractTable::GetTileGroupLayout(); - // Create a tile group with that partitioning + // Its a TempTable, so we don't need to make the Layout persistent std::shared_ptr tile_group( AbstractTable::GetTileGroupWithLayout( - INVALID_OID, tile_group_id, column_map, TEMPTABLE_DEFAULT_SIZE)); + INVALID_OID, tile_group_id, default_layout_, TEMPTABLE_DEFAULT_SIZE)); PELOTON_ASSERT(tile_group.get()); tile_groups_.push_back(tile_group); diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 5a40f4ad5fd..69dac07e157 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -16,10 +16,11 @@ #include "catalog/manager.h" #include "common/container_tuple.h" +#include "common/internal_types.h" #include "common/logger.h" #include "common/platform.h" -#include "common/internal_types.h" #include "storage/abstract_table.h" +#include "storage/layout.h" #include "storage/tile.h" #include "storage/tile_group_header.h" #include "storage/tuple.h" @@ -31,25 +32,23 @@ namespace storage { TileGroup::TileGroup(BackendType backend_type, TileGroupHeader *tile_group_header, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, int tuple_count) + std::shared_ptr layout, int tuple_count) : database_id(INVALID_OID), table_id(INVALID_OID), tile_group_id(INVALID_OID), backend_type(backend_type), - tile_schemas(schemas), tile_group_header(tile_group_header), table(table), - num_tuple_slots(tuple_count), - column_map(column_map) { - tile_count = tile_schemas.size(); - - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + num_tuple_slots_(tuple_count), + tile_group_layout_(layout) { + tile_count_ = schemas.size(); + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { auto &manager = catalog::Manager::GetInstance(); oid_t tile_id = manager.GetNextTileId(); std::shared_ptr tile(storage::TileFactory::GetTile( backend_type, database_id, table_id, tile_group_id, tile_id, - tile_group_header, tile_schemas[tile_itr], this, tuple_count)); + tile_group_header, schemas[tile_itr], this, tuple_count)); // Add a reference to the tile in the tile group tiles.push_back(tile); @@ -103,22 +102,21 @@ oid_t TileGroup::GetActiveTupleCount() const { */ void TileGroup::CopyTuple(const Tuple *tuple, const oid_t &tuple_slot_id) { LOG_TRACE("Tile Group Id :: %u status :: %u out of %u slots ", tile_group_id, - tuple_slot_id, num_tuple_slots); + tuple_slot_id, num_tuple_slots_); oid_t tile_column_count; oid_t column_itr = 0; - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tile_schemas[tile_itr]; - tile_column_count = schema.GetColumnCount(); - + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); + const catalog::Schema *schema = tile->GetSchema(); + tile_column_count = schema->GetColumnCount(); char *tile_tuple_location = tile->GetTupleLocation(tuple_slot_id); PELOTON_ASSERT(tile_tuple_location); // NOTE:: Only a tuple wrapper - storage::Tuple tile_tuple(&schema, tile_tuple_location); + storage::Tuple tile_tuple(schema, tile_tuple_location); for (oid_t tile_column_itr = 0; tile_column_itr < tile_column_count; tile_column_itr++) { @@ -138,7 +136,7 @@ oid_t TileGroup::InsertTuple(const Tuple *tuple) { oid_t tuple_slot_id = tile_group_header->GetNextEmptyTupleSlot(); LOG_TRACE("Tile Group Id :: %u status :: %u out of %u slots ", tile_group_id, - tuple_slot_id, num_tuple_slots); + tuple_slot_id, num_tuple_slots_); // No more slots if (tuple_slot_id == INVALID_OID) { @@ -185,22 +183,21 @@ oid_t TileGroup::InsertTupleFromRecovery(cid_t commit_id, oid_t tuple_slot_id, } LOG_TRACE("Tile Group Id :: %u status :: %u out of %u slots ", tile_group_id, - tuple_slot_id, num_tuple_slots); + tuple_slot_id, num_tuple_slots_); oid_t tile_column_count; oid_t column_itr = 0; - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tile_schemas[tile_itr]; - tile_column_count = schema.GetColumnCount(); - + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); + const catalog::Schema *schema = tile->GetSchema(); + tile_column_count = schema->GetColumnCount(); char *tile_tuple_location = tile->GetTupleLocation(tuple_slot_id); PELOTON_ASSERT(tile_tuple_location); // NOTE:: Only a tuple wrapper - storage::Tuple tile_tuple(&schema, tile_tuple_location); + storage::Tuple tile_tuple(schema, tile_tuple_location); for (oid_t tile_column_itr = 0; tile_column_itr < tile_column_count; tile_column_itr++) { @@ -285,22 +282,21 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, if (status == false) return INVALID_OID; LOG_TRACE("Tile Group Id :: %u status :: %u out of %u slots ", tile_group_id, - tuple_slot_id, num_tuple_slots); + tuple_slot_id, num_tuple_slots_); oid_t tile_column_count; oid_t column_itr = 0; - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tile_schemas[tile_itr]; - tile_column_count = schema.GetColumnCount(); - + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); + const catalog::Schema *schema = tile->GetSchema(); + tile_column_count = schema->GetColumnCount(); char *tile_tuple_location = tile->GetTupleLocation(tuple_slot_id); PELOTON_ASSERT(tile_tuple_location); // NOTE:: Only a tuple wrapper - storage::Tuple tile_tuple(&schema, tile_tuple_location); + storage::Tuple tile_tuple(schema, tile_tuple_location); for (oid_t tile_column_itr = 0; tile_column_itr < tile_column_count; tile_column_itr++) { @@ -319,22 +315,11 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, return tuple_slot_id; } -oid_t TileGroup::GetTileIdFromColumnId(oid_t column_id) { - oid_t tile_column_id, tile_offset; - LocateTileAndColumn(column_id, tile_offset, tile_column_id); - return tile_offset; -} - -oid_t TileGroup::GetTileColumnId(oid_t column_id) { - oid_t tile_column_id, tile_offset; - LocateTileAndColumn(column_id, tile_offset, tile_column_id); - return tile_column_id; -} - type::Value TileGroup::GetValue(oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - LocateTileAndColumn(column_id, tile_offset, tile_column_id); + tile_group_layout_->LocateTileAndColumn(column_id, tile_offset, + tile_column_id); return GetTile(tile_offset)->GetValue(tuple_id, tile_column_id); } @@ -342,37 +327,18 @@ void TileGroup::SetValue(type::Value &value, oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - LocateTileAndColumn(column_id, tile_offset, tile_column_id); + tile_group_layout_->LocateTileAndColumn(column_id, tile_offset, + tile_column_id); GetTile(tile_offset)->SetValue(value, tuple_id, tile_column_id); } std::shared_ptr TileGroup::GetTileReference( const oid_t tile_offset) const { - PELOTON_ASSERT(tile_offset < tile_count); + PELOTON_ASSERT(tile_offset < tile_count_); return tiles[tile_offset]; } -double TileGroup::GetSchemaDifference( - const storage::column_map_type &new_column_map) { - double theta = 0; - size_t capacity = column_map.size(); - double diff = 0; - - for (oid_t col_itr = 0; col_itr < capacity; col_itr++) { - auto &old_col = column_map.at(col_itr); - auto &new_col = new_column_map.at(col_itr); - - // The tile don't match - if (old_col.first != new_col.first) diff++; - } - - // compute diff - theta = diff / capacity; - - return theta; -} - void TileGroup::Sync() { // Sync the tile group data by syncing all the underlying tiles for (auto tile : tiles) { @@ -391,9 +357,10 @@ const std::string TileGroup::GetInfo() const { << peloton::GETINFO_DOUBLE_STAR << std::endl; os << "Database[" << database_id << "] // "; os << "Table[" << table_id << "] " << std::endl; + os << "Layout[" << *tile_group_layout_ << std::endl; os << (*tile_group_header) << std::endl; - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { Tile *tile = GetTile(tile_itr); if (tile != nullptr) { os << std::endl << (*tile); diff --git a/src/storage/tile_group_factory.cpp b/src/storage/tile_group_factory.cpp index f050d58e6df..2c0da483619 100644 --- a/src/storage/tile_group_factory.cpp +++ b/src/storage/tile_group_factory.cpp @@ -27,14 +27,18 @@ namespace storage { TileGroup *TileGroupFactory::GetTileGroup( oid_t database_id, oid_t table_id, oid_t tile_group_id, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, int tuple_count) { + std::shared_ptr layout, int tuple_count) { // Allocate the data on appropriate backend BackendType backend_type = BackendType::MM; // logging::LoggingUtil::GetBackendType(peloton_logging_mode); + // Ensure that the layout of the new TileGroup is not null. + if (layout == nullptr) { + throw NullPointerException("Layout of the TileGroup must be non-null."); + } TileGroupHeader *tile_header = new TileGroupHeader(backend_type, tuple_count); TileGroup *tile_group = new TileGroup(backend_type, tile_header, table, - schemas, column_map, tuple_count); + schemas, layout, tuple_count); tile_header->SetTileGroup(tile_group); diff --git a/src/tuning/index_tuner.cpp b/src/tuning/index_tuner.cpp index ca96907e605..cbd1bd57926 100644 --- a/src/tuning/index_tuner.cpp +++ b/src/tuning/index_tuner.cpp @@ -598,7 +598,7 @@ void IndexTuner::BootstrapTPCC(const std::string &path) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto table = catalog->GetTableWithName( - database_name, std::string(DEFUALT_SCHEMA_NAME), table_name, txn); + database_name, std::string(DEFAULT_SCHEMA_NAME), table_name, txn); txn_manager.CommitTransaction(txn); PELOTON_ASSERT(table != nullptr); for (auto &sample : samples) { diff --git a/src/tuning/layout_tuner.cpp b/src/tuning/layout_tuner.cpp index b189ecb85cd..a48fc64e564 100644 --- a/src/tuning/layout_tuner.cpp +++ b/src/tuning/layout_tuner.cpp @@ -16,9 +16,11 @@ #include #include +#include "catalog/catalog.h" #include "catalog/schema.h" #include "common/logger.h" #include "common/timer.h" +#include "concurrency/transaction_manager_factory.h" #include "storage/data_table.h" namespace peloton { @@ -29,9 +31,7 @@ LayoutTuner& LayoutTuner::GetInstance() { return layout_tuner; } -LayoutTuner::LayoutTuner() { - // Nothing to do here ! -} +LayoutTuner::LayoutTuner() {} LayoutTuner::~LayoutTuner() {} @@ -45,42 +45,6 @@ void LayoutTuner::Start() { LOG_INFO("Started layout tuner"); } -/** - * @brief Print information from column map, used to inspect layout generated - * from clusterer. - * - * @param column_map The column map to be printed. - */ -std::string LayoutTuner::GetColumnMapInfo(const column_map_type& column_map) { - std::stringstream ss; - std::map> tile_column_map; - - // Construct a tile_id => [col_ids] map - for (auto itr = column_map.begin(); itr != column_map.end(); itr++) { - oid_t col_id = itr->first; - oid_t tile_id = itr->second.first; - - if (tile_column_map.find(tile_id) == tile_column_map.end()) { - tile_column_map[tile_id] = {}; - } - - tile_column_map[tile_id].push_back(col_id); - } - - // Construct a string from tile_col_map - for (auto itr = tile_column_map.begin(); itr != tile_column_map.end(); - itr++) { - oid_t tile_id = itr->first; - ss << tile_id << ": "; - for (oid_t col_id : itr->second) { - ss << col_id << " "; - } - ss << " :: "; - } - - return ss.str(); -} - Sample GetClustererSample(const Sample& sample, oid_t column_count) { // Copy over the sample @@ -112,7 +76,7 @@ Sample GetClustererSample(const Sample& sample, oid_t column_count) { return clusterer_sample; } -void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { +bool LayoutTuner::UpdateDefaultPartition(storage::DataTable *table) { oid_t column_count = table->GetSchema()->GetColumnCount(); // Set up clusterer @@ -123,7 +87,9 @@ void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { // Check if we have any samples if (samples.empty()) { - return; + LOG_DEBUG("Table[%u] contains no LayoutSamples. Layout not tuned.", + table->GetOid()); + return false; } for (auto sample : samples) { @@ -142,12 +108,27 @@ void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { table->ClearLayoutSamples(); // Desired number of tiles - auto layout = clusterer.GetPartitioning(tile_count); - - LOG_TRACE("%s", GetColumnMapInfo(layout).c_str()); + auto column_map = clusterer.GetPartitioning(tile_count); + auto table_oid = table->GetOid(); + auto database_oid = table->GetDatabaseOid(); // Update table layout - table->SetDefaultLayout(layout); + // Since we need the layout to be updated in the catalog, + // Start a transaction before updating the default layout. + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto *txn = txn_manager.BeginTransaction(); + auto catalog = catalog::Catalog::GetInstance(); + if (catalog->CreateDefaultLayout(database_oid, table_oid, column_map, txn) == + nullptr) { + txn_manager.AbortTransaction(txn); + LOG_DEBUG("Layout Update to failed."); + return false; + } + txn_manager.CommitTransaction(txn); + + UNUSED_ATTRIBUTE auto layout = table->GetDefaultLayout(); + LOG_TRACE("Updated Layout: %s", layout.GetInfo().c_str()); + return true; } void LayoutTuner::Tune() { @@ -164,7 +145,8 @@ void LayoutTuner::Tune() { table->TransformTileGroup(tile_group_offset, theta); // Update partitioning periodically - UpdateDefaultPartition(table); + // TODO Lin/Tianyu - Add Failure Handling/Retry logic. + UNUSED_ATTRIBUTE bool update_result = UpdateDefaultPartition(table); // Sleep a bit std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration)); diff --git a/test/binder/binder_test.cpp b/test/binder/binder_test.cpp index b82df1ec72a..b5b266c21cf 100644 --- a/test/binder/binder_test.cpp +++ b/test/binder/binder_test.cpp @@ -129,11 +129,11 @@ TEST_F(BinderCorrectnessTest, SelectStatementTest) { catalog_ptr->GetDatabaseWithName(default_database_name, txn)->GetOid(); oid_t tableA_oid = catalog_ptr ->GetTableWithName(default_database_name, - DEFUALT_SCHEMA_NAME, "a", txn) + DEFAULT_SCHEMA_NAME, "a", txn) ->GetOid(); oid_t tableB_oid = catalog_ptr ->GetTableWithName(default_database_name, - DEFUALT_SCHEMA_NAME, "b", txn) + DEFAULT_SCHEMA_NAME, "b", txn) ->GetOid(); txn_manager.CommitTransaction(txn); @@ -262,7 +262,7 @@ TEST_F(BinderCorrectnessTest, DeleteStatementTest) { catalog_ptr->GetDatabaseWithName(default_database_name, txn)->GetOid(); oid_t tableB_oid = catalog_ptr ->GetTableWithName(default_database_name, - DEFUALT_SCHEMA_NAME, "b", txn) + DEFAULT_SCHEMA_NAME, "b", txn) ->GetOid(); string deleteSQL = "DELETE FROM b WHERE 1 = b1 AND b2 = 'str'"; diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index 3e6b5e15872..36af866b958 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -15,15 +15,16 @@ #include "catalog/database_catalog.h" #include "catalog/database_metrics_catalog.h" #include "catalog/index_catalog.h" +#include "catalog/layout_catalog.h" #include "catalog/query_metrics_catalog.h" #include "catalog/system_catalogs.h" #include "catalog/table_catalog.h" #include "common/harness.h" #include "common/logger.h" #include "concurrency/transaction_manager_factory.h" +#include "sql/testing_sql_util.h" #include "storage/storage_manager.h" #include "type/ephemeral_pool.h" -#include "sql/testing_sql_util.h" namespace peloton { namespace test { @@ -79,11 +80,11 @@ TEST_F(CatalogTests, CreatingTable) { new catalog::Schema({id_column, name_column})); catalog::Catalog::GetInstance()->CreateTable( - "emp_db", DEFUALT_SCHEMA_NAME, "emp_table", std::move(table_schema), txn); - catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFUALT_SCHEMA_NAME, + "emp_db", DEFAULT_SCHEMA_NAME, "emp_table", std::move(table_schema), txn); + catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema_2), txn); - catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFUALT_SCHEMA_NAME, + catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFAULT_SCHEMA_NAME, "salary_table", std::move(table_schema_3), txn); // insert random tuple into DATABASE_METRICS_CATALOG and check @@ -112,7 +113,7 @@ TEST_F(CatalogTests, CreatingTable) { EXPECT_EQ('a', *param1.buf); // check colum object EXPECT_EQ("name", catalog::Catalog::GetInstance() - ->GetTableObject("emp_db", DEFUALT_SCHEMA_NAME, + ->GetTableObject("emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn) ->GetColumnObject(1) ->GetColumnName()); @@ -124,7 +125,7 @@ TEST_F(CatalogTests, TableObject) { auto txn = txn_manager.BeginTransaction(); auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn); + "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); auto index_objects = table_object->GetIndexObjects(); auto column_objects = table_object->GetColumnObjects(); @@ -158,7 +159,7 @@ TEST_F(CatalogTests, TableObject) { bool update_result = pg_table->UpdateVersionId(1, department_table_oid, txn); // get version id after update, invalidate old cache table_object = catalog::Catalog::GetInstance()->GetTableObject( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn); + "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); uint32_t version_oid = table_object->GetVersionId(); EXPECT_NE(department_table_oid, INVALID_OID); EXPECT_EQ(update_result, true); @@ -234,23 +235,27 @@ TEST_F(CatalogTests, DroppingTable) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - // NOTE: everytime we create a database, there will be 8 catalog tables inside + // NOTE: everytime we create a database, there will be 9 catalog tables + // inside. Additionally, we create 3 tables for the test. + oid_t expected_table_count = CATALOG_TABLES_COUNT + 3; EXPECT_EQ( - 11, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); EXPECT_NE(nullptr, database_object); - catalog::Catalog::GetInstance()->DropTable("emp_db", DEFUALT_SCHEMA_NAME, + catalog::Catalog::GetInstance()->DropTable("emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); database_object = catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); EXPECT_NE(nullptr, database_object); auto department_table_object = - database_object->GetTableObject("department_table", DEFUALT_SCHEMA_NAME); + database_object->GetTableObject("department_table", DEFAULT_SCHEMA_NAME); + // Decrement expected_table_count to account for the dropped table. + expected_table_count--; EXPECT_EQ( - 10, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -259,30 +264,31 @@ TEST_F(CatalogTests, DroppingTable) { // Try to drop again txn = txn_manager.BeginTransaction(); EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn), + "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn), CatalogException); - // EXPECT_EQ( - 10, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); // Drop a table that does not exist txn = txn_manager.BeginTransaction(); EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable( - "emp_db", DEFUALT_SCHEMA_NAME, "void_table", txn), + "emp_db", DEFAULT_SCHEMA_NAME, "void_table", txn), CatalogException); EXPECT_EQ( - 10, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); // Drop the other table txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropTable("emp_db", DEFUALT_SCHEMA_NAME, + catalog::Catalog::GetInstance()->DropTable("emp_db", DEFAULT_SCHEMA_NAME, "emp_table", txn); + // Account for the dropped table. + expected_table_count--; EXPECT_EQ( - 9, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); } @@ -303,5 +309,109 @@ TEST_F(CatalogTests, DroppingCatalog) { EXPECT_NE(nullptr, catalog); } +TEST_F(CatalogTests, LayoutCatalogTest) { + // This test creates a table, changes its layout. + // Create another additional layout. + // Ensure that default is not changed. + // Drops layout and verifies that the default_layout is reset. + // It also queries pg_layout to ensure that the entry is removed. + + auto db_name = "temp_db"; + auto table_name = "temp_table"; + auto catalog = catalog::Catalog::GetInstance(); + // Create database. + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + EXPECT_EQ(ResultType::SUCCESS, catalog->CreateDatabase(db_name, txn)); + + // Create table. + auto val0 = catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), + "val0", true); + auto val1 = catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), + "val1", true); + auto val2 = catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), + "val2", true); + auto val3 = catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), + "val3", true); + std::unique_ptr table_schema( + new catalog::Schema({val0, val1, val2, val3})); + EXPECT_EQ(ResultType::SUCCESS, + catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn)); + txn_manager.CommitTransaction(txn); + + txn = txn_manager.BeginTransaction(); + auto database_oid = + catalog->GetDatabaseObject(db_name, txn)->GetDatabaseOid(); + auto table_object = + catalog->GetTableObject(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); + auto table_oid = table_object->GetTableOid(); + auto table = + catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); + txn_manager.CommitTransaction(txn); + + // Change default layout. + + std::map> default_map; + default_map[0] = std::make_pair(0, 0); + default_map[1] = std::make_pair(0, 1); + default_map[2] = std::make_pair(1, 0); + default_map[3] = std::make_pair(1, 1); + + txn = txn_manager.BeginTransaction(); + auto default_layout = + catalog->CreateDefaultLayout(database_oid, table_oid, default_map, txn); + txn_manager.CommitTransaction(txn); + EXPECT_NE(nullptr, default_layout); + + // Create additional layout. + std::map> non_default_map; + non_default_map[0] = std::make_pair(0, 0); + non_default_map[1] = std::make_pair(0, 1); + non_default_map[2] = std::make_pair(1, 0); + non_default_map[3] = std::make_pair(1, 1); + + txn = txn_manager.BeginTransaction(); + auto other_layout = + catalog->CreateLayout(database_oid, table_oid, non_default_map, txn); + txn_manager.CommitTransaction(txn); + + // Check that the default layout is still the same. + EXPECT_NE(*other_layout.get(), table->GetDefaultLayout()); + + // Drop the default layout. + auto default_layout_oid = default_layout->GetOid(); + txn = txn_manager.BeginTransaction(); + EXPECT_EQ(ResultType::SUCCESS, catalog->DropLayout(database_oid, table_oid, + default_layout_oid, txn)); + txn_manager.CommitTransaction(txn); + + // Check that default layout is reset and set to row_store. + EXPECT_NE(*default_layout.get(), table->GetDefaultLayout()); + EXPECT_TRUE(table->GetDefaultLayout().IsRowStore()); + + // Query pg_layout to ensure that the entry is dropped + txn = txn_manager.BeginTransaction(); + auto pg_layout = catalog->GetSystemCatalogs(database_oid)->GetLayoutCatalog(); + EXPECT_EQ(nullptr, + pg_layout->GetLayoutWithOid(table_oid, default_layout_oid, txn)); + + // The additional layout must be present in pg_layout + auto other_layout_oid = other_layout->GetOid(); + EXPECT_EQ( + *(other_layout.get()), + *(pg_layout->GetLayoutWithOid(table_oid, other_layout_oid, txn).get())); + txn_manager.CommitTransaction(txn); + + // Drop database + txn = txn_manager.BeginTransaction(); + catalog->DropDatabaseWithName(db_name, txn); + txn_manager.CommitTransaction(txn); +} + } // namespace test } // namespace peloton diff --git a/test/catalog/constraints_test.cpp b/test/catalog/constraints_test.cpp index d37f4f7081e..dfd8b015a81 100644 --- a/test/catalog/constraints_test.cpp +++ b/test/catalog/constraints_test.cpp @@ -237,10 +237,10 @@ TEST_F(ConstraintsTests, UNIQUETest) { new catalog::Schema({column1, column2})); std::string table_name("TEST_TABLE"); catalog::Catalog::GetInstance()->CreateTable(DEFAULT_DB_NAME, - DEFUALT_SCHEMA_NAME, table_name, + DEFAULT_SCHEMA_NAME, table_name, std::move(table_schema), txn); storage::DataTable *table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); // table->AddUNIQUEIndex(); diff --git a/test/catalog/manager_test.cpp b/test/catalog/manager_test.cpp index a0112f41f46..7b5ec2161ac 100644 --- a/test/catalog/manager_test.cpp +++ b/test/catalog/manager_test.cpp @@ -13,9 +13,10 @@ #include "common/harness.h" -#include "common/macros.h" #include "catalog/manager.h" #include "catalog/schema.h" +#include "common/macros.h" +#include "storage/layout.h" #include "storage/tile_group.h" #include "storage/tile_group_factory.h" @@ -48,12 +49,14 @@ void AddTileGroup(UNUSED_ATTRIBUTE uint64_t thread_id) { std::unique_ptr schema1(new catalog::Schema(columns)); schemas.push_back(*schema1); - std::map> column_map; - column_map[0] = std::make_pair(0, 0); + std::shared_ptr layout = + std::make_shared(columns.size()); for (oid_t txn_itr = 0; txn_itr < 100; txn_itr++) { - std::unique_ptr tile_group(storage::TileGroupFactory::GetTileGroup( - INVALID_OID, INVALID_OID, INVALID_OID, nullptr, schemas, column_map, 3)); + std::unique_ptr tile_group( + storage::TileGroupFactory::GetTileGroup(INVALID_OID, INVALID_OID, + INVALID_OID, nullptr, schemas, + layout, 3)); } } diff --git a/test/codegen/bloom_filter_test.cpp b/test/codegen/bloom_filter_test.cpp index bf9962ca834..ec632255dc9 100644 --- a/test/codegen/bloom_filter_test.cpp +++ b/test/codegen/bloom_filter_test.cpp @@ -213,7 +213,7 @@ TEST_F(BloomFilterCodegenTest, PerformanceTest) { int curr_size = 0; std::vector numbers; std::unordered_set number_set; - auto *table1 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto *table1 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table1_name, txn); while (curr_size < table1_target_size) { // Find a unique random number @@ -234,7 +234,7 @@ TEST_F(BloomFilterCodegenTest, PerformanceTest) { LOG_INFO("Finish populating test1"); // Load the inner table which contains twice tuples as the outer table - auto *table2 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto *table2 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table2_name, txn); unsigned outer_table_cardinality = numbers.size() * outer_to_inner_ratio; for (unsigned i = 0; i < outer_table_cardinality; i++) { @@ -334,7 +334,7 @@ void BloomFilterCodegenTest::CreateTable(std::string table_name, int tuple_size, } auto *catalog = catalog::Catalog::GetInstance(); catalog->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, std::unique_ptr(new catalog::Schema(cols)), txn); } diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index 881e24a6996..99fda04bb2b 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -18,6 +18,7 @@ #include "expression/operator_expression.h" #include "planner/seq_scan_plan.h" #include "storage/storage_manager.h" +#include "storage/table_factory.h" #include "codegen/testing_codegen_util.h" @@ -35,6 +36,50 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { CreateAndLoadAllColsTable(); } + void ScanLayoutTable(oid_t tuples_per_tilegroup, oid_t tilegroup_count, + oid_t column_count) { + auto table = GetLayoutTable(); + oid_t tuple_count = tuples_per_tilegroup * tilegroup_count; + + ///////////////////////////////////////////////////////// + // Do a seq scan on the table with the given layout + ///////////////////////////////////////////////////////// + + // Column ids to be scanned. + std::vector column_ids; + for (oid_t col_id = 0; col_id < column_count; col_id++) { + column_ids.push_back(col_id); + } + + // Setup the scan plan node + planner::SeqScanPlan scan(table, nullptr, column_ids); + + // Do binding + planner::BindingContext context; + scan.PerformBinding(context); + + // Printing consumer + codegen::BufferingConsumer buffer{column_ids, context}; + + // COMPILE and execute + CompileAndExecute(scan, buffer); + + // Check that we got all the results + const auto &results = buffer.GetOutputTuples(); + EXPECT_EQ(results.size(), tuple_count); + + for (oid_t tuple_id = 0; tuple_id < tuple_count; tuple_id++) { + auto &tuple = results[tuple_id]; + int tuple_id_value = tuple_id; + for (oid_t col_id = 0; col_id < column_count; col_id++) { + auto value = + type::ValueFactory::GetIntegerValue(tuple_id_value + col_id); + EXPECT_EQ(CmpBool::CmpTrue, + tuple.GetValue(col_id).CompareEquals(value)); + } + } + } + uint32_t NumRowsInTestTable() const { return num_rows_to_insert; } oid_t TestTableId() { return test_table_oids[0]; } @@ -67,11 +112,11 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { std::unique_ptr schema{new catalog::Schema(cols)}; // Insert table in catalog - catalog->CreateTable(test_db_name, DEFUALT_SCHEMA_NAME, all_cols_table_name, + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, all_cols_table_name, std::move(schema), txn); all_cols_table = catalog->GetTableWithName( - test_db_name, DEFUALT_SCHEMA_NAME, all_cols_table_name, txn); + test_db_name, DEFAULT_SCHEMA_NAME, all_cols_table_name, txn); auto *table_schema = all_cols_table->GetSchema(); // Insert one row where all columns are NULL @@ -584,5 +629,207 @@ TEST_F(TableScanTranslatorTest, ScanWithModuloPredicate) { type::ValueFactory::GetIntegerValue(1))); } +TEST_F(TableScanTranslatorTest, ScanRowLayout) { + // + // Creates a table with LayoutType::ROW and + // invokes the TableScanTranslator + // + uint32_t tuples_per_tilegroup = 100; + uint32_t tilegroup_count = 5; + uint32_t column_count = 100; + bool is_inlined = true; + CreateAndLoadTableWithLayout(LayoutType::ROW, tuples_per_tilegroup, + tilegroup_count, column_count, is_inlined); + ScanLayoutTable(tuples_per_tilegroup, tilegroup_count, column_count); +} + +TEST_F(TableScanTranslatorTest, ScanColumnLayout) { + // + // Creates a table with LayoutType::COLUMN and + // invokes the TableScanTranslator + // + uint32_t tuples_per_tilegroup = 100; + uint32_t tilegroup_count = 5; + uint32_t column_count = 100; + bool is_inlined = true; + CreateAndLoadTableWithLayout(LayoutType::COLUMN, tuples_per_tilegroup, + tilegroup_count, column_count, is_inlined); + ScanLayoutTable(tuples_per_tilegroup, tilegroup_count, column_count); +} + +TEST_F(TableScanTranslatorTest, MultiLayoutScan) { + // + // Creates a table with LayoutType::ROW + // Sets the default_layout_ as LayoutType::COLUMN + // Inserts tuples + // Sets the default_layout_ as LayoutType::HYBRID + // Inserts some more tuples + // invokes the TableScanTranslator + // + + const int tuples_per_tilegroup = 100; + const int col_count = 6; + const bool is_inlined = true; + uint32_t tuple_count = 100; + + ///////////////////////////////////////////////////////// + // Define the schema. + ///////////////////////////////////////////////////////// + + std::vector columns; + + for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { + auto column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "FIELD" + std::to_string(col_itr), is_inlined); + + columns.push_back(column); + } + + std::unique_ptr table_schema = + std::unique_ptr(new catalog::Schema(columns)); + std::string table_name("MULTI_LAYOUT_TABLE"); + + ///////////////////////////////////////////////////////// + // Create table. + ///////////////////////////////////////////////////////// + + bool is_catalog = false; + auto *catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + const bool allocate = true; + auto txn = txn_manager.BeginTransaction(); + + // Insert table in catalog + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn, is_catalog, + tuples_per_tilegroup, LayoutType::ROW); + // Get table reference + auto table = catalog->GetTableWithName(test_db_name, DEFAULT_SCHEMA_NAME, + table_name, txn); + txn_manager.EndTransaction(txn); + + ///////////////////////////////////////////////////////// + // Reset default_layout_ to LayoutType::COLUMN + ///////////////////////////////////////////////////////// + table->ResetDefaultLayout(LayoutType::COLUMN); + + ///////////////////////////////////////////////////////// + // Load in 100 tuples + ///////////////////////////////////////////////////////// + txn = txn_manager.BeginTransaction(); + auto table_schema_ptr = table->GetSchema(); + auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); + + for (oid_t row_id = 0; row_id < tuple_count; row_id++) { + int populate_value = row_id; + + storage::Tuple tuple(table_schema_ptr, allocate); + + for (oid_t col_id = 0; col_id < col_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(populate_value + col_id); + tuple.SetValue(col_id, value, testing_pool); + } + + ItemPointer *index_entry_ptr = nullptr; + ItemPointer tuple_slot_id = + table->InsertTuple(&tuple, txn, &index_entry_ptr); + + EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); + EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); + + txn_manager.PerformInsert(txn, tuple_slot_id, index_entry_ptr); + } + + txn_manager.CommitTransaction(txn); + + ///////////////////////////////////////////////////////// + // Set default_layout_ to LayoutType::HYBRID + ///////////////////////////////////////////////////////// + // Populate column_map with HYBRID layout + column_map_type column_map; + column_map[0] = std::make_pair(0, 0); + column_map[1] = std::make_pair(1, 0); + column_map[2] = std::make_pair(1, 1); + column_map[3] = std::make_pair(2, 0); + column_map[4] = std::make_pair(2, 1); + column_map[5] = std::make_pair(2, 2); + + auto database_oid = table->GetDatabaseOid(); + auto table_oid = table->GetOid(); + + txn = txn_manager.BeginTransaction(); + auto layout = + catalog->CreateDefaultLayout(database_oid, table_oid, column_map, txn); + EXPECT_NE(nullptr, layout); + txn_manager.CommitTransaction(txn); + + ///////////////////////////////////////////////////////// + // Load in 200 tuples + ///////////////////////////////////////////////////////// + tuple_count = 200; + oid_t prev_tuple_count = 100; + txn = txn_manager.BeginTransaction(); + for (oid_t row_id = 0; row_id < tuple_count; row_id++) { + int populate_value = row_id + prev_tuple_count; + + storage::Tuple tuple(table_schema_ptr, allocate); + + for (oid_t col_id = 0; col_id < col_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(populate_value + col_id); + tuple.SetValue(col_id, value, testing_pool); + } + + ItemPointer *index_entry_ptr = nullptr; + ItemPointer tuple_slot_id = + table->InsertTuple(&tuple, txn, &index_entry_ptr); + + EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); + EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); + + txn_manager.PerformInsert(txn, tuple_slot_id, index_entry_ptr); + } + + txn_manager.CommitTransaction(txn); + + ///////////////////////////////////////////////////////// + // Do a seq scan on the table + ///////////////////////////////////////////////////////// + + // Reset Tuple Count + tuple_count = tuple_count + prev_tuple_count; + // Column ids to be scanned. + std::vector column_ids; + for (oid_t col_id = 0; col_id < col_count; col_id++) { + column_ids.push_back(col_id); + } + + // Setup the scan plan node + planner::SeqScanPlan scan(table, nullptr, column_ids); + + // Do binding + planner::BindingContext context; + scan.PerformBinding(context); + + // Printing consumer + codegen::BufferingConsumer buffer{column_ids, context}; + + // COMPILE and execute + CompileAndExecute(scan, buffer); + + // Check that we got all the results + const auto &results = buffer.GetOutputTuples(); + EXPECT_EQ(results.size(), tuple_count); + + for (oid_t tuple_id = 0; tuple_id < tuple_count; tuple_id++) { + auto &tuple = results[tuple_id]; + int tuple_id_value = tuple_id; + for (oid_t col_id = 0; col_id < col_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(tuple_id_value + col_id); + EXPECT_EQ(CmpBool::CmpTrue, tuple.GetValue(col_id).CompareEquals(value)); + } + } +} + } // namespace test } // namespace peloton diff --git a/test/codegen/testing_codegen_util.cpp b/test/codegen/testing_codegen_util.cpp index fe8f6c26440..ff05144978b 100644 --- a/test/codegen/testing_codegen_util.cpp +++ b/test/codegen/testing_codegen_util.cpp @@ -32,7 +32,8 @@ namespace test { // PELOTON CODEGEN TEST //===----------------------------------------------------------------------===// -PelotonCodeGenTest::PelotonCodeGenTest(oid_t tuples_per_tilegroup) { +PelotonCodeGenTest::PelotonCodeGenTest(oid_t tuples_per_tilegroup, + peloton::LayoutType layout_type) { auto *catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); @@ -41,9 +42,10 @@ PelotonCodeGenTest::PelotonCodeGenTest(oid_t tuples_per_tilegroup) { catalog->CreateDatabase(test_db_name, txn); test_db = catalog->GetDatabaseWithName(test_db_name, txn); // Create test table - CreateTestTables(txn, tuples_per_tilegroup); + CreateTestTables(txn, tuples_per_tilegroup, layout_type); txn_manager.CommitTransaction(txn); + layout_table = nullptr; } PelotonCodeGenTest::~PelotonCodeGenTest() { @@ -103,27 +105,28 @@ std::unique_ptr PelotonCodeGenTest::CreateTestSchema( // Create all the test tables, but don't load any data void PelotonCodeGenTest::CreateTestTables(concurrency::TransactionContext *txn, - oid_t tuples_per_tilegroup) { + oid_t tuples_per_tilegroup, + peloton::LayoutType layout_type) { auto *catalog = catalog::Catalog::GetInstance(); for (int i = 0; i < 4; i++) { auto table_schema = CreateTestSchema(); - catalog->CreateTable(test_db_name, DEFUALT_SCHEMA_NAME, test_table_names[i], + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, test_table_names[i], std::move(table_schema), txn, false, - tuples_per_tilegroup); + tuples_per_tilegroup, layout_type); test_table_oids.push_back(catalog ->GetTableObject(test_db_name, - DEFUALT_SCHEMA_NAME, + DEFAULT_SCHEMA_NAME, test_table_names[i], txn) ->GetTableOid()); } for (int i = 4; i < 5; i++) { auto table_schema = CreateTestSchema(true); - catalog->CreateTable(test_db_name, DEFUALT_SCHEMA_NAME, test_table_names[i], + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, test_table_names[i], std::move(table_schema), txn, false, - tuples_per_tilegroup); + tuples_per_tilegroup, layout_type); test_table_oids.push_back(catalog ->GetTableObject(test_db_name, - DEFUALT_SCHEMA_NAME, + DEFAULT_SCHEMA_NAME, test_table_names[i], txn) ->GetTableOid()); } @@ -175,6 +178,80 @@ void PelotonCodeGenTest::LoadTestTable(oid_t table_id, uint32_t num_rows, txn_manager.CommitTransaction(txn); } +void PelotonCodeGenTest::CreateAndLoadTableWithLayout( + peloton::LayoutType layout_type, uint32_t tuples_per_tilegroup, + uint32_t tile_group_count, uint32_t column_count, bool is_inlined) { + uint32_t tuple_count = tuples_per_tilegroup * tile_group_count; + ///////////////////////////////////////////////////////// + // Define the schema. + ///////////////////////////////////////////////////////// + + std::vector columns; + + for (oid_t col_itr = 0; col_itr <= column_count; col_itr++) { + auto column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "FIELD" + std::to_string(col_itr), is_inlined); + + columns.push_back(column); + } + + std::unique_ptr table_schema = + std::unique_ptr(new catalog::Schema(columns)); + std::string table_name("LAYOUT_TABLE"); + + ///////////////////////////////////////////////////////// + // Create table. + ///////////////////////////////////////////////////////// + + bool is_catalog = false; + auto *catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + const bool allocate = true; + auto txn = txn_manager.BeginTransaction(); + + // Insert table in catalog + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn, is_catalog, + tuples_per_tilegroup, layout_type); + // Get table reference + layout_table = catalog->GetTableWithName(test_db_name, DEFAULT_SCHEMA_NAME, + table_name, txn); + txn_manager.EndTransaction(txn); + + ///////////////////////////////////////////////////////// + // Load in the data + ///////////////////////////////////////////////////////// + + // Insert tuples into tile_group. + + txn = txn_manager.BeginTransaction(); + auto table_schema_ptr = layout_table->GetSchema(); + auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); + + for (oid_t row_id = 0; row_id < tuple_count; row_id++) { + int populate_value = row_id; + + storage::Tuple tuple(table_schema_ptr, allocate); + + for (oid_t col_id = 0; col_id <= column_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(populate_value + col_id); + tuple.SetValue(col_id, value, testing_pool); + } + + ItemPointer *index_entry_ptr = nullptr; + ItemPointer tuple_slot_id = + layout_table->InsertTuple(&tuple, txn, &index_entry_ptr); + + EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); + EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); + + txn_manager.PerformInsert(txn, tuple_slot_id, index_entry_ptr); + } + + txn_manager.CommitTransaction(txn); +} + void PelotonCodeGenTest::ExecuteSync( codegen::Query &query, std::unique_ptr executor_context, diff --git a/test/executor/create_index_test.cpp b/test/executor/create_index_test.cpp index 034eca553fa..47bbd509307 100644 --- a/test/executor/create_index_test.cpp +++ b/test/executor/create_index_test.cpp @@ -205,7 +205,7 @@ TEST_F(CreateIndexTests, CreatingIndex) { txn = txn_manager.BeginTransaction(); auto target_table_ = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); // Expected 2 , Primary key index + created index EXPECT_EQ(target_table_->GetIndexCount(), 2); diff --git a/test/executor/create_test.cpp b/test/executor/create_test.cpp index 0faede0508d..214f32fa44d 100644 --- a/test/executor/create_test.cpp +++ b/test/executor/create_test.cpp @@ -99,7 +99,7 @@ TEST_F(CreateTests, CreatingTable) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("department_table", DEFUALT_SCHEMA_NAME, + planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); @@ -141,7 +141,7 @@ TEST_F(CreateTests, CreatingUDFs) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("accounts", DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node("accounts", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -243,7 +243,7 @@ TEST_F(CreateTests, CreatingTrigger) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("accounts", DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node("accounts", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -333,7 +333,7 @@ TEST_F(CreateTests, CreatingTrigger) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -373,7 +373,7 @@ TEST_F(CreateTests, CreatingTriggerWithoutWhen) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("accounts", DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node("accounts", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -421,7 +421,7 @@ TEST_F(CreateTests, CreatingTriggerWithoutWhen) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -462,7 +462,7 @@ TEST_F(CreateTests, CreatingTriggerInCatalog) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("accounts", DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node("accounts", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -502,7 +502,7 @@ TEST_F(CreateTests, CreatingTriggerInCatalog) { // check whether the trigger catalog table contains this new trigger auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); auto trigger_list = catalog::Catalog::GetInstance() ->GetSystemCatalogs(table_object->GetDatabaseOid()) diff --git a/test/executor/delete_test.cpp b/test/executor/delete_test.cpp index f081b766720..648944c8fe9 100644 --- a/test/executor/delete_test.cpp +++ b/test/executor/delete_test.cpp @@ -116,7 +116,7 @@ TEST_F(DeleteTests, VariousOperations) { new catalog::Schema({id_column, name_column})); std::unique_ptr context( new executor::ExecutorContext(txn)); - planner::CreatePlan node("department_table", DEFUALT_SCHEMA_NAME, + planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); executor::CreateExecutor create_executor(&node, context.get()); @@ -125,7 +125,7 @@ TEST_F(DeleteTests, VariousOperations) { LOG_INFO("Table created!"); storage::DataTable *table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); diff --git a/test/executor/drop_test.cpp b/test/executor/drop_test.cpp index 19e6c1293da..d90f06cbcb1 100644 --- a/test/executor/drop_test.cpp +++ b/test/executor/drop_test.cpp @@ -96,29 +96,33 @@ TEST_F(DropTests, DroppingTable) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_2", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_2", std::move(table_schema2), txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - // NOTE: everytime we create a database, there will be 8 catalog tables inside + // NOTE: everytime we create a database, there will be 9 catalog tables + // inside. In this test case, we have created two additional tables. + oid_t expeected_table_count = CATALOG_TABLES_COUNT + 2; EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() .size(), - 10); + expeected_table_count); // Now dropping the table using the executor - catalog->DropTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); + // Account for the dropped table. + expeected_table_count--; EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() .size(), - 9); + expeected_table_count); // free the database just created catalog->DropDatabaseWithName(TEST_DB_NAME, txn); @@ -148,7 +152,7 @@ TEST_F(DropTests, DroppingTrigger) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); @@ -180,7 +184,7 @@ TEST_F(DropTests, DroppingTrigger) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -212,12 +216,12 @@ TEST_F(DropTests, DroppingTrigger) { // Now dropping the table using the executer txn = txn_manager.BeginTransaction(); - catalog->DropTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); - EXPECT_EQ(8, (int)catalog::Catalog::GetInstance() - ->GetDatabaseObject(TEST_DB_NAME, txn) - ->GetTableObjects() - .size()); + EXPECT_EQ(CATALOG_TABLES_COUNT, (int)catalog::Catalog::GetInstance() + ->GetDatabaseObject(TEST_DB_NAME, txn) + ->GetTableObjects() + .size()); txn_manager.CommitTransaction(txn); // free the database just created @@ -250,18 +254,18 @@ TEST_F(DropTests, DroppingIndexByName) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_01", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); auto source_table = catalog->GetTableWithName( - TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_01", txn); + TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", txn); oid_t col_id = source_table->GetSchema()->GetColumnID(id_column.column_name); std::vector source_col_ids; source_col_ids.push_back(col_id); std::string index_name1 = "Testing_Drop_Index_By_Name"; - catalog->CreateIndex(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_01", + catalog->CreateIndex(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", source_col_ids, index_name1, false, IndexType::BWTREE, txn); txn_manager.CommitTransaction(txn); @@ -276,25 +280,25 @@ TEST_F(DropTests, DroppingIndexByName) { ->GetSystemCatalogs(database_object->GetDatabaseOid()) ->GetIndexCatalog(); auto index_object = - pg_index->GetIndexObject(index_name1, DEFUALT_SCHEMA_NAME, txn); + pg_index->GetIndexObject(index_name1, DEFAULT_SCHEMA_NAME, txn); EXPECT_NE(nullptr, index_object); // Check the effect of drop // Most major check in this test case // Now dropping the index using the DropIndex functionality catalog->DropIndex(database_object->GetDatabaseOid(), index_object->GetIndexOid(), txn); - EXPECT_EQ(pg_index->GetIndexObject(index_name1, DEFUALT_SCHEMA_NAME, txn), + EXPECT_EQ(pg_index->GetIndexObject(index_name1, DEFAULT_SCHEMA_NAME, txn), nullptr); txn_manager.CommitTransaction(txn); // Drop the table just created txn = txn_manager.BeginTransaction(); // Check the effect of drop index - EXPECT_EQ(pg_index->GetIndexObject(index_name1, DEFUALT_SCHEMA_NAME, txn), + EXPECT_EQ(pg_index->GetIndexObject(index_name1, DEFAULT_SCHEMA_NAME, txn), nullptr); // Now dropping the table - catalog->DropTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_01", + catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", txn); txn_manager.CommitTransaction(txn); diff --git a/test/executor/insert_test.cpp b/test/executor/insert_test.cpp index 4499eb6f53e..489b871121d 100644 --- a/test/executor/insert_test.cpp +++ b/test/executor/insert_test.cpp @@ -53,11 +53,11 @@ TEST_F(InsertTests, InsertRecord) { txn = txn_manager.BeginTransaction(); catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "TEST_TABLE", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "TEST_TABLE", std::move(table_schema), txn); auto table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "TEST_TABLE", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "TEST_TABLE", txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); diff --git a/test/executor/logical_tile_test.cpp b/test/executor/logical_tile_test.cpp index baf3d342262..0f225e65963 100644 --- a/test/executor/logical_tile_test.cpp +++ b/test/executor/logical_tile_test.cpp @@ -98,10 +98,14 @@ TEST_F(LogicalTileTests, TileMaterializationTest) { std::shared_ptr tile_group( TestingExecutorUtil::CreateTileGroup(tuple_count)); + std::vector tile_schemas; + for (unsigned int i = 0; i < tile_group->NumTiles(); i++) { + tile_schemas.push_back(tile_group->GetTile(i)->GetSchema()); + } + // Create tuple schema from tile schemas. - std::vector &tile_schemas = tile_group->GetTileSchemas(); std::unique_ptr schema( - catalog::Schema::AppendSchemaList(tile_schemas)); + catalog::Schema::AppendSchemaPtrList(tile_schemas)); // Create tuples and insert them into tile group. const bool allocate = true; @@ -156,10 +160,7 @@ TEST_F(LogicalTileTests, TileMaterializationTest) { logical_tile->AddPositionList(std::move(position_list1)); logical_tile->AddPositionList(std::move(position_list2)); - PELOTON_ASSERT(tile_schemas.size() == 2); - catalog::Schema *schema1 = &tile_schemas[0]; - catalog::Schema *schema2 = &tile_schemas[1]; - oid_t column_count = schema2->GetColumnCount(); + oid_t column_count = tile_schemas[1]->GetColumnCount(); for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { logical_tile->AddColumn(base_tile_ref, column_itr, column_itr); } @@ -185,12 +186,12 @@ TEST_F(LogicalTileTests, TileMaterializationTest) { logical_tile->AddPositionList(std::move(position_list3)); logical_tile->AddPositionList(std::move(position_list4)); - oid_t column_count1 = schema1->GetColumnCount(); + oid_t column_count1 = tile_schemas[0]->GetColumnCount(); for (oid_t column_itr = 0; column_itr < column_count1; column_itr++) { logical_tile->AddColumn(base_tile_ref1, column_itr, column_itr); } - oid_t column_count2 = schema2->GetColumnCount(); + oid_t column_count2 = tile_schemas[1]->GetColumnCount(); for (oid_t column_itr = 0; column_itr < column_count2; column_itr++) { logical_tile->AddColumn(base_tile_ref2, column_itr, column_count1 + column_itr); diff --git a/test/executor/seq_scan_test.cpp b/test/executor/seq_scan_test.cpp index 0e9f11d6a87..0649aa38e63 100644 --- a/test/executor/seq_scan_test.cpp +++ b/test/executor/seq_scan_test.cpp @@ -83,25 +83,29 @@ storage::DataTable *CreateTable() { column_map1[1] = std::make_pair(0, 1); column_map1[2] = std::make_pair(1, 0); column_map1[3] = std::make_pair(1, 1); + std::shared_ptr layout1 = + std::make_shared(column_map1); std::map> column_map2; column_map2[0] = std::make_pair(0, 0); column_map2[1] = std::make_pair(1, 0); column_map2[2] = std::make_pair(1, 1); column_map2[3] = std::make_pair(1, 2); + std::shared_ptr layout2 = + std::make_shared(column_map2); // Create tile groups. table->AddTileGroup(std::shared_ptr( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), table.get(), - schemas1, column_map1, tuple_count))); + schemas1, layout1, tuple_count))); table->AddTileGroup(std::shared_ptr( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), table.get(), - schemas2, column_map2, tuple_count))); + schemas2, layout2, tuple_count))); TestingExecutorUtil::PopulateTiles(table->GetTileGroup(0), tuple_count); TestingExecutorUtil::PopulateTiles(table->GetTileGroup(1), tuple_count); diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index b9aff4693e6..cd923c11ed8 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -29,7 +29,9 @@ #include "index/index_factory.h" #include "storage/data_table.h" #include "storage/database.h" +#include "storage/layout.h" #include "storage/table_factory.h" +#include "storage/tile.h" #include "storage/tile_group.h" #include "storage/tile_group_factory.h" #include "storage/tuple.h" @@ -169,11 +171,15 @@ std::shared_ptr TestingExecutorUtil::CreateTileGroup( column_map[2] = std::make_pair(1, 0); column_map[3] = std::make_pair(1, 1); + std::shared_ptr layout = + std::shared_ptr( + new const storage::Layout(column_map)); + std::shared_ptr tile_group_ptr( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, - column_map, tuple_count)); + layout, tuple_count)); catalog::Manager::GetInstance().AddTileGroup(tile_group_ptr->GetTileGroupId(), tile_group_ptr); @@ -252,10 +258,15 @@ void TestingExecutorUtil::PopulateTable(storage::DataTable *table, int num_rows, */ void TestingExecutorUtil::PopulateTiles( std::shared_ptr tile_group, int num_rows) { + size_t tile_count = tile_group->GetTileCount(); // Create tuple schema from tile schemas. - std::vector &tile_schemas = tile_group->GetTileSchemas(); + std::vector tile_schemas; + for (oid_t tile_id = 0; tile_id < tile_count; tile_id++) { + tile_schemas.push_back(tile_group->GetTile(tile_id)->GetSchema()); + } + std::unique_ptr schema( - catalog::Schema::AppendSchemaList(tile_schemas)); + catalog::Schema::AppendSchemaPtrList(tile_schemas)); // Ensure that the tile group is as expected. PELOTON_ASSERT(schema->GetColumnCount() == 4); @@ -409,6 +420,31 @@ storage::DataTable *TestingExecutorUtil::CreateTable( return table; } +storage::DataTable *TestingExecutorUtil::CreateTableUpdateCatalog( + int tuples_per_tilegroup_count, std::string &db_name) { + auto table_schema = std::unique_ptr( + new catalog::Schema({GetColumnInfo(0), GetColumnInfo(1), GetColumnInfo(2), + GetColumnInfo(3)})); + std::string table_name("test_table"); + + bool is_catalog = false; + auto *catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + // Insert table in catalog + catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn, is_catalog, + tuples_per_tilegroup_count); + txn_manager.CommitTransaction(txn); + + txn = txn_manager.BeginTransaction(); + auto table = + catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); + txn_manager.CommitTransaction(txn); + + return table; +} + /** * @brief Convenience method to create table for test. * diff --git a/test/executor/update_test.cpp b/test/executor/update_test.cpp index f8f3c12d0a2..04c42162cdc 100644 --- a/test/executor/update_test.cpp +++ b/test/executor/update_test.cpp @@ -90,24 +90,32 @@ storage::DataTable *CreateTable() { column_map1[2] = std::make_pair(1, 0); column_map1[3] = std::make_pair(1, 1); + std::shared_ptr layout1 = + std::shared_ptr( + new const storage::Layout(column_map1)); + std::map> column_map2; column_map2[0] = std::make_pair(0, 0); column_map2[1] = std::make_pair(1, 0); column_map2[2] = std::make_pair(1, 1); column_map2[3] = std::make_pair(1, 2); + std::shared_ptr layout2 = + std::shared_ptr( + new const storage::Layout(column_map1)); + // Create tile groups. table->AddTileGroup(std::shared_ptr( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), table.get(), - schemas1, column_map1, tuple_count))); + schemas1, layout1, tuple_count))); table->AddTileGroup(std::shared_ptr( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), table.get(), - schemas2, column_map2, tuple_count))); + schemas2, layout2, tuple_count))); TestingExecutorUtil::PopulateTiles(table->GetTileGroup(0), tuple_count); TestingExecutorUtil::PopulateTiles(table->GetTileGroup(1), tuple_count); @@ -176,7 +184,7 @@ TEST_F(UpdateTests, UpdatingOld) { new catalog::Schema({id_column, manager_id_column, name_column})); std::unique_ptr context( new executor::ExecutorContext(txn)); - planner::CreatePlan node("department_table", DEFUALT_SCHEMA_NAME, + planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); executor::CreateExecutor create_executor(&node, context.get()); @@ -186,7 +194,7 @@ TEST_F(UpdateTests, UpdatingOld) { LOG_INFO("Table created!"); storage::DataTable *table = catalog->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); txn_manager.CommitTransaction(txn); // Inserting a tuple end-to-end diff --git a/test/include/catalog/testing_constraints_util.h b/test/include/catalog/testing_constraints_util.h index 30a9b658223..1c96b995fa9 100644 --- a/test/include/catalog/testing_constraints_util.h +++ b/test/include/catalog/testing_constraints_util.h @@ -132,13 +132,13 @@ class TestingConstraintsUtil { // Create table. txn = txn_manager.BeginTransaction(); auto result = - catalog->CreateTable(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, + catalog->CreateTable(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, std::move(table_schema), txn, false); txn_manager.CommitTransaction(txn); EXPECT_EQ(ResultType::SUCCESS, result); txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); EXPECT_NE(nullptr, table); diff --git a/test/include/codegen/testing_codegen_util.h b/test/include/codegen/testing_codegen_util.h index 7e5cd3ee20f..8234dfb0d2b 100644 --- a/test/include/codegen/testing_codegen_util.h +++ b/test/include/codegen/testing_codegen_util.h @@ -52,7 +52,8 @@ class PelotonCodeGenTest : public PelotonTest { "table4", "table5"}; std::vector test_table_oids; - PelotonCodeGenTest(oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP); + PelotonCodeGenTest(oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP, + peloton::LayoutType layout_type = LayoutType::ROW); virtual ~PelotonCodeGenTest(); @@ -64,6 +65,9 @@ class PelotonCodeGenTest : public PelotonTest { return *GetDatabase().GetTableWithOid(static_cast(table_id)); } + // Get the layout table + storage::DataTable *GetLayoutTable() const { return layout_table; } + // Create the schema (common among all tables) catalog::Column GetTestColumn(uint32_t col_id) const; @@ -71,14 +75,20 @@ class PelotonCodeGenTest : public PelotonTest { bool add_primary = false) const; // Create the test tables - void CreateTestTables( - concurrency::TransactionContext *txn, - oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP); + void CreateTestTables(concurrency::TransactionContext *txn, + oid_t tuples_per_tilegroup, + peloton::LayoutType layout_type); // Load the given table with the given number of rows void LoadTestTable(oid_t table_id, uint32_t num_rows, bool insert_nulls = false); + // Load tables with the specified layout + void CreateAndLoadTableWithLayout(peloton::LayoutType layout_type, + oid_t tuples_per_tilegroup, + oid_t tile_group_count, oid_t column_count, + bool is_inlined); + static void ExecuteSync( codegen::Query &query, std::unique_ptr executor_context, @@ -117,6 +127,7 @@ class PelotonCodeGenTest : public PelotonTest { private: storage::Database *test_db; + storage::DataTable *layout_table; }; //===----------------------------------------------------------------------===// diff --git a/test/include/executor/testing_executor_util.h b/test/include/executor/testing_executor_util.h index 1c1ff85aea9..5d9dc14b5dd 100644 --- a/test/include/executor/testing_executor_util.h +++ b/test/include/executor/testing_executor_util.h @@ -75,6 +75,14 @@ class TestingExecutorUtil { int tuples_per_tilegroup_count = TESTS_TUPLES_PER_TILEGROUP, bool indexes = true, oid_t table_oid = INVALID_OID); + /** + * @brief Creates a basic table and adds its entry to the catalog + * + * @return A pointer to the DataTable created. + */ + static storage::DataTable *CreateTableUpdateCatalog( + int tuples_per_tilegroup_count, std::string &db_name); + /** @brief Creates a basic table with allocated and populated tuples */ static storage::DataTable *CreateAndPopulateTable(); diff --git a/test/optimizer/old_optimizer_test.cpp b/test/optimizer/old_optimizer_test.cpp index 1069ab87909..92949cc8521 100644 --- a/test/optimizer/old_optimizer_test.cpp +++ b/test/optimizer/old_optimizer_test.cpp @@ -171,7 +171,7 @@ TEST_F(OldOptimizerTests, UpdateDelWithIndexScanTest) { txn = txn_manager.BeginTransaction(); auto target_table_ = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); // Expected 1 , Primary key index + created index EXPECT_EQ(target_table_->GetIndexCount(), 2); txn_manager.CommitTransaction(txn); diff --git a/test/optimizer/optimizer_test.cpp b/test/optimizer/optimizer_test.cpp index dae410999d6..8b5ed1e0ec7 100644 --- a/test/optimizer/optimizer_test.cpp +++ b/test/optimizer/optimizer_test.cpp @@ -119,12 +119,14 @@ TEST_F(OptimizerTests, HashJoinTest) { LOG_INFO("Table Created"); traffic_cop.CommitQueryHelper(); - // NOTE: everytime we create a database, there will be 8 catalog tables inside + // NOTE: everytime we create a database, there will be 9 catalog tables inside + // Additionally, we also created a table for the test. + oid_t expected_table_count = CATALOG_TABLES_COUNT + 1; txn = txn_manager.BeginTransaction(); EXPECT_EQ(catalog::Catalog::GetInstance() ->GetDatabaseWithName(DEFAULT_DB_NAME, txn) ->GetTableCount(), - 1 + CATALOG_TABLES_COUNT); + expected_table_count); traffic_cop.SetTcopTxnState(txn); LOG_INFO("Creating table"); @@ -155,11 +157,13 @@ TEST_F(OptimizerTests, HashJoinTest) { LOG_INFO("Table Created"); traffic_cop.CommitQueryHelper(); + // Account for table created. + expected_table_count++; txn = txn_manager.BeginTransaction(); EXPECT_EQ(catalog::Catalog::GetInstance() ->GetDatabaseWithName(DEFAULT_DB_NAME, txn) ->GetTableCount(), - 2 + CATALOG_TABLES_COUNT); + expected_table_count); // Inserting a tuple to table_a traffic_cop.SetTcopTxnState(txn); diff --git a/test/optimizer/selectivity_test.cpp b/test/optimizer/selectivity_test.cpp index 0f5e4a9ab8c..cbb8df08d2c 100644 --- a/test/optimizer/selectivity_test.cpp +++ b/test/optimizer/selectivity_test.cpp @@ -72,7 +72,7 @@ TEST_F(SelectivityTests, RangeSelectivityTest) { txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); auto database = catalog->GetDatabaseWithName(DEFAULT_DB_NAME, txn); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, TEST_TABLE_NAME, txn); txn_manager.CommitTransaction(txn); oid_t db_id = database->GetOid(); @@ -181,7 +181,7 @@ TEST_F(SelectivityTests, EqualSelectivityTest) { txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); auto database = catalog->GetDatabaseWithName(DEFAULT_DB_NAME, txn); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, TEST_TABLE_NAME, txn); txn_manager.CommitTransaction(txn); oid_t db_id = database->GetOid(); diff --git a/test/optimizer/table_stats_collector_test.cpp b/test/optimizer/table_stats_collector_test.cpp index 7cafebe3e9e..e98f1a54bf9 100644 --- a/test/optimizer/table_stats_collector_test.cpp +++ b/test/optimizer/table_stats_collector_test.cpp @@ -60,7 +60,7 @@ TEST_F(TableStatsCollectorTests, SingleColumnTableTest) { } txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); txn_manager.CommitTransaction(txn); TableStatsCollector stats{table}; @@ -111,7 +111,7 @@ TEST_F(TableStatsCollectorTests, MultiColumnTableTest) { } txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); txn_manager.CommitTransaction(txn); TableStatsCollector stats{table}; diff --git a/test/optimizer/tuple_samples_storage_test.cpp b/test/optimizer/tuple_samples_storage_test.cpp index 2103d63549c..72ed769f270 100644 --- a/test/optimizer/tuple_samples_storage_test.cpp +++ b/test/optimizer/tuple_samples_storage_test.cpp @@ -85,7 +85,7 @@ TEST_F(TupleSamplesStorageTests, AddSamplesTableTest) { data_table->GetDatabaseOid(), data_table->GetOid()); txn = txn_manager.BeginTransaction(); storage::DataTable *samples_table = catalog->GetTableWithName( - SAMPLES_DB_NAME, DEFUALT_SCHEMA_NAME, samples_table_name, txn); + SAMPLES_DB_NAME, DEFAULT_SCHEMA_NAME, samples_table_name, txn); txn_manager.CommitTransaction(txn); EXPECT_TRUE(samples_table != nullptr); diff --git a/test/planner/plan_util_test.cpp b/test/planner/plan_util_test.cpp index 8455e98efac..77df6f54e88 100644 --- a/test/planner/plan_util_test.cpp +++ b/test/planner/plan_util_test.cpp @@ -54,10 +54,10 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "test_table", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", std::move(table_schema), txn); auto source_table = catalog->GetTableWithName( - TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "test_table", txn); + TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", txn); EXPECT_NE(source_table, nullptr); txn_manager.CommitTransaction(txn); @@ -67,7 +67,7 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { source_col_ids.push_back(col_id); // create index on 'id' - catalog->CreateIndex(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "test_table", + catalog->CreateIndex(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", source_col_ids, "test_id_idx", false, IndexType::BWTREE, txn); @@ -75,7 +75,7 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { col_id = source_table->GetSchema()->GetColumnID(fname_column.column_name); source_col_ids.push_back(col_id); - catalog->CreateIndex(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "test_table", + catalog->CreateIndex(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", source_col_ids, "test_fname_idx", false, IndexType::BWTREE, txn); txn_manager.CommitTransaction(txn); @@ -91,7 +91,7 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { // 1) id // 2) id and first_name auto table_object = - db_object->GetTableObject("test_table", DEFUALT_SCHEMA_NAME); + db_object->GetTableObject("test_table", DEFAULT_SCHEMA_NAME); EXPECT_NE(table_object, nullptr); oid_t id_idx_oid = table_object->GetIndexObject("test_id_idx")->GetIndexOid(); oid_t fname_idx_oid = @@ -198,14 +198,14 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { new catalog::Schema({id_column, fname_column, lname_column})); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_COLUMNS, DEFUALT_SCHEMA_NAME, "test_table", + catalog->CreateTable(TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); // Obtain ids for the table and columns txn = txn_manager.BeginTransaction(); auto source_table = catalog->GetTableWithName( - TEST_DB_COLUMNS, DEFUALT_SCHEMA_NAME, "test_table", txn); + TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table", txn); txn_manager.CommitTransaction(txn); oid_t table_id = source_table->GetOid(); @@ -228,14 +228,14 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { std::unique_ptr job_table_schema( new catalog::Schema({age_column, job_column, pid_column})); - catalog->CreateTable(TEST_DB_COLUMNS, DEFUALT_SCHEMA_NAME, "test_table_job", + catalog->CreateTable(TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table_job", std::move(job_table_schema), txn); txn_manager.CommitTransaction(txn); // Obtain ids for the table and columns txn = txn_manager.BeginTransaction(); auto source_table_job = catalog->GetTableWithName( - TEST_DB_COLUMNS, DEFUALT_SCHEMA_NAME, "test_table_job", txn); + TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table_job", txn); oid_t table_job_id = source_table_job->GetOid(); oid_t age_col_oid = source_table_job->GetSchema()->GetColumnID(age_column.column_name); diff --git a/test/planner/planner_test.cpp b/test/planner/planner_test.cpp index 7b263bccd79..5fd1709eaca 100644 --- a/test/planner/planner_test.cpp +++ b/test/planner/planner_test.cpp @@ -85,7 +85,7 @@ TEST_F(PlannerTest, DeletePlanTestParameter) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); @@ -100,7 +100,7 @@ TEST_F(PlannerTest, DeletePlanTestParameter) { ExpressionType::COMPARE_EQUAL, tuple_expr, parameter_expr); auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); // Create delete plan std::unique_ptr delete_plan( @@ -149,7 +149,7 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); @@ -159,7 +159,7 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { auto table_name = std::string("department_table"); auto database_name = DEFAULT_DB_NAME; auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - database_name, DEFUALT_SCHEMA_NAME, table_name, txn); + database_name, DEFAULT_SCHEMA_NAME, table_name, txn); auto schema = target_table->GetSchema(); TargetList tlist; @@ -246,7 +246,7 @@ TEST_F(PlannerTest, InsertPlanTestParameter) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); auto ret = catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); if (ret != ResultType::SUCCESS) LOG_TRACE("create table failed"); txn_manager.CommitTransaction(txn); @@ -276,7 +276,7 @@ TEST_F(PlannerTest, InsertPlanTestParameter) { std::unique_ptr(parameter_expr_2)); auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); std::unique_ptr insert_plan( new planner::InsertPlan(target_table, &insert_statement->columns, @@ -320,7 +320,7 @@ TEST_F(PlannerTest, InsertPlanTestParameterColumns) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); @@ -354,7 +354,7 @@ TEST_F(PlannerTest, InsertPlanTestParameterColumns) { std::unique_ptr(parameter_expr_2)); auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); std::unique_ptr insert_plan( new planner::InsertPlan(target_table, &insert_statement->columns, diff --git a/test/sql/drop_sql_test.cpp b/test/sql/drop_sql_test.cpp index 64f6af5f001..90ebbe69bf0 100644 --- a/test/sql/drop_sql_test.cpp +++ b/test/sql/drop_sql_test.cpp @@ -41,7 +41,7 @@ TEST_F(DropSQLTests, DropTableTest) { txn = txn_manager.BeginTransaction(); try { table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "test", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); } catch (CatalogException &e) { table = nullptr; } @@ -77,7 +77,7 @@ TEST_F(DropSQLTests, DropTableTest) { txn = txn_manager.BeginTransaction(); try { table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "test", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); } catch (CatalogException &e) { txn_manager.CommitTransaction(txn); table = nullptr; @@ -114,7 +114,7 @@ TEST_F(DropSQLTests, DropIndexTest) { std::shared_ptr index; txn = txn_manager.BeginTransaction(); try { - index = pg_index->GetIndexObject("idx", DEFUALT_SCHEMA_NAME, txn); + index = pg_index->GetIndexObject("idx", DEFAULT_SCHEMA_NAME, txn); } catch (CatalogException &e) { index = nullptr; @@ -128,7 +128,7 @@ TEST_F(DropSQLTests, DropIndexTest) { // Check if index is not in catalog txn = txn_manager.BeginTransaction(); - index = pg_index->GetIndexObject("idx", DEFUALT_SCHEMA_NAME, txn); + index = pg_index->GetIndexObject("idx", DEFAULT_SCHEMA_NAME, txn); EXPECT_EQ(index, nullptr); // Free the database just created diff --git a/test/sql/optimizer_sql_test.cpp b/test/sql/optimizer_sql_test.cpp index a81734c529d..3855c015e20 100644 --- a/test/sql/optimizer_sql_test.cpp +++ b/test/sql/optimizer_sql_test.cpp @@ -332,7 +332,7 @@ TEST_F(OptimizerSQLTests, DDLSqlTest) { auto txn = txn_manager.BeginTransaction(); // using transaction to get table from catalog auto table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "test2", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test2", txn); EXPECT_NE(nullptr, table); auto cols = table->GetSchema()->GetColumns(); EXPECT_EQ(3, cols.size()); @@ -354,7 +354,7 @@ TEST_F(OptimizerSQLTests, DDLSqlTest) { txn = txn_manager.BeginTransaction(); EXPECT_THROW(catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "test2", txn), + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test2", txn), peloton::Exception); txn_manager.CommitTransaction(txn); } diff --git a/test/statistics/stats_test.cpp b/test/statistics/stats_test.cpp index a53775e67a6..ef3c7da6cba 100644 --- a/test/statistics/stats_test.cpp +++ b/test/statistics/stats_test.cpp @@ -136,16 +136,16 @@ TEST_F(StatsTests, MultiThreadStatsTest) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); catalog->CreateDatabase("emp_db", txn); - catalog::Catalog::GetInstance()->CreateTable( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", - std::move(table_schema), txn); + catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFAULT_SCHEMA_NAME, + "department_table", + std::move(table_schema), txn); // Create multiple stat worker threads int num_threads = 8; storage::Database *database = catalog->GetDatabaseWithName("emp_db", txn); storage::DataTable *table = catalog->GetTableWithName( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn); + "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); txn_manager.CommitTransaction(txn); LaunchParallelTest(num_threads, TransactionTest, database, table); // Wait for aggregation to finish diff --git a/test/statistics/testing_stats_util.cpp b/test/statistics/testing_stats_util.cpp index 7b0f87852fe..5c087e4aba4 100644 --- a/test/statistics/testing_stats_util.cpp +++ b/test/statistics/testing_stats_util.cpp @@ -125,7 +125,7 @@ void TestingStatsUtil::CreateTable(bool has_primary_key) { auto txn = txn_manager.BeginTransaction(); std::unique_ptr context( new executor::ExecutorContext(txn)); - planner::CreatePlan node("department_table", DEFUALT_SCHEMA_NAME, "emp_db", + planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, "emp_db", std::move(table_schema), CreateType::TABLE); executor::CreateExecutor create_executor(&node, context.get()); create_executor.Init(); diff --git a/test/storage/data_table_test.cpp b/test/storage/data_table_test.cpp index e85337fecda..d64ec8bcb90 100644 --- a/test/storage/data_table_test.cpp +++ b/test/storage/data_table_test.cpp @@ -42,7 +42,7 @@ TEST_F(DataTableTests, TransformTileGroupTest) { txn_manager.CommitTransaction(txn); // Create the new column map - storage::column_map_type column_map; + column_map_type column_map; column_map[0] = std::make_pair(0, 0); column_map[1] = std::make_pair(0, 1); column_map[2] = std::make_pair(1, 0); diff --git a/test/storage/database_test.cpp b/test/storage/database_test.cpp index b60919a79d3..427d4c220e3 100644 --- a/test/storage/database_test.cpp +++ b/test/storage/database_test.cpp @@ -63,11 +63,16 @@ TEST_F(DatabaseTests, AddDropTableTest) { database->AddTable(data_table.get()); // NOTE: everytime we create a database, there will be 8 catalog tables inside - EXPECT_TRUE(database->GetTableCount() == 1 + CATALOG_TABLES_COUNT); + // Addtionally, we created a table for the test + oid_t expected_table_count = CATALOG_TABLES_COUNT + 1; + + EXPECT_EQ(expected_table_count, database->GetTableCount()); database->DropTableWithOid(table_oid); - EXPECT_TRUE(database->GetTableCount() == CATALOG_TABLES_COUNT); + // Account for the dropped table. + expected_table_count--; + EXPECT_EQ(expected_table_count, database->GetTableCount()); data_table.release(); diff --git a/test/storage/tile_group_test.cpp b/test/storage/tile_group_test.cpp index 5411d1be2de..4aca1ecb536 100644 --- a/test/storage/tile_group_test.cpp +++ b/test/storage/tile_group_test.cpp @@ -84,11 +84,14 @@ TEST_F(TileGroupTests, BasicTest) { column_map[2] = std::make_pair(1, 0); column_map[3] = std::make_pair(1, 1); + std::shared_ptr layout = + std::make_shared(column_map); + std::shared_ptr tile_group( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, - column_map, 4)); + layout, 4)); catalog::Manager::GetInstance().AddTileGroup(tile_group->GetTileGroupId(), tile_group); @@ -212,11 +215,14 @@ TEST_F(TileGroupTests, StressTest) { column_map[2] = std::make_pair(1, 0); column_map[3] = std::make_pair(1, 1); + std::shared_ptr layout = + std::make_shared(column_map); + std::shared_ptr tile_group( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, - column_map, 10000)); + layout, 10000)); catalog::Manager::GetInstance().AddTileGroup(tile_group->GetTileGroupId(), tile_group); @@ -232,11 +238,14 @@ TEST_F(TileGroupTests, StressTest) { // std::vector schemas; // // // SCHEMA -// catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), +// catalog::Column column1(type::TypeId::INTEGER, +// type::Type::GetTypeSize(type::TypeId::INTEGER), // "A", true); -// catalog::Column column2(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), +// catalog::Column column2(type::TypeId::INTEGER, +// type::Type::GetTypeSize(type::TypeId::INTEGER), // "B", true); -// catalog::Column column3(type::TypeId::TINYINT, type::Type::GetTypeSize(type::TypeId::TINYINT), +// catalog::Column column3(type::TypeId::TINYINT, +// type::Type::GetTypeSize(type::TypeId::TINYINT), // "C", true); // catalog::Column column4(type::TypeId::VARCHAR, 50, "D", false); // @@ -272,11 +281,14 @@ TEST_F(TileGroupTests, StressTest) { // column_map[2] = std::make_pair(1, 0); // column_map[3] = std::make_pair(1, 1); // +// std::shared_ptr layout = +// std::make_shared(column_map); +// // std::shared_ptr tile_group = // storage::TileGroupFactory::GetTileGroup( // INVALID_OID, INVALID_OID, // TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, -// column_map, 3); +// layout, 3); // catalog::Manager::GetInstance().AddTileGroup(tile_group->GetTileGroupId(), // tile_group); // @@ -371,19 +383,15 @@ TEST_F(TileGroupTests, TileCopyTest) { column_names.push_back(tile_column_names); const int tuple_count = 4; - storage::column_map_type column_map; - // default column map - auto col_count = schema->GetColumnCount(); - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(0, col_itr); - } + std::shared_ptr layout = + std::make_shared(schema->GetColumnCount()); std::shared_ptr tile_group( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, - column_map, tuple_count)); + layout, tuple_count)); catalog::Manager::GetInstance().AddTileGroup(tile_group->GetTileGroupId(), tile_group); diff --git a/test/trigger/trigger_test.cpp b/test/trigger/trigger_test.cpp index ea22d4deca5..c9c5b238ca3 100644 --- a/test/trigger/trigger_test.cpp +++ b/test/trigger/trigger_test.cpp @@ -56,7 +56,7 @@ class TriggerTests : public PelotonTest { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node(table_name, DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node(table_name, DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -73,7 +73,7 @@ class TriggerTests : public PelotonTest { auto txn = txn_manager.BeginTransaction(); auto table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, std::string(table_name), txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, std::string(table_name), txn); std::unique_ptr context( new executor::ExecutorContext(txn)); @@ -148,7 +148,7 @@ class TriggerTests : public PelotonTest { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(trigger_number, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -276,7 +276,7 @@ TEST_F(TriggerTests, BeforeAndAfterRowInsertTriggers) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -363,7 +363,7 @@ TEST_F(TriggerTests, AfterStatmentInsertTriggers) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -467,7 +467,7 @@ TEST_F(TriggerTests, OtherTypesTriggers) { auto txn = txn_manager.BeginTransaction(); storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); trigger::TriggerList *new_trigger_list = target_table->GetTriggerList(); diff --git a/test/tuning/layout_tuner_test.cpp b/test/tuning/layout_tuner_test.cpp index 108bfe41ae6..521ff92cc78 100644 --- a/test/tuning/layout_tuner_test.cpp +++ b/test/tuning/layout_tuner_test.cpp @@ -23,7 +23,7 @@ #include "concurrency/transaction_manager_factory.h" #include "storage/data_table.h" -#include "storage/tile_group.h" +#include "storage/layout.h" namespace peloton { namespace test { @@ -38,14 +38,17 @@ TEST_F(LayoutTunerTests, BasicTest) { const int tuple_count = TESTS_TUPLES_PER_TILEGROUP; + std::string db_name = "test_db"; + TestingExecutorUtil::InitializeDatabase(db_name); + + auto data_table = + TestingExecutorUtil::CreateTableUpdateCatalog(tuple_count, db_name); + // Create a table and populate it auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - std::unique_ptr data_table( - TestingExecutorUtil::CreateTable(tuple_count, false)); - TestingExecutorUtil::PopulateTable(data_table.get(), tuple_count, false, - false, - true, txn); + TestingExecutorUtil::PopulateTable(data_table, tuple_count, false, false, + true, txn); txn_manager.CommitTransaction(txn); // Check column count @@ -56,12 +59,11 @@ TEST_F(LayoutTunerTests, BasicTest) { tuning::LayoutTuner &layout_tuner = tuning::LayoutTuner::GetInstance(); // Attach table to index tuner - layout_tuner.AddTable(data_table.get()); + layout_tuner.AddTable(data_table); // Check old default tile group layout auto old_default_layout = data_table->GetDefaultLayout(); - LOG_INFO("Layout: %s", - layout_tuner.GetColumnMapInfo(old_default_layout).c_str()); + LOG_INFO("Layout: %s", old_default_layout.GetColumnMapInfo().c_str()); // Start layout tuner layout_tuner.Start(); @@ -107,25 +109,30 @@ TEST_F(LayoutTunerTests, BasicTest) { // Check new default tile group layout auto new_default_layout = data_table->GetDefaultLayout(); - LOG_INFO("Layout: %s", - layout_tuner.GetColumnMapInfo(new_default_layout).c_str()); + LOG_INFO("Layout: %s", new_default_layout.GetColumnMapInfo().c_str()); // Ensure that the layout has been changed EXPECT_NE(new_default_layout, old_default_layout); // Check the new default table layout - column_count = new_default_layout.size(); + column_count = new_default_layout.GetColumnCount(); EXPECT_EQ(column_count, 4); - auto first_column_tile = new_default_layout[0].first; - auto second_column_tile = new_default_layout[1].first; - auto third_column_tile = new_default_layout[2].first; - auto fourth_column_tile = new_default_layout[3].first; - EXPECT_EQ(first_column_tile, 0); - EXPECT_EQ(second_column_tile, 0); - EXPECT_EQ(third_column_tile, 0); - EXPECT_EQ(fourth_column_tile, 1); - + // Check the tile corresponding to each column. + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(0), 0); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(1), 0); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(2), 0); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(3), 1); + + // Check the per tile stats of the new layout + // The layout must contain 2 tiles with the following stats + // 0 -> 3 + // 1 -> 1 + auto layout_stats = new_default_layout.GetLayoutStats(); + EXPECT_EQ(layout_stats[0], 3); + EXPECT_EQ(layout_stats[1], 1); + + TestingExecutorUtil::DeleteDatabase(db_name); } } // namespace test