From c361ddd0388a85c9207e32f2b7db84152968c272 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Thu, 12 Sep 2024 09:40:12 +0000 Subject: [PATCH 01/21] Put TColumnShard* object to TDBWrapper --- ydb/core/tablet_flat/flat_executor.cpp | 2 +- ydb/core/tablet_flat/tablet_flat_executor.h | 9 +++++++-- .../transaction/tx_gc_insert_table.cpp | 2 +- .../blobs_action/transaction/tx_write.cpp | 9 +++++++++ .../blobs_action/transaction/tx_write_index.cpp | 2 +- ydb/core/tx/columnshard/columnshard__init.cpp | 2 +- .../transactions/tx_data_from_source.cpp | 2 +- ydb/core/tx/columnshard/engines/db_wrapper.cpp | 5 +++++ ydb/core/tx/columnshard/engines/db_wrapper.h | 15 +++++++++++++-- .../normalizer/portion/broken_blobs.cpp | 2 +- .../tx/columnshard/normalizer/portion/clean.cpp | 2 +- .../tx/columnshard/normalizer/portion/portion.cpp | 2 +- ydb/core/tx/columnshard/operations/write.cpp | 4 ++-- .../transactions/operators/long_tx_write.h | 4 ++-- 14 files changed, 46 insertions(+), 16 deletions(-) diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index ac3f48916a0e..3436ebafc18a 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -1704,7 +1704,7 @@ void TExecutor::ExecuteTransaction(TAutoPtr seat, const TActorContext &ct PrivatePageCache->ResetTouchesAndToLoad(true); TPageCollectionTxEnv env(*Database, *PrivatePageCache); - TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId, seat->Self->TxSpan); + TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId, seat->Self->TxSpan, Owner); txc.NotEnoughMemory(seat->NotEnoughMemoryCount); Database->Begin(Stamp(), env); diff --git a/ydb/core/tablet_flat/tablet_flat_executor.h b/ydb/core/tablet_flat/tablet_flat_executor.h index 439d181362d5..726b14ca9a3a 100644 --- a/ydb/core/tablet_flat/tablet_flat_executor.h +++ b/ydb/core/tablet_flat/tablet_flat_executor.h @@ -23,6 +23,10 @@ namespace NTable { namespace NTabletFlatExecutor { +namespace NFlatExecutorSetup { + struct ITablet; +} + class TTransactionContext; class TExecutor; struct TPageCollectionTxEnv; @@ -206,7 +210,7 @@ class TTransactionContext : public TTxMemoryProviderBase { public: TTransactionContext(ui64 tablet, ui32 gen, ui32 step, NTable::TDatabase &db, IExecuting &env, - ui64 memoryLimit, ui64 taskId, NWilson::TSpan &transactionSpan) + ui64 memoryLimit, ui64 taskId, NWilson::TSpan &transactionSpan, NFlatExecutorSetup::ITablet *owner) : TTxMemoryProviderBase(memoryLimit, taskId) , Tablet(tablet) , Generation(gen) @@ -214,6 +218,7 @@ class TTransactionContext : public TTxMemoryProviderBase { , Env(env) , DB(db) , TransactionSpan(transactionSpan) + , Owner(owner) {} ~TTransactionContext() {} @@ -247,6 +252,7 @@ class TTransactionContext : public TTxMemoryProviderBase { NTable::TDatabase &DB; NWilson::TSpan &TransactionSpan; NWilson::TSpan TransactionExecutionSpan; + NFlatExecutorSetup::ITablet* Owner; private: bool Rescheduled_ = false; @@ -280,7 +286,6 @@ enum class ETerminationReason { MemoryLimitExceeded = 1, }; - class ITransaction : TNonCopyable { public: using TTransactionContext = NTabletFlatExecutor::TTransactionContext; diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp index 15a05e7108a7..e0d6b0fcb9a3 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NColumnShard { bool TTxInsertTableCleanup::Execute(TTransactionContext& txc, const TActorContext& /*ctx*/) { TMemoryProfileGuard mpg("TTxInsertTableCleanup::Execute"); TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, txc.Owner); NIceDb::TNiceDb db(txc.DB); Self->TryAbortWrites(db, dbTable, std::move(WriteIdsToAbort)); diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp index 579a2e5eaa14..7f120d82059f 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp @@ -11,6 +11,15 @@ bool TTxWrite::InsertOneBlob(TTransactionContext& txc, const NOlap::TWideSeriali TBlobGroupSelector dsGroupSelector(Self->Info()); NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); + + const auto& writeMeta = batch.GetAggregation().GetWriteMeta(); + meta.SetModificationType(TEnumOperator::SerializeToProto(writeMeta.GetModificationType())); + *meta.MutableSchemaSubset() = batch.GetAggregation().GetSchemaSubset().SerializeToProto(); + auto schemeVersion = batch.GetAggregation().GetSchemaVersion(); + auto tableSchema = Self->TablesManager.GetPrimaryIndex()->GetVersionedIndex().GetSchemaVerified(schemeVersion); + + auto userData = std::make_shared(writeMeta.GetTableId(), blobRange, meta, tableSchema->GetVersion(), batch->GetData()); + NOlap::TInsertedData insertData(writeId, userData); bool ok = Self->InsertTable->Insert(dbTable, std::move(insertData)); if (ok) { Self->UpdateInsertTableCounters(); diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp index 57a1eee50146..3da42cd4bf9b 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp @@ -20,7 +20,7 @@ bool TTxWriteIndex::Execute(TTransactionContext& txc, const TActorContext& ctx) Y_ABORT_UNLESS(Ev->Get()->IndexInfo->GetLastSchema()->GetSnapshot() <= snapshot); TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbWrap(txc.DB, &dsGroupSelector); + NOlap::TDbWrapper dbWrap(txc.DB, &dsGroupSelector, txc.Owner); AFL_VERIFY(Self->TablesManager.MutablePrimaryIndex().ApplyChangesOnExecute(dbWrap, changes, snapshot)); LOG_S_DEBUG(TxPrefix() << "(" << changes->TypeString() << ") apply" << TxSuffix()); NOlap::TWriteIndexContext context(&txc.DB, dbWrap, Self->MutableIndexAs()); diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index 8ce503cdacba..6d63f8f705cd 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -100,7 +100,7 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx) NIceDb::TNiceDb db(txc.DB); TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, txc.Owner); { ACFL_DEBUG("step", "TInsertTable::Load_Start"); TMemoryProfileGuard g("TTxInit/InsertTable"); diff --git a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp index 0cde35b73dec..433dd79fb2ca 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NOlap::NDataSharing { bool TTxDataFromSource::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { using namespace NKikimr::NColumnShard; - TDbWrapper dbWrapper(txc.DB, nullptr); + TDbWrapper dbWrapper(txc.DB, nullptr, txc.Owner); auto& index = Self->TablesManager.MutablePrimaryIndexAsVerified(); { ui64* lastPortionPtr = index.GetLastPortionPointer(); diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index b5c8e5e4ea58..3340f5568684 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -3,6 +3,7 @@ #include "portions/constructor.h" #include #include +#include #include namespace NKikimr::NOlap { @@ -221,4 +222,8 @@ TConclusion>> TD return result; } +NColumnShard::TColumnShard* GetColumnShard(NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner) { + return dynamic_cast(owner); +} + } diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h index 50958b6fca29..a5f35905b898 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.h +++ b/ydb/core/tx/columnshard/engines/db_wrapper.h @@ -4,6 +4,10 @@ #include #include +namespace NColumnShard { +class TColumnShard; +} + namespace NKikimrTxColumnShard { class TIndexPortionMeta; } @@ -56,12 +60,18 @@ class IDbWrapper { virtual TConclusion>> LoadGranulesShardingInfo() = 0; }; +NColumnShard::TColumnShard* GetColumnShard(NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner); + class TDbWrapper : public IDbWrapper { public: - TDbWrapper(NTable::TDatabase& db, const IBlobGroupSelector* dsGroupSelector) + TDbWrapper(NTable::TDatabase& db, const IBlobGroupSelector* dsGroupSelector, NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner) : Database(db) , DsGroupSelector(dsGroupSelector) - {} + , CS(GetColumnShard(owner)) + { + AFL_VERIFY(owner != nullptr); + AFL_VERIFY(CS != nullptr); + } void Insert(const TInsertedData& data) override; void Commit(const TCommittedData& data) override; @@ -92,6 +102,7 @@ class TDbWrapper : public IDbWrapper { private: NTable::TDatabase& Database; const IBlobGroupSelector* DsGroupSelector; + NColumnShard::TColumnShard* CS; }; } diff --git a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp index 238638d9596a..b2d891e3b82b 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp @@ -22,7 +22,7 @@ class TNormalizerResult : public INormalizerChanges { bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& normController) const override { NOlap::TBlobManagerDb blobManagerDb(txc.DB); - TDbWrapper db(txc.DB, nullptr); + TDbWrapper db(txc.DB, nullptr, txc.Owner); for (auto&& [_, portionInfo] : BrokenPortions) { auto schema = Schemas->FindPtr(portionInfo->GetPortionId()); AFL_VERIFY(!!schema)("portion_id", portionInfo->GetPortionId()); diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp index d1e00669f8b3..f8aa8cb9b432 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp @@ -23,7 +23,7 @@ class TBlobsRemovingResult : public INormalizerChanges { NOlap::TBlobManagerDb blobManagerDb(txc.DB); RemovingAction->OnExecuteTxAfterRemoving(blobManagerDb, true); - TDbWrapper db(txc.DB, nullptr); + TDbWrapper db(txc.DB, nullptr, txc.Owner); for (auto&& portion : Portions) { AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("message", "remove lost portion")("path_id", portion->GetPathId())("portion_id", portion->GetPortionId()); portion->RemoveFromDatabase(db); diff --git a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp index 739715f44125..5a49acaf62df 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp @@ -20,7 +20,7 @@ class TPortionsNormalizer::TNormalizerResult : public INormalizerChanges { bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& /* normController */) const override { using namespace NColumnShard; - TDbWrapper db(txc.DB, nullptr); + TDbWrapper db(txc.DB, nullptr, txc.Owner); for (auto&& portionInfo : Portions) { auto schema = Schemas->FindPtr(portionInfo->GetPortionId()); diff --git a/ydb/core/tx/columnshard/operations/write.cpp b/ydb/core/tx/columnshard/operations/write.cpp index 1068e7167413..89af347ef8dd 100644 --- a/ydb/core/tx/columnshard/operations/write.cpp +++ b/ydb/core/tx/columnshard/operations/write.cpp @@ -44,7 +44,7 @@ void TWriteOperation::CommitOnExecute(TColumnShard& owner, NTabletFlatExecutor:: Y_ABORT_UNLESS(Status == EOperationStatus::Prepared); TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, &owner); for (auto gWriteId : InsertWriteIds) { auto pathExists = [&](ui64 pathId) { @@ -108,7 +108,7 @@ void TWriteOperation::AbortOnExecute(TColumnShard& owner, NTabletFlatExecutor::T Y_ABORT_UNLESS(Status == EOperationStatus::Prepared); TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, &owner); THashSet writeIds; writeIds.insert(InsertWriteIds.begin(), InsertWriteIds.end()); diff --git a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h index 45b642c8e98a..30fdf9b60149 100644 --- a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h +++ b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h @@ -55,7 +55,7 @@ namespace NKikimr::NColumnShard { bool ProgressOnExecute(TColumnShard& owner, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc) override { TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, &owner); auto pathExists = [&](ui64 pathId) { return owner.TablesManager.HasTable(pathId); @@ -88,7 +88,7 @@ namespace NKikimr::NColumnShard { AFL_VERIFY(owner.RemoveLongTxWrite(db, writeId, GetTxId())); } TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, &owner); owner.InsertTable->Abort(dbTable, WriteIds); return true; } From edc66e1f5f545ed8bd52b732e51fa031f2320cec Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Thu, 12 Sep 2024 11:28:27 +0000 Subject: [PATCH 02/21] Maintain version ref counts --- ydb/core/tx/columnshard/columnshard_impl.h | 3 +++ ydb/core/tx/columnshard/engines/column_engine.h | 10 ++++++++++ .../tx/columnshard/engines/column_engine_logs.cpp | 13 +++++++++++++ .../tx/columnshard/engines/column_engine_logs.h | 1 + ydb/core/tx/columnshard/engines/db_wrapper.cpp | 8 ++++++++ .../tx/columnshard/engines/portions/constructor.h | 8 ++++++++ ydb/core/tx/columnshard/tables_manager.h | 9 +++++++++ 7 files changed, 52 insertions(+) diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index aa6ef920ece7..33f3def4fecf 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -53,6 +53,7 @@ class TChangesWithAppend; class TCompactColumnEngineChanges; class TInsertColumnEngineChanges; class TStoragesManager; +class TDbWrapper; namespace NReader { class TTxScan; @@ -186,6 +187,8 @@ class TColumnShard friend class NOlap::NReader::TTxInternalScan; friend class NOlap::NReader::NPlain::TIndexScannerConstructor; + friend class NOlap::TDbWrapper; + class TStoragesManager; friend class TTxController; diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index 2c616c06e32d..f360571191f8 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -298,6 +298,16 @@ class IColumnEngine { return TSnapshot::Zero(); } virtual void OnTieringModified(const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) = 0; + + ui32 VersionAddRef(ui64 version, i32 diff) { + ui32& refCount = VersionCounts[version]; + refCount += diff; + LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount - diff << " to " << refCount); + return refCount; + } + +private: + THashMap VersionCounts; }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 4218fc19cf5e..c296aa1ea648 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -26,6 +26,16 @@ namespace NKikimr::NOlap { +TColumnEngineForLogs::TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager) + : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) + , StoragesManager(storagesManager) + , TabletId(tabletId) + , LastPortion(0) + , LastGranule(0) +{ + ActualizationController = std::make_shared(); +} + TColumnEngineForLogs::TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema) : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) @@ -207,6 +217,9 @@ bool TColumnEngineForLogs::LoadColumns(IDbWrapper& db) { const TIndexInfo& indexInfo = portion.GetSchema(VersionedIndex)->GetIndexInfo(); AFL_VERIFY(portion.MutableMeta().LoadMetadata(metaProto, indexInfo)); AFL_VERIFY(constructors.AddConstructorVerified(std::move(portion))); + if (portion.HasSchemaVersion()) { + VersionAddRef(portion.GetSchemaVersion(), 1); + } })) { return false; } diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 7b515c26f40c..2388d02f85dc 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -83,6 +83,7 @@ class TColumnEngineForLogs : public IColumnEngine { TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema); TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema); + TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager); virtual void OnTieringModified(const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) override; diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 3340f5568684..3f110a51068c 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -65,10 +65,12 @@ void TDbWrapper::WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRe } void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { + LOG_S_CRIT("Writing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); NIceDb::TNiceDb db(Database); auto metaProto = portion.GetMeta().SerializeToProto(); using IndexPortions = NColumnShard::Schema::IndexPortions; auto removeSnapshot = portion.GetRemoveSnapshotOptional(); + CS->TablesManager.VersionAddRef(portion.GetSchemaVersionVerified(), 1); db.Table().Key(portion.GetPathId(), portion.GetPortion()).Update( NIceDb::TUpdate(portion.GetSchemaVersionVerified()), NIceDb::TUpdate(portion.GetShardingVersionDef(0)), @@ -78,8 +80,13 @@ void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { } void TDbWrapper::ErasePortion(const NOlap::TPortionInfo& portion) { + LOG_S_CRIT("Erasing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); NIceDb::TNiceDb db(Database); using IndexPortions = NColumnShard::Schema::IndexPortions; + ui32 refCount = CS->TablesManager.VersionAddRef(portion.GetSchemaVersionVerified(), -1); + if (refCount == 0) { + LOG_S_CRIT("Ref count is set to 0 for version " << portion.GetSchemaVersionVerified() << " need to delete"); + } db.Table().Key(portion.GetPathId(), portion.GetPortion()).Delete(); } @@ -125,6 +132,7 @@ bool TDbWrapper::LoadPortions(const std::function() << " portion id " << rowset.GetValue() << " schema version " << rowset.GetValue() << " database " << (ui64)&Database); NOlap::TPortionInfoConstructor portion(rowset.GetValue(), rowset.GetValue()); portion.SetSchemaVersion(rowset.GetValue()); if (rowset.HaveValue() && rowset.GetValue()) { diff --git a/ydb/core/tx/columnshard/engines/portions/constructor.h b/ydb/core/tx/columnshard/engines/portions/constructor.h index 94b285255c72..f70d0c637a80 100644 --- a/ydb/core/tx/columnshard/engines/portions/constructor.h +++ b/ydb/core/tx/columnshard/engines/portions/constructor.h @@ -193,6 +193,14 @@ class TPortionInfoConstructor { ShardingVersion = version; } + bool HasSchemaVersion() const { + return SchemaVersion.has_value(); + } + + ui64 GetSchemaVersion() const { + return SchemaVersion.value(); + } + void SetRemoveSnapshot(const TSnapshot& snap) { AFL_VERIFY(!RemoveSnapshot); if (snap.Valid()) { diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 01643fe8db9a..241d0636c492 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -6,6 +6,7 @@ #include "engines/column_engine.h" #include +#include #include #include #include @@ -249,6 +250,14 @@ class TTablesManager { bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json); [[nodiscard]] std::unique_ptr CreateAddShardingInfoTx(TColumnShard& owner, const ui64 pathId, const ui64 versionId, const NSharding::TGranuleShardingLogicContainer& tabletShardingLogic) const; + + ui32 VersionAddRef(ui64 schemaVersion, i32 diff) { + if (!PrimaryIndex) { + PrimaryIndex = std::make_unique(TabletId, StoragesManager); + } + return PrimaryIndex->VersionAddRef(schemaVersion, diff); + } + }; } From 42fcd945f00241774e82d67d3409133a9e2783f8 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Thu, 12 Sep 2024 13:35:57 +0000 Subject: [PATCH 03/21] Keep version counts in TColumnShard rather than in IColumnEngine --- ydb/core/tx/columnshard/columnshard_impl.h | 13 +++++++++++++ .../tx/columnshard/engines/column_engine.h | 10 ---------- .../columnshard/engines/column_engine_logs.cpp | 3 ++- ydb/core/tx/columnshard/engines/db_wrapper.cpp | 4 ++-- ydb/core/tx/columnshard/engines/db_wrapper.h | 18 +++++++++++++----- ydb/core/tx/columnshard/tables_manager.h | 8 -------- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 33f3def4fecf..27d3d303beb1 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -137,6 +137,9 @@ using ITransaction = NTabletFlatExecutor::ITransaction; template using TTransactionBase = NTabletFlatExecutor::TTransactionBase; +class TVersionCounts : public THashMap { +}; + class TColumnShard : public TActor , public NTabletFlatExecutor::TTabletExecutedFlat @@ -623,6 +626,16 @@ class TColumnShard } TColumnShard(TTabletStorageInfo* info, const TActorId& tablet); + + ui32 VersionAddRef(ui64 version, i32 diff) { + ui32& refCount = VersionCounts[version]; + refCount += diff; + LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount - diff << " to " << refCount << " this " << (ui64)this); + return refCount; + } + +private: + TVersionCounts VersionCounts; }; } diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index f360571191f8..2c616c06e32d 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -298,16 +298,6 @@ class IColumnEngine { return TSnapshot::Zero(); } virtual void OnTieringModified(const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) = 0; - - ui32 VersionAddRef(ui64 version, i32 diff) { - ui32& refCount = VersionCounts[version]; - refCount += diff; - LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount - diff << " to " << refCount); - return refCount; - } - -private: - THashMap VersionCounts; }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index c296aa1ea648..e0b33ec4e9a0 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -218,7 +219,7 @@ bool TColumnEngineForLogs::LoadColumns(IDbWrapper& db) { AFL_VERIFY(portion.MutableMeta().LoadMetadata(metaProto, indexInfo)); AFL_VERIFY(constructors.AddConstructorVerified(std::move(portion))); if (portion.HasSchemaVersion()) { - VersionAddRef(portion.GetSchemaVersion(), 1); + db.CS->VersionAddRef(portion.GetSchemaVersion(), 1); } })) { return false; diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 3f110a51068c..549336bb1079 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -70,7 +70,7 @@ void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { auto metaProto = portion.GetMeta().SerializeToProto(); using IndexPortions = NColumnShard::Schema::IndexPortions; auto removeSnapshot = portion.GetRemoveSnapshotOptional(); - CS->TablesManager.VersionAddRef(portion.GetSchemaVersionVerified(), 1); + CS->VersionAddRef(portion.GetSchemaVersionVerified(), 1); db.Table().Key(portion.GetPathId(), portion.GetPortion()).Update( NIceDb::TUpdate(portion.GetSchemaVersionVerified()), NIceDb::TUpdate(portion.GetShardingVersionDef(0)), @@ -83,7 +83,7 @@ void TDbWrapper::ErasePortion(const NOlap::TPortionInfo& portion) { LOG_S_CRIT("Erasing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); NIceDb::TNiceDb db(Database); using IndexPortions = NColumnShard::Schema::IndexPortions; - ui32 refCount = CS->TablesManager.VersionAddRef(portion.GetSchemaVersionVerified(), -1); + ui32 refCount = CS->VersionAddRef(portion.GetSchemaVersionVerified(), -1); if (refCount == 0) { LOG_S_CRIT("Ref count is set to 0 for version " << portion.GetSchemaVersionVerified() << " need to delete"); } diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h index a5f35905b898..776c72a48a93 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.h +++ b/ydb/core/tx/columnshard/engines/db_wrapper.h @@ -30,8 +30,16 @@ class IColumnEngine; class TPortionInfo; class TPortionInfoConstructor; +NColumnShard::TColumnShard* GetColumnShard(NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner); + class IDbWrapper { public: + IDbWrapper() = default; + IDbWrapper(NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner) + : CS(GetColumnShard(owner)) + { + } + virtual ~IDbWrapper() = default; virtual void Insert(const TInsertedData& data) = 0; @@ -58,16 +66,17 @@ class IDbWrapper { virtual void WriteCounter(ui32 counterId, ui64 value) = 0; virtual bool LoadCounters(const std::function& callback) = 0; virtual TConclusion>> LoadGranulesShardingInfo() = 0; -}; -NColumnShard::TColumnShard* GetColumnShard(NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner); +public: + NColumnShard::TColumnShard* CS = nullptr; +}; class TDbWrapper : public IDbWrapper { public: TDbWrapper(NTable::TDatabase& db, const IBlobGroupSelector* dsGroupSelector, NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner) - : Database(db) + : IDbWrapper(owner) + , Database(db) , DsGroupSelector(dsGroupSelector) - , CS(GetColumnShard(owner)) { AFL_VERIFY(owner != nullptr); AFL_VERIFY(CS != nullptr); @@ -102,7 +111,6 @@ class TDbWrapper : public IDbWrapper { private: NTable::TDatabase& Database; const IBlobGroupSelector* DsGroupSelector; - NColumnShard::TColumnShard* CS; }; } diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 241d0636c492..c89189d888b6 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -250,14 +250,6 @@ class TTablesManager { bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json); [[nodiscard]] std::unique_ptr CreateAddShardingInfoTx(TColumnShard& owner, const ui64 pathId, const ui64 versionId, const NSharding::TGranuleShardingLogicContainer& tabletShardingLogic) const; - - ui32 VersionAddRef(ui64 schemaVersion, i32 diff) { - if (!PrimaryIndex) { - PrimaryIndex = std::make_unique(TabletId, StoragesManager); - } - return PrimaryIndex->VersionAddRef(schemaVersion, diff); - } - }; } From 1d76717650f0c9fff4c9c30e68cd918e7416a7bb Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Thu, 12 Sep 2024 14:31:14 +0000 Subject: [PATCH 04/21] Do not increase ref count if portion is already in the local db --- ydb/core/tx/columnshard/columnshard_impl.h | 41 ++++++++++++++++--- .../tx/columnshard/engines/db_wrapper.cpp | 4 +- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 27d3d303beb1..4553c75936c1 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -137,7 +137,35 @@ using ITransaction = NTabletFlatExecutor::ITransaction; template using TTransactionBase = NTabletFlatExecutor::TTransactionBase; -class TVersionCounts : public THashMap { +class TVersionCounts { +private: + THashMap PortionVersions; + THashMap VersionCounts; + +public: + void VersionAddRef(ui64 portion, ui64 version) { + ui64& curVer = PortionVersions[portion]; + if (curVer != 0) {// Portion is already in the local database, no need to increase ref count + AFL_VERIFY(version == curVer); + return; + } + curVer = version; + ui32& refCount = VersionCounts[version]; + refCount++; + LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount - 1 << " to " << refCount << " this " << (ui64)this); + } + + ui32 VersionRemoveRef(ui64 portion, ui64 version) { + auto iter = PortionVersions.find(portion); +// AFL_VERIFY(iter != PortionVersions.end()); + if (iter == PortionVersions.end()) { //Portion is already removed from local databae, no need to decrease ref count + return (ui32)-1; + } + PortionVersions.erase(iter); + ui32& refCount = VersionCounts[version]; + LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount << " to " << refCount - 1 << " this " << (ui64)this); + return --refCount; + } }; class TColumnShard @@ -627,11 +655,12 @@ class TColumnShard TColumnShard(TTabletStorageInfo* info, const TActorId& tablet); - ui32 VersionAddRef(ui64 version, i32 diff) { - ui32& refCount = VersionCounts[version]; - refCount += diff; - LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount - diff << " to " << refCount << " this " << (ui64)this); - return refCount; + void VersionAddRef(ui64 portion, ui64 version) { + VersionCounts.VersionAddRef(portion, version); + } + + ui32 VersionRemoveRef(ui64 portion, ui64 version) { + return VersionCounts.VersionRemoveRef(portion, version); } private: diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 549336bb1079..7c3cd87f5f7d 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -70,7 +70,7 @@ void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { auto metaProto = portion.GetMeta().SerializeToProto(); using IndexPortions = NColumnShard::Schema::IndexPortions; auto removeSnapshot = portion.GetRemoveSnapshotOptional(); - CS->VersionAddRef(portion.GetSchemaVersionVerified(), 1); + CS->VersionAddRef(portion.GetPortion(), portion.GetSchemaVersionVerified()); db.Table().Key(portion.GetPathId(), portion.GetPortion()).Update( NIceDb::TUpdate(portion.GetSchemaVersionVerified()), NIceDb::TUpdate(portion.GetShardingVersionDef(0)), @@ -83,7 +83,7 @@ void TDbWrapper::ErasePortion(const NOlap::TPortionInfo& portion) { LOG_S_CRIT("Erasing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); NIceDb::TNiceDb db(Database); using IndexPortions = NColumnShard::Schema::IndexPortions; - ui32 refCount = CS->VersionAddRef(portion.GetSchemaVersionVerified(), -1); + ui32 refCount = CS->VersionRemoveRef(portion.GetPortion(), portion.GetSchemaVersionVerified()); if (refCount == 0) { LOG_S_CRIT("Ref count is set to 0 for version " << portion.GetSchemaVersionVerified() << " need to delete"); } From ffc16cb9a277d85025ed1280c4c7e987efa4df5d Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Fri, 13 Sep 2024 09:33:45 +0000 Subject: [PATCH 05/21] Moved TTransactionContext::Owner initialization to classes inherited from TTransactionBase --- ydb/core/tablet_flat/flat_executor.cpp | 3 +- .../transaction/tx_write_index.cpp | 1 + ydb/core/tx/columnshard/columnshard__init.cpp | 2 + ydb/core/tx/columnshard/columnshard_impl.h | 78 +++++++++++++++++-- .../tx/columnshard/columnshard_schema.cpp | 2 + .../tx/columnshard/engines/db_wrapper.cpp | 1 + ydb/core/tx/columnshard/engines/db_wrapper.h | 4 +- .../engines/insert_table/insert_table.cpp | 9 ++- .../engines/insert_table/insert_table.h | 9 +++ .../normalizer/portion/broken_blobs.cpp | 1 + .../normalizer/portion/portion.cpp | 1 + 11 files changed, 99 insertions(+), 12 deletions(-) diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 3436ebafc18a..2a21963828d1 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -1704,7 +1704,8 @@ void TExecutor::ExecuteTransaction(TAutoPtr seat, const TActorContext &ct PrivatePageCache->ResetTouchesAndToLoad(true); TPageCollectionTxEnv env(*Database, *PrivatePageCache); - TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId, seat->Self->TxSpan, Owner); +// TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId, seat->Self->TxSpan, Owner); + TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId, seat->Self->TxSpan, nullptr); txc.NotEnoughMemory(seat->NotEnoughMemoryCount); Database->Begin(Stamp(), env); diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp index 3da42cd4bf9b..cf0de282921b 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp @@ -7,6 +7,7 @@ namespace NKikimr::NColumnShard { bool TTxWriteIndex::Execute(TTransactionContext& txc, const TActorContext& ctx) { + txc.Owner = Self; auto changes = Ev->Get()->IndexChanges; TMemoryProfileGuard mpg("TTxWriteIndex::Execute::" + changes->TypeString()); TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_BLOBS)("tablet_id", Self->TabletID())("external_task_id", changes->GetTaskIdentifier()); diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index 6d63f8f705cd..0f73fe195362 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -239,6 +239,7 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx) } bool TTxInit::Execute(TTransactionContext& txc, const TActorContext& ctx) { + txc.Owner = Self; NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "initialize_shard"); LOG_S_DEBUG("TTxInit.Execute at tablet " << Self->TabletID()); @@ -340,6 +341,7 @@ class TTxApplyNormalizer : public TTransactionBase { }; bool TTxApplyNormalizer::Execute(TTransactionContext& txc, const TActorContext&) { + txc.Owner = Self; NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("event", "initialize_shard"); AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("step", "TTxApplyNormalizer.Execute")("details", Self->NormalizerController.DebugString()); if (!Changes->ApplyOnExecute(txc, Self->NormalizerController)) { diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 4553c75936c1..f2963adfa0ed 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -45,6 +45,8 @@ #include #include +#include + namespace NKikimr::NOlap { class TCleanupPortionsColumnEngineChanges; class TCleanupTablesColumnEngineChanges; @@ -137,14 +139,50 @@ using ITransaction = NTabletFlatExecutor::ITransaction; template using TTransactionBase = NTabletFlatExecutor::TTransactionBase; +class TInsertKey { +public: + ui64 PlanStep; + ui64 TxId; + TString DedupId; + ui8 RecType; + +public: + TInsertKey() = default; + + TInsertKey(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType) + : PlanStep(planStep) + , TxId(txId) + , DedupId(dedupId) + , RecType(recType) + { + } + + bool operator==(const TInsertKey& other) const { + return (PlanStep == other.PlanStep) && (TxId == other.TxId) && (DedupId == other.DedupId) && (RecType == other.RecType); + } + + ui64 Hash() const { + return CombineHashes(PlanStep, CombineHashes(TxId, CombineHashes(hash()(DedupId), (ui64)RecType))); + } +}; + class TVersionCounts { private: + struct Hasher { + inline size_t operator()(const TInsertKey& key) const noexcept { + return key.Hash(); + } + }; + +private: + THashMap InsertVersions; THashMap PortionVersions; THashMap VersionCounts; public: - void VersionAddRef(ui64 portion, ui64 version) { - ui64& curVer = PortionVersions[portion]; + template + void VersionAddRef(Versions& versions, const Key& portion, ui64 version) { + ui64& curVer = versions[portion]; if (curVer != 0) {// Portion is already in the local database, no need to increase ref count AFL_VERIFY(version == curVer); return; @@ -155,17 +193,34 @@ class TVersionCounts { LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount - 1 << " to " << refCount << " this " << (ui64)this); } - ui32 VersionRemoveRef(ui64 portion, ui64 version) { - auto iter = PortionVersions.find(portion); -// AFL_VERIFY(iter != PortionVersions.end()); - if (iter == PortionVersions.end()) { //Portion is already removed from local databae, no need to decrease ref count + template + ui32 VersionRemoveRef(Versions& versions, const Key& portion, ui64 version) { + auto iter = versions.find(portion); + if (iter == versions.end()) { //Portion is already removed from local databae, no need to decrease ref count return (ui32)-1; } - PortionVersions.erase(iter); + versions.erase(iter); ui32& refCount = VersionCounts[version]; LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount << " to " << refCount - 1 << " this " << (ui64)this); return --refCount; } + + void VersionAddRef(ui64 portion, ui64 version) { + VersionAddRef(PortionVersions, portion, version); + } + + ui32 VersionRemoveRef(ui64 portion, ui64 version) { + return VersionRemoveRef(PortionVersions, portion, version); + } + + void VersionAddRef(const TInsertKey& key, ui64 version) { + VersionAddRef(InsertVersions, key, version); + } + + ui32 VersionRemoveRef(const TInsertKey& key, ui64 version) { + return VersionRemoveRef(InsertVersions, key, version); + } + }; class TColumnShard @@ -663,6 +718,15 @@ class TColumnShard return VersionCounts.VersionRemoveRef(portion, version); } + void VersionAddRef(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType, ui64 version) { + VersionCounts.VersionAddRef(TInsertKey(planStep, txId, dedupId, recType), version); + } + + ui32 VersionRemoveRef(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType, ui64 version) { + return VersionCounts.VersionRemoveRef(TInsertKey(planStep, txId, dedupId, recType), version); + } + + private: TVersionCounts VersionCounts; }; diff --git a/ydb/core/tx/columnshard/columnshard_schema.cpp b/ydb/core/tx/columnshard/columnshard_schema.cpp index c9a60029a892..17d6c282d068 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/columnshard_schema.cpp @@ -1,5 +1,6 @@ #include "columnshard_schema.h" #include "transactions/tx_controller.h" +#include "columnshard_impl.h" namespace NKikimr::NColumnShard { @@ -13,6 +14,7 @@ bool Schema::InsertTable_Load(NIceDb::TNiceDb& db, const IBlobGroupSelector* dsG NOlap::TInsertTableRecordLoadContext constructor; constructor.ParseFromDatabase(rowset); + insertTable.CS->VersionAddRef(constructor.GetPlanStep(), constructor.GetInsertWriteId(), constructor.PathId, constructor.GetDedupId(), (ui8)constructor.GetRecType(), constructor.SchemaVersion); switch (constructor.GetRecType()) { case Schema::EInsertTableIds::Inserted: insertTable.AddInserted(constructor.BuildInsertedOrAborted(dsGroupSelector), true); diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 7c3cd87f5f7d..ef298f3b2f85 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -41,6 +41,7 @@ void TDbWrapper::EraseAborted(const TInsertedData& data) { bool TDbWrapper::Load(TInsertTableAccessor& insertTable, const TInstant& loadTime) { NIceDb::TNiceDb db(Database); + insertTable.CS = CS; return NColumnShard::Schema::InsertTable_Load(db, DsGroupSelector, insertTable, loadTime); } diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h index 776c72a48a93..43aac2c311da 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.h +++ b/ydb/core/tx/columnshard/engines/db_wrapper.h @@ -78,8 +78,8 @@ class TDbWrapper : public IDbWrapper { , Database(db) , DsGroupSelector(dsGroupSelector) { - AFL_VERIFY(owner != nullptr); - AFL_VERIFY(CS != nullptr); +// AFL_VERIFY(owner != nullptr); +// AFL_VERIFY(CS != nullptr); } void Insert(const TInsertedData& data) override; diff --git a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp index 56a4f730a422..8a74f8d8eb08 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp +++ b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp @@ -1,9 +1,10 @@ #include "insert_table.h" -#include #include +#include #include #include +//#include namespace NKikimr::NOlap { @@ -17,7 +18,11 @@ bool TInsertTable::Insert(IDbWrapper& dbTable, TInsertedData&& data) { return false; } } - +/* +void TInsertTableAccessor::AddSchemaVersion(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType, ui64 schemaVersion) { + CS->VersionAddRec(planStep, txId, dedupId, recType, schemaVersion); +} +*/ TInsertionSummary::TCounters TInsertTable::Commit( IDbWrapper& dbTable, ui64 planStep, ui64 txId, const THashSet& writeIds, std::function pathExists) { Y_ABORT_UNLESS(!writeIds.empty()); diff --git a/ydb/core/tx/columnshard/engines/insert_table/insert_table.h b/ydb/core/tx/columnshard/engines/insert_table/insert_table.h index 1eb6835053b7..faf80778842f 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/insert_table.h +++ b/ydb/core/tx/columnshard/engines/insert_table/insert_table.h @@ -8,6 +8,10 @@ #include #include +namespace NKikimr::NColumnShard { +class TColumnShard; +} + namespace NKikimr::NOlap { class TPKRangesFilter; class IDbWrapper; @@ -46,6 +50,8 @@ class TInsertTableAccessor { } } + void AddSchemaVersion(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType, ui64 schemaVersion); + bool AddInserted(TInsertedData&& data, const bool load) { if (load) { AddBlobLink(data.GetBlobRange().BlobId); @@ -84,6 +90,9 @@ class TInsertTableAccessor { bool IsOverloadedByCommitted(const ui64 pathId) const { return Summary.IsOverloaded(pathId); } + +public: + NKikimr::NColumnShard::TColumnShard* CS; }; class TInsertTable: public TInsertTableAccessor { diff --git a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp index b2d891e3b82b..93f1f40174e4 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp @@ -28,6 +28,7 @@ class TNormalizerResult : public INormalizerChanges { AFL_VERIFY(!!schema)("portion_id", portionInfo->GetPortionId()); AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "portion_removed_as_broken")("portion_id", portionInfo->GetAddress().DebugString()); portionInfo->SetRemoveSnapshot(TSnapshot(1, 1)); + LOG_S_CRIT("Saving broken portion"); portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), false); } if (BrokenPortions.size()) { diff --git a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp index 5a49acaf62df..b71d82edd432 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp @@ -25,6 +25,7 @@ class TPortionsNormalizer::TNormalizerResult : public INormalizerChanges { for (auto&& portionInfo : Portions) { auto schema = Schemas->FindPtr(portionInfo->GetPortionId()); AFL_VERIFY(!!schema)("portion_id", portionInfo->GetPortionId()); + LOG_S_CRIT("Saving normalized"); portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), true); } return true; From dc8a1eeaf9f8978eb335f13ffdd0e561a09cc2a1 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Fri, 13 Sep 2024 11:15:00 +0000 Subject: [PATCH 06/21] InsertTable schema version counts are maintained --- .../blobs_action/transaction/tx_write.cpp | 1 + ydb/core/tx/columnshard/columnshard_impl.h | 35 ++++++++++--------- .../engines/column_engine_logs.cpp | 2 +- .../tx/columnshard/engines/db_wrapper.cpp | 10 ++++-- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp index 7f120d82059f..6b226afdd47b 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp @@ -41,6 +41,7 @@ bool TTxWrite::CommitOneBlob(TTransactionContext& txc, const NOlap::TWideSeriali } bool TTxWrite::Execute(TTransactionContext& txc, const TActorContext&) { + txc.Owner = Self; TMemoryProfileGuard mpg("TTxWrite::Execute"); NActors::TLogContextGuard logGuard = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_BLOBS)("tablet_id", Self->TabletID())("tx_state", "execute"); diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index f2963adfa0ed..c057e25571d7 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -143,26 +143,28 @@ class TInsertKey { public: ui64 PlanStep; ui64 TxId; + ui64 PathId; TString DedupId; ui8 RecType; public: TInsertKey() = default; - TInsertKey(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType) + TInsertKey(ui64 planStep, ui64 txId, ui64 pathId, const TString& dedupId, ui8 recType) : PlanStep(planStep) , TxId(txId) + , PathId(pathId) , DedupId(dedupId) , RecType(recType) { } bool operator==(const TInsertKey& other) const { - return (PlanStep == other.PlanStep) && (TxId == other.TxId) && (DedupId == other.DedupId) && (RecType == other.RecType); + return (PlanStep == other.PlanStep) && (TxId == other.TxId) && (PathId == other.PathId) && (DedupId == other.DedupId) && (RecType == other.RecType); } ui64 Hash() const { - return CombineHashes(PlanStep, CombineHashes(TxId, CombineHashes(hash()(DedupId), (ui64)RecType))); + return CombineHashes(PlanStep, CombineHashes(TxId, CombineHashes(PathId, CombineHashes(hash()(DedupId), (ui64)RecType)))); } }; @@ -176,7 +178,7 @@ class TVersionCounts { private: THashMap InsertVersions; - THashMap PortionVersions; + THashMap, ui64> PortionVersions; THashMap VersionCounts; public: @@ -184,6 +186,7 @@ class TVersionCounts { void VersionAddRef(Versions& versions, const Key& portion, ui64 version) { ui64& curVer = versions[portion]; if (curVer != 0) {// Portion is already in the local database, no need to increase ref count + LOG_S_CRIT("Schema version is already written"); AFL_VERIFY(version == curVer); return; } @@ -205,12 +208,12 @@ class TVersionCounts { return --refCount; } - void VersionAddRef(ui64 portion, ui64 version) { - VersionAddRef(PortionVersions, portion, version); + void VersionAddRef(ui64 portion, ui64 pathId, ui64 version) { + VersionAddRef(PortionVersions, std::pair(portion, pathId), version); } - ui32 VersionRemoveRef(ui64 portion, ui64 version) { - return VersionRemoveRef(PortionVersions, portion, version); + ui32 VersionRemoveRef(ui64 portion, ui64 pathId, ui64 version) { + return VersionRemoveRef(PortionVersions, std::pair(portion, pathId), version); } void VersionAddRef(const TInsertKey& key, ui64 version) { @@ -710,20 +713,20 @@ class TColumnShard TColumnShard(TTabletStorageInfo* info, const TActorId& tablet); - void VersionAddRef(ui64 portion, ui64 version) { - VersionCounts.VersionAddRef(portion, version); + void VersionAddRef(ui64 portion, ui64 pathId, ui64 version) { + VersionCounts.VersionAddRef(portion, pathId, version); } - ui32 VersionRemoveRef(ui64 portion, ui64 version) { - return VersionCounts.VersionRemoveRef(portion, version); + ui32 VersionRemoveRef(ui64 portion, ui64 pathId, ui64 version) { + return VersionCounts.VersionRemoveRef(portion, pathId, version); } - void VersionAddRef(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType, ui64 version) { - VersionCounts.VersionAddRef(TInsertKey(planStep, txId, dedupId, recType), version); + void VersionAddRef(ui64 planStep, ui64 txId, ui64 pathId, const TString& dedupId, ui8 recType, ui64 version) { + VersionCounts.VersionAddRef(TInsertKey(planStep, txId, pathId, dedupId, recType), version); } - ui32 VersionRemoveRef(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType, ui64 version) { - return VersionCounts.VersionRemoveRef(TInsertKey(planStep, txId, dedupId, recType), version); + ui32 VersionRemoveRef(ui64 planStep, ui64 txId, ui64 pathId, const TString& dedupId, ui8 recType, ui64 version) { + return VersionCounts.VersionRemoveRef(TInsertKey(planStep, txId, pathId, dedupId, recType), version); } diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index e0b33ec4e9a0..458dee83370c 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -219,7 +219,7 @@ bool TColumnEngineForLogs::LoadColumns(IDbWrapper& db) { AFL_VERIFY(portion.MutableMeta().LoadMetadata(metaProto, indexInfo)); AFL_VERIFY(constructors.AddConstructorVerified(std::move(portion))); if (portion.HasSchemaVersion()) { - db.CS->VersionAddRef(portion.GetSchemaVersion(), 1); + db.CS->VersionAddRef(portion.GetPortionIdVerified(), portion.GetPathId(), portion.GetSchemaVersion()); } })) { return false; diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index ef298f3b2f85..b8d97c4c4adf 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -10,31 +10,37 @@ namespace NKikimr::NOlap { void TDbWrapper::Insert(const TInsertedData& data) { NIceDb::TNiceDb db(Database); + CS->VersionAddRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data.GetSchemaVersion()); NColumnShard::Schema::InsertTable_Insert(db, data); } void TDbWrapper::Commit(const TCommittedData& data) { NIceDb::TNiceDb db(Database); + CS->VersionAddRef(data.GetSnapshot().GetPlanStep(), data.GetSnapshot().GetTxId(), data.GetPathId(), data.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, data.GetSchemaVersion()); NColumnShard::Schema::InsertTable_Commit(db, data); } void TDbWrapper::Abort(const TInsertedData& data) { NIceDb::TNiceDb db(Database); + CS->VersionAddRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data.GetSchemaVersion()); NColumnShard::Schema::InsertTable_Abort(db, data); } void TDbWrapper::EraseInserted(const TInsertedData& data) { NIceDb::TNiceDb db(Database); + CS->VersionRemoveRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data.GetSchemaVersion()); NColumnShard::Schema::InsertTable_EraseInserted(db, data); } void TDbWrapper::EraseCommitted(const TCommittedData& data) { NIceDb::TNiceDb db(Database); + CS->VersionRemoveRef(data.GetSnapshot().GetPlanStep(), data.GetSnapshot().GetTxId(), data.GetPathId(), data.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, data.GetSchemaVersion()); NColumnShard::Schema::InsertTable_EraseCommitted(db, data); } void TDbWrapper::EraseAborted(const TInsertedData& data) { NIceDb::TNiceDb db(Database); + CS->VersionRemoveRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data.GetSchemaVersion()); NColumnShard::Schema::InsertTable_EraseAborted(db, data); } @@ -71,7 +77,7 @@ void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { auto metaProto = portion.GetMeta().SerializeToProto(); using IndexPortions = NColumnShard::Schema::IndexPortions; auto removeSnapshot = portion.GetRemoveSnapshotOptional(); - CS->VersionAddRef(portion.GetPortion(), portion.GetSchemaVersionVerified()); + CS->VersionAddRef(portion.GetPortion(), portion.GetPathId(), portion.GetSchemaVersionVerified()); db.Table().Key(portion.GetPathId(), portion.GetPortion()).Update( NIceDb::TUpdate(portion.GetSchemaVersionVerified()), NIceDb::TUpdate(portion.GetShardingVersionDef(0)), @@ -84,7 +90,7 @@ void TDbWrapper::ErasePortion(const NOlap::TPortionInfo& portion) { LOG_S_CRIT("Erasing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); NIceDb::TNiceDb db(Database); using IndexPortions = NColumnShard::Schema::IndexPortions; - ui32 refCount = CS->VersionRemoveRef(portion.GetPortion(), portion.GetSchemaVersionVerified()); + ui32 refCount = CS->VersionRemoveRef(portion.GetPortion(), portion.GetPathId(), portion.GetSchemaVersionVerified()); if (refCount == 0) { LOG_S_CRIT("Ref count is set to 0 for version " << portion.GetSchemaVersionVerified() << " need to delete"); } From 1e08bfb4b029ef8204f479bda8d2cbaeda7a2a69 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Fri, 13 Sep 2024 15:59:40 +0000 Subject: [PATCH 07/21] Moved version count maintenance from TDbWrapper to calling functions --- .../transaction/tx_gc_insert_table.cpp | 2 +- .../transaction/tx_write_index.cpp | 2 +- ydb/core/tx/columnshard/columnshard__init.cpp | 6 ++--- ydb/core/tx/columnshard/columnshard_impl.cpp | 4 +-- .../tx/columnshard/columnshard_schema.cpp | 3 ++- ydb/core/tx/columnshard/columnshard_schema.h | 6 ++--- .../transactions/tx_data_from_source.cpp | 4 +-- .../engines/changes/cleanup_portions.cpp | 2 +- .../engines/changes/with_appended.cpp | 4 +-- .../engines/column_engine_logs.cpp | 14 ++++++----- .../columnshard/engines/column_engine_logs.h | 7 +++--- .../tx/columnshard/engines/db_wrapper.cpp | 23 +++++------------ ydb/core/tx/columnshard/engines/db_wrapper.h | 25 +++---------------- .../engines/insert_table/insert_table.cpp | 16 +++++++----- .../engines/insert_table/insert_table.h | 12 +++++++-- .../columnshard/engines/insert_table/ya.make | 2 ++ .../engines/portions/portion_info.cpp | 10 ++++++-- .../engines/portions/portion_info.h | 4 +-- .../normalizer/portion/broken_blobs.cpp | 5 ++-- .../columnshard/normalizer/portion/chunks.cpp | 3 ++- .../columnshard/normalizer/portion/clean.cpp | 5 ++-- .../normalizer/portion/normalizer.cpp | 3 ++- .../normalizer/portion/portion.cpp | 5 ++-- ydb/core/tx/columnshard/operations/write.cpp | 4 +-- ydb/core/tx/columnshard/tables_manager.cpp | 9 ++++--- ydb/core/tx/columnshard/tables_manager.h | 4 ++- .../transactions/operators/long_tx_write.h | 4 +-- 27 files changed, 95 insertions(+), 93 deletions(-) diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp index e0d6b0fcb9a3..15a05e7108a7 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_gc_insert_table.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NColumnShard { bool TTxInsertTableCleanup::Execute(TTransactionContext& txc, const TActorContext& /*ctx*/) { TMemoryProfileGuard mpg("TTxInsertTableCleanup::Execute"); TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, txc.Owner); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); NIceDb::TNiceDb db(txc.DB); Self->TryAbortWrites(db, dbTable, std::move(WriteIdsToAbort)); diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp index cf0de282921b..20225dcf3ea3 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write_index.cpp @@ -21,7 +21,7 @@ bool TTxWriteIndex::Execute(TTransactionContext& txc, const TActorContext& ctx) Y_ABORT_UNLESS(Ev->Get()->IndexInfo->GetLastSchema()->GetSnapshot() <= snapshot); TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbWrap(txc.DB, &dsGroupSelector, txc.Owner); + NOlap::TDbWrapper dbWrap(txc.DB, &dsGroupSelector); AFL_VERIFY(Self->TablesManager.MutablePrimaryIndex().ApplyChangesOnExecute(dbWrap, changes, snapshot)); LOG_S_DEBUG(TxPrefix() << "(" << changes->TypeString() << ") apply" << TxSuffix()); NOlap::TWriteIndexContext context(&txc.DB, dbWrap, Self->MutableIndexAs()); diff --git a/ydb/core/tx/columnshard/columnshard__init.cpp b/ydb/core/tx/columnshard/columnshard__init.cpp index 0f73fe195362..734e5abd2d00 100644 --- a/ydb/core/tx/columnshard/columnshard__init.cpp +++ b/ydb/core/tx/columnshard/columnshard__init.cpp @@ -100,11 +100,11 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx) NIceDb::TNiceDb db(txc.DB); TBlobGroupSelector dsGroupSelector(Self->Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, txc.Owner); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); { ACFL_DEBUG("step", "TInsertTable::Load_Start"); TMemoryProfileGuard g("TTxInit/InsertTable"); - auto localInsertTable = std::make_unique(); + auto localInsertTable = std::make_unique(Self); if (!localInsertTable->Load(db, dbTable, TAppData::TimeProvider->Now())) { ACFL_ERROR("step", "TInsertTable::Load_Fails"); return false; @@ -115,7 +115,7 @@ bool TTxInit::ReadEverything(TTransactionContext& txc, const TActorContext& ctx) { ACFL_DEBUG("step", "TTablesManager::Load_Start"); - TTablesManager tManagerLocal(Self->StoragesManager, Self->TabletID()); + TTablesManager tManagerLocal(Self->StoragesManager, Self->TabletID(), Self); { TMemoryProfileGuard g("TTxInit/TTablesManager"); if (!tManagerLocal.InitFromDB(db)) { diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 9f1c7a10859d..f433cffeccce 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -76,10 +76,10 @@ TColumnShard::TColumnShard(TTabletStorageInfo* info, const TActorId& tablet) , PeriodicWakeupActivationPeriod(NYDBTest::TControllers::GetColumnShardController()->GetPeriodicWakeupActivationPeriod()) , StatsReportInterval(NYDBTest::TControllers::GetColumnShardController()->GetStatsReportInterval()) , InFlightReadsTracker(StoragesManager, Counters.GetRequestsTracingCounters()) - , TablesManager(StoragesManager, info->TabletID) + , TablesManager(StoragesManager, info->TabletID, this) , Subscribers(std::make_shared(*this)) , PipeClientCache(NTabletPipe::CreateBoundedClientCache(new NTabletPipe::TBoundedClientCacheConfig(), GetPipeClientConfig())) - , InsertTable(std::make_unique()) + , InsertTable(std::make_unique(this)) , InsertTaskSubscription(NOlap::TInsertColumnEngineChanges::StaticTypeName(), Counters.GetSubscribeCounters()) , CompactTaskSubscription(NOlap::TCompactColumnEngineChanges::StaticTypeName(), Counters.GetSubscribeCounters()) , TTLTaskSubscription(NOlap::TTTLColumnEngineChanges::StaticTypeName(), Counters.GetSubscribeCounters()) diff --git a/ydb/core/tx/columnshard/columnshard_schema.cpp b/ydb/core/tx/columnshard/columnshard_schema.cpp index 17d6c282d068..03683d8b7b11 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/columnshard_schema.cpp @@ -14,7 +14,8 @@ bool Schema::InsertTable_Load(NIceDb::TNiceDb& db, const IBlobGroupSelector* dsG NOlap::TInsertTableRecordLoadContext constructor; constructor.ParseFromDatabase(rowset); - insertTable.CS->VersionAddRef(constructor.GetPlanStep(), constructor.GetInsertWriteId(), constructor.PathId, constructor.GetDedupId(), (ui8)constructor.GetRecType(), constructor.SchemaVersion); + LOG_S_CRIT("Loaded schema version " << constructor.GetSchemaVersion() << " planstep " << constructor.GetPlanStep() << " txid " << constructor.GetWriteTxId() << " dedup id " << constructor.GetDedupId() << " rec type " << (ui32)constructor.GetRecType()); + insertTable.CS->VersionAddRef(constructor.GetPlanStep(), constructor.GetWriteTxId(), constructor.GetPathId(), constructor.GetDedupId(), (ui8)constructor.GetRecType(), constructor.GetSchemaVersion()); switch (constructor.GetRecType()) { case Schema::EInsertTableIds::Inserted: insertTable.AddInserted(constructor.BuildInsertedOrAborted(dsGroupSelector), true); diff --git a/ydb/core/tx/columnshard/columnshard_schema.h b/ydb/core/tx/columnshard/columnshard_schema.h index 4f08426ddd70..947dc031038e 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.h +++ b/ydb/core/tx/columnshard/columnshard_schema.h @@ -984,11 +984,11 @@ class TInsertTableRecordLoadContext { private: NColumnShard::Schema::EInsertTableIds RecType; ui64 PlanStep; - ui64 WriteTxId; TInsertWriteId InsertWriteId; - ui64 PathId; + YDB_ACCESSOR_DEF(ui64, WriteTxId); + YDB_ACCESSOR_DEF(ui64, PathId); YDB_ACCESSOR_DEF(TString, DedupId); - ui64 SchemaVersion; + YDB_ACCESSOR_DEF(ui64, SchemaVersion); TString BlobIdString; std::optional BlobId; TString MetadataString; diff --git a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp index 433dd79fb2ca..8d83de7b4cee 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp @@ -6,7 +6,7 @@ namespace NKikimr::NOlap::NDataSharing { bool TTxDataFromSource::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { using namespace NKikimr::NColumnShard; - TDbWrapper dbWrapper(txc.DB, nullptr, txc.Owner); + TDbWrapper dbWrapper(txc.DB, nullptr); auto& index = Self->TablesManager.MutablePrimaryIndexAsVerified(); { ui64* lastPortionPtr = index.GetLastPortionPointer(); @@ -21,7 +21,7 @@ bool TTxDataFromSource::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, THashMap> sharedBlobIds; for (auto&& i : PortionsByPathId) { for (auto&& p : i.second.GetPortions()) { - p.SaveToDatabase(dbWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); + p.SaveToDatabase(dbWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false, static_cast(txc.Owner)); } } NIceDb::TNiceDb db(txc.DB); diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp index 7917b77682b9..6764e2cd4c7a 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp @@ -22,7 +22,7 @@ void TCleanupPortionsColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TC } THashMap> blobIdsByStorage; for (auto&& p : PortionsToDrop) { - p.RemoveFromDatabase(context.DBWrapper); + p.RemoveFromDatabase(context.DBWrapper, self); p.FillBlobIdsByStorage(blobIdsByStorage, context.EngineLogs.GetVersionedIndex()); pathIds.emplace(p.GetPathId()); } diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index 24d44eb34587..7c87d08cc26d 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -15,7 +15,7 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, Y_ABORT_UNLESS(!portionInfo.Empty()); Y_ABORT_UNLESS(portionInfo.HasRemoveSnapshot()); AFL_VERIFY(usedPortionIds.emplace(portionInfo.GetPortionId()).second)("portion_info", portionInfo.DebugString(true)); - portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); + portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false, self); } const auto predRemoveDroppedTable = [self](const TWritePortionInfoWithBlobsResult& item) { auto& portionInfo = item.GetPortionResult(); @@ -30,7 +30,7 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, for (auto& portionInfoWithBlobs : AppendedPortions) { auto& portionInfo = portionInfoWithBlobs.GetPortionResult(); AFL_VERIFY(usedPortionIds.emplace(portionInfo.GetPortionId()).second)("portion_info", portionInfo.DebugString(true)); - portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); + portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false, self); } } diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 458dee83370c..0c33469caeb7 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -26,7 +26,7 @@ #include namespace NKikimr::NOlap { - +/* TColumnEngineForLogs::TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager) : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) , StoragesManager(storagesManager) @@ -36,26 +36,28 @@ TColumnEngineForLogs::TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr< { ActualizationController = std::make_shared(); } - +*/ TColumnEngineForLogs::TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, - const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema) + const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema, NColumnShard::TColumnShard* cs) : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) , StoragesManager(storagesManager) , TabletId(tabletId) , LastPortion(0) , LastGranule(0) + , CS(cs) { ActualizationController = std::make_shared(); RegisterSchemaVersion(snapshot, schema); } TColumnEngineForLogs::TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, - const TSnapshot& snapshot, TIndexInfo&& schema) + const TSnapshot& snapshot, TIndexInfo&& schema, NColumnShard::TColumnShard* cs) : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) , StoragesManager(storagesManager) , TabletId(tabletId) , LastPortion(0) - , LastGranule(0) { + , LastGranule(0) + , CS(cs) { ActualizationController = std::make_shared(); RegisterSchemaVersion(snapshot, std::move(schema)); } @@ -219,7 +221,7 @@ bool TColumnEngineForLogs::LoadColumns(IDbWrapper& db) { AFL_VERIFY(portion.MutableMeta().LoadMetadata(metaProto, indexInfo)); AFL_VERIFY(constructors.AddConstructorVerified(std::move(portion))); if (portion.HasSchemaVersion()) { - db.CS->VersionAddRef(portion.GetPortionIdVerified(), portion.GetPathId(), portion.GetSchemaVersion()); + CS->VersionAddRef(portion.GetPortionIdVerified(), portion.GetPathId(), portion.GetSchemaVersion()); } })) { return false; diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 2388d02f85dc..119a748fda6d 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -81,9 +81,9 @@ class TColumnEngineForLogs : public IColumnEngine { ADD, }; - TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema); - TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema); - TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager); + TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema, NColumnShard::TColumnShard* cs); + TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema, NColumnShard::TColumnShard* cs); +// TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager); virtual void OnTieringModified(const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) override; @@ -189,6 +189,7 @@ class TColumnEngineForLogs : public IColumnEngine { TColumnEngineStats Counters; ui64 LastPortion; ui64 LastGranule; + NColumnShard::TColumnShard* CS = nullptr; TSnapshot LastSnapshot = TSnapshot::Zero(); bool Loaded = false; diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index b8d97c4c4adf..65b04b626276 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -3,51 +3,49 @@ #include "portions/constructor.h" #include #include -#include #include namespace NKikimr::NOlap { void TDbWrapper::Insert(const TInsertedData& data) { NIceDb::TNiceDb db(Database); - CS->VersionAddRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data.GetSchemaVersion()); + LOG_S_CRIT("Writing inserted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_Insert(db, data); } void TDbWrapper::Commit(const TCommittedData& data) { NIceDb::TNiceDb db(Database); - CS->VersionAddRef(data.GetSnapshot().GetPlanStep(), data.GetSnapshot().GetTxId(), data.GetPathId(), data.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, data.GetSchemaVersion()); + LOG_S_CRIT("Writing committed data, schema version " << data.GetSchemaVersion() << " planstep " << data.GetSnapshot().GetPlanStep() << " path id " << data.GetPathId() << " write id " << data.GetSnapshot() << " dedup id " << data.GetDedupId() << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_Commit(db, data); } void TDbWrapper::Abort(const TInsertedData& data) { NIceDb::TNiceDb db(Database); - CS->VersionAddRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data.GetSchemaVersion()); + LOG_S_CRIT("Writing aborted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_Abort(db, data); } void TDbWrapper::EraseInserted(const TInsertedData& data) { NIceDb::TNiceDb db(Database); - CS->VersionRemoveRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data.GetSchemaVersion()); + LOG_S_CRIT("Erasing inserted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_EraseInserted(db, data); } void TDbWrapper::EraseCommitted(const TCommittedData& data) { NIceDb::TNiceDb db(Database); - CS->VersionRemoveRef(data.GetSnapshot().GetPlanStep(), data.GetSnapshot().GetTxId(), data.GetPathId(), data.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, data.GetSchemaVersion()); + LOG_S_CRIT("Erasing committed data, schema version " << data.GetSchemaVersion() << " planstep " << data.GetSnapshot().GetPlanStep() << " path id " << data.GetPathId() << " write id " << data.GetSnapshot() << " dedup id " << data.GetDedupId() << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_EraseCommitted(db, data); } void TDbWrapper::EraseAborted(const TInsertedData& data) { NIceDb::TNiceDb db(Database); - CS->VersionRemoveRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data.GetSchemaVersion()); + LOG_S_CRIT("Erasing aborted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_EraseAborted(db, data); } bool TDbWrapper::Load(TInsertTableAccessor& insertTable, const TInstant& loadTime) { NIceDb::TNiceDb db(Database); - insertTable.CS = CS; return NColumnShard::Schema::InsertTable_Load(db, DsGroupSelector, insertTable, loadTime); } @@ -77,7 +75,6 @@ void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { auto metaProto = portion.GetMeta().SerializeToProto(); using IndexPortions = NColumnShard::Schema::IndexPortions; auto removeSnapshot = portion.GetRemoveSnapshotOptional(); - CS->VersionAddRef(portion.GetPortion(), portion.GetPathId(), portion.GetSchemaVersionVerified()); db.Table().Key(portion.GetPathId(), portion.GetPortion()).Update( NIceDb::TUpdate(portion.GetSchemaVersionVerified()), NIceDb::TUpdate(portion.GetShardingVersionDef(0)), @@ -90,10 +87,6 @@ void TDbWrapper::ErasePortion(const NOlap::TPortionInfo& portion) { LOG_S_CRIT("Erasing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); NIceDb::TNiceDb db(Database); using IndexPortions = NColumnShard::Schema::IndexPortions; - ui32 refCount = CS->VersionRemoveRef(portion.GetPortion(), portion.GetPathId(), portion.GetSchemaVersionVerified()); - if (refCount == 0) { - LOG_S_CRIT("Ref count is set to 0 for version " << portion.GetSchemaVersionVerified() << " need to delete"); - } db.Table().Key(portion.GetPathId(), portion.GetPortion()).Delete(); } @@ -237,8 +230,4 @@ TConclusion>> TD return result; } -NColumnShard::TColumnShard* GetColumnShard(NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner) { - return dynamic_cast(owner); -} - } diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h index 43aac2c311da..50958b6fca29 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.h +++ b/ydb/core/tx/columnshard/engines/db_wrapper.h @@ -4,10 +4,6 @@ #include #include -namespace NColumnShard { -class TColumnShard; -} - namespace NKikimrTxColumnShard { class TIndexPortionMeta; } @@ -30,16 +26,8 @@ class IColumnEngine; class TPortionInfo; class TPortionInfoConstructor; -NColumnShard::TColumnShard* GetColumnShard(NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner); - class IDbWrapper { public: - IDbWrapper() = default; - IDbWrapper(NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner) - : CS(GetColumnShard(owner)) - { - } - virtual ~IDbWrapper() = default; virtual void Insert(const TInsertedData& data) = 0; @@ -66,21 +54,14 @@ class IDbWrapper { virtual void WriteCounter(ui32 counterId, ui64 value) = 0; virtual bool LoadCounters(const std::function& callback) = 0; virtual TConclusion>> LoadGranulesShardingInfo() = 0; - -public: - NColumnShard::TColumnShard* CS = nullptr; }; class TDbWrapper : public IDbWrapper { public: - TDbWrapper(NTable::TDatabase& db, const IBlobGroupSelector* dsGroupSelector, NTabletFlatExecutor::NFlatExecutorSetup::ITablet *owner) - : IDbWrapper(owner) - , Database(db) + TDbWrapper(NTable::TDatabase& db, const IBlobGroupSelector* dsGroupSelector) + : Database(db) , DsGroupSelector(dsGroupSelector) - { -// AFL_VERIFY(owner != nullptr); -// AFL_VERIFY(CS != nullptr); - } + {} void Insert(const TInsertedData& data) override; void Commit(const TCommittedData& data) override; diff --git a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp index 8a74f8d8eb08..cb77b59219f4 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp +++ b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp @@ -4,13 +4,14 @@ #include #include #include -//#include +#include namespace NKikimr::NOlap { bool TInsertTable::Insert(IDbWrapper& dbTable, TInsertedData&& data) { if (auto* dataPtr = Summary.AddInserted(std::move(data))) { AddBlobLink(dataPtr->GetBlobRange().BlobId); + CS->VersionAddRef(0, (ui64)dataPtr->GetInsertWriteId(), dataPtr->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, dataPtr->GetSchemaVersion()); dbTable.Insert(*dataPtr); return true; } else { @@ -18,11 +19,7 @@ bool TInsertTable::Insert(IDbWrapper& dbTable, TInsertedData&& data) { return false; } } -/* -void TInsertTableAccessor::AddSchemaVersion(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType, ui64 schemaVersion) { - CS->VersionAddRec(planStep, txId, dedupId, recType, schemaVersion); -} -*/ + TInsertionSummary::TCounters TInsertTable::Commit( IDbWrapper& dbTable, ui64 planStep, ui64 txId, const THashSet& writeIds, std::function pathExists) { Y_ABORT_UNLESS(!writeIds.empty()); @@ -39,6 +36,7 @@ TInsertionSummary::TCounters TInsertTable::Commit( counters.RawBytes += data->GetMeta().GetRawBytes(); counters.Bytes += data->BlobSize(); + CS->VersionRemoveRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data->GetSchemaVersion()); dbTable.EraseInserted(*data); const ui64 pathId = data->GetPathId(); @@ -48,12 +46,14 @@ TInsertionSummary::TCounters TInsertTable::Commit( AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "commit_insertion")("path_id", data->GetPathId())( "blob_range", data->GetBlobRange().ToString()); auto committed = data->Commit(planStep, txId); + CS->VersionAddRef(committed.GetSnapshot().GetPlanStep(), committed.GetSnapshot().GetTxId(), committed.GetPathId(), committed.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, committed.GetSchemaVersion()); dbTable.Commit(committed); pathInfo->AddCommitted(std::move(committed)); } else { AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "abort_insertion")("path_id", data->GetPathId())( "blob_range", data->GetBlobRange().ToString()); + CS->VersionAddRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data->GetSchemaVersion()); dbTable.Abort(*data); Summary.AddAborted(std::move(*data)); } @@ -86,6 +86,8 @@ void TInsertTable::Abort(IDbWrapper& dbTable, const THashSet& wr if (std::optional data = Summary.ExtractInserted(writeId)) { AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "abort_insertion")("path_id", data->GetPathId())( "blob_range", data->GetBlobRange().ToString())("write_id", writeId); + CS->VersionRemoveRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data->GetSchemaVersion()); + CS->VersionAddRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data->GetSchemaVersion()); dbTable.EraseInserted(*data); dbTable.Abort(*data); Summary.AddAborted(std::move(*data)); @@ -100,6 +102,7 @@ THashSet TInsertTable::OldWritesToAbort(const TInstant& now) con void TInsertTable::EraseCommittedOnExecute( IDbWrapper& dbTable, const TCommittedData& data, const std::shared_ptr& blobsAction) { if (Summary.HasCommitted(data)) { + CS->VersionRemoveRef(data.GetSnapshot().GetPlanStep(), data.GetSnapshot().GetTxId(), data.GetPathId(), data.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, data.GetSchemaVersion()); dbTable.EraseCommitted(data); RemoveBlobLinkOnExecute(data.GetBlobRange().BlobId, blobsAction); } @@ -114,6 +117,7 @@ void TInsertTable::EraseCommittedOnComplete(const TCommittedData& data) { void TInsertTable::EraseAbortedOnExecute( IDbWrapper& dbTable, const TInsertedData& data, const std::shared_ptr& blobsAction) { if (Summary.HasAborted(data.GetInsertWriteId())) { + CS->VersionRemoveRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data.GetSchemaVersion()); dbTable.EraseAborted(data); RemoveBlobLinkOnExecute(data.GetBlobRange().BlobId, blobsAction); } diff --git a/ydb/core/tx/columnshard/engines/insert_table/insert_table.h b/ydb/core/tx/columnshard/engines/insert_table/insert_table.h index faf80778842f..8f0526f7d185 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/insert_table.h +++ b/ydb/core/tx/columnshard/engines/insert_table/insert_table.h @@ -29,6 +29,11 @@ class TInsertTableAccessor { bool RemoveBlobLinkOnComplete(const TUnifiedBlobId& blobId); public: + TInsertTableAccessor(NColumnShard::TColumnShard* cs = nullptr) + : CS(cs) + { + } + void ErasePath(const ui64 pathId) { Summary.ErasePath(pathId); } @@ -50,8 +55,6 @@ class TInsertTableAccessor { } } - void AddSchemaVersion(ui64 planStep, ui64 txId, const TString& dedupId, ui8 recType, ui64 schemaVersion); - bool AddInserted(TInsertedData&& data, const bool load) { if (load) { AddBlobLink(data.GetBlobRange().BlobId); @@ -101,6 +104,11 @@ class TInsertTable: public TInsertTableAccessor { TInsertWriteId LastWriteId = TInsertWriteId{ 0 }; public: + TInsertTable(NColumnShard::TColumnShard* cs = nullptr) + : TInsertTableAccessor(cs) + { + } + static constexpr const TDuration WaitCommitDelay = TDuration::Minutes(10); static constexpr ui64 CleanupPackageSize = 10000; diff --git a/ydb/core/tx/columnshard/engines/insert_table/ya.make b/ydb/core/tx/columnshard/engines/insert_table/ya.make index e6fde75077d5..684ffa6674c0 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/ya.make +++ b/ydb/core/tx/columnshard/engines/insert_table/ya.make @@ -16,6 +16,8 @@ PEERDIR( ydb/core/protos ydb/core/formats/arrow ydb/core/tablet_flat + ydb/core/tx/columnshard/data_sharing/protos + ydb/core/tx/columnshard/blobs_action/protos ) END() diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index 6652bf9c4c5f..d15eaeed87c7 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -1,5 +1,6 @@ #include "portion_info.h" #include "constructor.h" +#include #include #include #include @@ -130,7 +131,11 @@ std::vector TPortionInfo::GetColumnChunksP return result; } -void TPortionInfo::RemoveFromDatabase(IDbWrapper& db) const { +void TPortionInfo::RemoveFromDatabase(IDbWrapper& db, NColumnShard::TColumnShard* cs) const { + ui32 refCount = cs->VersionRemoveRef(GetPortion(), GetPathId(), GetSchemaVersionVerified()); + if (refCount == 0) { + LOG_S_CRIT("Ref count is set to 0 for version " << GetSchemaVersionVerified() << " need to delete"); + } db.ErasePortion(*this); for (auto& record : Records) { db.EraseColumn(*this, record); @@ -140,8 +145,9 @@ void TPortionInfo::RemoveFromDatabase(IDbWrapper& db) const { } } -void TPortionInfo::SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta) const { +void TPortionInfo::SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta, NColumnShard::TColumnShard* cs) const { FullValidation(); + cs->VersionAddRef(GetPortion(), GetPathId(), GetSchemaVersionVerified()); db.WritePortion(*this); if (!saveOnlyMeta) { for (auto& record : Records) { diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 1591765f83e6..0fba888a2f7e 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -260,9 +260,9 @@ class TPortionInfo { return PathId; } - void RemoveFromDatabase(IDbWrapper& db) const; + void RemoveFromDatabase(IDbWrapper& db, NColumnShard::TColumnShard* cs) const; - void SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta) const; + void SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta, NColumnShard::TColumnShard* cs) const; bool OlderThen(const TPortionInfo& info) const { return RecordSnapshotMin() < info.RecordSnapshotMin(); diff --git a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp index 93f1f40174e4..4ae8e36cdf63 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -22,14 +23,14 @@ class TNormalizerResult : public INormalizerChanges { bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& normController) const override { NOlap::TBlobManagerDb blobManagerDb(txc.DB); - TDbWrapper db(txc.DB, nullptr, txc.Owner); + TDbWrapper db(txc.DB, nullptr); for (auto&& [_, portionInfo] : BrokenPortions) { auto schema = Schemas->FindPtr(portionInfo->GetPortionId()); AFL_VERIFY(!!schema)("portion_id", portionInfo->GetPortionId()); AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "portion_removed_as_broken")("portion_id", portionInfo->GetAddress().DebugString()); portionInfo->SetRemoveSnapshot(TSnapshot(1, 1)); LOG_S_CRIT("Saving broken portion"); - portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), false); + portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), false, static_cast(txc.Owner)); } if (BrokenPortions.size()) { TStringBuilder sb; diff --git a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp index f42f38061e45..252da8ba58bc 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/chunks.cpp @@ -3,6 +3,7 @@ #include #include +#include #include @@ -135,7 +136,7 @@ TConclusion> TChunksNormalizer::DoInit(const return tasks; } - TTablesManager tablesManager(controller.GetStoragesManager(), 0); + TTablesManager tablesManager(controller.GetStoragesManager(), 0, static_cast(txc.Owner)); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TChunksNormalizer")("error", "can't initialize tables manager"); return TConclusionStatus::Fail("Can't load index"); diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp index f8aa8cb9b432..f1c02e593bd1 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -23,10 +24,10 @@ class TBlobsRemovingResult : public INormalizerChanges { NOlap::TBlobManagerDb blobManagerDb(txc.DB); RemovingAction->OnExecuteTxAfterRemoving(blobManagerDb, true); - TDbWrapper db(txc.DB, nullptr, txc.Owner); + TDbWrapper db(txc.DB, nullptr); for (auto&& portion : Portions) { AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("message", "remove lost portion")("path_id", portion->GetPathId())("portion_id", portion->GetPortionId()); - portion->RemoveFromDatabase(db); + portion->RemoveFromDatabase(db, static_cast(txc.Owner)); } return true; } diff --git a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp index 63cea8b19952..73850669e9ae 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/normalizer.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace NKikimr::NOlap { @@ -22,7 +23,7 @@ TConclusion> TPortionsNormalizerBase::DoInit( return TConclusionStatus::Fail("Not ready"); } - NColumnShard::TTablesManager tablesManager(controller.GetStoragesManager(), 0); + NColumnShard::TTablesManager tablesManager(controller.GetStoragesManager(), 0, static_cast(txc.Owner)); if (!tablesManager.InitFromDB(db)) { ACFL_TRACE("normalizer", "TPortionsNormalizer")("error", "can't initialize tables manager"); return TConclusionStatus::Fail("Can't load index"); diff --git a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp index b71d82edd432..fcd958a89ad3 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -20,13 +21,13 @@ class TPortionsNormalizer::TNormalizerResult : public INormalizerChanges { bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& /* normController */) const override { using namespace NColumnShard; - TDbWrapper db(txc.DB, nullptr, txc.Owner); + TDbWrapper db(txc.DB, nullptr); for (auto&& portionInfo : Portions) { auto schema = Schemas->FindPtr(portionInfo->GetPortionId()); AFL_VERIFY(!!schema)("portion_id", portionInfo->GetPortionId()); LOG_S_CRIT("Saving normalized"); - portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), true); + portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), true, static_cast(txc.Owner)); } return true; } diff --git a/ydb/core/tx/columnshard/operations/write.cpp b/ydb/core/tx/columnshard/operations/write.cpp index 89af347ef8dd..1068e7167413 100644 --- a/ydb/core/tx/columnshard/operations/write.cpp +++ b/ydb/core/tx/columnshard/operations/write.cpp @@ -44,7 +44,7 @@ void TWriteOperation::CommitOnExecute(TColumnShard& owner, NTabletFlatExecutor:: Y_ABORT_UNLESS(Status == EOperationStatus::Prepared); TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, &owner); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); for (auto gWriteId : InsertWriteIds) { auto pathExists = [&](ui64 pathId) { @@ -108,7 +108,7 @@ void TWriteOperation::AbortOnExecute(TColumnShard& owner, NTabletFlatExecutor::T Y_ABORT_UNLESS(Status == EOperationStatus::Prepared); TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, &owner); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); THashSet writeIds; writeIds.insert(InsertWriteIds.begin(), InsertWriteIds.end()); diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 6d657cebcb6c..7c245650fa4d 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -178,7 +178,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { "version", schemaInfo.GetSchema().GetVersion()); if (!PrimaryIndex) { PrimaryIndex = std::make_unique( - TabletId, StoragesManager, preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema()); + TabletId, StoragesManager, preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema(), CS); } else { PrimaryIndex->RegisterSchemaVersion(preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema()); } @@ -284,7 +284,7 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho Schema::SaveSchemaPresetVersionInfo(db, presetId, version, versionInfo); if (versionInfo.HasSchema()) { if (!PrimaryIndex) { - PrimaryIndex = std::make_unique(TabletId, StoragesManager, version, schema); + PrimaryIndex = std::make_unique(TabletId, StoragesManager, version, schema, CS); for (auto&& i : Tables) { PrimaryIndex->RegisterTable(i.first); } @@ -338,9 +338,10 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& table.AddVersion(version); } -TTablesManager::TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId) +TTablesManager::TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId, NColumnShard::TColumnShard* cs) : StoragesManager(storagesManager) - , TabletId(tabletId) { + , TabletId(tabletId) + , CS(cs) { } bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const ui64 pathId) const { diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index c89189d888b6..097dd3be1960 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -150,8 +150,10 @@ class TTablesManager { std::unique_ptr PrimaryIndex; std::shared_ptr StoragesManager; ui64 TabletId = 0; + NColumnShard::TColumnShard* CS = nullptr; + public: - TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId); + TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId, NColumnShard::TColumnShard* cs); bool TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, const ui64 pathId) const; bool TryFinalizeDropPathOnComplete(const ui64 pathId); diff --git a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h index 30fdf9b60149..45b642c8e98a 100644 --- a/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h +++ b/ydb/core/tx/columnshard/transactions/operators/long_tx_write.h @@ -55,7 +55,7 @@ namespace NKikimr::NColumnShard { bool ProgressOnExecute(TColumnShard& owner, const NOlap::TSnapshot& version, NTabletFlatExecutor::TTransactionContext& txc) override { TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, &owner); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); auto pathExists = [&](ui64 pathId) { return owner.TablesManager.HasTable(pathId); @@ -88,7 +88,7 @@ namespace NKikimr::NColumnShard { AFL_VERIFY(owner.RemoveLongTxWrite(db, writeId, GetTxId())); } TBlobGroupSelector dsGroupSelector(owner.Info()); - NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector, &owner); + NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); owner.InsertTable->Abort(dbTable, WriteIds); return true; } From 81a5da5a885e8f970e2f528e92292a0d9f21e3eb Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Fri, 13 Sep 2024 20:47:16 +0000 Subject: [PATCH 08/21] Fixed tests --- .../engines/insert_table/insert_table.cpp | 16 ++++++++++++---- .../engines/portions/portion_info.cpp | 4 +++- .../tx/columnshard/engines/ut/ut_logs_engine.cpp | 14 +++++++------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp index cb77b59219f4..98db7acc9b43 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp +++ b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp @@ -11,7 +11,9 @@ namespace NKikimr::NOlap { bool TInsertTable::Insert(IDbWrapper& dbTable, TInsertedData&& data) { if (auto* dataPtr = Summary.AddInserted(std::move(data))) { AddBlobLink(dataPtr->GetBlobRange().BlobId); - CS->VersionAddRef(0, (ui64)dataPtr->GetInsertWriteId(), dataPtr->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, dataPtr->GetSchemaVersion()); + if (CS != nullptr) { // Can happen in tests + CS->VersionAddRef(0, (ui64)dataPtr->GetInsertWriteId(), dataPtr->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, dataPtr->GetSchemaVersion()); + } dbTable.Insert(*dataPtr); return true; } else { @@ -36,7 +38,9 @@ TInsertionSummary::TCounters TInsertTable::Commit( counters.RawBytes += data->GetMeta().GetRawBytes(); counters.Bytes += data->BlobSize(); - CS->VersionRemoveRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data->GetSchemaVersion()); + if (CS != nullptr) { // Can happen in tests + CS->VersionRemoveRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data->GetSchemaVersion()); + } dbTable.EraseInserted(*data); const ui64 pathId = data->GetPathId(); @@ -46,14 +50,18 @@ TInsertionSummary::TCounters TInsertTable::Commit( AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "commit_insertion")("path_id", data->GetPathId())( "blob_range", data->GetBlobRange().ToString()); auto committed = data->Commit(planStep, txId); - CS->VersionAddRef(committed.GetSnapshot().GetPlanStep(), committed.GetSnapshot().GetTxId(), committed.GetPathId(), committed.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, committed.GetSchemaVersion()); + if (CS != nullptr) { // Can happen in tests + CS->VersionAddRef(committed.GetSnapshot().GetPlanStep(), committed.GetSnapshot().GetTxId(), committed.GetPathId(), committed.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, committed.GetSchemaVersion()); + } dbTable.Commit(committed); pathInfo->AddCommitted(std::move(committed)); } else { AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "abort_insertion")("path_id", data->GetPathId())( "blob_range", data->GetBlobRange().ToString()); - CS->VersionAddRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data->GetSchemaVersion()); + if (CS != nullptr) { // Can happen in tests + CS->VersionAddRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data->GetSchemaVersion()); + } dbTable.Abort(*data); Summary.AddAborted(std::move(*data)); } diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index d15eaeed87c7..b5d23b85d30b 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -147,7 +147,9 @@ void TPortionInfo::RemoveFromDatabase(IDbWrapper& db, NColumnShard::TColumnShard void TPortionInfo::SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta, NColumnShard::TColumnShard* cs) const { FullValidation(); - cs->VersionAddRef(GetPortion(), GetPathId(), GetSchemaVersionVerified()); + if (cs != nullptr) { // Can happen in tests + cs->VersionAddRef(GetPortion(), GetPathId(), GetSchemaVersionVerified()); + } db.WritePortion(*this); if (!saveOnlyMeta) { for (auto& record : Records) { diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index 38dc6ffa044d..f6f5175388bb 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -444,7 +444,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { // PlanStep, TxId, PathId, DedupId, BlobId, Data, [Metadata] // load TSnapshot indexSnapshot(1, 1); - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); for (auto&& i : paths) { engine.RegisterTable(i); } @@ -533,7 +533,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui32 step = 1000; TSnapshot indexSnapshot(1, 1); - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); engine.RegisterTable(pathId); engine.Load(db); @@ -635,7 +635,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui64 planStep = 1; TSnapshot indexSnapshot(1, 1); - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); engine.RegisterTable(pathId); engine.Load(db); @@ -660,7 +660,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } { // check it's overloaded after reload - TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo)); + TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo), nullptr); tmpEngine.RegisterTable(pathId); tmpEngine.Load(db); } @@ -691,7 +691,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } { // check it's not overloaded after reload - TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo)); + TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo), nullptr); tmpEngine.RegisterTable(pathId); tmpEngine.Load(db); } @@ -711,7 +711,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui64 planStep = 1; TSnapshot indexSnapshot(1, 1); { - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); engine.RegisterTable(pathId); engine.Load(db); @@ -789,7 +789,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } { // load - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo)); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); engine.RegisterTable(pathId); engine.Load(db); From 890af11351e02419d77b0a01187ec010930d523a Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Mon, 16 Sep 2024 09:33:54 +0000 Subject: [PATCH 09/21] Moved versions counting down from SaveToDatabase and RemoveToDatabase to calling functions --- .../destination/transactions/tx_data_from_source.cpp | 4 +++- .../tx/columnshard/engines/changes/cleanup_portions.cpp | 6 +++++- ydb/core/tx/columnshard/engines/changes/with_appended.cpp | 7 +++++-- ydb/core/tx/columnshard/engines/portions/portion_info.cpp | 8 ++++++-- ydb/core/tx/columnshard/engines/portions/portion_info.h | 4 ++-- .../tx/columnshard/normalizer/portion/broken_blobs.cpp | 8 ++++++-- ydb/core/tx/columnshard/normalizer/portion/clean.cpp | 4 +++- ydb/core/tx/columnshard/normalizer/portion/portion.cpp | 4 +++- 8 files changed, 33 insertions(+), 12 deletions(-) diff --git a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp index 8d83de7b4cee..a9e3fa6ec10c 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp @@ -5,6 +5,8 @@ namespace NKikimr::NOlap::NDataSharing { bool TTxDataFromSource::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { + LOG_S_CRIT("SaveToDatabase from TTxDataFromSource"); + exit(1); using namespace NKikimr::NColumnShard; TDbWrapper dbWrapper(txc.DB, nullptr); auto& index = Self->TablesManager.MutablePrimaryIndexAsVerified(); @@ -21,7 +23,7 @@ bool TTxDataFromSource::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, THashMap> sharedBlobIds; for (auto&& i : PortionsByPathId) { for (auto&& p : i.second.GetPortions()) { - p.SaveToDatabase(dbWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false, static_cast(txc.Owner)); + p.SaveToDatabase(dbWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); } } NIceDb::TNiceDb db(txc.DB); diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp index 6764e2cd4c7a..c2c003d942f8 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp @@ -22,7 +22,11 @@ void TCleanupPortionsColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TC } THashMap> blobIdsByStorage; for (auto&& p : PortionsToDrop) { - p.RemoveFromDatabase(context.DBWrapper, self); + p.RemoveFromDatabase(context.DBWrapper); + ui32 refCount = self->VersionRemoveRef(p.GetPortion(), p.GetPathId(), p.GetSchemaVersionVerified()); + if (refCount == 0) { + LOG_S_CRIT("Ref count is set to 0 for version " << p.GetSchemaVersionVerified() << " need to delete"); + } p.FillBlobIdsByStorage(blobIdsByStorage, context.EngineLogs.GetVersionedIndex()); pathIds.emplace(p.GetPathId()); } diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index 7c87d08cc26d..17b449495dbd 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -9,13 +9,15 @@ namespace NKikimr::NOlap { void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) { + LOG_S_CRIT("SaveToDatabase from TChangesWithAppend"); THashSet usedPortionIds; auto schemaPtr = context.EngineLogs.GetVersionedIndex().GetLastSchema(); for (auto& [_, portionInfo] : PortionsToRemove) { Y_ABORT_UNLESS(!portionInfo.Empty()); Y_ABORT_UNLESS(portionInfo.HasRemoveSnapshot()); AFL_VERIFY(usedPortionIds.emplace(portionInfo.GetPortionId()).second)("portion_info", portionInfo.DebugString(true)); - portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false, self); + portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); + self->VersionAddRef(portionInfo.GetPortion(), portionInfo.GetPathId(), portionInfo.GetSchemaVersionVerified()); } const auto predRemoveDroppedTable = [self](const TWritePortionInfoWithBlobsResult& item) { auto& portionInfo = item.GetPortionResult(); @@ -30,7 +32,8 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, for (auto& portionInfoWithBlobs : AppendedPortions) { auto& portionInfo = portionInfoWithBlobs.GetPortionResult(); AFL_VERIFY(usedPortionIds.emplace(portionInfo.GetPortionId()).second)("portion_info", portionInfo.DebugString(true)); - portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false, self); + portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); + self->VersionAddRef(portionInfo.GetPortion(), portionInfo.GetPathId(), portionInfo.GetSchemaVersionVerified()); } } diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index b5d23b85d30b..985cec8e2e46 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -131,11 +131,13 @@ std::vector TPortionInfo::GetColumnChunksP return result; } -void TPortionInfo::RemoveFromDatabase(IDbWrapper& db, NColumnShard::TColumnShard* cs) const { +void TPortionInfo::RemoveFromDatabase(IDbWrapper& db) const { +/* ui32 refCount = cs->VersionRemoveRef(GetPortion(), GetPathId(), GetSchemaVersionVerified()); if (refCount == 0) { LOG_S_CRIT("Ref count is set to 0 for version " << GetSchemaVersionVerified() << " need to delete"); } +*/ db.ErasePortion(*this); for (auto& record : Records) { db.EraseColumn(*this, record); @@ -145,11 +147,13 @@ void TPortionInfo::RemoveFromDatabase(IDbWrapper& db, NColumnShard::TColumnShard } } -void TPortionInfo::SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta, NColumnShard::TColumnShard* cs) const { +void TPortionInfo::SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta) const { FullValidation(); +/* if (cs != nullptr) { // Can happen in tests cs->VersionAddRef(GetPortion(), GetPathId(), GetSchemaVersionVerified()); } +*/ db.WritePortion(*this); if (!saveOnlyMeta) { for (auto& record : Records) { diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 0fba888a2f7e..1591765f83e6 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -260,9 +260,9 @@ class TPortionInfo { return PathId; } - void RemoveFromDatabase(IDbWrapper& db, NColumnShard::TColumnShard* cs) const; + void RemoveFromDatabase(IDbWrapper& db) const; - void SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta, NColumnShard::TColumnShard* cs) const; + void SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta) const; bool OlderThen(const TPortionInfo& info) const { return RecordSnapshotMin() < info.RecordSnapshotMin(); diff --git a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp index 4ae8e36cdf63..4dc28697c7e2 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp @@ -21,8 +21,10 @@ class TNormalizerResult : public INormalizerChanges { {} bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& normController) const override { + LOG_S_CRIT("SaveToDatabase from Broken blobs Normalizer"); + exit(1); NOlap::TBlobManagerDb blobManagerDb(txc.DB); - + TDbWrapper db(txc.DB, nullptr); for (auto&& [_, portionInfo] : BrokenPortions) { auto schema = Schemas->FindPtr(portionInfo->GetPortionId()); @@ -30,7 +32,7 @@ class TNormalizerResult : public INormalizerChanges { AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "portion_removed_as_broken")("portion_id", portionInfo->GetAddress().DebugString()); portionInfo->SetRemoveSnapshot(TSnapshot(1, 1)); LOG_S_CRIT("Saving broken portion"); - portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), false, static_cast(txc.Owner)); + portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), false); } if (BrokenPortions.size()) { TStringBuilder sb; @@ -159,6 +161,8 @@ INormalizerTask::TPtr TNormalizer::BuildTask(std::vector TNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { + LOG_S_CRIT("SaveToDatabase from Broken blobs Normalizer init"); + exit(1); return true; } diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp index f1c02e593bd1..e0fd7bdfa5d5 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp @@ -27,7 +27,7 @@ class TBlobsRemovingResult : public INormalizerChanges { TDbWrapper db(txc.DB, nullptr); for (auto&& portion : Portions) { AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("message", "remove lost portion")("path_id", portion->GetPathId())("portion_id", portion->GetPortionId()); - portion->RemoveFromDatabase(db, static_cast(txc.Owner)); + portion->RemoveFromDatabase(db); } return true; } @@ -86,6 +86,8 @@ INormalizerTask::TPtr TCleanPortionsNormalizer::BuildTask(std::vector TCleanPortionsNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { + LOG_S_CRIT("TCleanPortionsNormalizer init"); + exit(1); return true; } diff --git a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp index fcd958a89ad3..a4cb558cb81b 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp @@ -20,6 +20,7 @@ class TPortionsNormalizer::TNormalizerResult : public INormalizerChanges { {} bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& /* normController */) const override { + LOG_S_CRIT("SaveToDatabase from Portions normalizer"); using namespace NColumnShard; TDbWrapper db(txc.DB, nullptr); @@ -27,7 +28,7 @@ class TPortionsNormalizer::TNormalizerResult : public INormalizerChanges { auto schema = Schemas->FindPtr(portionInfo->GetPortionId()); AFL_VERIFY(!!schema)("portion_id", portionInfo->GetPortionId()); LOG_S_CRIT("Saving normalized"); - portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), true, static_cast(txc.Owner)); + portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), true); } return true; } @@ -46,6 +47,7 @@ INormalizerTask::TPtr TPortionsNormalizer::BuildTask(std::vector TPortionsNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext& txc) { + LOG_S_CRIT("SaveToDatabase from Portions normalizer init"); using namespace NColumnShard; NIceDb::TNiceDb db(txc.DB); From 5e35b8697fbc1014f27b21bac34fab5661e1582d Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Mon, 16 Sep 2024 12:01:20 +0000 Subject: [PATCH 10/21] Unused version schema is removed from local database --- ydb/core/tx/columnshard/columnshard_impl.h | 3 +++ .../engines/changes/cleanup_portions.cpp | 2 ++ .../tx/columnshard/normalizer/portion/clean.cpp | 9 ++++++++- ydb/core/tx/columnshard/tables_manager.cpp | 15 +++++++++++++++ ydb/core/tx/columnshard/tables_manager.h | 10 ++++++++++ 5 files changed, 38 insertions(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index c057e25571d7..05f1d8dbe2c1 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -729,6 +729,9 @@ class TColumnShard return VersionCounts.VersionRemoveRef(TInsertKey(planStep, txId, pathId, dedupId, recType), version); } + void RemoveUnusedSchemaVersion(NTable::TDatabase* database, ui64 version) { + TablesManager.RemoveUnusedSchemaVersion(database, version); + } private: TVersionCounts VersionCounts; diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp index c2c003d942f8..47ca9fc95a17 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp @@ -23,9 +23,11 @@ void TCleanupPortionsColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TC THashMap> blobIdsByStorage; for (auto&& p : PortionsToDrop) { p.RemoveFromDatabase(context.DBWrapper); + LOG_S_CRIT("Removing portion from cleanup"); ui32 refCount = self->VersionRemoveRef(p.GetPortion(), p.GetPathId(), p.GetSchemaVersionVerified()); if (refCount == 0) { LOG_S_CRIT("Ref count is set to 0 for version " << p.GetSchemaVersionVerified() << " need to delete"); + self->TablesManager.RemoveUnusedSchemaVersion(context.DB, p.GetSchemaVersionVerified()); } p.FillBlobIdsByStorage(blobIdsByStorage, context.EngineLogs.GetVersionedIndex()); pathIds.emplace(p.GetPathId()); diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp index e0fd7bdfa5d5..f38fbc21c457 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp @@ -28,6 +28,13 @@ class TBlobsRemovingResult : public INormalizerChanges { for (auto&& portion : Portions) { AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("message", "remove lost portion")("path_id", portion->GetPathId())("portion_id", portion->GetPortionId()); portion->RemoveFromDatabase(db); + LOG_S_CRIT("Removing portion from normalizer"); + NColumnShard::TColumnShard* cs = static_cast(txc.Owner); + ui32 refCount = cs->VersionRemoveRef(portion->GetPortion(), portion->GetPathId(), portion->GetSchemaVersionVerified()); + if (refCount == 0) { + LOG_S_CRIT("Ref count is set to 0 for version " << portion->GetSchemaVersionVerified() << " need to delete"); + cs->RemoveUnusedSchemaVersion(&txc.DB, portion->GetSchemaVersionVerified()); + } } return true; } @@ -87,7 +94,7 @@ INormalizerTask::TPtr TCleanPortionsNormalizer::BuildTask(std::vector TCleanPortionsNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { LOG_S_CRIT("TCleanPortionsNormalizer init"); - exit(1); +// exit(1); return true; } diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 7c245650fa4d..c81be5c7eb0d 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -153,6 +153,10 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { TSchemaPreset::TSchemaPresetVersionInfo info; Y_ABORT_UNLESS(info.ParseFromString(rowset.GetValue())); + auto& key = VersionToKey[info.GetSchema().GetVersion()]; + key.PlanStep = version.GetPlanStep(); + key.TxId = version.GetTxId(); + key.Id = id; AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "load_preset")("preset_id", id)("snapshot", version)("version", info.HasSchema() ? info.GetSchema().GetVersion() : -1); preset.AddVersion(version, info); if (!rowset.Next()) { @@ -272,6 +276,13 @@ bool TTablesManager::RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIc return true; } +void TTablesManager::RemoveUnusedSchemaVersion(NTable::TDatabase* database, ui64 version) { + auto iter = VersionToKey.find(version); + AFL_VERIFY(iter != VersionToKey.end()); + NIceDb::TNiceDb db(*database); + db.Table().Key(iter->second.Id, iter->second.PlanStep, iter->second.TxId).Delete(); +} + void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db, std::shared_ptr& manager) { Y_ABORT_UNLESS(SchemaPresetsIds.contains(presetId)); @@ -283,6 +294,10 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho Schema::SaveSchemaPresetVersionInfo(db, presetId, version, versionInfo); if (versionInfo.HasSchema()) { + auto& key = VersionToKey[versionInfo.GetSchema().GetVersion()]; + key.Id = presetId; + key.PlanStep = version.GetPlanStep(); + key.TxId = version.GetTxId(); if (!PrimaryIndex) { PrimaryIndex = std::make_unique(TabletId, StoragesManager, version, schema, CS); for (auto&& i : Tables) { diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 097dd3be1960..788801b1f258 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -142,6 +142,14 @@ class TTableInfo { }; class TTablesManager { +private: + class TSchemaKey { + public: + ui64 PlanStep; + ui64 TxId; + ui32 Id; + }; + private: THashMap Tables; THashSet SchemaPresetsIds; @@ -151,6 +159,7 @@ class TTablesManager { std::shared_ptr StoragesManager; ui64 TabletId = 0; NColumnShard::TColumnShard* CS = nullptr; + THashMap VersionToKey; public: TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId, NColumnShard::TColumnShard* cs); @@ -247,6 +256,7 @@ class TTablesManager { void RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db); bool RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIceDb::TNiceDb& db); + void RemoveUnusedSchemaVersion(NTable::TDatabase* database, ui64 version); void AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db, std::shared_ptr& manager); void AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr& manager); bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json); From 59c183c4aa5e361d5122333253780a3a67c87ad4 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Mon, 16 Sep 2024 14:41:17 +0000 Subject: [PATCH 11/21] Unused schema versions are removed from local db and memory --- .../columnshard/engines/changes/cleanup_portions.cpp | 4 ++-- .../tx/columnshard/engines/changes/with_appended.cpp | 10 +++++++--- ydb/core/tx/columnshard/engines/column_engine.h | 3 +++ ydb/core/tx/columnshard/engines/column_engine_logs.h | 4 ++++ .../engines/scheme/versions/versioned_index.cpp | 9 +++++++++ .../engines/scheme/versions/versioned_index.h | 1 + ydb/core/tx/columnshard/tables_manager.cpp | 3 +++ 7 files changed, 29 insertions(+), 5 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp index 47ca9fc95a17..e07546d17f2f 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp @@ -23,14 +23,14 @@ void TCleanupPortionsColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TC THashMap> blobIdsByStorage; for (auto&& p : PortionsToDrop) { p.RemoveFromDatabase(context.DBWrapper); + p.FillBlobIdsByStorage(blobIdsByStorage, context.EngineLogs.GetVersionedIndex()); + pathIds.emplace(p.GetPathId()); LOG_S_CRIT("Removing portion from cleanup"); ui32 refCount = self->VersionRemoveRef(p.GetPortion(), p.GetPathId(), p.GetSchemaVersionVerified()); if (refCount == 0) { LOG_S_CRIT("Ref count is set to 0 for version " << p.GetSchemaVersionVerified() << " need to delete"); self->TablesManager.RemoveUnusedSchemaVersion(context.DB, p.GetSchemaVersionVerified()); } - p.FillBlobIdsByStorage(blobIdsByStorage, context.EngineLogs.GetVersionedIndex()); - pathIds.emplace(p.GetPathId()); } for (auto&& i : blobIdsByStorage) { auto action = BlobsAction.GetRemoving(i.first); diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index 17b449495dbd..e0c1b1e2cd39 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -9,7 +9,7 @@ namespace NKikimr::NOlap { void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) { - LOG_S_CRIT("SaveToDatabase from TChangesWithAppend"); +// LOG_S_CRIT("SaveToDatabase from TChangesWithAppend"); THashSet usedPortionIds; auto schemaPtr = context.EngineLogs.GetVersionedIndex().GetLastSchema(); for (auto& [_, portionInfo] : PortionsToRemove) { @@ -17,7 +17,9 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, Y_ABORT_UNLESS(portionInfo.HasRemoveSnapshot()); AFL_VERIFY(usedPortionIds.emplace(portionInfo.GetPortionId()).second)("portion_info", portionInfo.DebugString(true)); portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); - self->VersionAddRef(portionInfo.GetPortion(), portionInfo.GetPathId(), portionInfo.GetSchemaVersionVerified()); + if (self != nullptr) { // nullptr can happen in tests + self->VersionAddRef(portionInfo.GetPortion(), portionInfo.GetPathId(), portionInfo.GetSchemaVersionVerified()); + } } const auto predRemoveDroppedTable = [self](const TWritePortionInfoWithBlobsResult& item) { auto& portionInfo = item.GetPortionResult(); @@ -33,7 +35,9 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, auto& portionInfo = portionInfoWithBlobs.GetPortionResult(); AFL_VERIFY(usedPortionIds.emplace(portionInfo.GetPortionId()).second)("portion_info", portionInfo.DebugString(true)); portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); - self->VersionAddRef(portionInfo.GetPortion(), portionInfo.GetPathId(), portionInfo.GetSchemaVersionVerified()); + if (self != nullptr) { // nullptr can happen in tests + self->VersionAddRef(portionInfo.GetPortion(), portionInfo.GetPathId(), portionInfo.GetSchemaVersionVerified()); + } } } diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index 2c616c06e32d..91d50568bba5 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -298,6 +298,9 @@ class IColumnEngine { return TSnapshot::Zero(); } virtual void OnTieringModified(const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) = 0; + + virtual void RemoveSchemaVersion(ui64) { + } }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 119a748fda6d..c87a539a3e86 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -202,6 +202,10 @@ class TColumnEngineForLogs : public IColumnEngine { bool ErasePortion(const TPortionInfo& portionInfo, bool updateStats = true); void UpdatePortionStats(const TPortionInfo& portionInfo, EStatsUpdateType updateType = EStatsUpdateType::DEFAULT, const TPortionInfo* exPortionInfo = nullptr); void UpdatePortionStats(TColumnEngineStats& engineStats, const TPortionInfo& portionInfo, EStatsUpdateType updateType, const TPortionInfo* exPortionInfo = nullptr) const; + + void RemoveSchemaVersion(ui64 version) override { + VersionedIndex.RemoveVersion(version); + } }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp index d9a858f349c3..43a58c4902ea 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp @@ -6,6 +6,15 @@ namespace NKikimr::NOlap { +void TVersionedIndex::RemoveVersion(ui64 version) { + auto itVersion = SnapshotByVersion.find(version); + AFL_VERIFY(itVersion != SnapshotByVersion.end()); + auto itSnap = Snapshots.find(itVersion->second->GetSnapshot()); + AFL_VERIFY(itSnap != Snapshots.end()); + SnapshotByVersion.erase(itVersion); + Snapshots.erase(itSnap); +} + const TIndexInfo* TVersionedIndex::AddIndex(const TSnapshot& snapshot, TIndexInfo&& indexInfo) { if (Snapshots.empty()) { PrimaryKey = indexInfo.GetPrimaryKey(); diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h index fe554a790d8f..b252d3ef7536 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h @@ -108,6 +108,7 @@ class TVersionedIndex { return PrimaryKey; } + void RemoveVersion(ui64 version); const TIndexInfo* AddIndex(const TSnapshot& snapshot, TIndexInfo&& indexInfo); bool LoadShardingInfo(IDbWrapper& db); diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index c81be5c7eb0d..558d4cfae782 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -281,6 +281,9 @@ void TTablesManager::RemoveUnusedSchemaVersion(NTable::TDatabase* database, ui64 AFL_VERIFY(iter != VersionToKey.end()); NIceDb::TNiceDb db(*database); db.Table().Key(iter->second.Id, iter->second.PlanStep, iter->second.TxId).Delete(); + if (PrimaryIndex) { + PrimaryIndex->RemoveSchemaVersion(version); + } } void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db, std::shared_ptr& manager) { From 0591e59580272ea746d53f0d286e3c4b0d0c5703 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Tue, 17 Sep 2024 12:28:19 +0000 Subject: [PATCH 12/21] Remove unused schema from local db during commit rather than immediately --- .../engines/changes/cleanup_portions.cpp | 14 ++++++++------ .../columnshard/engines/changes/with_appended.cpp | 8 ++++++-- .../columnshard/engines/portions/portion_info.cpp | 11 ----------- .../tx/columnshard/normalizer/portion/clean.cpp | 14 ++++++++------ ydb/core/tx/columnshard/tables_manager.cpp | 6 +++--- 5 files changed, 25 insertions(+), 28 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp index e07546d17f2f..abe2e916a9b9 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp @@ -25,12 +25,14 @@ void TCleanupPortionsColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TC p.RemoveFromDatabase(context.DBWrapper); p.FillBlobIdsByStorage(blobIdsByStorage, context.EngineLogs.GetVersionedIndex()); pathIds.emplace(p.GetPathId()); - LOG_S_CRIT("Removing portion from cleanup"); - ui32 refCount = self->VersionRemoveRef(p.GetPortion(), p.GetPathId(), p.GetSchemaVersionVerified()); - if (refCount == 0) { - LOG_S_CRIT("Ref count is set to 0 for version " << p.GetSchemaVersionVerified() << " need to delete"); - self->TablesManager.RemoveUnusedSchemaVersion(context.DB, p.GetSchemaVersionVerified()); - } + context.DB->OnCommit([self, portion = p.GetPortion(), pathId = p.GetPathId(), schema = p.GetSchemaVersionVerified(), db = context.DB]() { + LOG_S_CRIT("Removing portion from cleanup"); + ui32 refCount = self->VersionRemoveRef(portion, pathId, schema); + if (refCount == 0) { + LOG_S_CRIT("Ref count is set to 0 for version " << schema << " need to delete"); + self->TablesManager.RemoveUnusedSchemaVersion(db, schema); + } + }); } for (auto&& i : blobIdsByStorage) { auto action = BlobsAction.GetRemoving(i.first); diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index e0c1b1e2cd39..14824b655215 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -18,7 +18,9 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, AFL_VERIFY(usedPortionIds.emplace(portionInfo.GetPortionId()).second)("portion_info", portionInfo.DebugString(true)); portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); if (self != nullptr) { // nullptr can happen in tests - self->VersionAddRef(portionInfo.GetPortion(), portionInfo.GetPathId(), portionInfo.GetSchemaVersionVerified()); + context.DB->OnCommit([self, portion = portionInfo.GetPortion(), pathId = portionInfo.GetPathId(), schema = portionInfo.GetSchemaVersionVerified()]() { + self->VersionAddRef(portion, pathId, schema); + }); } } const auto predRemoveDroppedTable = [self](const TWritePortionInfoWithBlobsResult& item) { @@ -36,7 +38,9 @@ void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, AFL_VERIFY(usedPortionIds.emplace(portionInfo.GetPortionId()).second)("portion_info", portionInfo.DebugString(true)); portionInfo.SaveToDatabase(context.DBWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); if (self != nullptr) { // nullptr can happen in tests - self->VersionAddRef(portionInfo.GetPortion(), portionInfo.GetPathId(), portionInfo.GetSchemaVersionVerified()); + context.DB->OnCommit([self, portion = portionInfo.GetPortion(), pathId = portionInfo.GetPathId(), schema = portionInfo.GetSchemaVersionVerified()]() { + self->VersionAddRef(portion, pathId, schema); + }); } } } diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp index 985cec8e2e46..dce24cd57b28 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.cpp +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.cpp @@ -132,12 +132,6 @@ std::vector TPortionInfo::GetColumnChunksP } void TPortionInfo::RemoveFromDatabase(IDbWrapper& db) const { -/* - ui32 refCount = cs->VersionRemoveRef(GetPortion(), GetPathId(), GetSchemaVersionVerified()); - if (refCount == 0) { - LOG_S_CRIT("Ref count is set to 0 for version " << GetSchemaVersionVerified() << " need to delete"); - } -*/ db.ErasePortion(*this); for (auto& record : Records) { db.EraseColumn(*this, record); @@ -149,11 +143,6 @@ void TPortionInfo::RemoveFromDatabase(IDbWrapper& db) const { void TPortionInfo::SaveToDatabase(IDbWrapper& db, const ui32 firstPKColumnId, const bool saveOnlyMeta) const { FullValidation(); -/* - if (cs != nullptr) { // Can happen in tests - cs->VersionAddRef(GetPortion(), GetPathId(), GetSchemaVersionVerified()); - } -*/ db.WritePortion(*this); if (!saveOnlyMeta) { for (auto& record : Records) { diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp index f38fbc21c457..5af992370c2c 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp @@ -28,13 +28,15 @@ class TBlobsRemovingResult : public INormalizerChanges { for (auto&& portion : Portions) { AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("message", "remove lost portion")("path_id", portion->GetPathId())("portion_id", portion->GetPortionId()); portion->RemoveFromDatabase(db); - LOG_S_CRIT("Removing portion from normalizer"); NColumnShard::TColumnShard* cs = static_cast(txc.Owner); - ui32 refCount = cs->VersionRemoveRef(portion->GetPortion(), portion->GetPathId(), portion->GetSchemaVersionVerified()); - if (refCount == 0) { - LOG_S_CRIT("Ref count is set to 0 for version " << portion->GetSchemaVersionVerified() << " need to delete"); - cs->RemoveUnusedSchemaVersion(&txc.DB, portion->GetSchemaVersionVerified()); - } + txc.DB.OnCommit([cs, portion = portion->GetPortion(), pathId = portion->GetPathId(), schema = portion->GetSchemaVersionVerified(), db = &txc.DB]() { + LOG_S_CRIT("Removing portion from normalizer"); + ui32 refCount = cs->VersionRemoveRef(portion, pathId, schema); + if (refCount == 0) { + LOG_S_CRIT("Ref count is set to 0 for version " << schema << " need to delete"); + cs->RemoveUnusedSchemaVersion(db, schema); + } + }); } return true; } diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 558d4cfae782..510560aea361 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -281,9 +281,9 @@ void TTablesManager::RemoveUnusedSchemaVersion(NTable::TDatabase* database, ui64 AFL_VERIFY(iter != VersionToKey.end()); NIceDb::TNiceDb db(*database); db.Table().Key(iter->second.Id, iter->second.PlanStep, iter->second.TxId).Delete(); - if (PrimaryIndex) { - PrimaryIndex->RemoveSchemaVersion(version); - } +// if (PrimaryIndex) { +// PrimaryIndex->RemoveSchemaVersion(version); +// } } void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db, std::shared_ptr& manager) { From 2802c1bbc0d3318ea9b2e9620d5b1e6a8ec9fa9d Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Tue, 17 Sep 2024 14:39:22 +0000 Subject: [PATCH 13/21] Removing unused versions are performed in TDatabase::Commit --- ydb/core/tablet_flat/flat_executor.cpp | 3 +- ydb/core/tablet_flat/tablet_flat_executor.h | 5 ++-- ydb/core/tx/columnshard/columnshard_impl.h | 1 + ydb/core/tx/columnshard/engines/db_wrapper.h | 5 ++++ .../engines/insert_table/insert_table.cpp | 30 ++++++++++++++----- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/ydb/core/tablet_flat/flat_executor.cpp b/ydb/core/tablet_flat/flat_executor.cpp index 2a21963828d1..ac3f48916a0e 100644 --- a/ydb/core/tablet_flat/flat_executor.cpp +++ b/ydb/core/tablet_flat/flat_executor.cpp @@ -1704,8 +1704,7 @@ void TExecutor::ExecuteTransaction(TAutoPtr seat, const TActorContext &ct PrivatePageCache->ResetTouchesAndToLoad(true); TPageCollectionTxEnv env(*Database, *PrivatePageCache); -// TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId, seat->Self->TxSpan, Owner); - TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId, seat->Self->TxSpan, nullptr); + TTransactionContext txc(Owner->TabletID(), Generation(), Step(), *Database, env, seat->CurrentTxDataLimit, seat->TaskId, seat->Self->TxSpan); txc.NotEnoughMemory(seat->NotEnoughMemoryCount); Database->Begin(Stamp(), env); diff --git a/ydb/core/tablet_flat/tablet_flat_executor.h b/ydb/core/tablet_flat/tablet_flat_executor.h index 726b14ca9a3a..7269839b96aa 100644 --- a/ydb/core/tablet_flat/tablet_flat_executor.h +++ b/ydb/core/tablet_flat/tablet_flat_executor.h @@ -210,7 +210,7 @@ class TTransactionContext : public TTxMemoryProviderBase { public: TTransactionContext(ui64 tablet, ui32 gen, ui32 step, NTable::TDatabase &db, IExecuting &env, - ui64 memoryLimit, ui64 taskId, NWilson::TSpan &transactionSpan, NFlatExecutorSetup::ITablet *owner) + ui64 memoryLimit, ui64 taskId, NWilson::TSpan &transactionSpan) : TTxMemoryProviderBase(memoryLimit, taskId) , Tablet(tablet) , Generation(gen) @@ -218,7 +218,6 @@ class TTransactionContext : public TTxMemoryProviderBase { , Env(env) , DB(db) , TransactionSpan(transactionSpan) - , Owner(owner) {} ~TTransactionContext() {} @@ -252,7 +251,7 @@ class TTransactionContext : public TTxMemoryProviderBase { NTable::TDatabase &DB; NWilson::TSpan &TransactionSpan; NWilson::TSpan TransactionExecutionSpan; - NFlatExecutorSetup::ITablet* Owner; + NFlatExecutorSetup::ITablet* Owner = nullptr; private: bool Rescheduled_ = false; diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 05f1d8dbe2c1..a7731ceb43f8 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -204,6 +204,7 @@ class TVersionCounts { } versions.erase(iter); ui32& refCount = VersionCounts[version]; + AFL_VERIFY(refCount > 0); LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount << " to " << refCount - 1 << " this " << (ui64)this); return --refCount; } diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.h b/ydb/core/tx/columnshard/engines/db_wrapper.h index 50958b6fca29..9ddc572c7b84 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.h +++ b/ydb/core/tx/columnshard/engines/db_wrapper.h @@ -54,6 +54,7 @@ class IDbWrapper { virtual void WriteCounter(ui32 counterId, ui64 value) = 0; virtual bool LoadCounters(const std::function& callback) = 0; virtual TConclusion>> LoadGranulesShardingInfo() = 0; + virtual void OnCommit(std::function&&) {}; }; class TDbWrapper : public IDbWrapper { @@ -89,6 +90,10 @@ class TDbWrapper : public IDbWrapper { virtual TConclusion>> LoadGranulesShardingInfo() override; + virtual void OnCommit(std::function&& callback) override { + Database.OnCommit(callback); + } + private: NTable::TDatabase& Database; const IBlobGroupSelector* DsGroupSelector; diff --git a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp index 98db7acc9b43..e183d5f69f40 100644 --- a/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp +++ b/ydb/core/tx/columnshard/engines/insert_table/insert_table.cpp @@ -12,7 +12,9 @@ bool TInsertTable::Insert(IDbWrapper& dbTable, TInsertedData&& data) { if (auto* dataPtr = Summary.AddInserted(std::move(data))) { AddBlobLink(dataPtr->GetBlobRange().BlobId); if (CS != nullptr) { // Can happen in tests - CS->VersionAddRef(0, (ui64)dataPtr->GetInsertWriteId(), dataPtr->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, dataPtr->GetSchemaVersion()); + dbTable.OnCommit([cs = CS, writeId = (ui64)dataPtr->GetInsertWriteId(), pathId = dataPtr->GetPathId(), schema = dataPtr->GetSchemaVersion()]() { + cs->VersionAddRef(0, writeId, pathId, "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, schema); + }); } dbTable.Insert(*dataPtr); return true; @@ -39,7 +41,9 @@ TInsertionSummary::TCounters TInsertTable::Commit( counters.Bytes += data->BlobSize(); if (CS != nullptr) { // Can happen in tests - CS->VersionRemoveRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data->GetSchemaVersion()); + dbTable.OnCommit([cs = CS, writeId = (ui64)data->GetInsertWriteId(), pathId = data->GetPathId(), schema = data->GetSchemaVersion()]() { + cs->VersionRemoveRef(0, writeId, pathId, "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, schema); + }); } dbTable.EraseInserted(*data); @@ -51,7 +55,9 @@ TInsertionSummary::TCounters TInsertTable::Commit( "blob_range", data->GetBlobRange().ToString()); auto committed = data->Commit(planStep, txId); if (CS != nullptr) { // Can happen in tests - CS->VersionAddRef(committed.GetSnapshot().GetPlanStep(), committed.GetSnapshot().GetTxId(), committed.GetPathId(), committed.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, committed.GetSchemaVersion()); + dbTable.OnCommit([cs = CS, planStep = committed.GetSnapshot().GetPlanStep(), txId = committed.GetSnapshot().GetTxId(), pathId = committed.GetPathId(), dedupId = committed.GetDedupId(), schema = committed.GetSchemaVersion()]() { + cs->VersionAddRef(planStep, txId, pathId, dedupId, (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, schema); + }); } dbTable.Commit(committed); @@ -60,7 +66,9 @@ TInsertionSummary::TCounters TInsertTable::Commit( AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "abort_insertion")("path_id", data->GetPathId())( "blob_range", data->GetBlobRange().ToString()); if (CS != nullptr) { // Can happen in tests - CS->VersionAddRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data->GetSchemaVersion()); + dbTable.OnCommit([cs = CS, txId = (ui64)data->GetInsertWriteId(), pathId = data->GetPathId(), schema = data->GetSchemaVersion()]() { + cs->VersionAddRef(0, txId, pathId, "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, schema); + }); } dbTable.Abort(*data); Summary.AddAborted(std::move(*data)); @@ -94,8 +102,10 @@ void TInsertTable::Abort(IDbWrapper& dbTable, const THashSet& wr if (std::optional data = Summary.ExtractInserted(writeId)) { AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "abort_insertion")("path_id", data->GetPathId())( "blob_range", data->GetBlobRange().ToString())("write_id", writeId); - CS->VersionRemoveRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, data->GetSchemaVersion()); - CS->VersionAddRef(0, (ui64)data->GetInsertWriteId(), data->GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data->GetSchemaVersion()); + dbTable.OnCommit([cs = CS, txId = (ui64)data->GetInsertWriteId(), pathId = data->GetPathId(), schema = data->GetSchemaVersion()]() { + cs->VersionRemoveRef(0, txId, pathId, "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Inserted, schema); + cs->VersionAddRef(0, txId, pathId, "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, schema); + }); dbTable.EraseInserted(*data); dbTable.Abort(*data); Summary.AddAborted(std::move(*data)); @@ -110,7 +120,9 @@ THashSet TInsertTable::OldWritesToAbort(const TInstant& now) con void TInsertTable::EraseCommittedOnExecute( IDbWrapper& dbTable, const TCommittedData& data, const std::shared_ptr& blobsAction) { if (Summary.HasCommitted(data)) { - CS->VersionRemoveRef(data.GetSnapshot().GetPlanStep(), data.GetSnapshot().GetTxId(), data.GetPathId(), data.GetDedupId(), (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, data.GetSchemaVersion()); + dbTable.OnCommit([cs = CS, planStep = data.GetSnapshot().GetPlanStep(), txId = data.GetSnapshot().GetTxId(), pathId = data.GetPathId(), dedupId = data.GetDedupId(), schema = data.GetSchemaVersion()]() { + cs->VersionRemoveRef(planStep, txId, pathId, dedupId, (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Committed, schema); + }); dbTable.EraseCommitted(data); RemoveBlobLinkOnExecute(data.GetBlobRange().BlobId, blobsAction); } @@ -125,7 +137,9 @@ void TInsertTable::EraseCommittedOnComplete(const TCommittedData& data) { void TInsertTable::EraseAbortedOnExecute( IDbWrapper& dbTable, const TInsertedData& data, const std::shared_ptr& blobsAction) { if (Summary.HasAborted(data.GetInsertWriteId())) { - CS->VersionRemoveRef(0, (ui64)data.GetInsertWriteId(), data.GetPathId(), "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, data.GetSchemaVersion()); + dbTable.OnCommit([cs = CS, writeId = (ui64)data.GetInsertWriteId(), pathId = data.GetPathId(), schema = data.GetSchemaVersion()]() { + cs->VersionRemoveRef(0, writeId, pathId, "", (ui8)NKikimr::NColumnShard::Schema::EInsertTableIds::Aborted, schema); + }); dbTable.EraseAborted(data); RemoveBlobLinkOnExecute(data.GetBlobRange().BlobId, blobsAction); } From dfeeebbe611e21a049417fd58d924531f5a7a9d4 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Wed, 18 Sep 2024 15:17:57 +0000 Subject: [PATCH 14/21] Do not remove last schema version from memory and local database --- .../transactions/tx_data_from_source.cpp | 4 +++- .../tx/columnshard/engines/column_engine.h | 3 ++- .../columnshard/engines/column_engine_logs.h | 4 ++-- .../scheme/versions/versioned_index.cpp | 21 ++++++++++++++++++- .../engines/scheme/versions/versioned_index.h | 6 +++++- .../normalizer/portion/broken_blobs.cpp | 7 +++++-- .../columnshard/normalizer/portion/clean.cpp | 1 - ydb/core/tx/columnshard/tables_manager.cpp | 11 ++++++---- 8 files changed, 44 insertions(+), 13 deletions(-) diff --git a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp index a9e3fa6ec10c..8d2dc0f1277f 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp @@ -6,7 +6,6 @@ namespace NKikimr::NOlap::NDataSharing { bool TTxDataFromSource::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { LOG_S_CRIT("SaveToDatabase from TTxDataFromSource"); - exit(1); using namespace NKikimr::NColumnShard; TDbWrapper dbWrapper(txc.DB, nullptr); auto& index = Self->TablesManager.MutablePrimaryIndexAsVerified(); @@ -24,6 +23,9 @@ bool TTxDataFromSource::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, for (auto&& i : PortionsByPathId) { for (auto&& p : i.second.GetPortions()) { p.SaveToDatabase(dbWrapper, schemaPtr->GetIndexInfo().GetPKFirstColumnId(), false); + txc.DB.OnCommit([self = Self, portion = p.GetPortion(), pathId = p.GetPathId(), schema = p.GetSchemaVersionVerified()]() { + self->VersionAddRef(portion, pathId, schema); + }); } } NIceDb::TNiceDb db(txc.DB); diff --git a/ydb/core/tx/columnshard/engines/column_engine.h b/ydb/core/tx/columnshard/engines/column_engine.h index 91d50568bba5..a226a8e4b4b0 100644 --- a/ydb/core/tx/columnshard/engines/column_engine.h +++ b/ydb/core/tx/columnshard/engines/column_engine.h @@ -299,7 +299,8 @@ class IColumnEngine { } virtual void OnTieringModified(const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) = 0; - virtual void RemoveSchemaVersion(ui64) { + virtual bool RemoveSchemaVersion(ui64) { + return true; } }; diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index c87a539a3e86..9d55ea1ce48c 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -203,8 +203,8 @@ class TColumnEngineForLogs : public IColumnEngine { void UpdatePortionStats(const TPortionInfo& portionInfo, EStatsUpdateType updateType = EStatsUpdateType::DEFAULT, const TPortionInfo* exPortionInfo = nullptr); void UpdatePortionStats(TColumnEngineStats& engineStats, const TPortionInfo& portionInfo, EStatsUpdateType updateType, const TPortionInfo* exPortionInfo = nullptr) const; - void RemoveSchemaVersion(ui64 version) override { - VersionedIndex.RemoveVersion(version); + bool RemoveSchemaVersion(ui64 version) override { + return VersionedIndex.RemoveVersion(version); } }; diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp index 43a58c4902ea..00a835c8d762 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp @@ -6,7 +6,22 @@ namespace NKikimr::NOlap { -void TVersionedIndex::RemoveVersion(ui64 version) { +bool TVersionedIndex::RemoveVersion(ui64 version) { + if (SnapshotByVersion.size() > 0) { + ui64 lastVersion = SnapshotByVersion.rbegin()->first; + if (lastVersion == version) { // keep last version until greater version is added + if (LastNotDeletedVersion.has_value()) { + RemoveVersionNoCheck(version); + } + LastNotDeletedVersion = version; + return false; + } + } + RemoveVersionNoCheck(version); + return true; +} + +void TVersionedIndex::RemoveVersionNoCheck(ui64 version) { auto itVersion = SnapshotByVersion.find(version); AFL_VERIFY(itVersion != SnapshotByVersion.end()); auto itSnap = Snapshots.find(itVersion->second->GetSnapshot()); @@ -24,6 +39,10 @@ const TIndexInfo* TVersionedIndex::AddIndex(const TSnapshot& snapshot, TIndexInf const bool needActualization = indexInfo.GetSchemeNeedActualization(); auto newVersion = indexInfo.GetVersion(); + if (LastNotDeletedVersion.has_value() && (*LastNotDeletedVersion < newVersion)) { + RemoveVersionNoCheck(*LastNotDeletedVersion); + LastNotDeletedVersion.reset(); + } auto itVersion = SnapshotByVersion.emplace(newVersion, std::make_shared(std::move(indexInfo), snapshot)); if (!itVersion.second) { AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("message", "Skip registered version")("version", LastSchemaVersion); diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h index b252d3ef7536..0fcda1175fc4 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h @@ -30,6 +30,7 @@ class TVersionedIndex { std::map SnapshotByVersion; ui64 LastSchemaVersion = 0; std::optional SchemeVersionForActualization; + std::optional LastNotDeletedVersion; ISnapshotSchema::TPtr SchemeForActualization; public: @@ -108,9 +109,12 @@ class TVersionedIndex { return PrimaryKey; } - void RemoveVersion(ui64 version); + bool RemoveVersion(ui64 version); const TIndexInfo* AddIndex(const TSnapshot& snapshot, TIndexInfo&& indexInfo); bool LoadShardingInfo(IDbWrapper& db); + +private: + void RemoveVersionNoCheck(ui64 version); }; } // namespace NKikimr::NOlap diff --git a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp index 4dc28697c7e2..970418751e8a 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp @@ -22,7 +22,6 @@ class TNormalizerResult : public INormalizerChanges { bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& normController) const override { LOG_S_CRIT("SaveToDatabase from Broken blobs Normalizer"); - exit(1); NOlap::TBlobManagerDb blobManagerDb(txc.DB); TDbWrapper db(txc.DB, nullptr); @@ -33,6 +32,11 @@ class TNormalizerResult : public INormalizerChanges { portionInfo->SetRemoveSnapshot(TSnapshot(1, 1)); LOG_S_CRIT("Saving broken portion"); portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), false); + NColumnShard::TColumnShard* self = static_cast(txc.Owner); + AFL_VERIFY(self != nullptr); + txc.DB.OnCommit([self, portion = portionInfo->GetPortion(), pathId = portionInfo->GetPathId(), schema = portionInfo->GetSchemaVersionVerified()]() { + self->VersionAddRef(portion, pathId, schema); + }); } if (BrokenPortions.size()) { TStringBuilder sb; @@ -162,7 +166,6 @@ INormalizerTask::TPtr TNormalizer::BuildTask(std::vector TNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { LOG_S_CRIT("SaveToDatabase from Broken blobs Normalizer init"); - exit(1); return true; } diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp index 5af992370c2c..d23f4cfc30e1 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp @@ -96,7 +96,6 @@ INormalizerTask::TPtr TCleanPortionsNormalizer::BuildTask(std::vector TCleanPortionsNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { LOG_S_CRIT("TCleanPortionsNormalizer init"); -// exit(1); return true; } diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 510560aea361..993f4944fae7 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -280,10 +280,13 @@ void TTablesManager::RemoveUnusedSchemaVersion(NTable::TDatabase* database, ui64 auto iter = VersionToKey.find(version); AFL_VERIFY(iter != VersionToKey.end()); NIceDb::TNiceDb db(*database); - db.Table().Key(iter->second.Id, iter->second.PlanStep, iter->second.TxId).Delete(); -// if (PrimaryIndex) { -// PrimaryIndex->RemoveSchemaVersion(version); -// } + bool removed = false; + if (PrimaryIndex) { + removed = PrimaryIndex->RemoveSchemaVersion(version); + } + if (removed) { + db.Table().Key(iter->second.Id, iter->second.PlanStep, iter->second.TxId).Delete(); + } } void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db, std::shared_ptr& manager) { From f1a7ac896e1cb607b6e28e9608a29c4abb9e82d1 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Wed, 18 Sep 2024 15:25:14 +0000 Subject: [PATCH 15/21] Fixed version deletion --- .../tx/columnshard/engines/scheme/versions/versioned_index.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp index 00a835c8d762..fdcee6ce7a7a 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp @@ -11,7 +11,7 @@ bool TVersionedIndex::RemoveVersion(ui64 version) { ui64 lastVersion = SnapshotByVersion.rbegin()->first; if (lastVersion == version) { // keep last version until greater version is added if (LastNotDeletedVersion.has_value()) { - RemoveVersionNoCheck(version); + RemoveVersionNoCheck(*LastNotDeletedVersion); } LastNotDeletedVersion = version; return false; From f2e6bafebbbc19a2ad92fc933086056ba000282b Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Mon, 23 Sep 2024 14:57:54 +0000 Subject: [PATCH 16/21] Remove deleted last not deleted version from database when new version is added --- ydb/core/tx/columnshard/columnshard_impl.h | 2 ++ .../engines/column_engine_logs.cpp | 28 +++++++++---------- .../columnshard/engines/column_engine_logs.h | 6 ++-- .../scheme/versions/versioned_index.cpp | 1 + .../engines/scheme/versions/versioned_index.h | 4 ++- ydb/core/tx/columnshard/tables_manager.cpp | 16 +++++------ ydb/core/tx/columnshard/tables_manager.h | 14 +++++++++- 7 files changed, 43 insertions(+), 28 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index a7731ceb43f8..9a5f528a3953 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -56,6 +56,7 @@ class TCompactColumnEngineChanges; class TInsertColumnEngineChanges; class TStoragesManager; class TDbWrapper; +class TColumnEngineForLogs; namespace NReader { class TTxScan; @@ -278,6 +279,7 @@ class TColumnShard friend class NOlap::NReader::NPlain::TIndexScannerConstructor; friend class NOlap::TDbWrapper; + friend class NOlap::TColumnEngineForLogs; class TStoragesManager; friend class TTxController; diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp index 0c33469caeb7..13697b80e8db 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.cpp +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.cpp @@ -26,38 +26,29 @@ #include namespace NKikimr::NOlap { -/* -TColumnEngineForLogs::TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager) - : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) - , StoragesManager(storagesManager) - , TabletId(tabletId) - , LastPortion(0) - , LastGranule(0) -{ - ActualizationController = std::make_shared(); -} -*/ TColumnEngineForLogs::TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, - const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema, NColumnShard::TColumnShard* cs) + const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema, NColumnShard::TColumnShard* cs, NIceDb::TNiceDb* database) : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) , StoragesManager(storagesManager) , TabletId(tabletId) , LastPortion(0) , LastGranule(0) , CS(cs) + , Database(database) { ActualizationController = std::make_shared(); RegisterSchemaVersion(snapshot, schema); } TColumnEngineForLogs::TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, - const TSnapshot& snapshot, TIndexInfo&& schema, NColumnShard::TColumnShard* cs) + const TSnapshot& snapshot, TIndexInfo&& schema, NColumnShard::TColumnShard* cs, NIceDb::TNiceDb* database) : GranulesStorage(std::make_shared(SignalCounters, storagesManager)) , StoragesManager(storagesManager) , TabletId(tabletId) , LastPortion(0) , LastGranule(0) - , CS(cs) { + , CS(cs) + , Database(database) { ActualizationController = std::make_shared(); RegisterSchemaVersion(snapshot, std::move(schema)); } @@ -150,7 +141,16 @@ void TColumnEngineForLogs::RegisterSchemaVersion(const TSnapshot& snapshot, TInd switchOptimizer = !indexInfo.GetCompactionPlannerConstructor()->IsEqualTo(lastIndexInfo.GetCompactionPlannerConstructor()); } const bool isCriticalScheme = indexInfo.GetSchemeNeedActualization(); + auto lastNotDeleted = VersionedIndex.LastNotDeletedVersion; auto* indexInfoActual = VersionedIndex.AddIndex(snapshot, std::move(indexInfo)); + if (lastNotDeleted.has_value() && !VersionedIndex.LastNotDeletedVersion.has_value()) { + auto& versionToKey = CS->TablesManager.VersionToKey; + auto iter = versionToKey.find(*lastNotDeleted); + AFL_VERIFY(iter != versionToKey.end()); + for (auto& key: iter->second) { + Database->Table().Key(key.Id, key.PlanStep, key.TxId).Delete(); + } + } if (isCriticalScheme) { if (!ActualizationStarted) { ActualizationStarted = true; diff --git a/ydb/core/tx/columnshard/engines/column_engine_logs.h b/ydb/core/tx/columnshard/engines/column_engine_logs.h index 9d55ea1ce48c..3cee2e9a58dc 100644 --- a/ydb/core/tx/columnshard/engines/column_engine_logs.h +++ b/ydb/core/tx/columnshard/engines/column_engine_logs.h @@ -81,9 +81,8 @@ class TColumnEngineForLogs : public IColumnEngine { ADD, }; - TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema, NColumnShard::TColumnShard* cs); - TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema, NColumnShard::TColumnShard* cs); -// TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager); + TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, const NKikimrSchemeOp::TColumnTableSchema& schema, NColumnShard::TColumnShard* cs, NIceDb::TNiceDb* database); + TColumnEngineForLogs(ui64 tabletId, const std::shared_ptr& storagesManager, const TSnapshot& snapshot, TIndexInfo&& schema, NColumnShard::TColumnShard* cs, NIceDb::TNiceDb* database); virtual void OnTieringModified(const std::shared_ptr& manager, const NColumnShard::TTtl& ttl, const std::optional pathId) override; @@ -190,6 +189,7 @@ class TColumnEngineForLogs : public IColumnEngine { ui64 LastPortion; ui64 LastGranule; NColumnShard::TColumnShard* CS = nullptr; + NIceDb::TNiceDb* Database; TSnapshot LastSnapshot = TSnapshot::Zero(); bool Loaded = false; diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp index fdcee6ce7a7a..67e3909d94f4 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp @@ -40,6 +40,7 @@ const TIndexInfo* TVersionedIndex::AddIndex(const TSnapshot& snapshot, TIndexInf const bool needActualization = indexInfo.GetSchemeNeedActualization(); auto newVersion = indexInfo.GetVersion(); if (LastNotDeletedVersion.has_value() && (*LastNotDeletedVersion < newVersion)) { + LOG_S_CRIT("Removing schema version " << *LastNotDeletedVersion << " from memory, but not from db") RemoveVersionNoCheck(*LastNotDeletedVersion); LastNotDeletedVersion.reset(); } diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h index 0fcda1175fc4..b53482607e31 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.h @@ -30,9 +30,11 @@ class TVersionedIndex { std::map SnapshotByVersion; ui64 LastSchemaVersion = 0; std::optional SchemeVersionForActualization; - std::optional LastNotDeletedVersion; ISnapshotSchema::TPtr SchemeForActualization; +public: + std::optional LastNotDeletedVersion; + public: ISnapshotSchema::TPtr GetLastCriticalSchema() const { return SchemeForActualization; diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp index 993f4944fae7..6c11fc2b878c 100644 --- a/ydb/core/tx/columnshard/tables_manager.cpp +++ b/ydb/core/tx/columnshard/tables_manager.cpp @@ -154,9 +154,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { TSchemaPreset::TSchemaPresetVersionInfo info; Y_ABORT_UNLESS(info.ParseFromString(rowset.GetValue())); auto& key = VersionToKey[info.GetSchema().GetVersion()]; - key.PlanStep = version.GetPlanStep(); - key.TxId = version.GetTxId(); - key.Id = id; + key.emplace_back(id, version.GetPlanStep(), version.GetTxId()); AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "load_preset")("preset_id", id)("snapshot", version)("version", info.HasSchema() ? info.GetSchema().GetVersion() : -1); preset.AddVersion(version, info); if (!rowset.Next()) { @@ -182,7 +180,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) { "version", schemaInfo.GetSchema().GetVersion()); if (!PrimaryIndex) { PrimaryIndex = std::make_unique( - TabletId, StoragesManager, preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema(), CS); + TabletId, StoragesManager, preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema(), CS, &db); } else { PrimaryIndex->RegisterSchemaVersion(preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema()); } @@ -285,7 +283,9 @@ void TTablesManager::RemoveUnusedSchemaVersion(NTable::TDatabase* database, ui64 removed = PrimaryIndex->RemoveSchemaVersion(version); } if (removed) { - db.Table().Key(iter->second.Id, iter->second.PlanStep, iter->second.TxId).Delete(); + for (auto& key: iter->second) { + db.Table().Key(key.Id, key.PlanStep, key.TxId).Delete(); + } } } @@ -301,11 +301,9 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho Schema::SaveSchemaPresetVersionInfo(db, presetId, version, versionInfo); if (versionInfo.HasSchema()) { auto& key = VersionToKey[versionInfo.GetSchema().GetVersion()]; - key.Id = presetId; - key.PlanStep = version.GetPlanStep(); - key.TxId = version.GetTxId(); + key.emplace_back(presetId, version.GetPlanStep(), version.GetTxId()); if (!PrimaryIndex) { - PrimaryIndex = std::make_unique(TabletId, StoragesManager, version, schema, CS); + PrimaryIndex = std::make_unique(TabletId, StoragesManager, version, schema, CS, &db); for (auto&& i : Tables) { PrimaryIndex->RegisterTable(i.first); } diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index 788801b1f258..f2114fc09c4f 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -148,6 +148,16 @@ class TTablesManager { ui64 PlanStep; ui64 TxId; ui32 Id; + + public: + TSchemaKey() = default; + + TSchemaKey(ui32 id, ui64 planStep, ui64 txId) + : PlanStep(planStep) + , TxId(txId) + , Id(id) + { + } }; private: @@ -159,7 +169,9 @@ class TTablesManager { std::shared_ptr StoragesManager; ui64 TabletId = 0; NColumnShard::TColumnShard* CS = nullptr; - THashMap VersionToKey; + +public: + THashMap> VersionToKey; public: TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId, NColumnShard::TColumnShard* cs); From 4adeaf7d71c71e9a68675dc75dfe7dffb47cf35b Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Mon, 23 Sep 2024 15:19:32 +0000 Subject: [PATCH 17/21] Fixed wrong conflict resolution --- .../tx/columnshard/blobs_action/transaction/tx_write.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp index 6b226afdd47b..b16815a60b5e 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_write.cpp @@ -11,15 +11,6 @@ bool TTxWrite::InsertOneBlob(TTransactionContext& txc, const NOlap::TWideSeriali TBlobGroupSelector dsGroupSelector(Self->Info()); NOlap::TDbWrapper dbTable(txc.DB, &dsGroupSelector); - - const auto& writeMeta = batch.GetAggregation().GetWriteMeta(); - meta.SetModificationType(TEnumOperator::SerializeToProto(writeMeta.GetModificationType())); - *meta.MutableSchemaSubset() = batch.GetAggregation().GetSchemaSubset().SerializeToProto(); - auto schemeVersion = batch.GetAggregation().GetSchemaVersion(); - auto tableSchema = Self->TablesManager.GetPrimaryIndex()->GetVersionedIndex().GetSchemaVerified(schemeVersion); - - auto userData = std::make_shared(writeMeta.GetTableId(), blobRange, meta, tableSchema->GetVersion(), batch->GetData()); - NOlap::TInsertedData insertData(writeId, userData); bool ok = Self->InsertTable->Insert(dbTable, std::move(insertData)); if (ok) { Self->UpdateInsertTableCounters(); From f2dda7322436497a842ea77f6d480219e98a4d53 Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Mon, 23 Sep 2024 16:25:35 +0000 Subject: [PATCH 18/21] Fixed TColumnEngineForLogs tests build --- .../tx/columnshard/engines/ut/ut_logs_engine.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp index f6f5175388bb..e47617ba2c37 100644 --- a/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut/ut_logs_engine.cpp @@ -444,7 +444,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { // PlanStep, TxId, PathId, DedupId, BlobId, Data, [Metadata] // load TSnapshot indexSnapshot(1, 1); - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr, nullptr); for (auto&& i : paths) { engine.RegisterTable(i); } @@ -533,7 +533,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui32 step = 1000; TSnapshot indexSnapshot(1, 1); - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr, nullptr); engine.RegisterTable(pathId); engine.Load(db); @@ -635,7 +635,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui64 planStep = 1; TSnapshot indexSnapshot(1, 1); - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr, nullptr); engine.RegisterTable(pathId); engine.Load(db); @@ -660,7 +660,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } { // check it's overloaded after reload - TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo), nullptr); + TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo), nullptr, nullptr); tmpEngine.RegisterTable(pathId); tmpEngine.Load(db); } @@ -691,7 +691,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } { // check it's not overloaded after reload - TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo), nullptr); + TColumnEngineForLogs tmpEngine(0, CommonStoragesManager, TSnapshot::Zero(), TIndexInfo(tableInfo), nullptr, nullptr); tmpEngine.RegisterTable(pathId); tmpEngine.Load(db); } @@ -711,7 +711,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { ui64 planStep = 1; TSnapshot indexSnapshot(1, 1); { - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr, nullptr); engine.RegisterTable(pathId); engine.Load(db); @@ -789,7 +789,7 @@ Y_UNIT_TEST_SUITE(TColumnEngineTestLogs) { } { // load - TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr); + TColumnEngineForLogs engine(0, CommonStoragesManager, indexSnapshot, TIndexInfo(tableInfo), nullptr, nullptr); engine.RegisterTable(pathId); engine.Load(db); From 7d9b831f646b8a322c2210fa18d6bf6269ee58ba Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Tue, 24 Sep 2024 07:29:45 +0000 Subject: [PATCH 19/21] Fixed tests after rebase --- ydb/core/tx/columnshard/columnshard_schema.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/columnshard/columnshard_schema.cpp b/ydb/core/tx/columnshard/columnshard_schema.cpp index 03683d8b7b11..47f5bd00a022 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/columnshard_schema.cpp @@ -4,6 +4,10 @@ namespace NKikimr::NColumnShard { +ui64 GetPlanStep(const NOlap::TInsertTableRecordLoadContext& constructor) { + return constructor.GetRecType() == Schema::EInsertTableIds::Committed ? constructor.GetPlanStep() : 0; +} + bool Schema::InsertTable_Load(NIceDb::TNiceDb& db, const IBlobGroupSelector* dsGroupSelector, NOlap::TInsertTableAccessor& insertTable, const TInstant& /*loadTime*/) { auto rowset = db.Table().Select(); if (!rowset.IsReady()) { @@ -13,9 +17,10 @@ bool Schema::InsertTable_Load(NIceDb::TNiceDb& db, const IBlobGroupSelector* dsG while (!rowset.EndOfSet()) { NOlap::TInsertTableRecordLoadContext constructor; constructor.ParseFromDatabase(rowset); + ui64 planStep = GetPlanStep(constructor); - LOG_S_CRIT("Loaded schema version " << constructor.GetSchemaVersion() << " planstep " << constructor.GetPlanStep() << " txid " << constructor.GetWriteTxId() << " dedup id " << constructor.GetDedupId() << " rec type " << (ui32)constructor.GetRecType()); - insertTable.CS->VersionAddRef(constructor.GetPlanStep(), constructor.GetWriteTxId(), constructor.GetPathId(), constructor.GetDedupId(), (ui8)constructor.GetRecType(), constructor.GetSchemaVersion()); + LOG_S_CRIT("Loaded schema version " << constructor.GetSchemaVersion() << " planstep " << planStep << " txid " << constructor.GetWriteTxId() << " dedup id " << constructor.GetDedupId() << " rec type " << (ui32)constructor.GetRecType()); + insertTable.CS->VersionAddRef(planStep, constructor.GetWriteTxId(), constructor.GetPathId(), constructor.GetDedupId(), (ui8)constructor.GetRecType(), constructor.GetSchemaVersion()); switch (constructor.GetRecType()) { case Schema::EInsertTableIds::Inserted: insertTable.AddInserted(constructor.BuildInsertedOrAborted(dsGroupSelector), true); From 47efee48c786019f9eb65a455280989ff65db2fc Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Tue, 24 Sep 2024 09:45:44 +0000 Subject: [PATCH 20/21] Added possibility to quickly turn on and off logging --- ydb/core/tx/columnshard/columnshard_impl.h | 8 +++++--- .../tx/columnshard/columnshard_schema.cpp | 4 +++- ydb/core/tx/columnshard/common/log.h | 7 +++++++ .../engines/changes/cleanup_portions.cpp | 5 +++-- .../engines/changes/with_appended.cpp | 1 - .../tx/columnshard/engines/db_wrapper.cpp | 19 ++++++++++--------- .../scheme/versions/versioned_index.cpp | 4 +++- .../normalizer/portion/broken_blobs.cpp | 5 +---- .../columnshard/normalizer/portion/clean.cpp | 7 ++++--- .../normalizer/portion/portion.cpp | 3 --- ydb/core/tx/columnshard/tables_manager.h | 2 +- 11 files changed, 37 insertions(+), 28 deletions(-) create mode 100644 ydb/core/tx/columnshard/common/log.h diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 9a5f528a3953..007e3a4d758f 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -7,6 +7,8 @@ #include "columnshard_private_events.h" #include "tables_manager.h" +#include "common/log.h" + #include "blobs_action/events/delete_blobs.h" #include "bg_tasks/events/local.h" #include "transactions/tx_controller.h" @@ -187,14 +189,14 @@ class TVersionCounts { void VersionAddRef(Versions& versions, const Key& portion, ui64 version) { ui64& curVer = versions[portion]; if (curVer != 0) {// Portion is already in the local database, no need to increase ref count - LOG_S_CRIT("Schema version is already written"); + TEMPLOG("Schema version is already written"); AFL_VERIFY(version == curVer); return; } curVer = version; ui32& refCount = VersionCounts[version]; refCount++; - LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount - 1 << " to " << refCount << " this " << (ui64)this); + TEMPLOG("Ref count of schema version " << version << " changed from " << refCount - 1 << " to " << refCount << " this " << (ui64)this); } template @@ -206,7 +208,7 @@ class TVersionCounts { versions.erase(iter); ui32& refCount = VersionCounts[version]; AFL_VERIFY(refCount > 0); - LOG_S_CRIT("Ref count of schema version " << version << " changed from " << refCount << " to " << refCount - 1 << " this " << (ui64)this); + TEMPLOG("Ref count of schema version " << version << " changed from " << refCount << " to " << refCount - 1 << " this " << (ui64)this); return --refCount; } diff --git a/ydb/core/tx/columnshard/columnshard_schema.cpp b/ydb/core/tx/columnshard/columnshard_schema.cpp index 47f5bd00a022..9498d2f78d5a 100644 --- a/ydb/core/tx/columnshard/columnshard_schema.cpp +++ b/ydb/core/tx/columnshard/columnshard_schema.cpp @@ -2,6 +2,8 @@ #include "transactions/tx_controller.h" #include "columnshard_impl.h" +#include "common/log.h" + namespace NKikimr::NColumnShard { ui64 GetPlanStep(const NOlap::TInsertTableRecordLoadContext& constructor) { @@ -19,7 +21,7 @@ bool Schema::InsertTable_Load(NIceDb::TNiceDb& db, const IBlobGroupSelector* dsG constructor.ParseFromDatabase(rowset); ui64 planStep = GetPlanStep(constructor); - LOG_S_CRIT("Loaded schema version " << constructor.GetSchemaVersion() << " planstep " << planStep << " txid " << constructor.GetWriteTxId() << " dedup id " << constructor.GetDedupId() << " rec type " << (ui32)constructor.GetRecType()); + TEMPLOG("Loaded schema version " << constructor.GetSchemaVersion() << " planstep " << planStep << " txid " << constructor.GetWriteTxId() << " dedup id " << constructor.GetDedupId() << " rec type " << (ui32)constructor.GetRecType()); insertTable.CS->VersionAddRef(planStep, constructor.GetWriteTxId(), constructor.GetPathId(), constructor.GetDedupId(), (ui8)constructor.GetRecType(), constructor.GetSchemaVersion()); switch (constructor.GetRecType()) { case Schema::EInsertTableIds::Inserted: diff --git a/ydb/core/tx/columnshard/common/log.h b/ydb/core/tx/columnshard/common/log.h new file mode 100644 index 000000000000..a7a8ed1271c8 --- /dev/null +++ b/ydb/core/tx/columnshard/common/log.h @@ -0,0 +1,7 @@ +#pragma once + +#if 1 +#define TEMPLOG(x) LOG_S_CRIT(x) +#else +#define TEMPLOG(x) +#endif diff --git a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp index abe2e916a9b9..11370a7cf4a8 100644 --- a/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp +++ b/ydb/core/tx/columnshard/engines/changes/cleanup_portions.cpp @@ -1,5 +1,6 @@ #include "cleanup_portions.h" #include +#include #include #include #include @@ -26,10 +27,10 @@ void TCleanupPortionsColumnEngineChanges::DoWriteIndexOnExecute(NColumnShard::TC p.FillBlobIdsByStorage(blobIdsByStorage, context.EngineLogs.GetVersionedIndex()); pathIds.emplace(p.GetPathId()); context.DB->OnCommit([self, portion = p.GetPortion(), pathId = p.GetPathId(), schema = p.GetSchemaVersionVerified(), db = context.DB]() { - LOG_S_CRIT("Removing portion from cleanup"); + TEMPLOG("Removing portion from cleanup"); ui32 refCount = self->VersionRemoveRef(portion, pathId, schema); if (refCount == 0) { - LOG_S_CRIT("Ref count is set to 0 for version " << schema << " need to delete"); + TEMPLOG("Ref count is set to 0 for version " << schema << " need to delete"); self->TablesManager.RemoveUnusedSchemaVersion(db, schema); } }); diff --git a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp index 14824b655215..c229f1f3dd67 100644 --- a/ydb/core/tx/columnshard/engines/changes/with_appended.cpp +++ b/ydb/core/tx/columnshard/engines/changes/with_appended.cpp @@ -9,7 +9,6 @@ namespace NKikimr::NOlap { void TChangesWithAppend::DoWriteIndexOnExecute(NColumnShard::TColumnShard* self, TWriteIndexContext& context) { -// LOG_S_CRIT("SaveToDatabase from TChangesWithAppend"); THashSet usedPortionIds; auto schemaPtr = context.EngineLogs.GetVersionedIndex().GetLastSchema(); for (auto& [_, portionInfo] : PortionsToRemove) { diff --git a/ydb/core/tx/columnshard/engines/db_wrapper.cpp b/ydb/core/tx/columnshard/engines/db_wrapper.cpp index 65b04b626276..46b64f7fbf6d 100644 --- a/ydb/core/tx/columnshard/engines/db_wrapper.cpp +++ b/ydb/core/tx/columnshard/engines/db_wrapper.cpp @@ -3,43 +3,44 @@ #include "portions/constructor.h" #include #include +#include #include namespace NKikimr::NOlap { void TDbWrapper::Insert(const TInsertedData& data) { NIceDb::TNiceDb db(Database); - LOG_S_CRIT("Writing inserted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); + TEMPLOG("Writing inserted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_Insert(db, data); } void TDbWrapper::Commit(const TCommittedData& data) { NIceDb::TNiceDb db(Database); - LOG_S_CRIT("Writing committed data, schema version " << data.GetSchemaVersion() << " planstep " << data.GetSnapshot().GetPlanStep() << " path id " << data.GetPathId() << " write id " << data.GetSnapshot() << " dedup id " << data.GetDedupId() << " database " << (ui64)&Database); + TEMPLOG("Writing committed data, schema version " << data.GetSchemaVersion() << " planstep " << data.GetSnapshot().GetPlanStep() << " path id " << data.GetPathId() << " write id " << data.GetSnapshot() << " dedup id " << data.GetDedupId() << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_Commit(db, data); } void TDbWrapper::Abort(const TInsertedData& data) { NIceDb::TNiceDb db(Database); - LOG_S_CRIT("Writing aborted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); + TEMPLOG("Writing aborted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_Abort(db, data); } void TDbWrapper::EraseInserted(const TInsertedData& data) { NIceDb::TNiceDb db(Database); - LOG_S_CRIT("Erasing inserted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); + TEMPLOG("Erasing inserted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_EraseInserted(db, data); } void TDbWrapper::EraseCommitted(const TCommittedData& data) { NIceDb::TNiceDb db(Database); - LOG_S_CRIT("Erasing committed data, schema version " << data.GetSchemaVersion() << " planstep " << data.GetSnapshot().GetPlanStep() << " path id " << data.GetPathId() << " write id " << data.GetSnapshot() << " dedup id " << data.GetDedupId() << " database " << (ui64)&Database); + TEMPLOG("Erasing committed data, schema version " << data.GetSchemaVersion() << " planstep " << data.GetSnapshot().GetPlanStep() << " path id " << data.GetPathId() << " write id " << data.GetSnapshot() << " dedup id " << data.GetDedupId() << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_EraseCommitted(db, data); } void TDbWrapper::EraseAborted(const TInsertedData& data) { NIceDb::TNiceDb db(Database); - LOG_S_CRIT("Erasing aborted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); + TEMPLOG("Erasing aborted data, schema version " << data.GetSchemaVersion() << " planstep 0 " << " path id " << data.GetPathId() << " write id " << data.GetInsertWriteId() << " dedup id " << " database " << (ui64)&Database); NColumnShard::Schema::InsertTable_EraseAborted(db, data); } @@ -70,7 +71,7 @@ void TDbWrapper::WriteColumn(const NOlap::TPortionInfo& portion, const TColumnRe } void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { - LOG_S_CRIT("Writing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); + TEMPLOG("Writing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); NIceDb::TNiceDb db(Database); auto metaProto = portion.GetMeta().SerializeToProto(); using IndexPortions = NColumnShard::Schema::IndexPortions; @@ -84,7 +85,7 @@ void TDbWrapper::WritePortion(const NOlap::TPortionInfo& portion) { } void TDbWrapper::ErasePortion(const NOlap::TPortionInfo& portion) { - LOG_S_CRIT("Erasing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); + TEMPLOG("Erasing portion, schema version " << portion.GetSchemaVersionVerified() << " path id " << portion.GetPathId() << " portion id " << portion.GetPortion() << " database " << (ui64)&Database); NIceDb::TNiceDb db(Database); using IndexPortions = NColumnShard::Schema::IndexPortions; db.Table().Key(portion.GetPathId(), portion.GetPortion()).Delete(); @@ -132,7 +133,7 @@ bool TDbWrapper::LoadPortions(const std::function() << " portion id " << rowset.GetValue() << " schema version " << rowset.GetValue() << " database " << (ui64)&Database); + TEMPLOG("Loaded portion, path id " << rowset.GetValue() << " portion id " << rowset.GetValue() << " schema version " << rowset.GetValue() << " database " << (ui64)&Database); NOlap::TPortionInfoConstructor portion(rowset.GetValue(), rowset.GetValue()); portion.SetSchemaVersion(rowset.GetValue()); if (rowset.HaveValue() && rowset.GetValue()) { diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp index 67e3909d94f4..9a0281c113c7 100644 --- a/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/versions/versioned_index.cpp @@ -4,6 +4,8 @@ #include #include +#include + namespace NKikimr::NOlap { bool TVersionedIndex::RemoveVersion(ui64 version) { @@ -40,7 +42,7 @@ const TIndexInfo* TVersionedIndex::AddIndex(const TSnapshot& snapshot, TIndexInf const bool needActualization = indexInfo.GetSchemeNeedActualization(); auto newVersion = indexInfo.GetVersion(); if (LastNotDeletedVersion.has_value() && (*LastNotDeletedVersion < newVersion)) { - LOG_S_CRIT("Removing schema version " << *LastNotDeletedVersion << " from memory, but not from db") + TEMPLOG("Removing schema version " << *LastNotDeletedVersion << " from memory, but not from db") RemoveVersionNoCheck(*LastNotDeletedVersion); LastNotDeletedVersion.reset(); } diff --git a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp index 970418751e8a..4adeec2f2c04 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/broken_blobs.cpp @@ -21,7 +21,6 @@ class TNormalizerResult : public INormalizerChanges { {} bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& normController) const override { - LOG_S_CRIT("SaveToDatabase from Broken blobs Normalizer"); NOlap::TBlobManagerDb blobManagerDb(txc.DB); TDbWrapper db(txc.DB, nullptr); @@ -30,7 +29,6 @@ class TNormalizerResult : public INormalizerChanges { AFL_VERIFY(!!schema)("portion_id", portionInfo->GetPortionId()); AFL_CRIT(NKikimrServices::TX_COLUMNSHARD)("event", "portion_removed_as_broken")("portion_id", portionInfo->GetAddress().DebugString()); portionInfo->SetRemoveSnapshot(TSnapshot(1, 1)); - LOG_S_CRIT("Saving broken portion"); portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), false); NColumnShard::TColumnShard* self = static_cast(txc.Owner); AFL_VERIFY(self != nullptr); @@ -164,8 +162,7 @@ INormalizerTask::TPtr TNormalizer::BuildTask(std::vector(std::move(blobIds), std::move(portionByBlobId), schemas); } - TConclusion TNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { - LOG_S_CRIT("SaveToDatabase from Broken blobs Normalizer init"); +TConclusion TNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { return true; } diff --git a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp index d23f4cfc30e1..39fe16646075 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/clean.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/clean.cpp @@ -6,6 +6,8 @@ #include #include +#include + #include @@ -30,10 +32,10 @@ class TBlobsRemovingResult : public INormalizerChanges { portion->RemoveFromDatabase(db); NColumnShard::TColumnShard* cs = static_cast(txc.Owner); txc.DB.OnCommit([cs, portion = portion->GetPortion(), pathId = portion->GetPathId(), schema = portion->GetSchemaVersionVerified(), db = &txc.DB]() { - LOG_S_CRIT("Removing portion from normalizer"); + TEMPLOG("Removing portion from normalizer"); ui32 refCount = cs->VersionRemoveRef(portion, pathId, schema); if (refCount == 0) { - LOG_S_CRIT("Ref count is set to 0 for version " << schema << " need to delete"); + TEMPLOG("Ref count is set to 0 for version " << schema << " need to delete"); cs->RemoveUnusedSchemaVersion(db, schema); } }); @@ -95,7 +97,6 @@ INormalizerTask::TPtr TCleanPortionsNormalizer::BuildTask(std::vector TCleanPortionsNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext&) { - LOG_S_CRIT("TCleanPortionsNormalizer init"); return true; } diff --git a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp index a4cb558cb81b..9323566ddf31 100644 --- a/ydb/core/tx/columnshard/normalizer/portion/portion.cpp +++ b/ydb/core/tx/columnshard/normalizer/portion/portion.cpp @@ -20,14 +20,12 @@ class TPortionsNormalizer::TNormalizerResult : public INormalizerChanges { {} bool ApplyOnExecute(NTabletFlatExecutor::TTransactionContext& txc, const TNormalizationController& /* normController */) const override { - LOG_S_CRIT("SaveToDatabase from Portions normalizer"); using namespace NColumnShard; TDbWrapper db(txc.DB, nullptr); for (auto&& portionInfo : Portions) { auto schema = Schemas->FindPtr(portionInfo->GetPortionId()); AFL_VERIFY(!!schema)("portion_id", portionInfo->GetPortionId()); - LOG_S_CRIT("Saving normalized"); portionInfo->SaveToDatabase(db, (*schema)->GetIndexInfo().GetPKFirstColumnId(), true); } return true; @@ -47,7 +45,6 @@ INormalizerTask::TPtr TPortionsNormalizer::BuildTask(std::vector TPortionsNormalizer::DoInitImpl(const TNormalizationController&, NTabletFlatExecutor::TTransactionContext& txc) { - LOG_S_CRIT("SaveToDatabase from Portions normalizer init"); using namespace NColumnShard; NIceDb::TNiceDb db(txc.DB); diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h index f2114fc09c4f..f7bf115ba6b3 100644 --- a/ydb/core/tx/columnshard/tables_manager.h +++ b/ydb/core/tx/columnshard/tables_manager.h @@ -171,7 +171,7 @@ class TTablesManager { NColumnShard::TColumnShard* CS = nullptr; public: - THashMap> VersionToKey; + THashMap> VersionToKey; public: TTablesManager(const std::shared_ptr& storagesManager, const ui64 tabletId, NColumnShard::TColumnShard* cs); From 2aeba59aaffc04f8c4534328dc1bdd4bfb33d18f Mon Sep 17 00:00:00 2001 From: Alexander Avdonkin Date: Tue, 24 Sep 2024 12:38:18 +0000 Subject: [PATCH 21/21] Replace left LOG_S_CRIT to TEMPLOG --- ydb/core/tx/columnshard/common/log.h | 2 +- .../destination/transactions/tx_data_from_source.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ydb/core/tx/columnshard/common/log.h b/ydb/core/tx/columnshard/common/log.h index a7a8ed1271c8..97735268c276 100644 --- a/ydb/core/tx/columnshard/common/log.h +++ b/ydb/core/tx/columnshard/common/log.h @@ -1,6 +1,6 @@ #pragma once -#if 1 +#if 0 #define TEMPLOG(x) LOG_S_CRIT(x) #else #define TEMPLOG(x) diff --git a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp index 8d2dc0f1277f..7c530515a033 100644 --- a/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp +++ b/ydb/core/tx/columnshard/data_sharing/destination/transactions/tx_data_from_source.cpp @@ -1,11 +1,12 @@ #include "tx_data_from_source.h" #include #include +#include namespace NKikimr::NOlap::NDataSharing { bool TTxDataFromSource::DoExecute(NTabletFlatExecutor::TTransactionContext& txc, const TActorContext& /*ctx*/) { - LOG_S_CRIT("SaveToDatabase from TTxDataFromSource"); + TEMPLOG("SaveToDatabase from TTxDataFromSource"); using namespace NKikimr::NColumnShard; TDbWrapper dbWrapper(txc.DB, nullptr); auto& index = Self->TablesManager.MutablePrimaryIndexAsVerified();