From ed3dd725e1376f1e07db8744583b0c68efd46e2f Mon Sep 17 00:00:00 2001 From: Ilia Shakhov Date: Tue, 24 Sep 2024 09:42:20 +0000 Subject: [PATCH 1/4] Add delay before shutdown --- ydb/core/driver_lib/run/run.cpp | 4 ++++ ydb/core/driver_lib/run/run.h | 1 + ydb/core/protos/config.proto | 1 + 3 files changed, 6 insertions(+) diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 4dd4c1ed5900..582ce49abef2 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -504,6 +504,9 @@ static TString ReadFile(const TString& fileName) { void TKikimrRunner::InitializeGracefulShutdown(const TKikimrRunConfig& runConfig) { Y_UNUSED(runConfig); GracefulShutdownSupported = true; + if (runConfig.AppConfig.HasDelayBeforeShutdownSeconds()) { + DelayBeforeShutdown = TDuration::Seconds(runConfig.AppConfig.GetDelayBeforeShutdownSeconds()); + } } void TKikimrRunner::InitializeKqpController(const TKikimrRunConfig& runConfig) { @@ -1732,6 +1735,7 @@ void TKikimrRunner::KikimrStop(bool graceful) { if (EnabledGrpcService) { ActorSystem->Send(new IEventHandle(NGRpcService::CreateGrpcPublisherServiceActorId(), {}, new TEvents::TEvPoisonPill)); + Sleep(DelayBeforeShutdown); } TIntrusivePtr drainProgress(new TDrainProgress()); diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h index 0977da36b521..b9f1fe4d5ad9 100644 --- a/ydb/core/driver_lib/run/run.h +++ b/ydb/core/driver_lib/run/run.h @@ -43,6 +43,7 @@ class TKikimrRunner : public virtual TThrRefBase, private IGlobalObjectStorage { bool EnabledGrpcService = false; bool GracefulShutdownSupported = false; + TDuration DelayBeforeShutdown; THolder SqsHttp; THolder YdbDriver; diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index 7401e5fb5914..ad9aeea159b3 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1989,6 +1989,7 @@ message TAppConfig { optional TMemoryControllerConfig MemoryControllerConfig = 81; optional TGroupedMemoryLimiterConfig GroupedMemoryLimiterConfig = 82; optional NKikimrReplication.TReplicationDefaults ReplicationConfig = 83; + optional uint32 DelayBeforeShutdownSeconds = 108; repeated TNamedConfig NamedConfigs = 100; optional string ClusterYamlConfig = 101; From 057743b09ecbdbfb70e2b840fcbea67c37a16dbb Mon Sep 17 00:00:00 2001 From: Ilia Shakhov Date: Thu, 26 Sep 2024 13:39:27 +0000 Subject: [PATCH 2/4] Add min delay before shutdown --- ydb/core/driver_lib/run/run.cpp | 12 ++++++++---- ydb/core/driver_lib/run/run.h | 3 ++- ydb/core/protos/config.proto | 6 +++++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 582ce49abef2..557a08a048ab 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -502,10 +502,10 @@ static TString ReadFile(const TString& fileName) { } void TKikimrRunner::InitializeGracefulShutdown(const TKikimrRunConfig& runConfig) { - Y_UNUSED(runConfig); GracefulShutdownSupported = true; - if (runConfig.AppConfig.HasDelayBeforeShutdownSeconds()) { - DelayBeforeShutdown = TDuration::Seconds(runConfig.AppConfig.GetDelayBeforeShutdownSeconds()); + const auto& config = runConfig.AppConfig.GetShutdownConfig(); + if (config.HasMinDelayBeforeShutdownSeconds()) { + MinDelayBeforeShutdown = TDuration::Seconds(config.GetMinDelayBeforeShutdownSeconds()); } } @@ -1735,9 +1735,9 @@ void TKikimrRunner::KikimrStop(bool graceful) { if (EnabledGrpcService) { ActorSystem->Send(new IEventHandle(NGRpcService::CreateGrpcPublisherServiceActorId(), {}, new TEvents::TEvPoisonPill)); - Sleep(DelayBeforeShutdown); } + THPTimer timer; TIntrusivePtr drainProgress(new TDrainProgress()); if (AppData->FeatureFlags.GetEnableDrainOnShutdown() && GracefulShutdownSupported && ActorSystem) { drainProgress->OnSend(); @@ -1771,6 +1771,10 @@ void TKikimrRunner::KikimrStop(bool graceful) { } } + // Wait for a minimum delay to make sure that clients forget about this node + auto timeLeftBeforeShutdown = MinDelayBeforeShutdown - TDuration::Seconds(timer.Passed()); + Sleep(timeLeftBeforeShutdown); + if (ActorSystem) { ActorSystem->BroadcastToProxies([](const TActorId& proxyId) { return new IEventHandle(proxyId, {}, new TEvInterconnect::TEvTerminate); diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h index b9f1fe4d5ad9..a01767c91a2a 100644 --- a/ydb/core/driver_lib/run/run.h +++ b/ydb/core/driver_lib/run/run.h @@ -43,7 +43,8 @@ class TKikimrRunner : public virtual TThrRefBase, private IGlobalObjectStorage { bool EnabledGrpcService = false; bool GracefulShutdownSupported = false; - TDuration DelayBeforeShutdown; + TDuration MinDelayBeforeShutdown; + TDuration DelayBeforeShutdown2; THolder SqsHttp; THolder YdbDriver; diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index ad9aeea159b3..a989189b4f76 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1905,6 +1905,10 @@ message TMetadataCacheConfig { optional uint64 RefreshPeriodMs = 1 [default = 15000]; } +message TShutdownConfig { + optional uint32 MinDelayBeforeShutdownSeconds = 1; +} + message TLabel { optional string Name = 1; optional string Value = 2; @@ -1989,7 +1993,7 @@ message TAppConfig { optional TMemoryControllerConfig MemoryControllerConfig = 81; optional TGroupedMemoryLimiterConfig GroupedMemoryLimiterConfig = 82; optional NKikimrReplication.TReplicationDefaults ReplicationConfig = 83; - optional uint32 DelayBeforeShutdownSeconds = 108; + optional TShutdownConfig ShutdownConfig = 84; repeated TNamedConfig NamedConfigs = 100; optional string ClusterYamlConfig = 101; From eaf9dfa444e4e78cec7203a23386e8e7e0ab58f7 Mon Sep 17 00:00:00 2001 From: Ilia Shakhov Date: Thu, 26 Sep 2024 13:41:32 +0000 Subject: [PATCH 3/4] Little fix --- ydb/core/driver_lib/run/run.h | 1 - 1 file changed, 1 deletion(-) diff --git a/ydb/core/driver_lib/run/run.h b/ydb/core/driver_lib/run/run.h index a01767c91a2a..c84e17069c0c 100644 --- a/ydb/core/driver_lib/run/run.h +++ b/ydb/core/driver_lib/run/run.h @@ -44,7 +44,6 @@ class TKikimrRunner : public virtual TThrRefBase, private IGlobalObjectStorage { bool EnabledGrpcService = false; bool GracefulShutdownSupported = false; TDuration MinDelayBeforeShutdown; - TDuration DelayBeforeShutdown2; THolder SqsHttp; THolder YdbDriver; From bcefb4ba4e67848779270e69b8a551d237a9db81 Mon Sep 17 00:00:00 2001 From: Ilia Shakhov Date: Thu, 26 Sep 2024 14:33:38 +0000 Subject: [PATCH 4/4] Add additional check --- ydb/core/driver_lib/run/run.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ydb/core/driver_lib/run/run.cpp b/ydb/core/driver_lib/run/run.cpp index 557a08a048ab..3d7e42f382e9 100644 --- a/ydb/core/driver_lib/run/run.cpp +++ b/ydb/core/driver_lib/run/run.cpp @@ -1772,8 +1772,10 @@ void TKikimrRunner::KikimrStop(bool graceful) { } // Wait for a minimum delay to make sure that clients forget about this node - auto timeLeftBeforeShutdown = MinDelayBeforeShutdown - TDuration::Seconds(timer.Passed()); - Sleep(timeLeftBeforeShutdown); + auto passedTime = TDuration::Seconds(timer.Passed()); + if (MinDelayBeforeShutdown > passedTime) { + Sleep(MinDelayBeforeShutdown - passedTime); + } if (ActorSystem) { ActorSystem->BroadcastToProxies([](const TActorId& proxyId) {