Skip to content

Commit

Permalink
moved parameters for registration from server to storage config + add…
Browse files Browse the repository at this point in the history 
…ed registration tests for nbs (#1859)

* moved parameters for registration from server to storage config + added registration tests for nbs

* update

* update

* update

* fix ut

* update

* update

* update

* fix ut

* remove extra new lines
  • Loading branch information
@yegorskii
yegorskii committed Sep 10, 2024
1 parent 9fb3192 commit f0aab4b
Show file tree
Hide file tree
Showing 33 changed files with 841 additions and 234 deletions.
3 changes: 3 additions & 0 deletions cloud/blockstore/config/server.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ message TServerConfig
optional bool NvmfInitiatorEnabled = 31;

// Node type
// deprecated. moved to storage config.
optional string NodeType = 32;

// Endpoint storage details.
Expand All @@ -118,6 +119,7 @@ message TServerConfig
optional TAffinity NbdAffinity = 38;

// Dynamic node registration params.
// The following 3 are deprecated in favor of params in storage config
optional uint32 NodeRegistrationMaxAttempts = 39;
optional uint32 NodeRegistrationTimeout = 40; // in ms
optional uint32 NodeRegistrationErrorTimeout = 41; // in ms
Expand Down Expand Up @@ -196,6 +198,7 @@ message TServerConfig
optional bool EndpointStorageNotImplementedErrorIsFatal = 116;

// auth token for node registration via ydb discovery api.
// deprecated. moved to storage config.
optional string NodeRegistrationToken = 117;

// How many seconds will the external vhost server continue to work after
Expand Down
20 changes: 20 additions & 0 deletions cloud/blockstore/config/storage.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ syntax = "proto2";
package NCloud.NBlockStore.NProto;

import "cloud/storage/core/protos/authorization_mode.proto";
import "cloud/storage/core/protos/certificate.proto";

option go_package = "a.yandex-team.ru/cloud/blockstore/config";

Expand Down Expand Up @@ -1007,4 +1008,23 @@ message TStorageServiceConfig

// Resync range, if scrubbing found a mismatch.
optional bool ResyncRangeAfterScrubbing = 376;

// Delay that DR uses to wait unavailable agents to register before allowing
// infra to deploy a host.
optional uint32 IdleAgentDeployByCmsDelay = 377;

// Dynamic node registration params.
optional uint32 NodeRegistrationMaxAttempts = 378;
optional uint32 NodeRegistrationTimeout = 379; // in ms
optional uint32 NodeRegistrationErrorTimeout = 380; // in ms

// auth token for node registration via ydb discovery api.
optional string NodeRegistrationToken = 381;

// TLS node registration details.
optional string NodeRegistrationRootCertsFile = 382;
optional NCloud.NProto.TCertificate NodeRegistrationCert = 383;

// Node type
optional string NodeType = 384;
}
19 changes: 10 additions & 9 deletions cloud/blockstore/libs/daemon/ydb/bootstrap.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,17 +217,18 @@ void TBootstrapYdb::InitKikimrService()
Log,
PostponedCriticalEvents);

const auto& cert = Configs->StorageConfig->GetNodeRegistrationCert();

NCloud::NStorage::TNodeRegistrationSettings settings {
.MaxAttempts =
Configs->ServerConfig->GetNodeRegistrationMaxAttempts(),
.ErrorTimeout = Configs->ServerConfig->GetNodeRegistrationErrorTimeout(),
.RegistrationTimeout = Configs->ServerConfig->GetNodeRegistrationTimeout(),
.PathToGrpcCaFile = Configs->ServerConfig->GetRootCertsFile(),
.PathToGrpcCertFile = GetCertFileFromConfig(*Configs->ServerConfig),
.PathToGrpcPrivateKeyFile = GetCertPrivateKeyFileFromConfig(
*Configs->ServerConfig),
.NodeRegistrationToken = Configs->ServerConfig->GetNodeRegistrationToken(),
.NodeType = Configs->ServerConfig->GetNodeType(),
Configs->StorageConfig->GetNodeRegistrationMaxAttempts(),
.ErrorTimeout = Configs->StorageConfig->GetNodeRegistrationErrorTimeout(),
.RegistrationTimeout = Configs->StorageConfig->GetNodeRegistrationTimeout(),
.PathToGrpcCaFile = Configs->StorageConfig->GetNodeRegistrationRootCertsFile(),
.PathToGrpcCertFile = cert.CertFile,
.PathToGrpcPrivateKeyFile = cert.CertPrivateKeyFile,
.NodeRegistrationToken = Configs->StorageConfig->GetNodeRegistrationToken(),
.NodeType = Configs->StorageConfig->GetNodeType(),
};

NCloud::NStorage::TRegisterDynamicNodeOptions registerOpts {
Expand Down
70 changes: 62 additions & 8 deletions cloud/blockstore/libs/daemon/ydb/config_initializer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ void TConfigInitializerYdb::InitStorageConfig()
ParseProtoTextFromFileRobust(Options->StorageConfig, storageConfig);
}

AdaptNodeRegistrationParams(storageConfig);

SetupStorageConfig(storageConfig);

if (Options->SchemeShardDir) {
Expand Down Expand Up @@ -262,8 +264,14 @@ void TConfigInitializerYdb::ApplyStorageServiceConfig(const TString& text)
NProto::TStorageServiceConfig storageConfig;
ParseProtoTextFromStringRobust(text, storageConfig);

AdaptNodeRegistrationParams(storageConfig);

SetupStorageConfig(storageConfig);

if (Options->TemporaryServer) {
storageConfig.SetDisableManuallyPreemptedVolumesTracking(true);
}

StorageConfig = std::make_shared<NStorage::TStorageConfig>(
storageConfig,
FeaturesConfig);
Expand Down Expand Up @@ -332,10 +340,22 @@ void TConfigInitializerYdb::ApplyComputeClientConfig(const TString& text)

void TConfigInitializerYdb::ApplyCustomCMSConfigs(const NKikimrConfig::TAppConfig& config)
{
THashMap<TString, ui32> configs;

for (int i = 0; i < config.GetNamedConfigs().size(); ++i) {
TStringBuf name = config.GetNamedConfigs(i).GetName();

if (!name.SkipPrefix("Cloud.NBS.")) {
continue;
}

configs.emplace(name, i);
}

using TSelf = TConfigInitializerYdb;
using TApplyFn = void (TSelf::*)(const TString&);

const THashMap<TString, TApplyFn> map {
const TVector<std::pair<TString, TApplyFn>> configHandlers {
{ "ActorSystemConfig", &TSelf::ApplyActorSystemConfig },
{ "AuthConfig", &TSelf::ApplyAuthConfig },
{ "DiagnosticsConfig", &TSelf::ApplyDiagnosticsConfig },
Expand All @@ -357,16 +377,50 @@ void TConfigInitializerYdb::ApplyCustomCMSConfigs(const NKikimrConfig::TAppConfi
{ "ComputeClientConfig", &TSelf::ApplyComputeClientConfig },
};

for (auto& item : config.GetNamedConfigs()) {
TStringBuf name = item.GetName();
if (!name.SkipPrefix("Cloud.NBS.")) {
for (const auto& handler: configHandlers) {
auto it = configs.find(handler.first);
if (it == configs.end()) {
continue;
}

auto it = map.find(name);
if (it != map.end()) {
std::invoke(it->second, this, item.GetConfig());
}
std::invoke(
handler.second,
this,
config.GetNamedConfigs(it->second).GetConfig());
}
}

void TConfigInitializerYdb::AdaptNodeRegistrationParams(
NProto::TStorageServiceConfig& config)
{
if (!ServerConfig || !ServerConfig->GetServerConfig()) {
return;
}

const auto* serverConfig = ServerConfig->GetServerConfig();

if (!config.GetNodeRegistrationMaxAttempts()) {
config.SetNodeRegistrationMaxAttempts(
serverConfig->GetNodeRegistrationMaxAttempts());
}

if (!config.GetNodeRegistrationTimeout()) {
config.SetNodeRegistrationTimeout(
serverConfig->GetNodeRegistrationTimeout());
}

if (!config.GetNodeRegistrationErrorTimeout()) {
config.SetNodeRegistrationErrorTimeout(
serverConfig->GetNodeRegistrationErrorTimeout());
}

if (!config.GetNodeRegistrationToken()) {
config.SetNodeRegistrationToken(
serverConfig->GetNodeRegistrationToken());
}

if (!config.GetNodeType()) {
config.SetNodeType(serverConfig->GetNodeType());
}
}

Expand Down
1 change: 1 addition & 0 deletions cloud/blockstore/libs/daemon/ydb/config_initializer.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ struct TConfigInitializerYdb final

private:
void SetupStorageConfig(NProto::TStorageServiceConfig& config) const;
void AdaptNodeRegistrationParams(NProto::TStorageServiceConfig& config);

void ApplyDiagnosticsConfig(const TString& text);
void ApplyDiscoveryServiceConfig(const TString& text);
Expand Down
151 changes: 151 additions & 0 deletions cloud/blockstore/libs/daemon/ydb/config_initializer_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,157 @@ Y_UNIT_TEST_SUITE(TConfigInitializerTest)
UNIT_ASSERT(ci.KikimrConfig->GetFeatureFlags().GetEnableVPatch());
}
}

Y_UNIT_TEST(ShouldAdaptNodeRegistrationParams)
{
TTempDir dir;

auto serverConfigPath = dir.Path() / "nbs-server.txt";

auto serverConfigStr = R"(ServerConfig {
NodeRegistrationMaxAttempts: 100
NodeRegistrationTimeout: 200
NodeRegistrationErrorTimeout: 300
NodeRegistrationToken: "xyz"
NodeType: "abc"
}
)";

TOFStream(serverConfigPath.GetPath()).Write(serverConfigStr);

auto storageConfigPath = dir.Path() / "nbs-storage.txt";

TOFStream(storageConfigPath.GetPath()).Write("");

auto options = CreateOptions();
options->ServerConfig = serverConfigPath.GetPath();
options->StorageConfig = storageConfigPath.GetPath();

auto ci = TConfigInitializerYdb(std::move(options));
ci.InitServerConfig();
ci.InitStorageConfig();

const auto& proto = ci.StorageConfig->GetStorageConfigProto();

UNIT_ASSERT_VALUES_EQUAL(100, proto.GetNodeRegistrationMaxAttempts());
UNIT_ASSERT_VALUES_EQUAL(200, proto.GetNodeRegistrationTimeout());
UNIT_ASSERT_VALUES_EQUAL(300, proto.GetNodeRegistrationErrorTimeout());
UNIT_ASSERT_VALUES_EQUAL("xyz", proto.GetNodeRegistrationToken());
UNIT_ASSERT_VALUES_EQUAL("abc", proto.GetNodeType());
}

Y_UNIT_TEST(ShouldNotReplaceNodeRegistrationParamsInStorageConfig)
{
TTempDir dir;

auto serverConfigPath = dir.Path() / "nbs-server.txt";

auto serverConfigStr = R"(ServerConfig {
NodeRegistrationMaxAttempts: 100
NodeRegistrationTimeout: 200
NodeRegistrationErrorTimeout: 300
NodeRegistrationToken: "xyz"
NodeType: "abc"
}
)";

TOFStream(serverConfigPath.GetPath()).Write(serverConfigStr);

auto storageConfigPath = dir.Path() / "nbs-storage.txt";

auto storageConfigStr = R"(
NodeType: "123"
)";

TOFStream(storageConfigPath.GetPath()).Write(storageConfigStr);

auto options = CreateOptions();
options->ServerConfig = serverConfigPath.GetPath();
options->StorageConfig = storageConfigPath.GetPath();

auto ci = TConfigInitializerYdb(std::move(options));
ci.InitServerConfig();
ci.InitStorageConfig();

const auto& proto = ci.StorageConfig->GetStorageConfigProto();

UNIT_ASSERT_VALUES_EQUAL(100, proto.GetNodeRegistrationMaxAttempts());
UNIT_ASSERT_VALUES_EQUAL(200, proto.GetNodeRegistrationTimeout());
UNIT_ASSERT_VALUES_EQUAL(300, proto.GetNodeRegistrationErrorTimeout());
UNIT_ASSERT_VALUES_EQUAL("xyz", proto.GetNodeRegistrationToken());
UNIT_ASSERT_VALUES_EQUAL("123", proto.GetNodeType());
}

Y_UNIT_TEST(ShouldAdaptNodeRegistrationParamsWhebLoadingFromCms)
{
auto ci = TConfigInitializerYdb(CreateOptions());
ci.InitStorageConfig();

NKikimrConfig::TAppConfig appCfg;
auto* serverCfg = appCfg.MutableNamedConfigs()->Add();

serverCfg->SetName("Cloud.NBS.ServerAppConfig");
auto serverConfigStr = R"(ServerConfig {
NodeRegistrationMaxAttempts: 100
NodeRegistrationTimeout: 200
NodeRegistrationErrorTimeout: 300
NodeRegistrationToken: "xyz"
NodeType: "abc"
}
)";
serverCfg->SetConfig(serverConfigStr);

auto* storageCfg = appCfg.MutableNamedConfigs()->Add();
storageCfg->SetName("Cloud.NBS.StorageServiceConfig");
storageCfg->SetConfig("");

ci.ApplyCustomCMSConfigs(appCfg);

const auto& proto = ci.StorageConfig->GetStorageConfigProto();

UNIT_ASSERT_VALUES_EQUAL(100, proto.GetNodeRegistrationMaxAttempts());
UNIT_ASSERT_VALUES_EQUAL(200, proto.GetNodeRegistrationTimeout());
UNIT_ASSERT_VALUES_EQUAL(300, proto.GetNodeRegistrationErrorTimeout());
UNIT_ASSERT_VALUES_EQUAL("xyz", proto.GetNodeRegistrationToken());
UNIT_ASSERT_VALUES_EQUAL("abc", proto.GetNodeType());
}

Y_UNIT_TEST(ShouldNotReplaceNodeRegistrationParamsInStorageConfigWithCms)
{
auto ci = TConfigInitializerYdb(CreateOptions());
ci.InitStorageConfig();

NKikimrConfig::TAppConfig appCfg;
auto* serverCfg = appCfg.MutableNamedConfigs()->Add();

serverCfg->SetName("Cloud.NBS.ServerAppConfig");
auto serverConfigStr = R"(ServerConfig {
NodeRegistrationMaxAttempts: 100
NodeRegistrationTimeout: 200
NodeRegistrationErrorTimeout: 300
NodeRegistrationToken: "xyz"
NodeType: "abc"
}
)";
serverCfg->SetConfig(serverConfigStr);

auto* storageCfg = appCfg.MutableNamedConfigs()->Add();
storageCfg->SetName("Cloud.NBS.StorageServiceConfig");
auto storageConfigStr = R"(
NodeType: "123"
)";
storageCfg->SetConfig(storageConfigStr);

ci.ApplyCustomCMSConfigs(appCfg);

const auto& proto = ci.StorageConfig->GetStorageConfigProto();

UNIT_ASSERT_VALUES_EQUAL(100, proto.GetNodeRegistrationMaxAttempts());
UNIT_ASSERT_VALUES_EQUAL(200, proto.GetNodeRegistrationTimeout());
UNIT_ASSERT_VALUES_EQUAL(300, proto.GetNodeRegistrationErrorTimeout());
UNIT_ASSERT_VALUES_EQUAL("xyz", proto.GetNodeRegistrationToken());
UNIT_ASSERT_VALUES_EQUAL("123", proto.GetNodeType());
}
}

} // namespace NCloud::NBlockStore::NServer
19 changes: 10 additions & 9 deletions cloud/blockstore/libs/disk_agent/bootstrap.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,17 +289,18 @@ void TBootstrap::InitKikimrService()
Configs->InitDiagnosticsConfig();
Configs->InitSpdkEnvConfig();

const auto& cert = Configs->StorageConfig->GetNodeRegistrationCert();

NCloud::NStorage::TNodeRegistrationSettings settings {
.MaxAttempts =
Configs->ServerConfig->GetNodeRegistrationMaxAttempts(),
.ErrorTimeout = Configs->ServerConfig->GetNodeRegistrationErrorTimeout(),
.RegistrationTimeout = Configs->ServerConfig->GetNodeRegistrationTimeout(),
.PathToGrpcCaFile = Configs->ServerConfig->GetRootCertsFile(),
.PathToGrpcCertFile = GetCertFileFromConfig(*Configs->ServerConfig),
.PathToGrpcPrivateKeyFile = GetCertPrivateKeyFileFromConfig(
*Configs->ServerConfig),
.NodeRegistrationToken = Configs->ServerConfig->GetNodeRegistrationToken(),
.NodeType = Configs->Options->NodeType,
Configs->StorageConfig->GetNodeRegistrationMaxAttempts(),
.ErrorTimeout = Configs->StorageConfig->GetNodeRegistrationErrorTimeout(),
.RegistrationTimeout = Configs->StorageConfig->GetNodeRegistrationTimeout(),
.PathToGrpcCaFile = Configs->StorageConfig->GetNodeRegistrationRootCertsFile(),
.PathToGrpcCertFile = cert.CertFile,
.PathToGrpcPrivateKeyFile = cert.CertPrivateKeyFile,
.NodeRegistrationToken = Configs->StorageConfig->GetNodeRegistrationToken(),
.NodeType = Configs->StorageConfig->GetNodeType(),
};

NCloud::NStorage::TRegisterDynamicNodeOptions registerOpts {
Expand Down
Loading

0 comments on commit f0aab4b

Please sign in to comment.