From 0a25c6a80491e0e456d57224447dde0f69127b32 Mon Sep 17 00:00:00 2001 From: aviv Date: Wed, 17 Apr 2024 11:26:48 +0300 Subject: [PATCH] RavenDB-17793 : Prefixed Sharding : RavenDB-17793 : address PR comments RavenDB-17793 Add sharding prefixes step to database creator RavenDB-17793 : fixes after rebase RavenDB-17793 : address PR comments - throw instead of debug asserts, add more info to error messages RavenDB-17793 : add tests cases for (1) querying with shard context (2) prefixes should get precedence over anchoring when choosing bucket for id RavenDB-17793 : address PR comments - avoid doing remote calls within a read tx RavenDB-17793 : fix failing tests RavenDB-17793 : minor refactoring + cleanup RavenDB-17793 : remove BlockPrefixedSharding from ShardingStore RavenDB-17793 : fix bug in DeletePrefixedSettingCommand - need to remove all associated bucket ranges for this prefix (do not rely on number of shards in prefix setting) RavenDB-17793 : add more test scenarios RavenDB-17793 : do not allow to add prefix setting with no shards RavenDB-17793 : pass prefix to StartBucketMigrationCommand so that we can find the prefix setting by name (sorted) instead of searching by bucket range start (needs to be ordered first) RavenDB-17793 : when adding a new prefix, reuse old bucket ranges (i.e. bucket ranges that were given to a cretin prefix which was removed) if possible RavenDB-17793 : do not allow to remove shard from database if this shard is part of any prefix settings RavenDB-17793 : update prefix setting - don't allow to remove shard if there are bucket ranges mapped to this shards. update command should not modify BucketRanges at all RavenDB-17793 : throw on attempt to move bucket from prefixed range if destination shard is not a part of shards list in the prefix setting RavenDB-17793 : use binary search for lookups in prefixed list RavenDB-17793 : fix bug in sharded restore - do not fill PrefixedSharding on AddDatabaseCommand with no RaftCommandIndex when coming from restore (in restore we take the prefixed setting as is from smuggler record) RavenDB-17793 : prefixed settings list should be sorted in a descending manner, in order to support having a prefix of prefix RavenDB-17793 : ensure that prefixes settings cannot be modified via database record. add dedicated operation for updating prefix configuration --- .../AddPrefixedShardingSettingOperation.cs | 32 + .../DeletePrefixedShardingSettingOperation.cs | 38 + .../Sharding/PrefixedShardingCommand.cs | 53 + .../Sharding/PrefixedShardingSetting.cs | 9 + .../Sharding/ShardingConfiguration.cs | 13 + .../UpdatePrefixedShardingSettingOperation.cs | 33 + .../Handlers/Admin/AdminShardingHandler.cs | 19 + .../Admin/ShardedAdminShardingHandler.cs | 195 +- .../ClusterCommandsVersionManager.cs | 7 +- .../ServerWide/ClusterStateMachine.cs | 25 +- .../Commands/DeleteDatabaseCommand.cs | 13 +- .../AddPrefixedShardingSettingCommand.cs | 76 + .../Sharding/CreateNewShardCommand.cs | 1 - .../DeletePrefixedShardingSettingCommand.cs | 59 + .../Sharding/StartBucketMigrationCommand.cs | 29 +- .../UpdatePrefixedShardingSettingCommand.cs | 37 + .../ServerWide/JsonDeserializationCluster.cs | 7 +- src/Raven.Server/ServerWide/ServerStore.cs | 7 +- .../Sharding/PrefixSettingComparer.cs | 22 + .../Sharding/RawShardingConfiguration.cs | 2 + src/Raven.Server/ServerWide/ShardingStore.cs | 149 +- src/Raven.Server/Utils/ShardHelper.cs | 2 - .../Web/System/AdminDatabasesHandler.cs | 11 +- src/Raven.Server/Web/System/DatabaseHelper.cs | 9 +- .../resources/createDatabaseCommand.ts | 3 +- .../CreateDatabaseFromBackupStepBasicInfo.tsx | 1 + .../create/regular/CreateDatabaseRegular.tsx | 10 + .../regular/createDatabaseRegularDataUtils.ts | 15 + .../createDatabaseRegularValidation.ts | 40 + .../CreateDatabaseRegularStepBasicInfo.tsx | 1 + ...abaseRegularStepReplicationAndSharding.tsx | 21 +- .../regular/steps/StepShardingPrefixes.tsx | 160 ++ test/FastTests/Client/RavenCommandTest.cs | 3 +- .../DocumentsMigrationTests.cs | 1 - test/SlowTests/Sharding/PrefixedSharding.cs | 2539 ++++++++++++++--- .../ShardedSubscriptionSlowTests.cs | 7 +- .../RavenTestBase.ReshardingTestBase.cs | 27 +- .../RavenTestBase.Sharding.cs | 11 +- 38 files changed, 3169 insertions(+), 518 deletions(-) create mode 100644 src/Raven.Client/ServerWide/Sharding/AddPrefixedShardingSettingOperation.cs create mode 100644 src/Raven.Client/ServerWide/Sharding/DeletePrefixedShardingSettingOperation.cs create mode 100644 src/Raven.Client/ServerWide/Sharding/PrefixedShardingCommand.cs create mode 100644 src/Raven.Client/ServerWide/Sharding/UpdatePrefixedShardingSettingOperation.cs create mode 100644 src/Raven.Server/ServerWide/Commands/Sharding/AddPrefixedShardingSettingCommand.cs create mode 100644 src/Raven.Server/ServerWide/Commands/Sharding/DeletePrefixedShardingSettingCommand.cs create mode 100644 src/Raven.Server/ServerWide/Commands/Sharding/UpdatePrefixedShardingSettingCommand.cs create mode 100644 src/Raven.Server/ServerWide/Sharding/PrefixSettingComparer.cs create mode 100644 src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/StepShardingPrefixes.tsx diff --git a/src/Raven.Client/ServerWide/Sharding/AddPrefixedShardingSettingOperation.cs b/src/Raven.Client/ServerWide/Sharding/AddPrefixedShardingSettingOperation.cs new file mode 100644 index 000000000000..bc7b54850063 --- /dev/null +++ b/src/Raven.Client/ServerWide/Sharding/AddPrefixedShardingSettingOperation.cs @@ -0,0 +1,32 @@ +using System; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Http; +using Sparrow.Json; + +namespace Raven.Client.ServerWide.Sharding +{ + public sealed class AddPrefixedShardingSettingOperation : IMaintenanceOperation + { + private readonly PrefixedShardingSetting _setting; + + public AddPrefixedShardingSettingOperation(PrefixedShardingSetting setting) + { + _setting = setting ?? throw new ArgumentNullException(nameof(setting)); + } + + public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext ctx) + { + return new AddPrefixedShardingSettingCommand(conventions, _setting); + } + + private sealed class AddPrefixedShardingSettingCommand : PrefixedShardingCommand + { + protected override PrefixedCommandType CommandType => PrefixedCommandType.Add; + + public AddPrefixedShardingSettingCommand(DocumentConventions conventions, PrefixedShardingSetting setting) : base(conventions, setting) + { + } + } + } +} diff --git a/src/Raven.Client/ServerWide/Sharding/DeletePrefixedShardingSettingOperation.cs b/src/Raven.Client/ServerWide/Sharding/DeletePrefixedShardingSettingOperation.cs new file mode 100644 index 000000000000..b0a24971ab6a --- /dev/null +++ b/src/Raven.Client/ServerWide/Sharding/DeletePrefixedShardingSettingOperation.cs @@ -0,0 +1,38 @@ +using System; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Http; +using Sparrow.Json; + +namespace Raven.Client.ServerWide.Sharding +{ + public sealed class DeletePrefixedShardingSettingOperation : IMaintenanceOperation + { + private readonly string _prefix; + + public DeletePrefixedShardingSettingOperation(string prefix) + { + if (string.IsNullOrEmpty(prefix)) + throw new ArgumentNullException(nameof(prefix)); + + _prefix = prefix; + } + + public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext ctx) + { + return new DeletePrefixedShardingSettingCommand(conventions, new PrefixedShardingSetting + { + Prefix = _prefix + }); + } + + private sealed class DeletePrefixedShardingSettingCommand : PrefixedShardingCommand + { + protected override PrefixedCommandType CommandType => PrefixedCommandType.Delete; + + public DeletePrefixedShardingSettingCommand(DocumentConventions conventions, PrefixedShardingSetting setting) : base(conventions, setting) + { + } + } + } +} diff --git a/src/Raven.Client/ServerWide/Sharding/PrefixedShardingCommand.cs b/src/Raven.Client/ServerWide/Sharding/PrefixedShardingCommand.cs new file mode 100644 index 000000000000..1a85fa81b23d --- /dev/null +++ b/src/Raven.Client/ServerWide/Sharding/PrefixedShardingCommand.cs @@ -0,0 +1,53 @@ +using System; +using System.Net.Http; +using Raven.Client.Documents.Conventions; +using Raven.Client.Http; +using Raven.Client.Json; +using Raven.Client.Util; +using Sparrow.Json; + +namespace Raven.Client.ServerWide.Sharding +{ + internal abstract class PrefixedShardingCommand : RavenCommand, IRaftCommand + { + private readonly DocumentConventions _conventions; + private readonly PrefixedShardingSetting _setting; + + protected abstract PrefixedCommandType CommandType { get; } + + protected PrefixedShardingCommand(DocumentConventions conventions, PrefixedShardingSetting setting) + { + _conventions = conventions; + _setting = setting; + } + + public override bool IsReadRequest => false; + + public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url) + { + url = $"{node.Url}/databases/{node.Database}/admin/sharding/prefixed"; + + return new HttpRequestMessage + { + Method = CommandType switch + { + PrefixedCommandType.Add => HttpMethod.Put, + PrefixedCommandType.Delete => HttpMethod.Delete, + PrefixedCommandType.Update => HttpMethod.Post, + _ => throw new ArgumentOutOfRangeException() + }, + Content = new BlittableJsonContent(async stream => await ctx.WriteAsync(stream, + DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(_setting, ctx)).ConfigureAwait(false), _conventions) + }; + } + + public string RaftUniqueRequestId { get; } = RaftIdGenerator.NewId(); + } + + internal enum PrefixedCommandType + { + Add = 1, + Delete = 2, + Update = 3 + } +} diff --git a/src/Raven.Client/ServerWide/Sharding/PrefixedShardingSetting.cs b/src/Raven.Client/ServerWide/Sharding/PrefixedShardingSetting.cs index 18d961fc483a..107b055f7b06 100644 --- a/src/Raven.Client/ServerWide/Sharding/PrefixedShardingSetting.cs +++ b/src/Raven.Client/ServerWide/Sharding/PrefixedShardingSetting.cs @@ -11,6 +11,15 @@ public sealed class PrefixedShardingSetting private byte[] _prefixBytesLowerCase; + public PrefixedShardingSetting() + { + } + + public PrefixedShardingSetting(string prefix) + { + Prefix = prefix; + } + internal byte[] PrefixBytesLowerCase => _prefixBytesLowerCase ??= Encoding.UTF8.GetBytes(Prefix.ToLower()); public List Shards { get; set; } diff --git a/src/Raven.Client/ServerWide/Sharding/ShardingConfiguration.cs b/src/Raven.Client/ServerWide/Sharding/ShardingConfiguration.cs index 669a24c3b7b3..aedfd6c1ffb4 100644 --- a/src/Raven.Client/ServerWide/Sharding/ShardingConfiguration.cs +++ b/src/Raven.Client/ServerWide/Sharding/ShardingConfiguration.cs @@ -44,4 +44,17 @@ internal bool HasActiveMigrations() return false; } + + internal bool DoesShardHavePrefixes(int shardNumber) => DoesShardHavePrefixes(Prefixed, shardNumber); + + internal static bool DoesShardHavePrefixes(List prefixes, int shardNumber) + { + foreach (var setting in prefixes) + { + if (setting.Shards.Contains(shardNumber)) + return true; + } + + return false; + } } diff --git a/src/Raven.Client/ServerWide/Sharding/UpdatePrefixedShardingSettingOperation.cs b/src/Raven.Client/ServerWide/Sharding/UpdatePrefixedShardingSettingOperation.cs new file mode 100644 index 000000000000..7e3bfeb4f626 --- /dev/null +++ b/src/Raven.Client/ServerWide/Sharding/UpdatePrefixedShardingSettingOperation.cs @@ -0,0 +1,33 @@ +using System; +using Raven.Client.Documents.Conventions; +using Raven.Client.Documents.Operations; +using Raven.Client.Http; +using Sparrow.Json; + +namespace Raven.Client.ServerWide.Sharding +{ + public sealed class UpdatePrefixedShardingSettingOperation : IMaintenanceOperation + { + private readonly PrefixedShardingSetting _setting; + + public UpdatePrefixedShardingSettingOperation(PrefixedShardingSetting setting) + { + _setting = setting ?? throw new ArgumentNullException(nameof(setting)); + } + + public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext ctx) + { + return new UpdatePrefixedShardingSettingCommand(conventions, _setting); + } + + private sealed class UpdatePrefixedShardingSettingCommand : PrefixedShardingCommand + { + public UpdatePrefixedShardingSettingCommand(DocumentConventions conventions, PrefixedShardingSetting setting) : base(conventions, setting) + { + } + + protected override PrefixedCommandType CommandType => PrefixedCommandType.Update; + + } + } +} diff --git a/src/Raven.Server/Documents/Handlers/Admin/AdminShardingHandler.cs b/src/Raven.Server/Documents/Handlers/Admin/AdminShardingHandler.cs index b470847b6b65..a5718068f9fd 100644 --- a/src/Raven.Server/Documents/Handlers/Admin/AdminShardingHandler.cs +++ b/src/Raven.Server/Documents/Handlers/Admin/AdminShardingHandler.cs @@ -1,6 +1,7 @@ using System; using System.Net; using System.Threading.Tasks; +using Raven.Client.Exceptions.Sharding; using Raven.Server.Documents.Sharding; using Raven.Server.Routing; using Raven.Server.Utils; @@ -22,6 +23,24 @@ public async Task ExecuteMoveDocuments() HttpContext.Response.StatusCode = (int)HttpStatusCode.NoContent; } + [RavenAction("/databases/*/admin/sharding/prefixed", "PUT", AuthorizationStatus.DatabaseAdmin)] + public Task AddPrefixedShardingSetting() + { + throw new NotSupportedInShardingException("This operation is not available from a specific shard"); + } + + [RavenAction("/databases/*/admin/sharding/prefixed", "DELETE", AuthorizationStatus.DatabaseAdmin)] + public Task DeletePrefixedShardingSetting() + { + throw new NotSupportedInShardingException("This operation is not available from a specific shard"); + } + + [RavenAction("/databases/*/admin/sharding/prefixed", "POST", AuthorizationStatus.DatabaseAdmin)] + public Task UpdatePrefixedShardingSetting() + { + throw new NotSupportedInShardingException("This operation is not available from a specific shard"); + } + private void ValidateShardDatabaseName() { if (ShardHelper.IsShardName(DatabaseName) == false) diff --git a/src/Raven.Server/Documents/Sharding/Handlers/Admin/ShardedAdminShardingHandler.cs b/src/Raven.Server/Documents/Sharding/Handlers/Admin/ShardedAdminShardingHandler.cs index a612baac019d..702a47d43ff3 100644 --- a/src/Raven.Server/Documents/Sharding/Handlers/Admin/ShardedAdminShardingHandler.cs +++ b/src/Raven.Server/Documents/Sharding/Handlers/Admin/ShardedAdminShardingHandler.cs @@ -1,6 +1,22 @@ -using System.Threading.Tasks; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Raven.Client.Documents.Commands; +using Raven.Client.Documents.Conventions; +using Raven.Client.Http; +using Raven.Client.ServerWide.Sharding; using Raven.Server.Documents.Sharding.Handlers.Processors; using Raven.Server.Routing; +using Raven.Server.ServerWide; +using Raven.Server.ServerWide.Commands.Sharding; +using Raven.Server.ServerWide.Context; +using Raven.Server.ServerWide.Sharding; +using Raven.Server.Utils; +using Sparrow.Json; +using ShardingConfiguration = Raven.Client.ServerWide.Sharding.ShardingConfiguration; namespace Raven.Server.Documents.Sharding.Handlers.Admin { @@ -13,5 +29,182 @@ public async Task ExecuteMoveDocuments() "This operation is available only from a specific shard")) await processor.ExecuteAsync(); } + + [RavenShardedAction("/databases/*/admin/sharding/prefixed", "PUT")] + public async Task AddPrefixedShardingSetting() + { + using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) + { + var json = await context.ReadForMemoryAsync(RequestBodyStream(), GetType().Name); + var setting = JsonDeserializationCluster.PrefixedShardingSetting(json); + var shardingConfiguration = ServerStore.Cluster.ReadShardingConfiguration(DatabaseName); + + ShardingStore.AssertValidPrefix(setting, shardingConfiguration); + + var exists = shardingConfiguration.Prefixed.BinarySearch(setting, PrefixedSettingComparer.Instance) >= 0; + if (exists) + throw new InvalidOperationException( + $"Prefix '{setting.Prefix}' already exists in {nameof(ShardingConfiguration)}.{nameof(ShardingConfiguration.Prefixed)}. Please use '{nameof(UpdatePrefixedShardingSettingOperation)}' operation."); + + string[] urls; + using (context.OpenReadTransaction()) + { + var clusterTopology = ServerStore.GetClusterTopology(context); + urls = shardingConfiguration.Orchestrator.Topology.Members.Select(clusterTopology.GetUrlFromTag).ToArray(); + } + + if (await AssertNoDocumentsStartingWith(context, setting.Prefix, urls) == false) + throw new InvalidOperationException( + $"Cannot add prefix '{setting.Prefix}' to {nameof(ShardingConfiguration)}.{nameof(ShardingConfiguration.Prefixed)}. " + + $"There are existing documents in database '{DatabaseName}' that start with '{setting.Prefix}'. " + + "In order to define sharding by prefix, you cannot have any documents in the database that starts with this prefix."); + + var cmd = new AddPrefixedShardingSettingCommand(setting, DatabaseName, GetRaftRequestIdFromQuery()); + var (raftIndex, _) = await ServerStore.SendToLeaderAsync(cmd); + + await DatabaseContext.ServerStore.WaitForExecutionOnRelevantNodesAsync(context, shardingConfiguration.Orchestrator.Topology.Members, raftIndex); + + HttpContext.Response.StatusCode = (int)HttpStatusCode.NoContent; + } + } + + [RavenShardedAction("/databases/*/admin/sharding/prefixed", "DELETE")] + public async Task DeletePrefixedShardingSetting() + { + using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) + { + var json = await context.ReadForMemoryAsync(RequestBodyStream(), GetType().Name); + var setting = JsonDeserializationCluster.PrefixedShardingSetting(json); + + var shardingConfiguration = ServerStore.Cluster.ReadShardingConfiguration(DatabaseName); + bool found = shardingConfiguration.Prefixed.BinarySearch(setting, PrefixedSettingComparer.Instance) >= 0; + if (found == false) + throw new InvalidDataException($"Prefix '{setting.Prefix}' wasn't found in sharding configuration"); + + string[] urls; + using (context.OpenReadTransaction()) + { + var clusterTopology = ServerStore.GetClusterTopology(context); + urls = shardingConfiguration.Orchestrator.Topology.Members.Select(clusterTopology.GetUrlFromTag).ToArray(); + } + + if (await AssertNoDocumentsStartingWith(context, setting.Prefix, urls) == false) + throw new InvalidOperationException( + $"Cannot remove prefix '{setting.Prefix}' from {nameof(ShardingConfiguration)}.{nameof(ShardingConfiguration.Prefixed)}. " + + $"There are existing documents in database '{DatabaseName}' that start with '{setting.Prefix}'. " + + "In order to remove a sharding by prefix setting, you cannot have any documents in the database that starts with this prefix."); + + var cmd = new DeletePrefixedShardingSettingCommand(setting, DatabaseName, GetRaftRequestIdFromQuery()); + var (raftIndex, _) = await ServerStore.SendToLeaderAsync(cmd); + + await DatabaseContext.ServerStore.WaitForExecutionOnRelevantNodesAsync(context, shardingConfiguration.Orchestrator.Topology.Members, raftIndex); + + HttpContext.Response.StatusCode = (int)HttpStatusCode.NoContent; + } + } + + [RavenShardedAction("/databases/*/admin/sharding/prefixed", "POST")] + public async Task UpdatePrefixedShardingSetting() + { + using (ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) + { + var json = await context.ReadForMemoryAsync(RequestBodyStream(), GetType().Name); + var setting = JsonDeserializationCluster.PrefixedShardingSetting(json); + + var shardingConfiguration = ServerStore.Cluster.ReadShardingConfiguration(DatabaseName); + + var location = shardingConfiguration.Prefixed.BinarySearch(setting, PrefixedSettingComparer.Instance); + if (location < 0) + throw new InvalidDataException($"Prefix '{setting.Prefix}' wasn't found in sharding configuration"); + + var oldSetting = shardingConfiguration.Prefixed[location]; + AssertValidShardsDistribution(oldSetting, setting, shardingConfiguration); + + var cmd = new UpdatePrefixedShardingSettingCommand(setting, DatabaseName, GetRaftRequestIdFromQuery()); + var (raftIndex, _) = await ServerStore.SendToLeaderAsync(cmd); + + await DatabaseContext.ServerStore.WaitForExecutionOnRelevantNodesAsync(context, shardingConfiguration.Orchestrator.Topology.Members, raftIndex); + + HttpContext.Response.StatusCode = (int)HttpStatusCode.NoContent; + } + } + + private static void AssertValidShardsDistribution(PrefixedShardingSetting oldSetting, PrefixedShardingSetting updatedSetting, ShardingConfiguration configuration) + { + var removedShards = oldSetting.Shards; + + foreach (var shard in updatedSetting.Shards) + { + if (oldSetting.Shards.Contains(shard)) + removedShards.Remove(shard); + else if (configuration.Shards.ContainsKey(shard) == false) + throw new InvalidDataException($"Cannot assign shard number {shard} to prefix {updatedSetting.Prefix}, " + + $"there's no shard '{shard}' in sharding topology!"); + } + + if (removedShards.Count <= 0) + return; + + // check that there are no bucket ranges mapped to these shards + + int index; + bool found = false; + var prefixBucketRangeStart = oldSetting.BucketRangeStart; + + for (index = 0; index < configuration.BucketRanges.Count; index++) + { + var range = configuration.BucketRanges[index]; + if (range.BucketRangeStart < prefixBucketRangeStart) + continue; + + if (range.BucketRangeStart == prefixBucketRangeStart) + found = true; + + break; + } + + if (found == false) + return; + + var shards = new List + { + configuration.BucketRanges[index++].ShardNumber + }; + + var nextPrefixedRangeStart = prefixBucketRangeStart + ShardHelper.NumberOfBuckets; + for (; index < configuration.BucketRanges.Count; index++) + { + var range = configuration.BucketRanges[index]; + if (range.BucketRangeStart < nextPrefixedRangeStart) + { + shards.Add(range.ShardNumber); + continue; + } + + break; + } + + foreach (var shard in removedShards) + { + if (shards.Contains(shard)) + throw new InvalidOperationException( + $"Cannot remove shard {shard} from '{updatedSetting.Prefix}' settings in {nameof(ShardingConfiguration)}.{nameof(ShardingConfiguration.Prefixed)}. " + + $"There are bucket ranges mapped to this shard. " + + "In order to remove a shard from a Prefixed setting, first you need to migrate all its buckets to another shard."); + + } + } + + private async Task AssertNoDocumentsStartingWith(JsonOperationContext context, string prefix, string[] urls, string database = null) + { + using (var requestExecutor = RequestExecutor.CreateForServer(urls, database ?? DatabaseName, ServerStore.Server.Certificate.Certificate, DocumentConventions.DefaultForServer)) + { + var command = new GetDocumentsCommand(requestExecutor.Conventions, startWith: prefix, + startAfter: null, matches: null, exclude: null, start: 0, pageSize: 1, metadataOnly: false); + + await requestExecutor.ExecuteAsync(command, context, sessionInfo: null); + return command.Result.Results.Length == 0; + } + } } } diff --git a/src/Raven.Server/ServerWide/ClusterCommandsVersionManager.cs b/src/Raven.Server/ServerWide/ClusterCommandsVersionManager.cs index ed6c3f125878..d2b74988bfdc 100644 --- a/src/Raven.Server/ServerWide/ClusterCommandsVersionManager.cs +++ b/src/Raven.Server/ServerWide/ClusterCommandsVersionManager.cs @@ -181,9 +181,14 @@ public sealed class ClusterCommandsVersionManager [nameof(UpdateQueueSinkCommand)] = 60_000, [nameof(RemoveQueueSinkProcessStateCommand)] = 60_000, [nameof(UpdateQueueSinkProcessStateCommand)] = 60_000, - + [nameof(EditDataArchivalCommand)] = 60_000, + [nameof(UpdateResponsibleNodeForTasksCommand)] = UpdateResponsibleNodeForTasksCommand.CommandVersion, + [nameof(AddPrefixedShardingSettingCommand)] = 61_000, + [nameof(DeletePrefixedShardingSettingCommand)] = 61_000, + [nameof(UpdatePrefixedShardingSettingCommand)] = 61_000 + }; public bool CanPutCommand(string command) diff --git a/src/Raven.Server/ServerWide/ClusterStateMachine.cs b/src/Raven.Server/ServerWide/ClusterStateMachine.cs index a796ac2569ea..b7d953d74315 100644 --- a/src/Raven.Server/ServerWide/ClusterStateMachine.cs +++ b/src/Raven.Server/ServerWide/ClusterStateMachine.cs @@ -71,6 +71,7 @@ using Voron.Data.Tables; using Voron.Impl; using Constants = Raven.Client.Constants; +using ShardingConfiguration = Raven.Client.ServerWide.Sharding.ShardingConfiguration; namespace Raven.Server.ServerWide { @@ -486,6 +487,9 @@ protected override void Apply(ClusterOperationContext context, BlittableJsonRead case nameof(SourceMigrationSendCompletedCommand): case nameof(DestinationMigrationConfirmCommand): case nameof(SourceMigrationCleanupCommand): + case nameof(AddPrefixedShardingSettingCommand): + case nameof(DeletePrefixedShardingSettingCommand): + case nameof(UpdatePrefixedShardingSettingCommand): UpdateDatabase(context, type, cmd, index, serverStore); break; @@ -1795,12 +1799,12 @@ private unsafe List AddDatabase(ClusterOperationContext context, string using (var oldDatabaseRecord = ReadRawDatabaseRecord(context, addDatabaseCommand.Name)) { VerifyUnchangedTasks(oldDatabaseRecord?.Raw); + VerifyUnchangedPrefixedSetting(oldDatabaseRecord?.Raw); shouldSetClientConfigEtag = ShouldSetClientConfigEtag(newDatabaseRecord, oldDatabaseRecord?.Raw); } VerifyIndexNames(newDatabaseRecord); VerifyCustomSorters(); - using (var databaseRecordAsJson = UpdateDatabaseRecordIfNeeded(databaseExists, shouldSetClientConfigEtag, index, addDatabaseCommand, newDatabaseRecord, context)) { UpdateValue(index, items, valueNameLowered, valueName, databaseRecordAsJson); @@ -1897,6 +1901,25 @@ void VerifyCustomSorters() throw new RachisInvalidOperationException("Custom sorting is not supported in sharding as of yet"); } + + void VerifyUnchangedPrefixedSetting(BlittableJsonReaderObject dbDoc) + { + if (dbDoc == null || addDatabaseCommand.Record.IsSharded == false || addDatabaseCommand.IsRestore) + return; + + BlittableJsonReaderArray prefixedSetting = null, newPrefixedSetting = null; + + if (dbDoc.TryGet(nameof(DatabaseRecord.Sharding), out BlittableJsonReaderObject shardingConfig)) + shardingConfig.TryGet(nameof(ShardingConfiguration.Prefixed), out prefixedSetting); + + if (newDatabaseRecord.TryGet(nameof(DatabaseRecord.Sharding), out BlittableJsonReaderObject newConfig)) + newConfig.TryGet(nameof(DatabaseRecord.Sharding.Prefixed), out newPrefixedSetting); + + if (newPrefixedSetting?.Length != prefixedSetting?.Length || + newPrefixedSetting?.Equals(prefixedSetting) == false) + throw new RachisInvalidOperationException($"Cannot update {nameof(ShardingConfiguration.Prefixed)} configuration with DatabaseRecord. " + + $"Please use a dedicated operation to update the {nameof(ShardingConfiguration.Prefixed)} configuration."); + } } } catch (Exception e) diff --git a/src/Raven.Server/ServerWide/Commands/DeleteDatabaseCommand.cs b/src/Raven.Server/ServerWide/Commands/DeleteDatabaseCommand.cs index a19f06550266..fc5bc19d1ff9 100644 --- a/src/Raven.Server/ServerWide/Commands/DeleteDatabaseCommand.cs +++ b/src/Raven.Server/ServerWide/Commands/DeleteDatabaseCommand.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using Raven.Client.Exceptions.Database; using Raven.Client.ServerWide; @@ -59,10 +60,14 @@ public override void UpdateDatabaseRecord(DatabaseRecord record, long etag) if (record.Sharding.Shards.TryGetValue(ShardNumber.Value, out var topology) == false) throw new RachisApplyException($"The requested shard '{ShardNumber}' doesn't exists in '{record.DatabaseName}'"); - if (topology.ReplicationFactor == 1 && record.Sharding.DoesShardHaveBuckets(ShardNumber.Value)) + if (topology.ReplicationFactor == 1) { - throw new RachisApplyException( - $"Database {DatabaseName} cannot be deleted because it is the last copy of shard {ShardNumber.Value} and it still contains buckets."); + if (record.Sharding.DoesShardHaveBuckets(ShardNumber.Value)) + throw new RachisApplyException( + $"Database {DatabaseName} cannot be deleted because it is the last copy of shard {ShardNumber.Value} and it still contains buckets."); + if (record.Sharding.DoesShardHavePrefixes(ShardNumber.Value)) + throw new InvalidOperationException( + $"Database {DatabaseName} cannot be deleted because it is the last copy of shard {ShardNumber.Value} and there are prefixes settings for this shard."); } RemoveDatabaseFromSingleNode(record, record.Sharding.Shards[ShardNumber.Value], node, shardNumber: ShardNumber, deletionInProgressStatus, etag); diff --git a/src/Raven.Server/ServerWide/Commands/Sharding/AddPrefixedShardingSettingCommand.cs b/src/Raven.Server/ServerWide/Commands/Sharding/AddPrefixedShardingSettingCommand.cs new file mode 100644 index 000000000000..2b08c6da4e3e --- /dev/null +++ b/src/Raven.Server/ServerWide/Commands/Sharding/AddPrefixedShardingSettingCommand.cs @@ -0,0 +1,76 @@ +using System.Collections.Generic; +using System.Linq; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Sharding; +using Raven.Server.ServerWide.Sharding; +using Raven.Server.Utils; +using Sparrow.Json.Parsing; + +namespace Raven.Server.ServerWide.Commands.Sharding +{ + public sealed class AddPrefixedShardingSettingCommand : UpdateDatabaseCommand + { + public PrefixedShardingSetting Setting; + + public AddPrefixedShardingSettingCommand() + { + } + + public AddPrefixedShardingSettingCommand(PrefixedShardingSetting setting, string database, string raftId) : base(database, raftId) + { + Setting = setting; + } + + public override void UpdateDatabaseRecord(DatabaseRecord record, long etag) + { + record.Sharding.Prefixed ??= new List(); + Setting.BucketRangeStart = GetNextPrefixedBucketRangeStart(record.Sharding.Prefixed); + + var rangeStart = Setting.BucketRangeStart; + var shards = Setting.Shards; + var step = ShardHelper.NumberOfBuckets / shards.Count; + + foreach (var shardNumber in shards.OrderBy(x => x)) + { + record.Sharding.BucketRanges.Add(new ShardBucketRange + { + ShardNumber = shardNumber, + BucketRangeStart = rangeStart + }); + rangeStart += step; + } + + var index = record.Sharding.Prefixed.BinarySearch(Setting, PrefixedSettingComparer.Instance); + record.Sharding.Prefixed.Insert(~index, Setting); + } + + private static int GetNextPrefixedBucketRangeStart(IEnumerable prefixes) + { + var prefixesByRangeStart = prefixes.OrderBy(x => x.BucketRangeStart).ToList(); + if (prefixesByRangeStart.Count == 0) + return ShardHelper.NumberOfBuckets; + + for (int i = 0; i < prefixesByRangeStart.Count; i++) + { + var currentRangeStart = prefixesByRangeStart[i].BucketRangeStart; + var nextRangeStart = i + 1 < prefixesByRangeStart.Count + ? prefixesByRangeStart[i + 1].BucketRangeStart + : int.MaxValue; + + var expectedNext = currentRangeStart + ShardHelper.NumberOfBuckets; + if (expectedNext < nextRangeStart) + { + // found gap + return expectedNext; + } + } + + return prefixesByRangeStart[^1].BucketRangeStart + ShardHelper.NumberOfBuckets; + } + + public override void FillJson(DynamicJsonValue json) + { + json[nameof(Setting)] = Setting.ToJson(); + } + } +} diff --git a/src/Raven.Server/ServerWide/Commands/Sharding/CreateNewShardCommand.cs b/src/Raven.Server/ServerWide/Commands/Sharding/CreateNewShardCommand.cs index c8d5e1b27c14..f299f85f9fed 100644 --- a/src/Raven.Server/ServerWide/Commands/Sharding/CreateNewShardCommand.cs +++ b/src/Raven.Server/ServerWide/Commands/Sharding/CreateNewShardCommand.cs @@ -34,7 +34,6 @@ public override void UpdateDatabaseRecord(DatabaseRecord record, long etag) if (record.Sharding.Shards.ContainsKey(ShardNumber)) throw new RachisApplyException($"Cannot add new shard {ShardNumber} to the database {DatabaseName} because it already exists."); - record.Sharding.Shards.Add(ShardNumber, Topology); } diff --git a/src/Raven.Server/ServerWide/Commands/Sharding/DeletePrefixedShardingSettingCommand.cs b/src/Raven.Server/ServerWide/Commands/Sharding/DeletePrefixedShardingSettingCommand.cs new file mode 100644 index 000000000000..cb9d205edcfd --- /dev/null +++ b/src/Raven.Server/ServerWide/Commands/Sharding/DeletePrefixedShardingSettingCommand.cs @@ -0,0 +1,59 @@ +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Sharding; +using Raven.Server.Rachis; +using Raven.Server.ServerWide.Sharding; +using Raven.Server.Utils; +using Sparrow.Json.Parsing; + +namespace Raven.Server.ServerWide.Commands.Sharding +{ + public sealed class DeletePrefixedShardingSettingCommand : UpdateDatabaseCommand + { + public PrefixedShardingSetting Setting; + + public DeletePrefixedShardingSettingCommand() + { + } + + public DeletePrefixedShardingSettingCommand(PrefixedShardingSetting setting, string database, string raftId) : base(database, raftId) + { + Setting = setting; + } + + public override void UpdateDatabaseRecord(DatabaseRecord record, long etag) + { + int prefixIndex = record.Sharding.Prefixed.BinarySearch(Setting, PrefixedSettingComparer.Instance); + if (prefixIndex < 0) + throw new RachisApplyException($"Prefixed setting '{Setting.Prefix}' was not found in sharding configuration"); + + var prefixRangeStart = record.Sharding.Prefixed[prefixIndex].BucketRangeStart; + var nextPrefixRangeStart = prefixRangeStart + ShardHelper.NumberOfBuckets; + + record.Sharding.Prefixed.RemoveAt(prefixIndex); + + var numberOfRangesToRemove = 0; + var firstRangeIndex = 0; + for (int i = 0; i < record.Sharding.BucketRanges.Count; i++) + { + var range = record.Sharding.BucketRanges[i]; + if (range.BucketRangeStart < prefixRangeStart) + continue; + + if (range.BucketRangeStart == prefixRangeStart) + firstRangeIndex = i; + + else if (range.BucketRangeStart >= nextPrefixRangeStart) + break; + + numberOfRangesToRemove++; + } + + record.Sharding.BucketRanges.RemoveRange(firstRangeIndex, numberOfRangesToRemove); + } + + public override void FillJson(DynamicJsonValue json) + { + json[nameof(Setting)] = Setting.ToJson(); + } + } +} diff --git a/src/Raven.Server/ServerWide/Commands/Sharding/StartBucketMigrationCommand.cs b/src/Raven.Server/ServerWide/Commands/Sharding/StartBucketMigrationCommand.cs index cb482928e560..3b57dc21b7fd 100644 --- a/src/Raven.Server/ServerWide/Commands/Sharding/StartBucketMigrationCommand.cs +++ b/src/Raven.Server/ServerWide/Commands/Sharding/StartBucketMigrationCommand.cs @@ -5,6 +5,7 @@ using Raven.Client.ServerWide.Sharding; using Raven.Server.Rachis; using Raven.Server.ServerWide.Context; +using Raven.Server.ServerWide.Sharding; using Raven.Server.Utils; using Sparrow.Json.Parsing; using Sparrow.Logging; @@ -18,6 +19,7 @@ public sealed class StartBucketMigrationCommand : UpdateDatabaseCommand public int? SourceShard; public int DestinationShard; public int Bucket; + public string Prefix; private ShardBucketMigration _migration; @@ -25,13 +27,14 @@ public StartBucketMigrationCommand() { } - public StartBucketMigrationCommand(int bucket, int destShard, string database, string raftId) : base(database, raftId) + public StartBucketMigrationCommand(int bucket, int destShard, string database, string prefix, string raftId) : base(database, raftId) { Bucket = bucket; DestinationShard = destShard; + Prefix = prefix; } - public StartBucketMigrationCommand(int bucket, int sourceShard, int destShard, string database, string raftId) : this(bucket, destShard, database, raftId) + public StartBucketMigrationCommand(int bucket, int sourceShard, int destShard, string database, string raftId) : this(bucket, destShard, database, prefix: null, raftId) { SourceShard = sourceShard; } @@ -55,8 +58,7 @@ public override void UpdateDatabaseRecord(DatabaseRecord record, long etag) } } - if (record.Sharding.Shards.ContainsKey(DestinationShard) == false) - throw new RachisApplyException($"Destination shard {DestinationShard} doesn't exists"); + AssertDestinationShardExists(record.Sharding); _migration = new ShardBucketMigration { @@ -114,11 +116,30 @@ private void ProcessSubscriptionsForMigration(ClusterOperationContext context, S } } + private void AssertDestinationShardExists(ShardingConfiguration shardingConfiguration) + { + if (shardingConfiguration.Shards.ContainsKey(DestinationShard) == false) + throw new RachisApplyException($"Database '{DatabaseName}' : Failed to start migration of bucket '{Bucket}'. Destination shard {DestinationShard} doesn't exist"); + + if (string.IsNullOrEmpty(Prefix)) + return; + + // prefixed bucket range + var index = shardingConfiguration.Prefixed.BinarySearch(new PrefixedShardingSetting(Prefix), PrefixedSettingComparer.Instance); + if (index < 0) + throw new RachisApplyException($"Database '{DatabaseName}' : Failed to start migration of bucket '{Bucket}'. Prefix {Prefix} doesn't exist"); + + var shards = shardingConfiguration.Prefixed[index].Shards; + if (shards == null || shards.Contains(DestinationShard) == false) + throw new RachisApplyException($"Database '{DatabaseName}' : Failed to start migration of bucket '{Bucket}'. Destination shard {DestinationShard} doesn't exist"); + } + public override void FillJson(DynamicJsonValue json) { json[nameof(SourceShard)] = SourceShard; json[nameof(DestinationShard)] = DestinationShard; json[nameof(Bucket)] = Bucket; + json[nameof(Prefix)] = Prefix; } } } diff --git a/src/Raven.Server/ServerWide/Commands/Sharding/UpdatePrefixedShardingSettingCommand.cs b/src/Raven.Server/ServerWide/Commands/Sharding/UpdatePrefixedShardingSettingCommand.cs new file mode 100644 index 000000000000..dcc1b3cdafc0 --- /dev/null +++ b/src/Raven.Server/ServerWide/Commands/Sharding/UpdatePrefixedShardingSettingCommand.cs @@ -0,0 +1,37 @@ +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Sharding; +using Raven.Server.Rachis; +using Raven.Server.ServerWide.Sharding; +using Sparrow.Json.Parsing; + +namespace Raven.Server.ServerWide.Commands.Sharding +{ + public sealed class UpdatePrefixedShardingSettingCommand : UpdateDatabaseCommand + { + public PrefixedShardingSetting Setting; + + public UpdatePrefixedShardingSettingCommand() + { + } + + public UpdatePrefixedShardingSettingCommand(PrefixedShardingSetting setting, string database, string raftId) : base(database, raftId) + { + Setting = setting; + } + + public override void UpdateDatabaseRecord(DatabaseRecord record, long etag) + { + var location = record.Sharding.Prefixed.BinarySearch(Setting, PrefixedSettingComparer.Instance); + if (location < 0) + throw new RachisApplyException($"Prefixed setting '{Setting.Prefix}' was not found in sharding configuration"); + + var oldSetting = record.Sharding.Prefixed[location]; + oldSetting.Shards = Setting.Shards; + } + + public override void FillJson(DynamicJsonValue json) + { + json[nameof(Setting)] = Setting.ToJson(); + } + } +} diff --git a/src/Raven.Server/ServerWide/JsonDeserializationCluster.cs b/src/Raven.Server/ServerWide/JsonDeserializationCluster.cs index 4df78468e656..d23d06bbda4e 100644 --- a/src/Raven.Server/ServerWide/JsonDeserializationCluster.cs +++ b/src/Raven.Server/ServerWide/JsonDeserializationCluster.cs @@ -76,6 +76,8 @@ internal sealed class JsonDeserializationCluster : JsonDeserializationBase public static readonly Func ServerWideBackupConfiguration = GenerateJsonDeserializationRoutine(); + public static readonly Func PrefixedShardingSetting = GenerateJsonDeserializationRoutine(); + public static readonly Func ServerWideExternalReplication = GenerateJsonDeserializationRoutine(); public static readonly Func ExternalReplicationState = GenerateJsonDeserializationRoutine(); @@ -285,7 +287,10 @@ internal sealed class JsonDeserializationCluster : JsonDeserializationBase [nameof(UpdateQueueSinkCommand)] = GenerateJsonDeserializationRoutine(), [nameof(UpdateQueueSinkProcessStateCommand)] = GenerateJsonDeserializationRoutine(), [nameof(RemoveQueueSinkProcessStateCommand)] = GenerateJsonDeserializationRoutine(), - [nameof(UpdateResponsibleNodeForTasksCommand)] = GenerateJsonDeserializationRoutine() + [nameof(UpdateResponsibleNodeForTasksCommand)] = GenerateJsonDeserializationRoutine(), + [nameof(AddPrefixedShardingSettingCommand)] = GenerateJsonDeserializationRoutine(), + [nameof(DeletePrefixedShardingSettingCommand)] = GenerateJsonDeserializationRoutine(), + [nameof(UpdatePrefixedShardingSettingCommand)] = GenerateJsonDeserializationRoutine() }; } } diff --git a/src/Raven.Server/ServerWide/ServerStore.cs b/src/Raven.Server/ServerWide/ServerStore.cs index 05de29ed1211..32568c931e4d 100644 --- a/src/Raven.Server/ServerWide/ServerStore.cs +++ b/src/Raven.Server/ServerWide/ServerStore.cs @@ -3010,12 +3010,7 @@ public void AssignNodesToDatabase(ClusterTopology clusterTopology, string name, throw new ConcurrencyException($"Database '{databaseName}' already exists!"); } - DatabaseHelper.FillDatabaseTopology(this, context, databaseName, record, replicationFactor, index); - - if (record.IsSharded) - { - await Sharding.UpdatePrefixedShardingIfNeeded(context, record); - } + DatabaseHelper.FillDatabaseTopology(this, context, databaseName, record, replicationFactor, index, isRestore); } var addDatabaseCommand = new AddDatabaseCommand(raftRequestId) diff --git a/src/Raven.Server/ServerWide/Sharding/PrefixSettingComparer.cs b/src/Raven.Server/ServerWide/Sharding/PrefixSettingComparer.cs new file mode 100644 index 000000000000..5c0704bc6d49 --- /dev/null +++ b/src/Raven.Server/ServerWide/Sharding/PrefixSettingComparer.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using Raven.Client.ServerWide.Sharding; + +namespace Raven.Server.ServerWide.Sharding +{ + public class PrefixedSettingComparer : IComparer + { + private PrefixedSettingComparer() + { + } + + public static readonly PrefixedSettingComparer Instance = new(); + + public int Compare(PrefixedShardingSetting x, PrefixedShardingSetting y) + { + // compare prefixes in descending order + return string.Compare(y?.Prefix, x?.Prefix, StringComparison.OrdinalIgnoreCase); + } + } + +} diff --git a/src/Raven.Server/ServerWide/Sharding/RawShardingConfiguration.cs b/src/Raven.Server/ServerWide/Sharding/RawShardingConfiguration.cs index e0511a4fe7c1..6a5a66ce93a8 100644 --- a/src/Raven.Server/ServerWide/Sharding/RawShardingConfiguration.cs +++ b/src/Raven.Server/ServerWide/Sharding/RawShardingConfiguration.cs @@ -230,4 +230,6 @@ public ShardingConfiguration MaterializedConfiguration } public bool DoesShardHaveBuckets(int shardNumber) => ShardingConfiguration.DoesShardHaveBuckets(BucketRanges, shardNumber); + + public bool DoesShardHavePrefixes(int shardNumber) => ShardingConfiguration.DoesShardHavePrefixes(Prefixed, shardNumber); } diff --git a/src/Raven.Server/ServerWide/ShardingStore.cs b/src/Raven.Server/ServerWide/ShardingStore.cs index e074bb775bb5..06a9a7c40a2c 100644 --- a/src/Raven.Server/ServerWide/ShardingStore.cs +++ b/src/Raven.Server/ServerWide/ShardingStore.cs @@ -8,7 +8,6 @@ using System.Security.Cryptography.X509Certificates; using System.Threading.Tasks; using JetBrains.Annotations; -using Raven.Client.Documents.Commands; using Raven.Client.Documents.Conventions; using Raven.Client.Http; using Raven.Client.ServerWide; @@ -16,18 +15,13 @@ using Raven.Client.Util; using Raven.Server.ServerWide.Commands; using Raven.Server.ServerWide.Commands.Sharding; -using Raven.Server.ServerWide.Context; using Raven.Server.Utils; -using Sparrow.Json; using Sparrow.Server.Collections; -using Sparrow.Utils; namespace Raven.Server.ServerWide { public sealed class ShardingStore { - public bool BlockPrefixedSharding = true; - private readonly ServerStore _serverStore; public bool ManualMigration = false; @@ -36,9 +30,12 @@ public ShardingStore([NotNull] ServerStore serverStore) _serverStore = serverStore ?? throw new ArgumentNullException(nameof(serverStore)); } - public Task<(long Index, object Result)> StartBucketMigration(string database, int bucket, int toShard, string raftId = null) + public Task<(long Index, object Result)> StartBucketMigration(string database, int bucket, int toShard, string raftId = null) => + StartBucketMigration(database, bucket, toShard, prefix: null, raftId); + + public Task<(long Index, object Result)> StartBucketMigration(string database, int bucket, int toShard, string prefix, string raftId) { - var cmd = new StartBucketMigrationCommand(bucket, toShard, database, raftId ?? RaftIdGenerator.NewId()); + var cmd = new StartBucketMigrationCommand(bucket, toShard, database, prefix, raftId ?? RaftIdGenerator.NewId()); return _serverStore.SendToLeaderAsync(cmd); } @@ -156,7 +153,7 @@ private ClusterChanges.DatabaseChangedDelegate CreateDelegateForReshardingStatus }; } - public void FillShardingConfiguration(DatabaseRecord record, ClusterTopology clusterTopology, long? index) + public void FillShardingConfiguration(DatabaseRecord record, ClusterTopology clusterTopology, long? index, bool isRestore) { var shardingConfiguration = record.Sharding; if (shardingConfiguration.BucketRanges == null || @@ -176,7 +173,7 @@ public void FillShardingConfiguration(DatabaseRecord record, ClusterTopology clu } } - if (index is null or 0) + if (isRestore == false && index is null or 0) { FillPrefixedSharding(shardingConfiguration); } @@ -212,29 +209,16 @@ public void FillShardingConfiguration(DatabaseRecord record, ClusterTopology clu } } - public async Task UpdatePrefixedShardingIfNeeded(ClusterOperationContext context, DatabaseRecord databaseRecord) - { - if (_serverStore.Cluster.DatabaseExists(context, databaseRecord.DatabaseName) == false) - return; - - var clusterTopology = _serverStore.GetClusterTopology(context); - var existingConfiguration = _serverStore.Cluster.ReadShardingConfiguration(context, databaseRecord.DatabaseName); - if (databaseRecord.Sharding.Prefixed.SequenceEqual(existingConfiguration.Prefixed)) - return; - - var urls = databaseRecord.Sharding.Orchestrator.Topology.Members.Select(clusterTopology.GetUrlFromTag).ToArray(); - using (var requestExecutor = RequestExecutor.CreateForServer(urls, databaseRecord.DatabaseName, _serverStore.Server.Certificate.Certificate, DocumentConventions.DefaultForServer)) - { - await HandlePrefixSettingsUpdate(context, databaseRecord, existingConfiguration.Prefixed, requestExecutor); - } - } - private static void FillPrefixedSharding(ShardingConfiguration shardingConfiguration) { if (shardingConfiguration.Prefixed is not { Count: > 0 }) return; - var start = ShardHelper.NumberOfBuckets; + shardingConfiguration.Prefixed = shardingConfiguration.Prefixed + .OrderByDescending(x => x.Prefix) + .ToList(); + + var start = ShardHelper.NumberOfBuckets; foreach (var setting in shardingConfiguration.Prefixed) { AddPrefixedBucketRange(setting, start, shardingConfiguration); @@ -244,24 +228,15 @@ private static void FillPrefixedSharding(ShardingConfiguration shardingConfigura private static void AddPrefixedBucketRange(PrefixedShardingSetting setting, int rangeStart, ShardingConfiguration shardingConfiguration) { - if (setting.Prefix.EndsWith('/') == false && setting.Prefix.EndsWith('-') == false) - throw new InvalidOperationException( - $"Cannot add prefix '{setting.Prefix}' to {nameof(ShardingConfiguration)}.{nameof(ShardingConfiguration.Prefixed)}. " + - "In order to define sharding by prefix, the prefix string must end with '/' or '-' characters."); + AssertValidPrefix(setting, shardingConfiguration); setting.BucketRangeStart = rangeStart; var shards = setting.Shards; var step = ShardHelper.NumberOfBuckets / shards.Count; - foreach (var shardNumber in shards) + foreach (var shardNumber in shards.OrderBy(x => x)) { - if (shardingConfiguration.Shards.ContainsKey(shardNumber) == false) - { - throw new InvalidDataException($"Cannot assign shard number {shardNumber} to prefix {setting.Prefix}, " + - $"there's no shard '{shardNumber}' in sharding topology!"); - } - shardingConfiguration.BucketRanges.Add(new ShardBucketRange { ShardNumber = shardNumber, @@ -271,96 +246,26 @@ private static void AddPrefixedBucketRange(PrefixedShardingSetting setting, int } } - - - private static async Task HandlePrefixSettingsUpdate(JsonOperationContext context, DatabaseRecord databaseRecord, List existingSettings, RequestExecutor requestExecutor) + internal static void AssertValidPrefix(PrefixedShardingSetting setting, ShardingConfiguration shardingConfiguration) { - DevelopmentHelper.ShardingToDo(DevelopmentHelper.TeamMember.Aviv, DevelopmentHelper.Severity.Minor, - "optimize this and reuse deleted bucket ranges"); - - var shardingConfiguration = databaseRecord.Sharding; - var safeToRemove = new List(); - var maxBucketRangeStart = 0; - - foreach (var existingSetting in existingSettings) - { - bool found = false; - foreach (var setting in shardingConfiguration.Prefixed) - { - if (setting.Prefix != existingSetting.Prefix) - continue; - - found = true; - - if (setting.Shards.SequenceEqual(existingSetting.Shards) == false) - { - // todo - - // assigned shards were changed for this prefix settings - // check if we can change it in Sharding.BucketRanges (no existing docs) - } - - setting.BucketRangeStart = existingSetting.BucketRangeStart; - if (maxBucketRangeStart < setting.BucketRangeStart) - maxBucketRangeStart = setting.BucketRangeStart; - - break; - } - - if (found) - continue; - - // existingSetting.Prefix was removed - if (await AssertNoDocsStartingWith(existingSetting.Prefix, context, requestExecutor) == false) - throw new InvalidOperationException( - $"Cannot remove prefix '{existingSetting.Prefix}' from {nameof(ShardingConfiguration)}.{nameof(ShardingConfiguration.Prefixed)}. " + - $"There are existing documents in database '{databaseRecord.DatabaseName}' that start with '{existingSetting.Prefix}'. " + - "In order to remove a sharding by prefix setting, you cannot have any documents in the database that starts with this prefix."); + if (setting.Prefix.EndsWith('/') == false && setting.Prefix.EndsWith('-') == false) + throw new InvalidOperationException( + $"Cannot add prefix '{setting.Prefix}' to {nameof(ShardingConfiguration)}.{nameof(ShardingConfiguration.Prefixed)}. " + + "In order to define sharding by prefix, the prefix string must end with '/' or '-' characters."); - safeToRemove.Add(existingSetting); - } + if (setting.Shards.Count == 0) + throw new InvalidOperationException( + $"Cannot add prefix '{setting.Prefix}' to {nameof(ShardingConfiguration)}.{nameof(ShardingConfiguration.Prefixed)}. " + + $"{nameof(PrefixedShardingSetting)}.{nameof(PrefixedShardingSetting.Shards)} cannot be empty."); - // remove deleted prefixes from Sharding.BucketRanges - foreach (var setting in safeToRemove) + foreach (var shardNumber in setting.Shards) { - for (int index = 0; index < shardingConfiguration.BucketRanges.Count; index++) + if (shardingConfiguration.Shards.ContainsKey(shardNumber) == false) { - var range = shardingConfiguration.BucketRanges[index]; - if (range.BucketRangeStart != setting.BucketRangeStart) - continue; - - shardingConfiguration.BucketRanges.RemoveRange(index, setting.Shards.Count); - break; + throw new InvalidDataException($"Cannot assign shard number {shardNumber} to prefix {setting.Prefix}, " + + $"there's no shard '{shardNumber}' in sharding topology!"); } } - - var start = maxBucketRangeStart + ShardHelper.NumberOfBuckets; - - // add new prefixed settings to Sharding.BucketRanges - foreach (var setting in shardingConfiguration.Prefixed) - { - if (setting.BucketRangeStart != 0) - continue; // already added to BucketRanges - - if (await AssertNoDocsStartingWith(setting.Prefix, context, requestExecutor) == false) - throw new InvalidOperationException( - $"Cannot add prefix '{setting.Prefix}' to {nameof(ShardingConfiguration)}.{nameof(ShardingConfiguration.Prefixed)}. " + - $"There are existing documents in database '{databaseRecord.DatabaseName}' that start with '{setting.Prefix}'. " + - "In order to define sharding by prefix, you cannot have any documents in the database that starts with this prefix."); - - AddPrefixedBucketRange(setting, start, shardingConfiguration); - start += ShardHelper.NumberOfBuckets; - } - - } - - private static async Task AssertNoDocsStartingWith(string prefix, JsonOperationContext context, RequestExecutor requestExecutor) - { - var command = new GetDocumentsCommand(requestExecutor.Conventions, startWith: prefix, - startAfter: null, matches: null, exclude: null, start: 0, pageSize: int.MaxValue, metadataOnly: false); - - await requestExecutor.ExecuteAsync(command, context, sessionInfo: null); - return command.Result.Results.Length == 0; } private static Dictionary GetNodesDistribution(ClusterTopology clusterTopology, Dictionary shards) diff --git a/src/Raven.Server/Utils/ShardHelper.cs b/src/Raven.Server/Utils/ShardHelper.cs index 417a0d4309c3..533025262a62 100644 --- a/src/Raven.Server/Utils/ShardHelper.cs +++ b/src/Raven.Server/Utils/ShardHelper.cs @@ -142,8 +142,6 @@ public static IEnumerable GetShardNames(string databaseName, IEnumerable public static int GetShardNumberFor(ShardingConfiguration configuration, int bucket) => FindBucketShard(configuration.BucketRanges, bucket); - public static int GetShardNumberFor(RawShardingConfiguration configuration, int bucket) => FindBucketShard(configuration.BucketRanges, bucket); - public static int GetShardNumberFor(ShardingConfiguration configuration, ByteStringContext allocator, string id) => GetShardNumberAndBucketFor(configuration, allocator, id).ShardNumber; public static int GetShardNumberFor(ShardingConfiguration configuration, ByteStringContext allocator, LazyStringValue id) => GetShardNumberAndBucketFor(configuration, allocator, id).ShardNumber; diff --git a/src/Raven.Server/Web/System/AdminDatabasesHandler.cs b/src/Raven.Server/Web/System/AdminDatabasesHandler.cs index 19b5bab9fe7e..e9509e625184 100644 --- a/src/Raven.Server/Web/System/AdminDatabasesHandler.cs +++ b/src/Raven.Server/Web/System/AdminDatabasesHandler.cs @@ -626,10 +626,15 @@ public async Task Delete() throw new InvalidOperationException($"Database '{databaseName}' doesn't reside on node '{node}' so it can't be deleted from it"); } - if (isShard && topology.ReplicationFactor == 1 && rawRecord.Sharding.DoesShardHaveBuckets(shardNumber)) + if (isShard && topology.ReplicationFactor == 1) { - throw new InvalidOperationException( - $"Database {databaseName} cannot be deleted because it is the last copy of shard {shardNumber} and it contains data that has not been migrated."); + if (rawRecord.Sharding.DoesShardHaveBuckets(shardNumber)) + throw new InvalidOperationException( + $"Database {databaseName} cannot be deleted because it is the last copy of shard {shardNumber} and it contains data that has not been migrated."); + if (rawRecord.Sharding.DoesShardHavePrefixes(shardNumber)) + throw new InvalidOperationException( + $"Database {databaseName} cannot be deleted because it is the last copy of shard {shardNumber} and there are prefixes settings for this shard. " + + $"In order to delete shard {shardNumber} from database {databaseName} you need to remove shard {shardNumber} from all prefixes settings in DatabaseRecord.Sharding.Prefixed."); } pendingDeletes.Add(node); diff --git a/src/Raven.Server/Web/System/DatabaseHelper.cs b/src/Raven.Server/Web/System/DatabaseHelper.cs index 21a6c922f98f..3f8d74e1152d 100644 --- a/src/Raven.Server/Web/System/DatabaseHelper.cs +++ b/src/Raven.Server/Web/System/DatabaseHelper.cs @@ -127,7 +127,9 @@ public static void Validate(string name, DatabaseRecord record, RavenConfigurati record.Topology?.ValidateTopology(record.DatabaseName); } } - public static void FillDatabaseTopology(ServerStore server, ClusterOperationContext context, string name, DatabaseRecord record, int replicationFactor, long? index) + + public static void FillDatabaseTopology(ServerStore server, ClusterOperationContext context, string name, DatabaseRecord record, int replicationFactor, + long? index, bool isRestore) { if (replicationFactor <= 0) throw new ArgumentException("Replication factor must be greater than 0."); @@ -147,10 +149,7 @@ public static void FillDatabaseTopology(ServerStore server, ClusterOperationCont if (record.IsSharded) { - server.Sharding.FillShardingConfiguration(record, clusterTopology, index); - - if (server.Sharding.BlockPrefixedSharding && record.Sharding.Prefixed is { Count: > 0 }) - throw new InvalidOperationException("Cannot use prefixed sharding, this feature is currently blocked"); + server.Sharding.FillShardingConfiguration(record, clusterTopology, index, isRestore); if (string.IsNullOrEmpty(record.Sharding.DatabaseId)) { diff --git a/src/Raven.Studio/typescript/commands/resources/createDatabaseCommand.ts b/src/Raven.Studio/typescript/commands/resources/createDatabaseCommand.ts index 8223357d59fc..37b78d78b2c9 100644 --- a/src/Raven.Studio/typescript/commands/resources/createDatabaseCommand.ts +++ b/src/Raven.Studio/typescript/commands/resources/createDatabaseCommand.ts @@ -10,7 +10,8 @@ export type CreateDatabaseDto = Pick< Shards: Record; Orchestrator: { Topology: Pick - } + }, + Prefixed?: Raven.Client.ServerWide.Sharding.ShardingConfiguration["Prefixed"] }; }; diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/steps/CreateDatabaseFromBackupStepBasicInfo.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/steps/CreateDatabaseFromBackupStepBasicInfo.tsx index 55a9a49d29aa..c32917f81c73 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/steps/CreateDatabaseFromBackupStepBasicInfo.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/formBackup/steps/CreateDatabaseFromBackupStepBasicInfo.tsx @@ -34,6 +34,7 @@ export default function CreateDatabaseFromBackupStepBasicInfo() { name="basicInfoStep.databaseName" id="DbName" placeholder="Database Name" + autoComplete="off" /> diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx index 20583030af02..62c8c7b51283 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/CreateDatabaseRegular.tsx @@ -11,6 +11,7 @@ import StepBasicInfo from "./steps/CreateDatabaseRegularStepBasicInfo"; import StepEncryption from "../../../../../../common/FormEncryption"; import StepReplicationAndSharding from "./steps/CreateDatabaseRegularStepReplicationAndSharding"; import StepNodeSelection from "./steps/CreateDatabaseRegularStepNodeSelection"; +import StepShardingPrefixes from "components/pages/resources/databases/partials/create/regular/steps/StepShardingPrefixes"; import StepPath from "../shared/CreateDatabaseStepDataDirectory"; import { databaseSelectors } from "components/common/shell/databaseSliceSelectors"; import { useAppSelector } from "components/store"; @@ -59,6 +60,8 @@ export default function CreateDatabaseRegular({ closeModal, changeCreateModeToBa isSharded: data.replicationAndShardingStep.isSharded, isManualReplication: data.replicationAndShardingStep.isManualReplication, isEncrypted: data.basicInfoStep.isEncrypted, + isPrefixesForShards: data.replicationAndShardingStep.isPrefixesForShards, + usedPrefixes: data.shardingPrefixesStep.prefixesForShards.map((x) => x.prefix), }, options ), @@ -231,6 +234,12 @@ function getActiveStepsList( active: formValues.replicationAndShardingStep.isManualReplication, isInvalid: !!errors.manualNodeSelectionStep, }, + { + id: "shardingPrefixesStep", + label: "Sharding Prefixes", + active: formValues.replicationAndShardingStep.isPrefixesForShards, + isInvalid: !!errors.shardingPrefixesStep, + }, { id: "dataDirectoryStep", label: "Data Directory", @@ -269,6 +278,7 @@ function getStepViews( ), replicationAndShardingStep: , manualNodeSelectionStep: , + shardingPrefixesStep: , dataDirectoryStep: ( { shardsCount: 1, isDynamicDistribution: false, isManualReplication: false, + isPrefixesForShards: false, }, manualNodeSelectionStep: { // if there is only one node, it should be selected by default nodes: allNodeTags.length === 1 ? allNodeTags : [], shards: [], }, + shardingPrefixesStep: { + prefixesForShards: [ + { + prefix: "", + shardNumbers: [], + }, + ], + }, dataDirectoryStep: { isDefault: true, directory: "", @@ -71,6 +80,12 @@ function mapToDto(formValues: FormData, allNodeTags: string[]): CreateDatabaseDt Members: selectedOrchestrators, }, }, + Prefixed: formValues.replicationAndShardingStep.isPrefixesForShards + ? formValues.shardingPrefixesStep.prefixesForShards.map((x) => ({ + Prefix: x.prefix, + Shards: x.shardNumbers, + })) + : null, } : null; diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts index c57cb351c117..48130b245a53 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/createDatabaseRegularValidation.ts @@ -25,6 +25,7 @@ const replicationAndShardingStepSchema = yup.object({ }), isDynamicDistribution: yup.boolean(), isManualReplication: yup.boolean(), + isPrefixesForShards: yup.boolean(), }); const manualNodeSelectionStepSchema = yup.object({ @@ -70,11 +71,50 @@ const manualNodeSelectionStepSchema = yup.object({ ), }); +const shardingPrefixesStepSchema = yup.object({ + prefixesForShards: yup + .array() + .of( + yup.object({ + prefix: yup.string().when("$isPrefixesForShards", { + is: true, + then: (schema) => + schema + .trim() + .strict() + .required() + .test( + "invalid-ending-character", + "The prefix must end with '/' or '-' characters", + (value) => { + return value.endsWith("/") || value.endsWith("-"); + } + ) + .test("unique-prefix", "Prefix must be unique", (value, ctx) => { + return ctx.options.context.usedPrefixes.filter((x: string) => x === value).length < 2; + }), + }), + shardNumbers: yup + .array() + .of(yup.number()) + .when("$isPrefixesForShards", { + is: true, + then: (schema) => schema.min(1, "Select at least one shard"), + }), + }) + ) + .when("$isPrefixesForShards", { + is: true, + then: (schema) => schema.min(1), + }), +}); + export const createDatabaseRegularSchema = yup.object({ basicInfoStep: basicInfoStepSchema, encryptionStep: encryptionStepSchema, replicationAndShardingStep: replicationAndShardingStepSchema, manualNodeSelectionStep: manualNodeSelectionStepSchema, + shardingPrefixesStep: shardingPrefixesStepSchema, dataDirectoryStep: dataDirectoryStepSchema, }); diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepBasicInfo.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepBasicInfo.tsx index aaab4eb875f5..c03c9d9c75e3 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepBasicInfo.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepBasicInfo.tsx @@ -52,6 +52,7 @@ export default function CreateDatabaseRegularStepBasicInfo() { name="basicInfoStep.databaseName" placeholder="Database Name" id="DbName" + autoComplete="off" /> diff --git a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepReplicationAndSharding.tsx b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepReplicationAndSharding.tsx index 16aa91c55fec..3418207f9291 100644 --- a/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepReplicationAndSharding.tsx +++ b/src/Raven.Studio/typescript/components/pages/resources/databases/partials/create/regular/steps/CreateDatabaseRegularStepReplicationAndSharding.tsx @@ -101,7 +101,7 @@ export default function CreateDatabaseRegularStepReplicationAndSharding() { - + Available nodes:{" "} @@ -181,7 +181,7 @@ export default function CreateDatabaseRegularStepReplicationAndSharding() { /> - + Number of shards @@ -194,6 +194,19 @@ export default function CreateDatabaseRegularStepReplicationAndSharding() { max="100" /> + + Add prefixes for shards +
+ + Manage document distribution by defining +
a prefix for document names + + @@ -224,7 +237,7 @@ export default function CreateDatabaseRegularStepReplicationAndSharding() { - + - + (); + + const { + shardingPrefixesStep: { prefixesForShards }, + replicationAndShardingStep: { shardsCount }, + } = useWatch({ control }); + + const { fields, append, remove, update } = useFieldArray({ + control, + name: "shardingPrefixesStep.prefixesForShards", + }); + + const toggleShard = (index: number, shardNumber: number) => { + const field = prefixesForShards[index]; + + const updatedShardNumbers = field.shardNumbers.includes(shardNumber) + ? field.shardNumbers.filter((x) => x !== shardNumber) + : [...field.shardNumbers, shardNumber]; + + update(index, { + prefix: field.prefix, + shardNumbers: updatedShardNumbers, + }); + trigger(`shardingPrefixesStep.prefixesForShards.${index}.shardNumbers`); + }; + + const allShardNumbers = [...new Array(shardsCount).keys()]; + + return ( +
+

Sharding Prefixes

+ + + + + + {allShardNumbers.map((shardNumber) => ( + + ))} + + + + + + {fields.map((field, index) => ( + + ))} + +
+ + {shardNumber} +
+ + + +
+ + Sharding prefixes can be defined only when creating a database, and cannot be modified. +
+ Make sure everything is set up correctly. + +
+
+ ); +} + +interface PrefixRowProps { + index: number; + field: FieldArrayWithId; + allShardNumbers: number[]; + toggleShard: (index: number, shardNumber: number) => void; + remove: (index: number) => void; +} + +function PrefixRow({ index, field, allShardNumbers, toggleShard, remove }: PrefixRowProps) { + const { control, formState, trigger } = useFormContext(); + + const { + shardingPrefixesStep: { prefixesForShards }, + } = useWatch({ control }); + + const prefix = prefixesForShards[index].prefix; + + // Trigger validation for all fields when prefix changes (check for duplicates) + useEffect(() => { + if (!prefix) { + return; + } + + trigger("shardingPrefixesStep.prefixesForShards"); + }, [prefix, trigger]); + + const shardsError = formState.errors.shardingPrefixesStep?.prefixesForShards?.[index]?.shardNumbers?.message; + + return ( + + + + + {allShardNumbers.map((shardNumber) => ( + + toggleShard(index, shardNumber)} + /> + + ))} + + {index !== 0 && ( + + )} + + + {shardsError && ( + <> + + + {shardsError} + + + )} + + + ); +} diff --git a/test/FastTests/Client/RavenCommandTest.cs b/test/FastTests/Client/RavenCommandTest.cs index df6fc46862a7..519ecd151b23 100644 --- a/test/FastTests/Client/RavenCommandTest.cs +++ b/test/FastTests/Client/RavenCommandTest.cs @@ -70,7 +70,8 @@ public void WhenCommandCanBeCheckedForFastestNode_ItCanRunInParallel() "AddDatabaseShardCommand", "GetNextServerOperationIdCommand", "KillServerOperationCommand", "ModifyDatabaseTopologyCommand", "DelayBackupCommand", "PutDatabaseClientConfigurationCommand", "PutDatabaseSettingsCommand", "PutDatabaseStudioConfigurationCommand", "GetTcpInfoForReplicationCommand", "AddQueueSinkCommand", "UpdateQueueSinkCommand", "ConfigureDataArchivalCommand", - "AdoptOrphanedRevisionsCommand" + "AdoptOrphanedRevisionsCommand", + "AddPrefixedShardingSettingCommand", "DeletePrefixedShardingSettingCommand", "UpdatePrefixedShardingSettingCommand" }.OrderBy(t => t); var commandBaseType = typeof(RavenCommand<>); diff --git a/test/SlowTests/Sharding/BucketMigration/DocumentsMigrationTests.cs b/test/SlowTests/Sharding/BucketMigration/DocumentsMigrationTests.cs index b7bf810a85cf..dcace6f4a2eb 100644 --- a/test/SlowTests/Sharding/BucketMigration/DocumentsMigrationTests.cs +++ b/test/SlowTests/Sharding/BucketMigration/DocumentsMigrationTests.cs @@ -76,7 +76,6 @@ public async Task DocumentsMigratorShouldWork_SimpleCase() [RavenFact(RavenTestCategory.Sharding)] public async Task DocumentsMigratorShouldWork_MultipleWrongBuckets() { - Server.ServerStore.Sharding.BlockPrefixedSharding = false; using var store = Sharding.GetDocumentStore(new Options { ModifyDatabaseRecord = record => diff --git a/test/SlowTests/Sharding/PrefixedSharding.cs b/test/SlowTests/Sharding/PrefixedSharding.cs index 13692dde9577..4f4f35a0a795 100644 --- a/test/SlowTests/Sharding/PrefixedSharding.cs +++ b/test/SlowTests/Sharding/PrefixedSharding.cs @@ -4,13 +4,16 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using FastTests; using Raven.Client.Documents; using Raven.Client.Documents.Operations.Backups; +using Raven.Client.Documents.Smuggler; using Raven.Client.Exceptions; using Raven.Client.ServerWide.Operations; using Raven.Client.ServerWide.Sharding; +using Raven.Client.Util; +using Raven.Server.Documents; using Raven.Server.Documents.Sharding; +using Raven.Server.Rachis; using Raven.Server.ServerWide.Context; using Raven.Server.Utils; using Raven.Tests.Core.Utils.Entities; @@ -21,16 +24,14 @@ using Voron; using Xunit; using Xunit.Abstractions; +using BucketStats = Raven.Server.Documents.Sharding.BucketStats; namespace SlowTests.Sharding; -public class PrefixedSharding : RavenTestBase +public class PrefixedSharding : ClusterTestBase { public PrefixedSharding(ITestOutputHelper output) : base(output) { - DoNotReuseServer(); - - Server.ServerStore.Sharding.BlockPrefixedSharding = false; } [RavenFact(RavenTestCategory.Sharding)] @@ -41,24 +42,24 @@ public async Task CanShardByDocumentsPrefix() ModifyDatabaseRecord = record => { record.Sharding ??= new ShardingConfiguration(); - record.Sharding.Prefixed = new List - { + record.Sharding.Prefixed = + [ new PrefixedShardingSetting { // range for 'eu/' is : // shard 0 : [1M, 2M] - Prefix = "eu/", - Shards = new List { 0 } + Prefix = "eu/", + Shards = [0] }, new PrefixedShardingSetting { // range for 'asia/' is : // shard 1 : [2M, 2.5M] // shard 2 : [2.5M, 3M] - Prefix = "asia/", - Shards = new List { 1, 2 } + Prefix = "asia/", + Shards = [1, 2] } - }; + ]; } }); @@ -101,7 +102,7 @@ public async Task CanShardByDocumentsPrefix() } } - var rand = new System.Random(2022_04_19); + var rand = new Random(2022_04_19); var prefixes = new[] { "us/", "eu/", "asia/", null }; int d = 0; @@ -152,23 +153,57 @@ public async Task ShouldThrowOnAttemptToAddPrefixThatDoesntEndWithSlashOrComma() { using var store = Sharding.GetDocumentStore(); - var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); - var shardingConfiguration = record.Sharding; - - shardingConfiguration.Prefixed ??= new List(); - shardingConfiguration.Prefixed.Add(new PrefixedShardingSetting + var task = store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting { Prefix = "asia", - Shards = new List { 1, 2 } - }); + Shards = [1, 2] + })); - var task = store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, replicationFactor: 1, record.Etag)); var e = await Assert.ThrowsAsync(async () => await task); Assert.Contains( "Cannot add prefix 'asia' to ShardingConfiguration.Prefixed. In order to define sharding by prefix, the prefix string must end with '/' or '-' characters", e.Message); } + [RavenFact(RavenTestCategory.Sharding)] + public async Task ShouldThrowOnPrefixSettingWithNoShards() + { + using var store = Sharding.GetDocumentStore(); + + await Assert.ThrowsAsync(async () => await store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [] + }))); + + await store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0] + })); + + Assert.Throws(() => + { + using var newStore = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [] + } + ]; + } + }); + }); + + + } + [RavenFact(RavenTestCategory.Sharding)] public async Task ShouldNotAllowToAddPrefixIfWeHaveDocsStartingWith() { @@ -177,14 +212,14 @@ public async Task ShouldNotAllowToAddPrefixIfWeHaveDocsStartingWith() ModifyDatabaseRecord = record => { record.Sharding ??= new ShardingConfiguration(); - record.Sharding.Prefixed = new List - { + record.Sharding.Prefixed = + [ new PrefixedShardingSetting { - Prefix = "eu/", - Shards = new List { 0 } + Prefix = "eu/", + Shards = [0] } - }; + ]; } }); @@ -199,16 +234,12 @@ public async Task ShouldNotAllowToAddPrefixIfWeHaveDocsStartingWith() await session.SaveChangesAsync(); } - var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); - var shardingConfiguration = record.Sharding; - - shardingConfiguration.Prefixed.Add(new PrefixedShardingSetting + var task = store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting { Prefix = "asia/", - Shards = new List { 1, 2 } - }); + Shards = [1, 2] + })); - var task = store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, replicationFactor: 1, record.Etag)); var e = await Assert.ThrowsAsync(async () => await task); Assert.Contains( $"Cannot add prefix 'asia/' to ShardingConfiguration.Prefixed. There are existing documents in database '{store.Database}' that start with 'asia/'", @@ -223,19 +254,19 @@ public async Task ShouldNotAllowToDeletePrefixIfWeHaveDocsStartingWith() ModifyDatabaseRecord = record => { record.Sharding ??= new ShardingConfiguration(); - record.Sharding.Prefixed = new List - { + record.Sharding.Prefixed = + [ new PrefixedShardingSetting { - Prefix = "asia/", - Shards = new List { 0 } + Prefix = "asia/", + Shards = [0] }, new PrefixedShardingSetting { - Prefix = "eu/", - Shards = new List { 1, 2 } + Prefix = "eu/", + Shards = [1, 2] } - }; + ]; } }); @@ -250,12 +281,7 @@ public async Task ShouldNotAllowToDeletePrefixIfWeHaveDocsStartingWith() await session.SaveChangesAsync(); } - var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); - var shardingConfiguration = record.Sharding; - - shardingConfiguration.Prefixed.RemoveAt(0); - - var task = store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, replicationFactor: 1, record.Etag)); + var task = store.Maintenance.SendAsync(new DeletePrefixedShardingSettingOperation("asia/")); var e = await Assert.ThrowsAsync(async () => await task); Assert.Contains( $"Cannot remove prefix 'asia/' from ShardingConfiguration.Prefixed. There are existing documents in database '{store.Database}' that start with 'asia/'", @@ -270,14 +296,14 @@ public async Task CanAddPrefixIfNoDocsStartingWith() ModifyDatabaseRecord = record => { record.Sharding ??= new ShardingConfiguration(); - record.Sharding.Prefixed = new List - { + record.Sharding.Prefixed = + [ new PrefixedShardingSetting { - Prefix = "eu/", - Shards = new List { 0 } + Prefix = "eu/", + Shards = [0] } - }; + ]; } }); @@ -294,22 +320,31 @@ public async Task CanAddPrefixIfNoDocsStartingWith() var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); var shardingConfiguration = record.Sharding; + Assert.Equal(4, shardingConfiguration.BucketRanges.Count); - shardingConfiguration.Prefixed.Add(new PrefixedShardingSetting + await store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting { Prefix = "asia/", - Shards = new List { 1, 2 } - }); - - Assert.Equal(4, shardingConfiguration.BucketRanges.Count); - - await store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, replicationFactor: 1, record.Etag)); + Shards = [1, 2] + })); shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); Assert.Equal(2, shardingConfiguration.Prefixed.Count); Assert.Equal(ShardHelper.NumberOfBuckets * 2, shardingConfiguration.Prefixed[1].BucketRangeStart); Assert.Equal(6, shardingConfiguration.BucketRanges.Count); + + // check that we can add prefixes even if none were defined in database creation + var newStore = Sharding.GetDocumentStore(); + await newStore.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 2] + })); + + shardingConfiguration = await Sharding.GetShardingConfigurationAsync(newStore); + Assert.Equal(1, shardingConfiguration.Prefixed.Count); + } [RavenFact(RavenTestCategory.Sharding)] @@ -320,14 +355,14 @@ public async Task CanDeletePrefixIfNoDocsStartingWith() ModifyDatabaseRecord = record => { record.Sharding ??= new ShardingConfiguration(); - record.Sharding.Prefixed = new List - { + record.Sharding.Prefixed = + [ new PrefixedShardingSetting { - Prefix = "eu/", - Shards = new List { 0, 1 } + Prefix = "eu/", + Shards = [0, 1] } - }; + ]; } }); @@ -344,12 +379,9 @@ public async Task CanDeletePrefixIfNoDocsStartingWith() var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); var shardingConfiguration = record.Sharding; - - shardingConfiguration.Prefixed.RemoveAt(0); - Assert.Equal(5, shardingConfiguration.BucketRanges.Count); - await store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, replicationFactor: 1, record.Etag)); + await store.Maintenance.SendAsync(new DeletePrefixedShardingSettingOperation("eu/")); shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); @@ -365,19 +397,19 @@ public async Task CanDeleteOnePrefixThenAddAnotherIfNoDocsStartingWith() ModifyDatabaseRecord = record => { record.Sharding ??= new ShardingConfiguration(); - record.Sharding.Prefixed = new List - { + record.Sharding.Prefixed = + [ new PrefixedShardingSetting { - Prefix = "eu/", - Shards = new List { 0, 1 } + Prefix = "eu/", + Shards = [0, 1] }, new PrefixedShardingSetting { - Prefix = "us/", - Shards = new List { 1, 2 } + Prefix = "us/", + Shards = [1, 2] } - }; + ]; } }); @@ -391,316 +423,230 @@ public async Task CanDeleteOnePrefixThenAddAnotherIfNoDocsStartingWith() await session.SaveChangesAsync(); } - + var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); var shardingConfiguration = record.Sharding; Assert.Equal(7, shardingConfiguration.BucketRanges.Count); // remove 'eu/' prefix - shardingConfiguration.Prefixed.RemoveAt(0); - await store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, replicationFactor: 1, record.Etag)); + await store.Maintenance.SendAsync(new DeletePrefixedShardingSettingOperation("eu/")); shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); Assert.Equal(1, shardingConfiguration.Prefixed.Count); Assert.Equal(5, shardingConfiguration.BucketRanges.Count); // add a new prefix - record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); - shardingConfiguration = record.Sharding; - shardingConfiguration.Prefixed.Add(new PrefixedShardingSetting + await store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting { Prefix = "africa/", - Shards = new List { 0, 2 } - }); + Shards = [0, 2] + })); - await store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(record, replicationFactor: 1, record.Etag)); shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); Assert.Equal(2, shardingConfiguration.Prefixed.Count); Assert.Equal(7, shardingConfiguration.BucketRanges.Count); - Assert.Equal(ShardHelper.NumberOfBuckets * 2, shardingConfiguration.BucketRanges[3].BucketRangeStart); - Assert.Equal(ShardHelper.NumberOfBuckets * 2.5, shardingConfiguration.BucketRanges[4].BucketRangeStart); - Assert.Equal(ShardHelper.NumberOfBuckets * 3, shardingConfiguration.BucketRanges[5].BucketRangeStart); - Assert.Equal(ShardHelper.NumberOfBuckets * 3.5, shardingConfiguration.BucketRanges[6].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets, shardingConfiguration.BucketRanges[3].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 1.5, shardingConfiguration.BucketRanges[4].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 2, shardingConfiguration.BucketRanges[5].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 2.5, shardingConfiguration.BucketRanges[6].BucketRangeStart); } [RavenFact(RavenTestCategory.BackupExportImport | RavenTestCategory.Sharding)] public async Task BackupAndRestoreShardedDatabase_ShouldPreservePrefixedSettingsAndBucketRanges() { - using (var store = Sharding.GetDocumentStore()) + using var store = Sharding.GetDocumentStore(new Options { - var databaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); - databaseRecord.Sharding.Prefixed ??= new List(); - databaseRecord.Sharding.Prefixed.Add(new PrefixedShardingSetting - { - Prefix = "users/", - Shards = new List { 0 } - }); - databaseRecord.Sharding.Prefixed.Add(new PrefixedShardingSetting - { - Prefix = "orders/", - Shards = new List { 1 } - }); - databaseRecord.Sharding.Prefixed.Add(new PrefixedShardingSetting + ModifyDatabaseRecord = databaseRecord => { - Prefix = "employees/", - Shards = new List { 2 } - }); + databaseRecord.Sharding ??= new ShardingConfiguration(); + databaseRecord.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0] + }, + new PrefixedShardingSetting + { + Prefix = "orders/", + Shards = [1] + }, + new PrefixedShardingSetting + { + Prefix = "employees/", + Shards = [2] + } + ]; + } + }); - await store.Maintenance.Server.SendAsync(new UpdateDatabaseOperation(databaseRecord, replicationFactor: 1, databaseRecord.Etag)); - var sharding = await Sharding.GetShardingConfigurationAsync(store); + var sharding = await Sharding.GetShardingConfigurationAsync(store); - Assert.Equal(6, sharding.BucketRanges.Count); - Assert.Equal(ShardHelper.NumberOfBuckets, sharding.BucketRanges[3].BucketRangeStart); - Assert.Equal(ShardHelper.NumberOfBuckets * 2, sharding.BucketRanges[4].BucketRangeStart); - Assert.Equal(ShardHelper.NumberOfBuckets * 3, sharding.BucketRanges[5].BucketRangeStart); + Assert.Equal(6, sharding.BucketRanges.Count); + Assert.Equal(ShardHelper.NumberOfBuckets, sharding.BucketRanges[3].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 2, sharding.BucketRanges[4].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 3, sharding.BucketRanges[5].BucketRangeStart); - using (var session = store.OpenAsyncSession()) + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) { - for (int i = 0; i < 10; i++) - { - await session.StoreAsync(new User(), $"users/{i}"); - await session.StoreAsync(new Order(), $"orders/{i}"); - await session.StoreAsync(new Employee(), $"employees/{i}"); - } - - await session.SaveChangesAsync(); + await session.StoreAsync(new User(), $"users/{i}"); + await session.StoreAsync(new Order(), $"orders/{i}"); + await session.StoreAsync(new Employee(), $"employees/{i}"); } - using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + await session.SaveChangesAsync(); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + for (int i = 0; i < 10; i++) { - for (int i = 0; i < 10; i++) - { - var doc = await session.LoadAsync($"users/{i}"); - Assert.NotNull(doc); - } + var doc = await session.LoadAsync($"users/{i}"); + Assert.NotNull(doc); } - using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + for (int i = 0; i < 10; i++) { - for (int i = 0; i < 10; i++) - { - var doc = await session.LoadAsync($"orders/{i}"); - Assert.NotNull(doc); - } + var doc = await session.LoadAsync($"orders/{i}"); + Assert.NotNull(doc); } - using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + for (int i = 0; i < 10; i++) { - for (int i = 0; i < 10; i++) - { - var doc = await session.LoadAsync($"employees/{i}"); - Assert.NotNull(doc); - } + var doc = await session.LoadAsync($"employees/{i}"); + Assert.NotNull(doc); } + } - var bucketStats = new Dictionary>(); - await foreach (var db in Sharding.GetShardsDocumentDatabaseInstancesFor(store)) + var bucketStats = new Dictionary>(); + await foreach (var db in Sharding.GetShardsDocumentDatabaseInstancesFor(store)) + { + using (db.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) + using (ctx.OpenReadTransaction()) { - using (db.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) - using (ctx.OpenReadTransaction()) - { - var stats = ShardedDocumentsStorage.GetBucketStatistics(ctx, 0, int.MaxValue).ToList(); - Assert.Equal(10, stats.Count); - bucketStats[db.ShardNumber] = stats; - } + var stats = ShardedDocumentsStorage.GetBucketStatistics(ctx, 0, int.MaxValue).ToList(); + Assert.Equal(10, stats.Count); + bucketStats[db.ShardNumber] = stats; } + } - var waitHandles = await Sharding.Backup.WaitForBackupToComplete(store); - var backupPath = NewDataPath(suffix: "BackupFolder"); - var config = Backup.CreateBackupConfiguration(backupPath); - - await Sharding.Backup.UpdateConfigurationAndRunBackupAsync(Server, store, config); - Assert.True(WaitHandle.WaitAll(waitHandles, TimeSpan.FromMinutes(1))); + var waitHandles = await Sharding.Backup.WaitForBackupToComplete(store); + var backupPath = NewDataPath(suffix: "BackupFolder"); + var config = Backup.CreateBackupConfiguration(backupPath); - var dirs = Directory.GetDirectories(backupPath); - Assert.Equal(3, dirs.Length); + await Sharding.Backup.UpdateConfigurationAndRunBackupAsync(Server, store, config); + Assert.True(WaitHandle.WaitAll(waitHandles, TimeSpan.FromMinutes(1))); - sharding = await Sharding.GetShardingConfigurationAsync(store); - var settings = Sharding.Backup.GenerateShardRestoreSettings(dirs, sharding); + var dirs = Directory.GetDirectories(backupPath); + Assert.Equal(3, dirs.Length); - // restore the database with a different name - var restoredDatabaseName = $"restored_database-{Guid.NewGuid()}"; - using (Sharding.Backup.ReadOnly(backupPath)) - using (Backup.RestoreDatabase(store, new RestoreBackupConfiguration - { - DatabaseName = restoredDatabaseName, - ShardRestoreSettings = settings + sharding = await Sharding.GetShardingConfigurationAsync(store); + var settings = Sharding.Backup.GenerateShardRestoreSettings(dirs, sharding); - }, timeout: TimeSpan.FromSeconds(60))) + // restore the database with a different name + var restoredDatabaseName = $"restored_database-{Guid.NewGuid()}"; + using (Sharding.Backup.ReadOnly(backupPath)) + using (Backup.RestoreDatabase(store, new RestoreBackupConfiguration + { + DatabaseName = restoredDatabaseName, + ShardRestoreSettings = settings + }, timeout: TimeSpan.FromSeconds(60))) + { + var newDatabaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(restoredDatabaseName)); + Assert.Equal(3, newDatabaseRecord.Sharding.Shards.Count); + Assert.Equal(3, newDatabaseRecord.Sharding.Prefixed.Count); + + var usersPrefixSetting = newDatabaseRecord.Sharding.Prefixed[0]; + Assert.Equal("users/", usersPrefixSetting.Prefix); + Assert.Equal(1, usersPrefixSetting.Shards.Count); + Assert.Equal(0, usersPrefixSetting.Shards[0]); + Assert.Equal(ShardHelper.NumberOfBuckets, usersPrefixSetting.BucketRangeStart); + + var ordersPrefixSetting = newDatabaseRecord.Sharding.Prefixed[1]; + Assert.Equal("orders/", ordersPrefixSetting.Prefix); + Assert.Equal(1, ordersPrefixSetting.Shards.Count); + Assert.Equal(1, ordersPrefixSetting.Shards[0]); + Assert.Equal(ShardHelper.NumberOfBuckets * 2, ordersPrefixSetting.BucketRangeStart); + + var employeesPrefixSetting = newDatabaseRecord.Sharding.Prefixed[2]; + Assert.Equal("employees/", employeesPrefixSetting.Prefix); + Assert.Equal(1, employeesPrefixSetting.Shards.Count); + Assert.Equal(2, employeesPrefixSetting.Shards[0]); + Assert.Equal(ShardHelper.NumberOfBuckets * 3, employeesPrefixSetting.BucketRangeStart); + + Assert.Equal(6, newDatabaseRecord.Sharding.BucketRanges.Count); + Assert.Equal(ShardHelper.NumberOfBuckets, newDatabaseRecord.Sharding.BucketRanges[3].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 2, newDatabaseRecord.Sharding.BucketRanges[4].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 3, newDatabaseRecord.Sharding.BucketRanges[5].BucketRangeStart); + + using (var session = store.OpenAsyncSession(database: restoredDatabaseName)) { - var newDatabaseRecord = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(restoredDatabaseName)); - Assert.Equal(3, newDatabaseRecord.Sharding.Shards.Count); - Assert.Equal(3, newDatabaseRecord.Sharding.Prefixed.Count); - - var usersPrefixSetting = newDatabaseRecord.Sharding.Prefixed[0]; - Assert.Equal("users/", usersPrefixSetting.Prefix); - Assert.Equal(1, usersPrefixSetting.Shards.Count); - Assert.Equal(0, usersPrefixSetting.Shards[0]); - Assert.Equal(ShardHelper.NumberOfBuckets, usersPrefixSetting.BucketRangeStart); - - var ordersPrefixSetting = newDatabaseRecord.Sharding.Prefixed[1]; - Assert.Equal("orders/", ordersPrefixSetting.Prefix); - Assert.Equal(1, ordersPrefixSetting.Shards.Count); - Assert.Equal(1, ordersPrefixSetting.Shards[0]); - Assert.Equal(ShardHelper.NumberOfBuckets * 2, ordersPrefixSetting.BucketRangeStart); - - var employeesPrefixSetting = newDatabaseRecord.Sharding.Prefixed[2]; - Assert.Equal("employees/", employeesPrefixSetting.Prefix); - Assert.Equal(1, employeesPrefixSetting.Shards.Count); - Assert.Equal(2, employeesPrefixSetting.Shards[0]); - Assert.Equal(ShardHelper.NumberOfBuckets * 3, employeesPrefixSetting.BucketRangeStart); - - Assert.Equal(6, sharding.BucketRanges.Count); - Assert.Equal(ShardHelper.NumberOfBuckets, sharding.BucketRanges[3].BucketRangeStart); - Assert.Equal(ShardHelper.NumberOfBuckets * 2, sharding.BucketRanges[4].BucketRangeStart); - Assert.Equal(ShardHelper.NumberOfBuckets * 3, sharding.BucketRanges[5].BucketRangeStart); - - using (var session = store.OpenAsyncSession(database: restoredDatabaseName)) - { - for (int i = 0; i < 10; i++) - { - var user = await session.LoadAsync($"users/{i}"); - Assert.NotNull(user); - - var order = await session.LoadAsync($"orders/{i}"); - Assert.NotNull(order); - - var employee = await session.LoadAsync($"employees/{i}"); - Assert.NotNull(employee); - } - } - - // assert valid bucket stats - await foreach (var db in Sharding.GetShardsDocumentDatabaseInstancesFor(restoredDatabaseName)) - { - using (db.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) - using (ctx.OpenReadTransaction()) - { - var stats = ShardedDocumentsStorage.GetBucketStatistics(ctx, 0, int.MaxValue).ToList(); - var originalStats = bucketStats[db.ShardNumber]; - - Assert.Equal(originalStats.Count, stats.Count); - for (int i = 0; i < stats.Count; i++) - { - Assert.Equal(originalStats[i].Bucket, stats[i].Bucket); - Assert.Equal(originalStats[i].NumberOfDocuments, stats[i].NumberOfDocuments); - } - } - } - - using (var session = store.OpenAsyncSession(database: restoredDatabaseName)) - { - await session.StoreAsync(new User(), "users/11"); - await session.StoreAsync(new Order(), "orders/11"); - await session.StoreAsync(new Employee(), "employees/11"); - - await session.SaveChangesAsync(); - } - - using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(restoredDatabaseName, 0))) + for (int i = 0; i < 10; i++) { - var user = await session.LoadAsync("users/11"); + var user = await session.LoadAsync($"users/{i}"); Assert.NotNull(user); - } - using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(restoredDatabaseName, 1))) - { - var order = await session.LoadAsync("orders/11"); + var order = await session.LoadAsync($"orders/{i}"); Assert.NotNull(order); - } - using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(restoredDatabaseName, 2))) - { - var employee = await session.LoadAsync("employees/11"); + var employee = await session.LoadAsync($"employees/{i}"); Assert.NotNull(employee); } } - } - } - [RavenFact(RavenTestCategory.Sharding)] - public async Task CanMoveOneBucketFromPrefixedRange() - { - using var store = Sharding.GetDocumentStore(new Options - { - ModifyDatabaseRecord = record => + // assert valid bucket stats + await foreach (var db in Sharding.GetShardsDocumentDatabaseInstancesFor(restoredDatabaseName)) { - record.Sharding ??= new ShardingConfiguration(); - record.Sharding.Prefixed = new List + using (db.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) + using (ctx.OpenReadTransaction()) + { + var stats = ShardedDocumentsStorage.GetBucketStatistics(ctx, 0, int.MaxValue).ToList(); + var originalStats = bucketStats[db.ShardNumber]; + + Assert.Equal(originalStats.Count, stats.Count); + for (int i = 0; i < stats.Count; i++) { - new PrefixedShardingSetting - { - // bucket range for 'users/' is : - // shard 0 : [1M, 1.5M] - // shard 1 : [1.5M, 2M] - Prefix = "users/", - Shards = new List { 0, 1 } - } - }; + Assert.Equal(originalStats[i].Bucket, stats[i].Bucket); + Assert.Equal(originalStats[i].NumberOfDocuments, stats[i].NumberOfDocuments); + } + } } - }); - const string id = "users/1"; - using (var session = store.OpenAsyncSession()) - { - var user = new User + using (var session = store.OpenAsyncSession(database: restoredDatabaseName)) { - Name = "Original shard" - }; - await session.StoreAsync(user, id); - await session.SaveChangesAsync(); - } - - var shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); - - Assert.Equal(5, shardingConfiguration.BucketRanges.Count); - Assert.Equal(ShardHelper.NumberOfBuckets, shardingConfiguration.Prefixed[0].BucketRangeStart); - - var bucket = await Sharding.GetBucketAsync(store, id); - - var originalLocation = ShardHelper.GetShardNumberFor(shardingConfiguration, bucket); - Assert.Contains(originalLocation, shardingConfiguration.Prefixed[0].Shards); - var newLocation = shardingConfiguration.Prefixed[0].Shards.Single(s => s != originalLocation); - - await Sharding.Resharding.MoveShardForId(store, id); - - shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); - Assert.Equal(7, shardingConfiguration.BucketRanges.Count); - - Assert.Equal(bucket, shardingConfiguration.BucketRanges[^2].BucketRangeStart); - Assert.Equal(newLocation, shardingConfiguration.BucketRanges[^2].ShardNumber); + await session.StoreAsync(new User(), "users/11"); + await session.StoreAsync(new Order(), "orders/11"); + await session.StoreAsync(new Employee(), "employees/11"); - Assert.Equal(bucket + 1, shardingConfiguration.BucketRanges[^1].BucketRangeStart); - Assert.Equal(originalLocation, shardingConfiguration.BucketRanges[^1].ShardNumber); - - using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, originalLocation))) - { - var user = await session.LoadAsync(id); - Assert.Null(user); - } - using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, newLocation))) - { - var user = await session.LoadAsync(id); - Assert.Equal("Original shard", user.Name); - } + await session.SaveChangesAsync(); + } - // the document will be written to the new location - using (var session = store.OpenAsyncSession()) - { - var user = await session.LoadAsync(id); - user.Name = "New shard"; - await session.SaveChangesAsync(); - } + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(restoredDatabaseName, 0))) + { + var user = await session.LoadAsync("users/11"); + Assert.NotNull(user); + } - using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, originalLocation))) - { - var user = await session.LoadAsync(id); - Assert.Null(user); - } + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(restoredDatabaseName, 1))) + { + var order = await session.LoadAsync("orders/11"); + Assert.NotNull(order); + } - using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, newLocation))) - { - var user = await session.LoadAsync(id); - Assert.Equal("New shard", user.Name); + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(restoredDatabaseName, 2))) + { + var employee = await session.LoadAsync("employees/11"); + Assert.NotNull(employee); + } } } @@ -712,19 +658,19 @@ public async Task CanGetBucketStats_Prefixed() ModifyDatabaseRecord = record => { record.Sharding ??= new ShardingConfiguration(); - record.Sharding.Prefixed = new List - { - new PrefixedShardingSetting() - { - Prefix = "Users/", - Shards = new List { 0 } - }, - new PrefixedShardingSetting() - { - Prefix = "Orders/", - Shards = new List { 1 } - } - }; + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "Users/", + Shards = [0] + }, + new PrefixedShardingSetting + { + Prefix = "Orders/", + Shards = [1] + } + ]; } })) { @@ -798,24 +744,25 @@ public void RavenDb_19737() ModifyDatabaseRecord = record => { record.Sharding ??= new ShardingConfiguration(); - record.Sharding.Prefixed = new List - { + record.Sharding.Prefixed = + [ new PrefixedShardingSetting { // range for 'eu/' is : // shard 0 : [1M, 2M] - Prefix = "eu/", - Shards = new List { 0 } + Prefix = "eu/", + Shards = [0] }, + new PrefixedShardingSetting { // range for 'asia/' is : // shard 1 : [2M, 2.5M] // shard 2 : [2.5M, 3M] - Prefix = "asia/", - Shards = new List { 1, 2 } + Prefix = "asia/", + Shards = [1, 2] } - }; + ]; } }); @@ -841,6 +788,1918 @@ public void RavenDb_19737() } } + [RavenFact(RavenTestCategory.Sharding)] + public async Task CanUpdatePrefixSetting() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "eu/", + Shards = [0, 1] + }, + new PrefixedShardingSetting + { + Prefix = "us/", + Shards = [1, 2] + } + ]; + } + }); + + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) + { + string id = "eu/users/" + i; + await session.StoreAsync(new Item(), id); + } + + await session.SaveChangesAsync(); + } + + var shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + var bucketRanges = shardingConfiguration.BucketRanges; + Assert.Equal(7, bucketRanges.Count); + + Assert.Equal(ShardHelper.NumberOfBuckets, shardingConfiguration.BucketRanges[3].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 1.5, shardingConfiguration.BucketRanges[4].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 2, shardingConfiguration.BucketRanges[5].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 2.5, shardingConfiguration.BucketRanges[6].BucketRangeStart); + + // update 'eu/' prefix setting : add shard #2 + + await store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "eu/", + Shards = new List { 0, 1, 2 } + })); + + shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(2, shardingConfiguration.Prefixed.Count); + Assert.Equal("eu/", shardingConfiguration.Prefixed[1].Prefix); + Assert.Equal(3, shardingConfiguration.Prefixed[1].Shards.Count); + + // shard #2 should get no bucket ranges for the 'eu/' prefix + Assert.Equal(bucketRanges.Count, shardingConfiguration.BucketRanges.Count); + for (int index = 0; index < bucketRanges.Count; index++) + { + ShardBucketRange oldRange = bucketRanges[index]; + var newRange = shardingConfiguration.BucketRanges[index]; + + Assert.Equal(oldRange.BucketRangeStart, newRange.BucketRangeStart); + Assert.Equal(oldRange.ShardNumber, newRange.ShardNumber); + } + + // attempt to remove shard #1 from 'us/' setting should throw + // we cannot remove shard #1 because there are bucket ranges mapped to shard #1 for this prefix + + await Assert.ThrowsAsync(async () => + await store.Maintenance.SendAsync( + new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "us/", Shards = [2] + }))); + + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task CanHandlePrefixOfPrefix() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0] + }, + new PrefixedShardingSetting + { + Prefix = "users/us/utah/", + Shards = [1] + }, + new PrefixedShardingSetting + { + Prefix = "users/us/", + Shards = [2] + } + ]; + } + }); + + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) + { + string id = "users/us/california/" + i; + await session.StoreAsync(new Item(), id); + } + + await session.SaveChangesAsync(); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/")).ToList(); + Assert.Equal(0, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/")).ToList(); + Assert.Equal(0, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/")).ToList(); + Assert.Equal(10, docs.Count); + } + + var shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(3, shardingConfiguration.Prefixed.Count); + + // should be sorted by descending + Assert.Equal("users/us/utah/", shardingConfiguration.Prefixed[0].Prefix); + Assert.Equal("users/us/", shardingConfiguration.Prefixed[1].Prefix); + Assert.Equal("users/", shardingConfiguration.Prefixed[2].Prefix); + + var bucketRanges = shardingConfiguration.BucketRanges; + Assert.Equal(6, bucketRanges.Count); + Assert.Equal(ShardHelper.NumberOfBuckets, bucketRanges[3].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 2, bucketRanges[4].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 3, bucketRanges[5].BucketRangeStart); + + Assert.Equal(ShardHelper.NumberOfBuckets, shardingConfiguration.Prefixed[0].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 2, shardingConfiguration.Prefixed[1].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 3, shardingConfiguration.Prefixed[2].BucketRangeStart); + + // add 'users/us/arizona/' prefix setting + await store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/us/arizona/", + Shards = [1] + })); + + shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(4, shardingConfiguration.Prefixed.Count); + + // should still be sorted + Assert.Equal("users/us/utah/", shardingConfiguration.Prefixed[0].Prefix); + Assert.Equal("users/us/arizona/", shardingConfiguration.Prefixed[1].Prefix); + Assert.Equal("users/us/", shardingConfiguration.Prefixed[2].Prefix); + Assert.Equal("users/", shardingConfiguration.Prefixed[3].Prefix); + + Assert.Equal(ShardHelper.NumberOfBuckets, shardingConfiguration.Prefixed[0].BucketRangeStart); + + // new prefix should be added at the end of BucketRanges + Assert.Equal(ShardHelper.NumberOfBuckets * 4, shardingConfiguration.Prefixed[1].BucketRangeStart); + + Assert.Equal(ShardHelper.NumberOfBuckets * 2, shardingConfiguration.Prefixed[2].BucketRangeStart); + Assert.Equal(ShardHelper.NumberOfBuckets * 3, shardingConfiguration.Prefixed[3].BucketRangeStart); + + // check bucket ranges + + var newBucketRanges = shardingConfiguration.BucketRanges; + Assert.Equal(7, newBucketRanges.Count); + + for (int i = 0; i < bucketRanges.Count; i++) + { + var oldRange = bucketRanges[i]; + var newRange = newBucketRanges[i]; + + Assert.Equal(oldRange.BucketRangeStart, newRange.BucketRangeStart); + Assert.Equal(oldRange.ShardNumber, newRange.ShardNumber); + } + + Assert.Equal(ShardHelper.NumberOfBuckets * 4, newBucketRanges[6].BucketRangeStart); + Assert.Equal(1, newBucketRanges[6].ShardNumber); + + // should all go to shard #1 + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) + { + string id = "users/us/arizona/" + i; + await session.StoreAsync(new Item(), id); + } + + await session.SaveChangesAsync(); + + WaitForUserToContinueTheTest(store); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/arizona/")).ToList(); + Assert.Equal(0, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/arizona/")).ToList(); + Assert.Equal(10, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/arizona/")).ToList(); + Assert.Equal(0, docs.Count); + } + + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task AfterAddingNewPrefixMatchingDocsShouldNotGoToWrongShard() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0] + }, + + new PrefixedShardingSetting + { + Prefix = "users/eu/", + Shards = [0] + }, + + new PrefixedShardingSetting + { + Prefix = "users/asia/", + Shards = [2] + }, + + new PrefixedShardingSetting + { + Prefix = "users/africa/", + Shards = [2] + } + ]; + } + }); + + await foreach (var shard in Sharding.GetShardsDocumentDatabaseInstancesFor(store)) + { + shard.ForTestingPurposes ??= new DocumentDatabase.TestingStuff + { + EnableWritesToTheWrongShard = true + }; + } + + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) + { + string id = "users/eu/" + i; + await session.StoreAsync(new Item(), id); + } + + await session.SaveChangesAsync(); + } + + // add 'users/us/' prefix setting + await store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/us/", + Shards = [1] + })); + + // should all go to shard #1 + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) + { + string id = "users/us/" + i; + await session.StoreAsync(new Item(), id); + } + + await session.SaveChangesAsync(); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/")).ToList(); + Assert.Equal(0, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/")).ToList(); + Assert.Equal(10, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/")).ToList(); + Assert.Equal(0, docs.Count); + } + + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task PrefixesOperationsShouldBeCaseInsensitive() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "Users/", + Shards = [0, 1] + }, + new PrefixedShardingSetting + { + Prefix = "Companies/", + Shards = [0, 1, 2] + } + ]; + } + }); + + + await store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1, 2] + })); + + var shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + + Assert.Equal(2, shardingConfiguration.Prefixed.Count); + Assert.Equal("Users/", shardingConfiguration.Prefixed[0].Prefix); + Assert.Equal(new[] { 0, 1, 2 }, shardingConfiguration.Prefixed[0].Shards); + + await store.Maintenance.SendAsync(new DeletePrefixedShardingSettingOperation("COMPANIES/")); + + shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + + Assert.Equal(1, shardingConfiguration.Prefixed.Count); + + await store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "products/", + Shards = new List { 2 } + })); + + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) + { + var id = $"Products/{i}"; + await session.StoreAsync(new Item(), id); + } + + await session.SaveChangesAsync(); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("PRODUCTS/")).ToList(); + Assert.Equal(0, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("PRODUCTS/")).ToList(); + Assert.Equal(0, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("PRODUCTS/")).ToList(); + Assert.Equal(10, docs.Count); + } + + var task = store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting() + { + Prefix = "Products/", + Shards = new List() { 1 } + })); + await Assert.ThrowsAsync(async () => await task); + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task UpdatePrefixesInCluster() + { + var cluster = await CreateRaftCluster(3, watcherCluster: true); + var options = Sharding.GetOptionsForCluster(cluster.Leader, shards: 3, shardReplicationFactor: 1, orchestratorReplicationFactor: 3); + options.ModifyDatabaseRecord += record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0] + }, + new PrefixedShardingSetting + { + Prefix = "users/us/utah/", + Shards = [1] + }, + new PrefixedShardingSetting + { + Prefix = "users/us/", + Shards = [2] + } + ]; + }; + using var store = GetDocumentStore(options); + + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) + { + await session.StoreAsync(new Item(), "users/eu/sweden/" + i); + await session.StoreAsync(new Item(), "users/us/utah/" + i); + await session.StoreAsync(new Item(), "users/us/california/" + i); + } + + await session.SaveChangesAsync(); + } + + //var stores = Cluster.GetDocumentStores() + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/")).ToList(); + Assert.Equal(10, docs.Count); + + foreach (var doc in docs) + { + Assert.StartsWith("users/eu/sweden/", doc.Id); + } + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/")).ToList(); + Assert.Equal(10, docs.Count); + + foreach (var doc in docs) + { + Assert.StartsWith("users/us/utah/", doc.Id); + } + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/")).ToList(); + Assert.Equal(10, docs.Count); + + foreach (var doc in docs) + { + Assert.StartsWith("users/us/california/", doc.Id); + } + } + + // add 'users/us/arizona/' prefix setting + await store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/us/arizona/", + Shards = new List { 1 } + })); + + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) + { + string id = "users/us/arizona/" + i; + await session.StoreAsync(new Item(), id); + } + + await session.SaveChangesAsync(); + } + + var stores = GetDocumentStores(cluster.Nodes, store.Database, disableTopologyUpdates: true); + foreach (var s in stores) + { + using (var session = s.OpenAsyncSession()) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/arizona/")).ToList(); + Assert.Equal(10, docs.Count); + } + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/arizona/")).ToList(); + Assert.Equal(0, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/arizona/")).ToList(); + Assert.Equal(10, docs.Count); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var docs = (await session.Advanced.LoadStartingWithAsync("users/us/arizona/")).ToList(); + Assert.Equal(0, docs.Count); + } + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task CanMoveOneBucketFromPrefixedRange() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + // bucket range for 'users/' is : + // shard 0 : [1M, 1.5M] + // shard 1 : [1.5M, 2M] + Prefix = "users/", + Shards = [0, 1] + } + ]; + } + }); + + const string id = "users/1"; + using (var session = store.OpenAsyncSession()) + { + var user = new User + { + Name = "Original shard" + }; + await session.StoreAsync(user, id); + await session.SaveChangesAsync(); + } + + var shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + + Assert.Equal(5, shardingConfiguration.BucketRanges.Count); + Assert.Equal(ShardHelper.NumberOfBuckets, shardingConfiguration.Prefixed[0].BucketRangeStart); + + var bucket = await Sharding.GetBucketAsync(store, id); + + var originalLocation = ShardHelper.GetShardNumberFor(shardingConfiguration, bucket); + Assert.Contains(originalLocation, shardingConfiguration.Prefixed[0].Shards); + var newLocation = shardingConfiguration.Prefixed[0].Shards.Single(s => s != originalLocation); + + await Sharding.Resharding.MoveShardForId(store, id); + + shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(7, shardingConfiguration.BucketRanges.Count); + + Assert.Equal(bucket, shardingConfiguration.BucketRanges[^2].BucketRangeStart); + Assert.Equal(newLocation, shardingConfiguration.BucketRanges[^2].ShardNumber); + + Assert.Equal(bucket + 1, shardingConfiguration.BucketRanges[^1].BucketRangeStart); + Assert.Equal(originalLocation, shardingConfiguration.BucketRanges[^1].ShardNumber); + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, originalLocation))) + { + var user = await session.LoadAsync(id); + Assert.Null(user); + } + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, newLocation))) + { + var user = await session.LoadAsync(id); + Assert.Equal("Original shard", user.Name); + } + + // the document will be written to the new location + using (var session = store.OpenAsyncSession()) + { + var user = await session.LoadAsync(id); + user.Name = "New shard"; + await session.SaveChangesAsync(); + } + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, originalLocation))) + { + var user = await session.LoadAsync(id); + Assert.Null(user); + } + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, newLocation))) + { + var user = await session.LoadAsync(id); + Assert.Equal("New shard", user.Name); + } + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task CanMoveOneBucketFromPrefixedRangeToNewShard() + { + var (_, leader) = await CreateRaftCluster(3, watcherCluster: true); + var options = Sharding.GetOptionsForCluster(leader, shards: 2, shardReplicationFactor: 2, orchestratorReplicationFactor: 2); + options.ModifyDatabaseRecord += record => + { + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "foo/", + Shards = [0] + } + ]; + + }; + + using (var store = GetDocumentStore(options)) + { + var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); + var shardTopology = record.Sharding.Shards[0]; + Assert.Equal(2, shardTopology.Members.Count); + Assert.Equal(0, shardTopology.Promotables.Count); + Assert.Equal(2, shardTopology.ReplicationFactor); + + //create new shard + var res = store.Maintenance.Server.Send(new AddDatabaseShardOperation(store.Database)); + var newShardNumber = res.ShardNumber; + Assert.Equal(2, newShardNumber); + Assert.Equal(2, res.ShardTopology.ReplicationFactor); + Assert.Equal(2, res.ShardTopology.AllNodes.Count()); + await Cluster.WaitForRaftIndexToBeAppliedInClusterAsync(res.RaftCommandIndex); + + await AssertWaitForValueAsync(async () => + { + record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); + return record.Sharding.Shards.Count; + }, 3); + + await AssertWaitForValueAsync(async () => + { + record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); + record.Sharding.Shards.TryGetValue(newShardNumber, out shardTopology); + return shardTopology?.Members?.Count; + }, 2); + + var nodesContainingNewShard = shardTopology.Members; + + foreach (var node in nodesContainingNewShard) + { + var serverWithNewShard = Servers.Single(x => x.ServerStore.NodeTag == node); + Assert.True(serverWithNewShard.ServerStore.DatabasesLandlord.DatabasesCache.TryGetValue(ShardHelper.ToShardName(store.Database, newShardNumber), out _)); + } + + var id = "foo/bar"; + var bucket = await Sharding.GetBucketAsync(store, id); + var originalDocShard = await Sharding.GetShardNumberForAsync(store, id); + Assert.Equal(0, originalDocShard); + + using (var session = store.OpenAsyncSession()) + { + session.Advanced.WaitForReplicationAfterSaveChanges(replicas: 1); + + var user = new User + { + Name = "Original shard" + }; + await session.StoreAsync(user, id); + await session.SaveChangesAsync(); + } + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, originalDocShard))) + { + var user = await session.LoadAsync(id); + Assert.NotNull(user); + } + + // first we need to add the new shard to the prefix setting + await store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "foo/", + Shards = [0, newShardNumber] + })); + + // move bucket + await Sharding.Resharding.MoveShardForId(store, id, newShardNumber); + + var exists = WaitForDocument(store, id, predicate: null, database: ShardHelper.ToShardName(store.Database, newShardNumber)); + Assert.True(exists); + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, newShardNumber))) + { + var user = await session.LoadAsync(id); + Assert.NotNull(user); + } + + // check bucket ranges + var sharding = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(5, sharding.BucketRanges.Count); + Assert.Equal(ShardHelper.NumberOfBuckets ,sharding.BucketRanges[2].BucketRangeStart); + Assert.Equal(bucket, sharding.BucketRanges[3].BucketRangeStart); + Assert.Equal(bucket + 1, sharding.BucketRanges[4].BucketRangeStart); + + // the document will be written to the new location + using (var session = store.OpenAsyncSession()) + { + var user = await session.LoadAsync(id); + user.Name = "New shard"; + await session.SaveChangesAsync(); + } + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, originalDocShard))) + { + var user = await session.LoadAsync(id); + Assert.Null(user); + } + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, newShardNumber))) + { + var user = await session.LoadAsync(id); + Assert.Equal("New shard", user.Name); + } + } + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task ShouldThrowOnAttemptToMovePrefixedBucketToShardNotInPrefixSetting() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1] + } + ]; + } + }); + + var shardingConfig = await Sharding.GetShardingConfigurationAsync(store); + var bucket = Sharding.GetBucket(shardingConfig, "users/1"); + + // shard #2 is not a part of Prefixed['users/'].Shards + await Assert.ThrowsAsync(async ()=> + await Server.ServerStore.Sharding.StartBucketMigration(store.Database, bucket, toShard : 2, prefix: "users/", RaftIdGenerator.NewId())); + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task ShardByDocumentsPrefixWithManyDocs_CanMoveBigBucketToNewShard() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0 , 1] + } + ]; + } + }); + + const string bigBucketId = "users/123"; + using (var bulk = store.BulkInsert()) + { + for (int i = 0; i < 100_000; i++) + { + var id = $"users/{i}"; + bulk.Store(new User(), id); + bulk.Store(new User(), $"{id}${bigBucketId}"); + } + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var numberOfDocs = (await session.Advanced.LoadStartingWithAsync("users/", pageSize: int.MaxValue)).Count(); + Assert.Equal(149724, numberOfDocs); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var numberOfDocs = (await session.Advanced.LoadStartingWithAsync("users/", pageSize: int.MaxValue)).Count(); + Assert.Equal(50276, numberOfDocs); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var numberOfDocs = (await session.Advanced.LoadStartingWithAsync("users/", pageSize: int.MaxValue)).Count(); + Assert.Equal(0, numberOfDocs); + } + + int bucket, shardNumber; + var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); + using (var allocator = new ByteStringContext(SharedMultipleUseFlag.None)) + (shardNumber, bucket) = ShardHelper.GetShardNumberAndBucketFor(record.Sharding, allocator, bigBucketId); + + var shard = await GetDocumentDatabaseInstanceFor(store, ShardHelper.ToShardName(store.Database, shardNumber)); + + using (shard.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) + using (ctx.OpenReadTransaction()) + { + var stats = ShardedDocumentsStorage.GetBucketStatisticsFor(ctx, bucket); + Assert.Equal(30303958, stats.Size); + Assert.Equal(100_001, stats.NumberOfDocuments); + } + + // add shard #2 to prefix setting + await store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1, 2] + })); + + // move big bucket to the newly added shard + await Sharding.Resharding.MoveShardForId(store, $"users/0${bigBucketId}", toShard: 2); + + // assert stats + using (shard.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) + using (ctx.OpenReadTransaction()) + { + var stats = ShardedDocumentsStorage.GetBucketStatisticsFor(ctx, bucket); + Assert.Equal(0, stats.NumberOfDocuments); + Assert.Equal(9988978, stats.Size); + + var tombsCount = shard.DocumentsStorage.GetNumberOfTombstones(ctx); + Assert.Equal(100_001, tombsCount); + + await shard.TombstoneCleaner.ExecuteCleanup(); + } + + using (shard.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) + using (ctx.OpenReadTransaction()) + { + var stats = ShardedDocumentsStorage.GetBucketStatisticsFor(ctx, bucket); + Assert.Null(stats); + + var tombsCount = shard.DocumentsStorage.GetNumberOfTombstones(ctx); + Assert.Equal(0, tombsCount); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var numberOfDocs = (await session.Advanced.LoadStartingWithAsync("users/", pageSize: int.MaxValue)).Count(); + Assert.Equal(49723, numberOfDocs); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var numberOfDocs = (await session.Advanced.LoadStartingWithAsync("users/", pageSize: int.MaxValue)).Count(); + Assert.Equal(50276, numberOfDocs); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var numberOfDocs = (await session.Advanced.LoadStartingWithAsync("users/", pageSize: int.MaxValue)).Count(); + Assert.Equal(100_001, numberOfDocs); + } + + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task CanMoveBucketFromPrefixedRangeWhileWriting() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1] + } + ]; + } + }); + + using (var bulk = store.BulkInsert()) + { + for (int i = 0; i < 1000; i++) + { + var id = $"users/{i}"; + bulk.Store(new User(), id); + } + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(538, numberOfDocs); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(462, numberOfDocs); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(0, numberOfDocs); + } + + // add shard #2 to prefix setting + await store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1, 2] + })); + + // move bucket to the newly added shard while writing + var docId = "users/1"; + var bucket = await Sharding.GetBucketAsync(store, docId); + var originalShardNumber = await Sharding.GetShardNumberForAsync(store, docId); + + var writes = Task.Run(async () => + { + using (var session = store.OpenAsyncSession()) + { + for (int i = 1000; i < 2000; i++) + { + var id = $"users/{i}${docId}"; + await session.StoreAsync(new User(), id); + } + + await session.SaveChangesAsync(); + } + }); + var bucketMigration = Sharding.Resharding.MoveShardForId(store, docId, toShard: 2); + + await Task.WhenAll(bucketMigration, writes); + + // assert bucket ranges + var shardingConfig = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(7, shardingConfig.BucketRanges.Count); + + Assert.Equal(ShardHelper.NumberOfBuckets, shardingConfig.BucketRanges[3].BucketRangeStart); + Assert.Equal(0, shardingConfig.BucketRanges[3].ShardNumber); + + Assert.Equal(ShardHelper.NumberOfBuckets * 1.5, shardingConfig.BucketRanges[4].BucketRangeStart); + Assert.Equal(1, shardingConfig.BucketRanges[4].ShardNumber); + + Assert.Equal(bucket, shardingConfig.BucketRanges[5].BucketRangeStart); + Assert.Equal(2, shardingConfig.BucketRanges[5].ShardNumber); + + Assert.Equal(bucket + 1, shardingConfig.BucketRanges[6].BucketRangeStart); + Assert.Equal(1, shardingConfig.BucketRanges[6].ShardNumber); + + // assert stats + var originalShard = await GetDocumentDatabaseInstanceFor(store, ShardHelper.ToShardName(store.Database, originalShardNumber)); + using (originalShard.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) + using (ctx.OpenReadTransaction()) + { + var stats = ShardedDocumentsStorage.GetBucketStatisticsFor(ctx, bucket); + Assert.Equal(0, stats.NumberOfDocuments); + } + + var newShard = await GetDocumentDatabaseInstanceFor(store, ShardHelper.ToShardName(store.Database, shard: 2)); + using (newShard.DocumentsStorage.ContextPool.AllocateOperationContext(out DocumentsOperationContext ctx)) + using (ctx.OpenReadTransaction()) + { + var stats = ShardedDocumentsStorage.GetBucketStatisticsFor(ctx, bucket); + Assert.Equal(1001, stats.NumberOfDocuments); + } + + // assert docs + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 0))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(538, numberOfDocs); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 1))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(461, numberOfDocs); + } + + using (var session = store.OpenAsyncSession(database: ShardHelper.ToShardName(store.Database, 2))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(1001, numberOfDocs); + } + + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task ShouldNotAllowToRemoveShardFromDbIfItHasPrefixesSettings() + { + var cluster = await CreateRaftCluster(numberOfNodes: 3, watcherCluster: true); + var options = Sharding.GetOptionsForCluster(cluster.Leader, shards: 2, shardReplicationFactor: 1, orchestratorReplicationFactor: 3); + options.ModifyDatabaseRecord += record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1] + } + ]; + }; + + using var store = Sharding.GetDocumentStore(options); + + using var session = store.OpenAsyncSession(); + for (int i = 0; i < 1000; i++) + { + var id = $"users/{i}"; + await session.StoreAsync(new User(), id); + } + await session.SaveChangesAsync(); + + // add shard #2 to database + var sharding = await Sharding.GetShardingConfigurationAsync(store); + var shardNodes = sharding.Shards.Select(kvp => kvp.Value.Members[0]); + var nodeNotInDbGroup = cluster.Nodes.SingleOrDefault(n => shardNodes.Contains(n.ServerStore.NodeTag) == false)?.ServerStore.NodeTag; + Assert.NotNull(nodeNotInDbGroup); + + var addShardRes = store.Maintenance.Server.Send(new AddDatabaseShardOperation(store.Database, [nodeNotInDbGroup])); + Assert.Equal(2, addShardRes.ShardNumber); + await Cluster.WaitForRaftIndexToBeAppliedInClusterAsync(addShardRes.RaftCommandIndex); + + await AssertWaitForValueAsync(async () => + { + sharding = await Sharding.GetShardingConfigurationAsync(store); + sharding.Shards.TryGetValue(2, out var topology); + return topology?.Members.Count; + }, expectedVal: 1); + + + // add shard #2 to 'users/' prefix setting + await store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1, 2] + })); + + // should not allow to delete shard #2 because it's part of 'users/' prefix setting + var deleteShardTask = store.Maintenance.Server.SendAsync(new DeleteDatabasesOperation(store.Database, shardNumber: 2, hardDelete: true, fromNode: nodeNotInDbGroup)); + await Assert.ThrowsAsync(async () => await deleteShardTask); + + // remove shard #2 from 'users/' prefix setting + // can be removed because shard #2 has no bucket ranges for this prefix + await store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1] + })); + + // now we should be able to delete shard #2 from database + var res = await store.Maintenance.Server.SendAsync(new DeleteDatabasesOperation(store.Database, shardNumber: 2, hardDelete: true, fromNode: nodeNotInDbGroup)); + await Cluster.WaitForRaftIndexToBeAppliedInClusterAsync(res.RaftCommandIndex); + + await AssertWaitForValueAsync(async () => + { + sharding = await Sharding.GetShardingConfigurationAsync(store); + return sharding.Shards.TryGetValue(2, out _); + }, expectedVal: false); + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task WhenAddingNewPrefixShouldFillBucketRangeGaps() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/a/", + Shards = [0] + }, + new PrefixedShardingSetting + { + Prefix = "users/b/", + Shards = [1] + }, + new PrefixedShardingSetting + { + Prefix = "users/c/", + Shards = [0, 1] + }, + new PrefixedShardingSetting + { + Prefix = "users/d/", + Shards = [0, 1, 2] + }, + ]; + } + }); + + var sharding = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(4, sharding.Prefixed.Count); + Assert.Equal("users/d/", sharding.Prefixed[0].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets, sharding.Prefixed[0].BucketRangeStart); + + Assert.Equal("users/c/", sharding.Prefixed[1].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets * 2, sharding.Prefixed[1].BucketRangeStart); + + Assert.Equal("users/b/", sharding.Prefixed[2].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets * 3, sharding.Prefixed[2].BucketRangeStart); + + Assert.Equal("users/a/", sharding.Prefixed[3].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets * 4, sharding.Prefixed[3].BucketRangeStart); + + // deleting 'users/c/' will create a gap in prefixes bucket range start (range 1M - 2M range is missing) + await store.Maintenance.SendAsync(new DeletePrefixedShardingSettingOperation("users/c/")); + sharding = await Sharding.GetShardingConfigurationAsync(store); + + Assert.Equal(3, sharding.Prefixed.Count); + Assert.Equal("users/d/", sharding.Prefixed[0].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets, sharding.Prefixed[0].BucketRangeStart); + + Assert.Equal("users/b/", sharding.Prefixed[1].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets * 3, sharding.Prefixed[1].BucketRangeStart); + + Assert.Equal("users/a/", sharding.Prefixed[2].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets * 4, sharding.Prefixed[2].BucketRangeStart); + + // add a new prefix, 1M - 2M range should be assigned to it + await store.Maintenance.SendAsync(new AddPrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/z/", + Shards = [1, 2] + })); + + sharding = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(4, sharding.Prefixed.Count); + Assert.Equal("users/z/", sharding.Prefixed[0].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets * 2, sharding.Prefixed[0].BucketRangeStart); + + Assert.Equal("users/d/", sharding.Prefixed[1].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets, sharding.Prefixed[1].BucketRangeStart); + + Assert.Equal("users/b/", sharding.Prefixed[2].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets * 3, sharding.Prefixed[2].BucketRangeStart); + + Assert.Equal("users/a/", sharding.Prefixed[3].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets * 4, sharding.Prefixed[3].BucketRangeStart); + + } + + [RavenTheory(RavenTestCategory.Sharding | RavenTestCategory.Etl)] + [RavenData(SearchEngineMode = RavenSearchEngineMode.All)] + public async Task ReshardingWithEtl_PrefixedSource(Options options) + { + using var srcStore = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1] + } + ]; + } + }); + + using var dstStore = GetDocumentStore(options); + Etl.AddEtl(srcStore, dstStore, "users", script: null); + + using (var bulk = srcStore.BulkInsert()) + { + for (int i = 0; i < 1000; i++) + { + var id = $"users/{i}"; + bulk.Store(new User(), id); + } + } + + await AssertWaitForValueAsync(async () => + { + using var session = dstStore.OpenAsyncSession(); + return await session.Query().CountAsync(); + }, expectedVal: 1000); + + + // add shard #2 to prefix setting + await srcStore.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1, 2] + })); + + var docId = "users/1"; + var writes = Task.Run(async () => + { + using (var session = srcStore.OpenAsyncSession()) + { + for (int i = 1000; i < 2000; i++) + { + var id = $"users/{i}${docId}"; + await session.StoreAsync(new User(), id); + } + + await session.SaveChangesAsync(); + } + }); + + await Sharding.Resharding.MoveShardForId(srcStore, docId, toShard: 2); + await writes; + + // assert docs + using (var session = srcStore.OpenAsyncSession(database: ShardHelper.ToShardName(srcStore.Database, 0))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(538, numberOfDocs); + } + + using (var session = srcStore.OpenAsyncSession(database: ShardHelper.ToShardName(srcStore.Database, 1))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(461, numberOfDocs); + } + + using (var session = srcStore.OpenAsyncSession(database: ShardHelper.ToShardName(srcStore.Database, 2))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(1001, numberOfDocs); + } + + await AssertWaitForValueAsync(async () => + { + using var session = dstStore.OpenAsyncSession(); + return await session.Query().CountAsync(); + }, expectedVal: 2000); + } + + [RavenTheory(RavenTestCategory.Sharding | RavenTestCategory.Etl)] + [RavenData(SearchEngineMode = RavenSearchEngineMode.All)] + public async Task ReshardingWithEtl_PrefixedDestination(Options options) + { + using var srcStore = GetDocumentStore(options); + using var dstStore = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1] + } + ]; + } + }); + + Etl.AddEtl(srcStore, dstStore, "users", script: null); + + using (var bulk = srcStore.BulkInsert()) + { + for (int i = 0; i < 1000; i++) + { + var id = $"users/{i}"; + bulk.Store(new User(), id); + } + } + + await AssertWaitForValueAsync(async () => + { + using var session = dstStore.OpenAsyncSession(); + return await session.Query().CountAsync(); + }, expectedVal: 1000); + + + // add shard #2 to prefix setting + await dstStore.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1, 2] + })); + + var docId = "users/1"; + var writes = Task.Run(() => + { + using (var bulk = srcStore.BulkInsert()) + { + for (int i = 1000; i < 2000; i++) + { + var id = $"users/{i}${docId}"; + bulk.Store(new User(), id); + } + } + }); + + await Sharding.Resharding.MoveShardForId(dstStore, docId, toShard: 2); + await writes; + + await AssertWaitForValueAsync(async () => + { + using var session = dstStore.OpenAsyncSession(); + return await session.Query().CountAsync(); + }, expectedVal: 2000); + + // assert docs + using (var session = dstStore.OpenAsyncSession(database: ShardHelper.ToShardName(dstStore.Database, 0))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(538, numberOfDocs); + } + + using (var session = dstStore.OpenAsyncSession(database: ShardHelper.ToShardName(dstStore.Database, 1))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(461, numberOfDocs); + } + + using (var session = dstStore.OpenAsyncSession(database: ShardHelper.ToShardName(dstStore.Database, 2))) + { + var numberOfDocs = await session.Query().CountAsync(); + Assert.Equal(1001, numberOfDocs); + } + } + + [RavenFact(RavenTestCategory.BackupExportImport | RavenTestCategory.Sharding)] + public async Task CanImportIncrementalIntoPrefixedShardedDatabase() + { + var backupPath = NewDataPath(suffix: "_BackupFolder"); + + using (var store1 = Sharding.GetDocumentStore(new Options() + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new(); + record.Sharding.Prefixed = [new PrefixedShardingSetting + { + Prefix = "Users/", + Shards = [0, 1] + }]; + + } + })) + using (var store2 = Sharding.GetDocumentStore(new Options() + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new(); + record.Sharding.Prefixed = [new PrefixedShardingSetting + { + Prefix = "Users/", + Shards = [1 , 2] + }]; + } + })) + { + var shardNumToDocIds = new Dictionary>(); + var dbRecord = await store1.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store1.Database)); + var shardedCtx = new ShardedDatabaseContext(Server.ServerStore, dbRecord); + + // generate data on store1, keep track of doc-ids per shard + using (var session = store1.OpenAsyncSession()) + using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) + { + for (int i = 0; i < 100; i++) + { + var user = new User { Name = i.ToString() }; + var id = $"users/{i}"; + + var shardNumber = shardedCtx.GetShardNumberFor(context, id); + if (shardNumToDocIds.TryGetValue(shardNumber, out var ids) == false) + { + shardNumToDocIds[shardNumber] = ids = new List(); + } + ids.Add(id); + + await session.StoreAsync(user, id); + } + + Assert.Equal(2, shardNumToDocIds.Count); + Assert.False(shardNumToDocIds.ContainsKey(2)); + + await session.SaveChangesAsync(); + } + + var waitHandles = await Sharding.Backup.WaitForBackupToComplete(store1); + + var config = Backup.CreateBackupConfiguration(backupPath, incrementalBackupFrequency: "* * * * *"); + await Sharding.Backup.UpdateConfigurationAndRunBackupAsync(Server, store1, config); + + Assert.True(WaitHandle.WaitAll(waitHandles, TimeSpan.FromMinutes(1))); + + // import + var dirs = Directory.GetDirectories(backupPath); + Assert.Equal(3, dirs.Length); + + foreach (var dir in dirs) + { + await store2.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(), dir); + } + + using (var session = store2.OpenAsyncSession(ShardHelper.ToShardName(store2.Database, 0))) + { + var docs = await session.Query().ToListAsync(); + Assert.Equal(0, docs.Count); + } + using (var session = store2.OpenAsyncSession(ShardHelper.ToShardName(store2.Database, 1))) + { + var docs = await session.Query().ToListAsync(); + Assert.Equal(shardNumToDocIds[0].Count, docs.Count); + + foreach (var doc in docs) + { + var id = doc.Id; + Assert.True(shardNumToDocIds[0].Contains(id)); + } + } + using (var session = store2.OpenAsyncSession(ShardHelper.ToShardName(store2.Database, 2))) + { + var docs = await session.Query().ToListAsync(); + Assert.Equal(shardNumToDocIds[1].Count, docs.Count); + + foreach (var doc in docs) + { + var id = doc.Id; + Assert.True(shardNumToDocIds[1].Contains(id)); + } + } + + // add more data to store1 + using (var session = store1.OpenAsyncSession()) + using (Server.ServerStore.ContextPool.AllocateOperationContext(out TransactionOperationContext context)) + { + for (int i = 100; i < 200; i++) + { + var user = new User { Name = i.ToString() }; + var id = $"users/{i}"; + + var shardNumber = shardedCtx.GetShardNumberFor(context, id); + shardNumToDocIds[shardNumber].Add(id); + + await session.StoreAsync(user, id); + } + + await session.SaveChangesAsync(); + } + + waitHandles = await Sharding.Backup.WaitForBackupToComplete(store1); + + await Sharding.Backup.UpdateConfigurationAndRunBackupAsync(Server, store1, config); + + Assert.True(WaitHandle.WaitAll(waitHandles, TimeSpan.FromMinutes(1))); + + // import + var newDirs = Directory.GetDirectories(backupPath).Except(dirs).ToList(); + Assert.Equal(3, newDirs.Count); + + foreach (var dir in newDirs) + { + await store2.Smuggler.ImportIncrementalAsync(new DatabaseSmugglerImportOptions(), dir); + } + + // assert + using (var session = store2.OpenAsyncSession(ShardHelper.ToShardName(store2.Database, 0))) + { + var docs = await session.Query().ToListAsync(); + Assert.Equal(0, docs.Count); + } + using (var session = store2.OpenAsyncSession(ShardHelper.ToShardName(store2.Database, 1))) + { + var docs = await session.Query().ToListAsync(); + Assert.Equal(shardNumToDocIds[0].Count, docs.Count); + + foreach (var doc in docs) + { + var id = doc.Id; + Assert.True(shardNumToDocIds[0].Contains(id)); + } + } + using (var session = store2.OpenAsyncSession(ShardHelper.ToShardName(store2.Database, 2))) + { + var docs = await session.Query().ToListAsync(); + Assert.Equal(shardNumToDocIds[1].Count, docs.Count); + + foreach (var doc in docs) + { + var id = doc.Id; + Assert.True(shardNumToDocIds[1].Contains(id)); + } + } + } + } + + [RavenFact(RavenTestCategory.BackupExportImport | RavenTestCategory.Sharding)] + public async Task CanBackupAndRestorePrefixedShardedDatabase_FromIncrementalBackup() + { + var backupPath = NewDataPath(suffix: "BackupFolder"); + var cluster = await CreateRaftCluster(3, watcherCluster: true); + + var options = Sharding.GetOptionsForCluster(cluster.Leader, shards: 3, shardReplicationFactor: 1, orchestratorReplicationFactor: 3); + options.ModifyDatabaseRecord += record => + { + record.Sharding.Prefixed = [new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1] + }]; + }; + + using (var store = Sharding.GetDocumentStore(options)) + { + using (var session = store.OpenAsyncSession()) + { + for (int i = 0; i < 10; i++) + { + await session.StoreAsync(new User(), $"users/{i}"); + } + + await session.SaveChangesAsync(); + } + + var waitHandles = await Sharding.Backup.WaitForBackupsToComplete(cluster.Nodes, store.Database); + + var config = Backup.CreateBackupConfiguration(backupPath); + var backupTaskId = await Sharding.Backup.UpdateConfigurationAndRunBackupAsync(cluster.Nodes, store, config, isFullBackup: false); + + Assert.True(WaitHandle.WaitAll(waitHandles, TimeSpan.FromMinutes(1))); + + // add more data + waitHandles = await Sharding.Backup.WaitForBackupsToComplete(cluster.Nodes, store.Database); + using (var session = store.OpenAsyncSession()) + { + for (int i = 10; i < 20; i++) + { + await session.StoreAsync(new User(), $"users/{i}"); + } + + await session.SaveChangesAsync(); + } + + // add shard #2 to prefix setting and move one bucket to the new shard + await store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1, 2] + })); + + await Sharding.Resharding.MoveShardForId(store, "users/11", toShard: 2); + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, 0))) + { + var count = await session.Query().CountAsync(); + Assert.Equal(9, count); + } + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, 1))) + { + var count = await session.Query().CountAsync(); + Assert.Equal(10, count); + } + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, 2))) + { + var count = await session.Query().CountAsync(); + Assert.Equal(1, count); + } + + await Sharding.Backup.RunBackupAsync(store.Database, backupTaskId, isFullBackup: false, cluster.Nodes); + Assert.True(WaitHandle.WaitAll(waitHandles, TimeSpan.FromMinutes(1))); + + var dirs = Directory.GetDirectories(backupPath); + Assert.Equal(cluster.Nodes.Count, dirs.Length); + + foreach (var dir in dirs) + { + var files = Directory.GetFiles(dir); + Assert.Equal(2, files.Length); + } + + var sharding = await Sharding.GetShardingConfigurationAsync(store); + var settings = Sharding.Backup.GenerateShardRestoreSettings(dirs, sharding); + + // restore the database with a different name + var restoredDatabaseName = $"restored_database-{Guid.NewGuid()}"; + using (Sharding.Backup.ReadOnly(backupPath)) + using (Backup.RestoreDatabase(store, new RestoreBackupConfiguration + { + DatabaseName = restoredDatabaseName, + ShardRestoreSettings = settings + }, timeout: TimeSpan.FromSeconds(60))) + { + var dbRec = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(restoredDatabaseName)); + Assert.Equal(3, dbRec.Sharding.Shards.Count); + + var shardNodes = new HashSet(); + foreach (var shardToTopology in dbRec.Sharding.Shards) + { + var shardTopology = shardToTopology.Value; + Assert.Equal(1, shardTopology.Members.Count); + Assert.Equal(sharding.Shards[shardToTopology.Key].Members[0], shardTopology.Members[0]); + Assert.True(shardNodes.Add(shardTopology.Members[0])); + } + + using (var session = store.OpenSession(restoredDatabaseName)) + { + for (int i = 0; i < 20; i++) + { + var doc = session.Load($"users/{i}"); + Assert.NotNull(doc); + } + } + + sharding = await Sharding.GetShardingConfigurationAsync(store, restoredDatabaseName); + + Assert.Equal(1, sharding.Prefixed.Count); + Assert.Equal("users/", sharding.Prefixed[0].Prefix); + Assert.Equal(ShardHelper.NumberOfBuckets, sharding.Prefixed[0].BucketRangeStart); + Assert.Equal(3, sharding.Prefixed[0].Shards.Count); + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(restoredDatabaseName, 0))) + { + var count = await session.Query().CountAsync(); + Assert.Equal(9, count); + } + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(restoredDatabaseName, 1))) + { + var count = await session.Query().CountAsync(); + Assert.Equal(10, count); + } + + using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(restoredDatabaseName, 2))) + { + var count = await session.Query().CountAsync(); + Assert.Equal(1, count); + } + } + } + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task DeletingPrefixAfterShardsDistributionHasBeenUpdated() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0] + } + ]; + } + }); + + var shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(4, shardingConfiguration.BucketRanges.Count); + + await store.Maintenance.SendAsync(new UpdatePrefixedShardingSettingOperation(new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1, 2] + })); + + shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(4, shardingConfiguration.BucketRanges.Count); + + await store.Maintenance.SendAsync(new DeletePrefixedShardingSettingOperation("users/")); + + shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + + Assert.Equal(3, shardingConfiguration.BucketRanges.Count); + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task DeletingPrefixAfterBucketMigration() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/", + Shards = [0, 1, 2] + } + ]; + } + }); + + var shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(6, shardingConfiguration.BucketRanges.Count); + + var id = "users/2"; + var originalShardForId = await Sharding.GetShardNumberForAsync(store, id); + + Assert.Equal(0, originalShardForId); + + using (var session = store.OpenAsyncSession()) + { + await session.StoreAsync(new User(), id); + await session.SaveChangesAsync(); + } + + await Sharding.Resharding.MoveShardForId(store, id, toShard: 2); + + shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(8, shardingConfiguration.BucketRanges.Count); + + using (var session = store.OpenAsyncSession()) + { + session.Delete(id); + await session.SaveChangesAsync(); + } + + // this should delete all 5 bucket ranges assigned for this prefix + await store.Maintenance.SendAsync(new DeletePrefixedShardingSettingOperation("users/")); + + shardingConfiguration = await Sharding.GetShardingConfigurationAsync(store); + Assert.Equal(3, shardingConfiguration.BucketRanges.Count); + } + + [RavenFact(RavenTestCategory.Sharding)] + public void PrefixedSharding_CanQueryWithSpecifiedShardContext() + { + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "users/us/", + Shards = [0] + }, + new PrefixedShardingSetting + { + Prefix = "users/eu/", + Shards = [1] + }, + new PrefixedShardingSetting + { + Prefix = "users/asia/", + Shards = [2] + } + ]; + } + }); + { + using (var session = store.OpenSession()) + { + session.Store(new User(), "users/us/1"); + session.Store(new User(), "users/us/2"); + session.Store(new User(), "users/us/3"); + + session.Store(new User(), "users/eu/1"); + session.Store(new User(), "users/eu/2"); + + session.Store(new User(), "users/asia/1"); + + + session.SaveChanges(); + } + + using (var session = store.OpenSession()) + { + var results = session.Query() + .Customize(x => x.ShardContext(s => s.ByDocumentId("users/us/1"))) + .ToList(); + + Assert.Equal(3, results.Count); + + var results2 = session.Query() + .Customize(x => x.ShardContext(s => s.ByDocumentIds(new[] { "users/us/1", "users/eu/1" }))) + .Select(x => x.Id) + .ToList(); + + Assert.Equal(5, results2.Count); + Assert.Contains("users/us/1", results2); + Assert.Contains("users/us/2", results2); + Assert.Contains("users/us/3", results2); + Assert.Contains("users/eu/1", results2); + Assert.Contains("users/eu/2", results2); + + var results3 = session.Query() + .Customize(x => x.ShardContext(s => s.ByDocumentId("users/asia/"))) + .ToList(); + + Assert.Equal(1, results3.Count); + } + + using (var session = store.OpenSession()) + { + var results = session.Advanced.DocumentQuery() + .ShardContext(s => s.ByDocumentId("users/us/1")) + .ToList(); + + Assert.Equal(3, results.Count); + + var results2 = session.Advanced.DocumentQuery() + .ShardContext(s => s.ByDocumentIds(new[] { "users/us/1", "users/eu/2" })) + .ToList(); + + Assert.Equal(5, results2.Count); + var results3 = session.Advanced.DocumentQuery() + .ShardContext(s => s.ByDocumentId("users/asia/1")) + .ToList(); + + Assert.Equal(1, results3.Count); + } + } + } + + [RavenFact(RavenTestCategory.Sharding)] + public async Task PrefixesShouldGetPrecedenceOverAnchoring() + { + const string companyId = "companies/1"; + const string relatedDocId = $"products/1${companyId}"; + const int productsShard = 0; + + using var store = Sharding.GetDocumentStore(new Options + { + ModifyDatabaseRecord = record => + { + record.Sharding ??= new ShardingConfiguration(); + record.Sharding.Prefixed = + [ + new PrefixedShardingSetting + { + Prefix = "products/", + Shards = [productsShard] + } + ]; + } + }); + { + using (var session = store.OpenSession()) + { + session.Store(new Company(), companyId); + session.SaveChanges(); + } + + var companyShardNumber = await Sharding.GetShardNumberForAsync(store, companyId); + Assert.Equal(1, companyShardNumber); + + using (var session = store.OpenSession()) + { + session.Store(new Product(), relatedDocId); + session.SaveChanges(); + } + + using (var session = store.OpenSession(ShardHelper.ToShardName(store.Database, companyShardNumber))) + { + var product = session.Query() + .FirstOrDefault(); + + Assert.Null(product); + } + + using (var session = store.OpenSession(ShardHelper.ToShardName(store.Database, productsShard))) + { + var product = session.Query() + .FirstOrDefault(); + + Assert.NotNull(product); + } + } + } + private class Item { #pragma warning disable CS0649 diff --git a/test/SlowTests/Sharding/Subscriptions/ShardedSubscriptionSlowTests.cs b/test/SlowTests/Sharding/Subscriptions/ShardedSubscriptionSlowTests.cs index a1a12b421782..53211b607627 100644 --- a/test/SlowTests/Sharding/Subscriptions/ShardedSubscriptionSlowTests.cs +++ b/test/SlowTests/Sharding/Subscriptions/ShardedSubscriptionSlowTests.cs @@ -821,9 +821,7 @@ public async Task RunningSubscriptionShouldJumpToNextChangeVectorIfItWasChangedB [InlineData(false)] public async Task CanUseSubscriptionWithDocumentIncludes(bool diff) { - DoNotReuseServer(); - Server.ServerStore.Sharding.BlockPrefixedSharding = false; - + var ops = diff ? new Options { @@ -1141,9 +1139,6 @@ public void CanCreateSubscriptionWithIncludeTimeSeries_All_LastRange(bool byTime [InlineData(false)] public void SubscriptionWithIncludeAllCountersOfDocumentAndOfRelatedDocument(bool diff) { - DoNotReuseServer(); - Server.ServerStore.Sharding.BlockPrefixedSharding = false; - var ops = diff ? new Options { diff --git a/test/Tests.Infrastructure/RavenTestBase.ReshardingTestBase.cs b/test/Tests.Infrastructure/RavenTestBase.ReshardingTestBase.cs index 94d8634d37ec..236d6ce5e600 100644 --- a/test/Tests.Infrastructure/RavenTestBase.ReshardingTestBase.cs +++ b/test/Tests.Infrastructure/RavenTestBase.ReshardingTestBase.cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Raven.Client.Documents; using Raven.Client.ServerWide.Operations; using Raven.Client.ServerWide.Sharding; +using Raven.Client.Util; using Raven.Server; using Raven.Server.Utils; using Sparrow.Json; @@ -29,20 +31,31 @@ public async Task StartMovingShardForId(IDocumentStore store, string id, in var record = await store.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(store.Database)); var bucket = _parent.Sharding.GetBucket(record.Sharding, id); - PrefixedShardingSetting prefixed = null; + PrefixedShardingSetting prefixedSetting = null; foreach (var setting in record.Sharding.Prefixed) { if (id.StartsWith(setting.Prefix, StringComparison.OrdinalIgnoreCase)) { - prefixed = setting; + prefixedSetting = setting; break; } - } + } var shardNumber = ShardHelper.GetShardNumberFor(record.Sharding, bucket); - var moveToShard = toShard ?? (prefixed != null - ? ShardingTestBase.GetNextSortedShardNumber(prefixed, shardNumber) - : ShardingTestBase.GetNextSortedShardNumber(record.Sharding.Shards, shardNumber)); + var shards = prefixedSetting != null ? prefixedSetting.Shards : record.Sharding.Shards.Keys.ToList(); + + int moveToShard; + if (toShard.HasValue) + { + if (shards.Contains(toShard.Value) == false) + throw new InvalidOperationException($"Cannot move bucket '{bucket}' from shard {shardNumber} to shard {toShard}. " + + $"Sharding topology does not contain shard {toShard}"); + moveToShard = toShard.Value; + } + else + { + moveToShard = ShardingTestBase.GetNextSortedShardNumber(shards, shardNumber); + } using (var session = store.OpenAsyncSession(ShardHelper.ToShardName(store.Database, shardNumber))) { @@ -53,7 +66,7 @@ public async Task StartMovingShardForId(IDocumentStore store, string id, in { try { - await server.ServerStore.Sharding.StartBucketMigration(store.Database, bucket, moveToShard); + await server.ServerStore.Sharding.StartBucketMigration(store.Database, bucket, moveToShard, prefix: prefixedSetting?.Prefix, raftId: RaftIdGenerator.NewId()); break; } catch diff --git a/test/Tests.Infrastructure/RavenTestBase.Sharding.cs b/test/Tests.Infrastructure/RavenTestBase.Sharding.cs index 8aa4a027e357..d8f41ef0898b 100644 --- a/test/Tests.Infrastructure/RavenTestBase.Sharding.cs +++ b/test/Tests.Infrastructure/RavenTestBase.Sharding.cs @@ -111,15 +111,12 @@ public Options GetOptionsForCluster(RavenServer leader, int shards, int shardRep return options; } - public static int GetNextSortedShardNumber(PrefixedShardingSetting prefixedShardingSetting, int shardNumber) - { - var shardsSorted = prefixedShardingSetting.Shards.OrderBy(x => x).ToArray(); - return GetNextSortedShardNumber(shardNumber, shardsSorted); - } + public static int GetNextSortedShardNumber(Dictionary shards, int shardNumber) => + GetNextSortedShardNumber(shards.Keys, shardNumber); - public static int GetNextSortedShardNumber(Dictionary shards, int shardNumber) + public static int GetNextSortedShardNumber(ICollection shards, int shardNumber) { - var shardsSorted = shards.Keys.OrderBy(x => x).ToArray(); + var shardsSorted = shards.OrderBy(x => x).ToArray(); return GetNextSortedShardNumber(shardNumber, shardsSorted); }