RavenDB-17793 : Prefixed Sharding :

RavenDB-17793 : address PR comments

RavenDB-17793 Add sharding prefixes step to database creator

RavenDB-17793 : fixes after rebase

RavenDB-17793 : address PR comments - throw instead of debug asserts, add more info to error messages

RavenDB-17793 : add tests cases for (1) querying with shard context (2) prefixes should get precedence over anchoring when choosing bucket for id

RavenDB-17793 : address PR comments - avoid doing remote calls within a read tx

RavenDB-17793 : fix failing tests

RavenDB-17793 : minor refactoring + cleanup

RavenDB-17793 : remove BlockPrefixedSharding from ShardingStore

RavenDB-17793 : fix bug in DeletePrefixedSettingCommand - need to remove all associated bucket ranges for this prefix (do not rely on number of shards in prefix setting)

RavenDB-17793 : add more test scenarios

RavenDB-17793 : do not allow to add prefix setting with no shards

RavenDB-17793 : pass prefix to StartBucketMigrationCommand so that we can find the prefix setting by name (sorted) instead of searching by bucket range start (needs to be ordered first)

RavenDB-17793 : when adding a new prefix, reuse old bucket ranges (i.e. bucket ranges that were given to a cretin prefix which was removed) if possible

RavenDB-17793 : do not allow to remove shard from database if this shard is part of any prefix settings

RavenDB-17793 : update prefix setting - don't allow to remove shard if there are bucket ranges mapped to this shards. update command should not modify BucketRanges at all

RavenDB-17793 : throw on attempt to move bucket from prefixed range if destination shard is not a part of shards list in the prefix setting

RavenDB-17793 : use binary search for lookups in prefixed list

RavenDB-17793 : fix bug in sharded restore - do not fill PrefixedSharding on AddDatabaseCommand with no RaftCommandIndex when coming from restore (in restore we take the prefixed setting as is from smuggler record)

RavenDB-17793 : prefixed settings list should be sorted in a descending manner, in order to support having a prefix of prefix

RavenDB-17793 : ensure that prefixes settings cannot be modified via database record. add dedicated operation for updating prefix configuration

src/Raven.Client/ServerWide/Sharding/AddPrefixedShardingSettingOperation.cs

@@ -0,0 +1,32 @@
+using System;
+using Raven.Client.Documents.Conventions;
+using Raven.Client.Documents.Operations;
+using Raven.Client.Http;
+using Sparrow.Json;
+
+namespace Raven.Client.ServerWide.Sharding
+{
+    public sealed class AddPrefixedShardingSettingOperation : IMaintenanceOperation
+    {
+        private readonly PrefixedShardingSetting _setting;
+
+        public AddPrefixedShardingSettingOperation(PrefixedShardingSetting setting)
+        {
+            _setting = setting ?? throw new ArgumentNullException(nameof(setting));
+        }
+
+        public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext ctx)
+        {
+            return new AddPrefixedShardingSettingCommand(conventions, _setting);
+        }
+
+        private sealed class AddPrefixedShardingSettingCommand : PrefixedShardingCommand
+        {
+            protected override PrefixedCommandType CommandType => PrefixedCommandType.Add;
+
+            public AddPrefixedShardingSettingCommand(DocumentConventions conventions, PrefixedShardingSetting setting) : base(conventions, setting)
+            {
+            }
+        }
+    }
+}

src/Raven.Client/ServerWide/Sharding/DeletePrefixedShardingSettingOperation.cs

@@ -0,0 +1,38 @@
+using System;
+using Raven.Client.Documents.Conventions;
+using Raven.Client.Documents.Operations;
+using Raven.Client.Http;
+using Sparrow.Json;
+
+namespace Raven.Client.ServerWide.Sharding
+{
+    public sealed class DeletePrefixedShardingSettingOperation : IMaintenanceOperation
+    {
+        private readonly string _prefix;
+
+        public DeletePrefixedShardingSettingOperation(string prefix)
+        {
+            if (string.IsNullOrEmpty(prefix))
+                throw new ArgumentNullException(nameof(prefix));
+
+            _prefix = prefix;
+        }
+
+        public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext ctx)
+        {
+            return new DeletePrefixedShardingSettingCommand(conventions, new PrefixedShardingSetting
+            {
+                Prefix = _prefix
+            });
+        }
+
+        private sealed class DeletePrefixedShardingSettingCommand : PrefixedShardingCommand
+        {
+            protected override PrefixedCommandType CommandType => PrefixedCommandType.Delete;
+
+            public DeletePrefixedShardingSettingCommand(DocumentConventions conventions, PrefixedShardingSetting setting) : base(conventions, setting)
+            {
+            }
+        }
+    }
+}

src/Raven.Client/ServerWide/Sharding/PrefixedShardingCommand.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Net.Http;
+using Raven.Client.Documents.Conventions;
+using Raven.Client.Http;
+using Raven.Client.Json;
+using Raven.Client.Util;
+using Sparrow.Json;
+
+namespace Raven.Client.ServerWide.Sharding
+{
+    internal abstract class PrefixedShardingCommand : RavenCommand, IRaftCommand
+    {
+        private readonly DocumentConventions _conventions;
+        private readonly PrefixedShardingSetting _setting;
+
+        protected abstract PrefixedCommandType CommandType { get; }
+
+        protected PrefixedShardingCommand(DocumentConventions conventions, PrefixedShardingSetting setting)
+        {
+            _conventions = conventions;
+            _setting = setting;
+        }
+
+        public override bool IsReadRequest => false;
+
+        public override HttpRequestMessage CreateRequest(JsonOperationContext ctx, ServerNode node, out string url)
+        {
+            url = $"{node.Url}/databases/{node.Database}/admin/sharding/prefixed";
+
+            return new HttpRequestMessage
+            {
+                Method = CommandType switch
+                {
+                    PrefixedCommandType.Add => HttpMethod.Put,
+                    PrefixedCommandType.Delete => HttpMethod.Delete,
+                    PrefixedCommandType.Update => HttpMethod.Post,
+                    _ => throw new ArgumentOutOfRangeException()
+                },
+                Content = new BlittableJsonContent(async stream => await ctx.WriteAsync(stream,
+                    DocumentConventions.Default.Serialization.DefaultConverter.ToBlittable(_setting, ctx)).ConfigureAwait(false), _conventions)
+            };
+        }
+
+        public string RaftUniqueRequestId { get; } = RaftIdGenerator.NewId();
+    }
+
+    internal enum PrefixedCommandType
+    {
+        Add = 1,
+        Delete = 2,
+        Update = 3
+    }
+}

src/Raven.Client/ServerWide/Sharding/PrefixedShardingSetting.cs

@@ -11,6 +11,15 @@ public sealed class PrefixedShardingSetting
 
     private byte[] _prefixBytesLowerCase;
 
+    public PrefixedShardingSetting()
+    {
+    }
+
+    public PrefixedShardingSetting(string prefix)
+    {
+        Prefix = prefix;
+    }
+
     internal byte[] PrefixBytesLowerCase => _prefixBytesLowerCase ??= Encoding.UTF8.GetBytes(Prefix.ToLower());
 
     public List<int> Shards { get; set; }

src/Raven.Client/ServerWide/Sharding/ShardingConfiguration.cs

@@ -44,4 +44,17 @@ internal bool HasActiveMigrations()
 
         return false;
     }
+
+    internal bool DoesShardHavePrefixes(int shardNumber) => DoesShardHavePrefixes(Prefixed, shardNumber);
+
+    internal static bool DoesShardHavePrefixes(List<PrefixedShardingSetting> prefixes, int shardNumber)
+    {
+        foreach (var setting in prefixes)
+        {
+            if (setting.Shards.Contains(shardNumber))
+                return true;
+        }
+
+        return false;
+    }
 }

src/Raven.Client/ServerWide/Sharding/UpdatePrefixedShardingSettingOperation.cs

@@ -0,0 +1,33 @@
+using System;
+using Raven.Client.Documents.Conventions;
+using Raven.Client.Documents.Operations;
+using Raven.Client.Http;
+using Sparrow.Json;
+
+namespace Raven.Client.ServerWide.Sharding
+{
+    public sealed class UpdatePrefixedShardingSettingOperation : IMaintenanceOperation
+    {
+        private readonly PrefixedShardingSetting _setting;
+
+        public UpdatePrefixedShardingSettingOperation(PrefixedShardingSetting setting)
+        {
+            _setting = setting ?? throw new ArgumentNullException(nameof(setting));
+        }
+
+        public RavenCommand GetCommand(DocumentConventions conventions, JsonOperationContext ctx)
+        {
+            return new UpdatePrefixedShardingSettingCommand(conventions, _setting);
+        }
+
+        private sealed class UpdatePrefixedShardingSettingCommand : PrefixedShardingCommand
+        {
+            public UpdatePrefixedShardingSettingCommand(DocumentConventions conventions, PrefixedShardingSetting setting) : base(conventions, setting)
+            {
+            }
+
+            protected override PrefixedCommandType CommandType => PrefixedCommandType.Update;
+
+        }
+    }
+}

src/Raven.Server/Documents/Handlers/Admin/AdminShardingHandler.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Net;
 using System.Threading.Tasks;
+using Raven.Client.Exceptions.Sharding;
 using Raven.Server.Documents.Sharding;
 using Raven.Server.Routing;
 using Raven.Server.Utils;
@@ -22,6 +23,24 @@ public async Task ExecuteMoveDocuments()
             HttpContext.Response.StatusCode = (int)HttpStatusCode.NoContent;
         }
 
+        [RavenAction("/databases/*/admin/sharding/prefixed", "PUT", AuthorizationStatus.DatabaseAdmin)]
+        public Task AddPrefixedShardingSetting()
+        {
+            throw new NotSupportedInShardingException("This operation is not available from a specific shard");
+        }
+
+        [RavenAction("/databases/*/admin/sharding/prefixed", "DELETE", AuthorizationStatus.DatabaseAdmin)]
+        public Task DeletePrefixedShardingSetting()
+        {
+            throw new NotSupportedInShardingException("This operation is not available from a specific shard");
+        }
+
+        [RavenAction("/databases/*/admin/sharding/prefixed", "POST", AuthorizationStatus.DatabaseAdmin)]
+        public Task UpdatePrefixedShardingSetting()
+        {
+            throw new NotSupportedInShardingException("This operation is not available from a specific shard");
+        }
+
         private void ValidateShardDatabaseName()
         {
             if (ShardHelper.IsShardName(DatabaseName) == false)