POC: Feat/make indexing more resiliant

src/Umbraco.Core/Configuration/Models/IndexingSettings.cs

@@ -15,4 +15,8 @@ public class IndexingSettings
     /// </summary>
     [DefaultValue(StaticExplicitlyIndexEachNestedProperty)]
     public bool ExplicitlyIndexEachNestedProperty { get; set; } = StaticExplicitlyIndexEachNestedProperty;
+    /// <summary>
+    /// Defines how many items to index in a single page
+    /// </summary>
+    public int IndexingPageSize { get; set; } = 10000;
 }

src/Umbraco.Infrastructure/DependencyInjection/UmbracoBuilder.Examine.cs

@@ -57,7 +57,7 @@ public static IUmbracoBuilder AddExamine(this IUmbracoBuilder builder)
         builder.Services.AddUnique<IDeliveryApiContentIndexFieldDefinitionBuilder, DeliveryApiContentIndexFieldDefinitionBuilder>();
         builder.Services.AddUnique<IDeliveryApiContentIndexHelper, DeliveryApiContentIndexHelper>();
         builder.Services.AddSingleton<IDeliveryApiIndexingHandler, DeliveryApiIndexingHandler>();
-
+        builder.Services.AddSingleton<IIndexRebuildStatusManager, IndexRebuildStatusManager>();
         builder.Services.AddSingleton<ExamineIndexRebuilder>();  //TODO remove in Umbraco 15. Only the interface should be in the service provider
 
         builder.Services.AddUnique<IDeliveryApiCompositeIdHandler, DeliveryApiCompositeIdHandler>();

src/Umbraco.Infrastructure/Examine/ContentIndexPopulator.cs

@@ -1,5 +1,10 @@
+using System.Text.RegularExpressions;
 using Examine;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+using Umbraco.Cms.Core.Configuration.Models;
+using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Persistence.Querying;
 using Umbraco.Cms.Core.Services;
@@ -14,6 +19,8 @@
 {
     private readonly IContentService _contentService;
     private readonly IValueSetBuilder<IContent> _contentValueSetBuilder;
+    private readonly IIndexRebuildStatusManager _indexRebuildStatusManager;
+    private IndexingSettings _indexingSettings;
     private readonly ILogger<ContentIndexPopulator> _logger;
     private readonly int? _parentId;
 
@@ -28,6 +35,7 @@
     /// <summary>
     ///     Default constructor to lookup all content data
     /// </summary>
+    [Obsolete("Use the constructor with IOptionsMonitor<IndexingSettings>")]
     public ContentIndexPopulator(
         ILogger<ContentIndexPopulator> logger,
         IContentService contentService,
@@ -40,13 +48,14 @@
     /// <summary>
     ///     Optional constructor allowing specifying custom query parameters
     /// </summary>
+    [Obsolete("Use the constructor with IOptionsMonitor<IndexingSettings>")]
     public ContentIndexPopulator(
         ILogger<ContentIndexPopulator> logger,
         bool publishedValuesOnly,
         int? parentId,
         IContentService contentService,
         IUmbracoDatabaseFactory umbracoDatabaseFactory,
-        IValueSetBuilder<IContent> contentValueSetBuilder)
+        IValueSetBuilder<IContent> contentValueSetBuilder) : this(logger, publishedValuesOnly, parentId, contentService, umbracoDatabaseFactory, contentValueSetBuilder, StaticServiceProvider.Instance.GetRequiredService<IOptionsMonitor<IndexingSettings>>())
     {
         _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
         _umbracoDatabaseFactory = umbracoDatabaseFactory ?? throw new ArgumentNullException(nameof(umbracoDatabaseFactory));
@@ -55,7 +64,39 @@
         _publishedValuesOnly = publishedValuesOnly;
         _parentId = parentId;
     }
+    public ContentIndexPopulator(
+        ILogger<ContentIndexPopulator> logger,
+        IContentService contentService,
+        IUmbracoDatabaseFactory umbracoDatabaseFactory,
+        IContentValueSetBuilder contentValueSetBuilder,
+        IOptionsMonitor<IndexingSettings> indexingSettings)
+        : this(logger, false, null, contentService, umbracoDatabaseFactory, contentValueSetBuilder,indexingSettings)
+    {
+    }
+    public ContentIndexPopulator(
+        ILogger<ContentIndexPopulator> logger,
+        bool publishedValuesOnly,
+        int? parentId,
+        IContentService contentService,
+        IUmbracoDatabaseFactory umbracoDatabaseFactory,
+        IValueSetBuilder<IContent> contentValueSetBuilder,
+        IOptionsMonitor<IndexingSettings> indexingSettings,
+        IIndexRebuildStatusManager indexRebuildStatusManager)
+    {
+        _contentService = contentService ?? throw new ArgumentNullException(nameof(contentService));
+        _umbracoDatabaseFactory = umbracoDatabaseFactory ?? throw new ArgumentNullException(nameof(umbracoDatabaseFactory));
+        _contentValueSetBuilder = contentValueSetBuilder ?? throw new ArgumentNullException(nameof(contentValueSetBuilder));
+        _indexRebuildStatusManager = indexRebuildStatusManager;
+        _indexingSettings = indexingSettings.CurrentValue;
+        indexingSettings.OnChange(change =>
+        {
+            _indexingSettings = change;
+        });
+        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        _publishedValuesOnly = publishedValuesOnly;
+        _parentId = parentId;
 
+    }
     private IQuery<IContent> PublishedQuery => _publishedQuery ??=
         _umbracoDatabaseFactory.SqlContext.Query<IContent>().Where(x => x.Published);
 
@@ -75,7 +116,7 @@
             return;
         }
 
-        const int pageSize = 10000;
+
         var pageIndex = 0;
 
         var contentParentId = -1;
@@ -86,33 +127,44 @@
 
         if (_publishedValuesOnly)
         {
-            IndexPublishedContent(contentParentId, pageIndex, pageSize, indexes);
+            IndexPublishedContent(contentParentId, pageIndex, _indexingSettings.IndexingPageSize, indexes);
         }
         else
         {
-            IndexAllContent(contentParentId, pageIndex, pageSize, indexes);
+            IndexAllContent(contentParentId, pageIndex, _indexingSettings.IndexingPageSize, indexes);
         }
     }
 
     protected void IndexAllContent(int contentParentId, int pageIndex, int pageSize, IReadOnlyList<IIndex> indexes)
     {
         IContent[] content;
-
+        var totalBatches = 0;
         do
         {
-            content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out _).ToArray();
+            content = _contentService.GetPagedDescendants(contentParentId, pageIndex, pageSize, out var total).ToArray();
+            if (totalBatches == 0)
+            {
+                totalBatches = (int)Math.Ceiling((decimal)total / pageSize);
+            }
 
             var valueSets = _contentValueSetBuilder.GetValueSets(content).ToArray();
 
             // ReSharper disable once PossibleMultipleEnumeration
             foreach (IIndex index in indexes)
             {
+                _indexRebuildStatusManager.UpdatePopulatorStatus(index.Name, GetType().Name, true, pageIndex, totalBatches);
+
                 index.IndexItems(valueSets);
             }
 
             pageIndex++;
         }
         while (content.Length == pageSize);
+
+        foreach (IIndex index in indexes)
+        {
+            _indexRebuildStatusManager.UpdatePopulatorStatus(index.Name, GetType().Name, false, totalBatches, totalBatches);
+        }
     }
 
     protected void IndexPublishedContent(int contentParentId, int pageIndex, int pageSize, IReadOnlyList<IIndex> indexes)

src/Umbraco.Infrastructure/Examine/ExamineIndexRebuilder.cs

@@ -1,9 +1,11 @@
 // Copyright (c) Umbraco.
 // See LICENSE for more details.
 
 using Examine;
+using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Umbraco.Cms.Core;
+using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Runtime;
 using Umbraco.Cms.Core.Services;
 using Umbraco.Cms.Infrastructure.HostedServices;
@@ -16,6 +18,7 @@
     private readonly IBackgroundTaskQueue _backgroundTaskQueue;
     private readonly IExamineManager _examineManager;
     private readonly ILogger<ExamineIndexRebuilder> _logger;
+    private readonly IIndexRebuildStatusManager _indexRebuildStatusManager;
     private readonly IMainDom _mainDom;
     private readonly IEnumerable<IIndexPopulator> _populators;
     private readonly object _rebuildLocker = new();
@@ -24,17 +27,29 @@
     /// <summary>
     ///     Initializes a new instance of the <see cref="ExamineIndexRebuilder" /> class.
     /// </summary>
+    [Obsolete("Use constructor with IIndexRebuildStatusManager. This is scheduled for removal in Umbraco 15.")]
     public ExamineIndexRebuilder(
         IMainDom mainDom,
         IRuntimeState runtimeState,
         ILogger<ExamineIndexRebuilder> logger,
         IExamineManager examineManager,
         IEnumerable<IIndexPopulator> populators,
+        IBackgroundTaskQueue backgroundTaskQueue) : this(mainDom, runtimeState, logger, StaticServiceProvider.Instance.GetRequiredService<IIndexRebuildStatusManager>(), examineManager, populators, backgroundTaskQueue)
+    {
+    }
+    public ExamineIndexRebuilder(
+        IMainDom mainDom,
+        IRuntimeState runtimeState,
+        ILogger<ExamineIndexRebuilder> logger,
+        IIndexRebuildStatusManager indexRebuildStatusManager,
+        IExamineManager examineManager,
+        IEnumerable<IIndexPopulator> populators,
         IBackgroundTaskQueue backgroundTaskQueue)
     {
         _mainDom = mainDom;
         _runtimeState = runtimeState;
         _logger = logger;
+        _indexRebuildStatusManager = indexRebuildStatusManager;
         _examineManager = examineManager;
         _populators = populators;
         _backgroundTaskQueue = backgroundTaskQueue;
@@ -147,7 +162,7 @@
                 {
                     throw new InvalidOperationException($"No index found with name {indexName}");
                 }
-
+                _indexRebuildStatusManager.SetReindexStatus(index, _populators.Where(x=>x.IsRegistered(index)));
                 index.CreateIndex(); // clear the index
                 foreach (IIndexPopulator populator in _populators)
                 {
@@ -175,22 +190,24 @@
         {
             Thread.Sleep(delay);
         }
+        IEnumerable<string>? rebuildedIndex = null;
 
         try
         {
             if (!Monitor.TryEnter(_rebuildLocker))
             {
                 _logger.LogWarning(
                     $"Call was made to {nameof(RebuildIndexes)} but the task runner for rebuilding is already running");
             }
             else
             {
                 // If an index exists but it has zero docs we'll consider it empty and rebuild
                 IIndex[] indexes = (onlyEmptyIndexes
                     ? _examineManager.Indexes.Where(x =>
                         !x.IndexExists() || (x is IIndexStats stats && stats.GetDocumentCount() == 0))
                     : _examineManager.Indexes).ToArray();
-
+                rebuildedIndex = indexes.Select(x => x.Name);
+                _indexRebuildStatusManager.SetRebuildingIndexStatus(rebuildedIndex, true);
                 if (indexes.Length == 0)
                 {
                     return;
@@ -218,10 +235,15 @@
                         _logger.LogError(e, "Index populating failed for populator {Populator}", populator.GetType());
                     }
                 }
+
             }
         }
         finally
         {
+            if (rebuildedIndex != null && rebuildedIndex.Any())
+            {
+                _indexRebuildStatusManager.SetRebuildingIndexStatus(rebuildedIndex, false);
+            }
             if (Monitor.IsEntered(_rebuildLocker))
             {
                 Monitor.Exit(_rebuildLocker);

src/Umbraco.Infrastructure/Examine/IIndexRebuildStatusManager.cs

@@ -0,0 +1,30 @@
+using Examine;
+
+namespace Umbraco.Cms.Infrastructure.Examine;
+
+public interface IIndexRebuildStatusManager
+{
+    IndexStatus? GetRebuildingIndexStatus(string index);
+    void SetReindexStatus(IIndex index, IEnumerable<IIndexPopulator> where);
+
+    public void UpdatePopulatorStatus(string index, string populator, bool isRunning, int currentBatch, int totalBatches);
+}
+
+public class IndexStatus
+{
+    public bool IsRebuilding { get; set; }
+    public IEnumerable<PopulatorStatus>? PopulatorStatuses { get; set; }
+}
+
+public class PopulatorStatus
+{
+    public PopulatorStatus(string name)
+    {
+        Name = name;
+    }
+
+    public string Name { get; set; }
+    public bool IsRunning { get; set; }
+    public int CurrentBatch { get; set; }
+    public int TotalBatches { get; set; }
+}

src/Umbraco.Infrastructure/Examine/IndexRebuildStatusManager.cs

@@ -0,0 +1,92 @@
+using Examine;
+
+namespace Umbraco.Cms.Infrastructure.Examine;
+
+/// <summary>
+///
+/// </summary>
+public class IndexRebuildStatusManager : IIndexRebuildStatusManager
+{
+    private readonly IExamineManager _examineManager;
+    IDictionary<string, IndexStatus> _rebuildingStatus = new Dictionary<string, IndexStatus>();
+
+    public IndexRebuildStatusManager(IExamineManager examineManager)
+    {
+        _examineManager = examineManager;
+        foreach (var index in examineManager.Indexes)
+        {
+            _rebuildingStatus.Add(index.Name, new IndexStatus());
+        }
+    }
+
+    public IndexStatus? GetRebuildingIndexStatus(string index) =>
+        _rebuildingStatus.TryGetValue(index, out var status) ? status : null;
+
+    public void SetReindexStatus(IIndex index, IEnumerable<IIndexPopulator> where)
+    {
+        if (!_rebuildingStatus.TryGetValue(index.Name, out var status))
+        {
+            status = new IndexStatus();
+            _rebuildingStatus.Add(index.Name, status);
+        }
+
+        status.IsRebuilding = true;
+        status.PopulatorStatuses = where.Select(x => new PopulatorStatus(x.GetType().Name));
+    }
+
+    public void UpdatePopulatorStatus(
+        string index,
+        string populator,
+        bool isRunning,
+        int currentBatch,
+        int totalBatches)
+    {
+        var examineIndex = _examineManager.GetIndex(index);
+
+        if (_rebuildingStatus.TryGetValue(index, out var status))
+        {
+            var populatorStatus = status?.PopulatorStatuses?.FirstOrDefault(x => x.Name == populator);
+            if (populatorStatus != null)
+            {
+                populatorStatus.IsRunning = isRunning;
+                populatorStatus.CurrentBatch = currentBatch;
+                populatorStatus.TotalBatches = totalBatches;
+            }
+
+            examineIndex.IndexItems(
+            [
+                new ValueSet(
+                    populator,
+                    "populator",
+                    new Dictionary<string, object>()
+                    {
+                        { "isRunning", isRunning },
+                        { "currentBatch", currentBatch },
+                        { "totalBatches", totalBatches }
+                    })
+            ]);
+            return;
+        }
+        var currentStatus= new IndexStatus();
+        _rebuildingStatus.Add(index,currentStatus);
+        var newPopulatorStatus = new PopulatorStatus(populator)
+        {
+            IsRunning = isRunning,
+            CurrentBatch = currentBatch,
+            TotalBatches = totalBatches
+        };
+        currentStatus.PopulatorStatuses = new List<PopulatorStatus> { newPopulatorStatus };
+        examineIndex.IndexItems(
+            [
+                new ValueSet(
+                    populator,
+                    "populator",
+                    new Dictionary<string, object>()
+                    {
+                        { "isRunning", isRunning },
+                        { "currentBatch", currentBatch },
+                        { "totalBatches", totalBatches }
+                    })
+            ]);
+    }
+}