Fix search engines issue with StreamRendering enabled (#9510)

src/Templates/Boilerplate/Bit.Boilerplate/src/Client/Boilerplate.Client.Web/wwwroot/service-worker.published.js

@@ -34,17 +34,7 @@ self.externalAssets = [
         "url": "/"
     },
     {
-        //#if (framework == "net9.0")
         url: "_framework/blazor.web.js"
-        //#else
-            //#if (IsInsideProjectTemplate == true)
-            /*
-            //#endif
-        url: "_framework/blazor.web.js"
-            //#if (IsInsideProjectTemplate == true)
-            */
-            //#endif
-        //#endif
     },
     {
         "url": "Boilerplate.Server.Web.styles.css"

src/Templates/Boilerplate/Bit.Boilerplate/src/Server/Boilerplate.Server.Web/Components/App.razor

@@ -1,6 +1,5 @@
 @using Boilerplate.Client.Web.Components
 @using Microsoft.AspNetCore.Components.Routing
-@attribute [StreamRendering(enabled: true)]
 @*+:cnd:noEmit*@
 
 @{
@@ -66,19 +65,21 @@
         <LoadingComponent @rendermode="null" />
     }
 
-    <Routes @rendermode=renderMode />
+    @if (HttpContext.Request.IsCrawlerClient())
+    {
+        // For StreamRenderingDisabledContainer, read comments in App.razor.cs
+        <StreamRenderingDisabledContainer>
+            <Routes @rendermode=renderMode />
+        </StreamRenderingDisabledContainer>
+    }
+    else
+    {
+        <StreamRenderingEnabledContainer>
+            <Routes @rendermode=renderMode />
+        </StreamRenderingEnabledContainer>
+    }
 
-    @*#if (framework == "net9.0")*@
-    <Script src="_framework/blazor.web.js?ver=9.0.0" autostart="false"></Script>
-    @*#else*@
-        @*#if (IsInsideProjectTemplate == true)*@
-        /*
-        @*#endif)*@
-    <Script src="_framework/blazor.web.js?ver=8.0.11" autostart="false"></Script>
-        @*#if (IsInsideProjectTemplate == true)*@
-        */
-        @*#endif)*@
-    @*#endif)*@
+    <Script src="_framework/blazor.web.js" autostart="false"></Script>
     <!-- Ensure that the version of `blazor.web.js` matches the version specified in `service-worker.published.js`.
             This alignment is necessary for the PWA functionality of the Blazor app to work correctly. -->
     @if (serverWebSettings.WebAppRender.PwaEnabled)

src/Templates/Boilerplate/Bit.Boilerplate/src/Server/Boilerplate.Server.Web/Components/App.razor.cs

@@ -1,6 +1,7 @@
 using Microsoft.AspNetCore.Components;
-using Microsoft.AspNetCore.Components.Web;
 using Microsoft.AspNetCore.Localization;
+using Microsoft.AspNetCore.Components.Web;
+using Microsoft.AspNetCore.Components.Rendering;
 
 namespace Boilerplate.Server.Web.Components;
 
@@ -24,3 +25,34 @@ protected override void OnInitialized()
         }
     }
 }
+
+/// <summary>
+/// Streaming pre-rendering enhances user experience (UX) and overall application performance. 
+/// However, it prevents search engines from accessing your pre-rendered dynamic content.
+/// To address this, the conditional logic in App.razor, leveraging <see cref="HttpRequestExtensions.IsCrawlerClient(HttpRequest)"/>, 
+/// disables streaming specifically for search engine crawlers, while maintaining the improved UX and performance for regular users.
+/// </summary>
+[StreamRendering(enabled: true)]
+public class StreamRenderingEnabledContainer : ComponentBase
+{
+    [Parameter] public RenderFragment? ChildContent { get; set; }
+
+    protected override void BuildRenderTree(RenderTreeBuilder builder)
+    {
+        builder.AddContent(0, ChildContent);
+    }
+}
+
+/// <summary>
+/// <inheritdoc cref="StreamRenderingEnabledContainer"/>
+/// </summary>
+[StreamRendering(enabled: false)]
+public class StreamRenderingDisabledContainer : ComponentBase
+{
+    [Parameter] public RenderFragment? ChildContent { get; set; }
+
+    protected override void BuildRenderTree(RenderTreeBuilder builder)
+    {
+        builder.AddContent(0, ChildContent);
+    }
+}

src/Templates/Boilerplate/Bit.Boilerplate/src/Server/Boilerplate.Server.Web/Extensions/HttpRequestExtensions.cs

@@ -1,5 +1,4 @@
-using Microsoft.Extensions.Primitives;
-using Microsoft.Net.Http.Headers;
+using Microsoft.Net.Http.Headers;
 
 namespace Microsoft.AspNetCore.Http;
 
@@ -18,4 +17,30 @@ internal static Uri GetBaseUrl(this HttpRequest req)
 
         return uriBuilder.Uri;
     }
+
+    public static bool IsCrawlerClient(this HttpRequest request)
+    {
+        var agent = GetLoweredUserAgent(request);
+
+        if (agent.Contains("google")) return true;
+
+        if (agent.Contains("bing")) return true;
+
+        if (agent.Contains("yahoo")) return true;
+
+        if (agent.Contains("duckduck")) return true;
+
+        if (agent.Contains("yandex")) return true;
+
+        return false;
+    }
+
+    private static string GetLoweredUserAgent(HttpRequest request)
+    {
+        var userAgent = request.Headers[HeaderNames.UserAgent].ToString();
+
+        if (string.IsNullOrEmpty(userAgent)) return string.Empty;
+
+        return userAgent.ToLowerInvariant();
+    }
 }