Updated crawl delay for bots

OpenUpSA · Aug 7, 2024 · 8059b67 · 8059b67
1 parent 59dba67
commit 8059b67
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/pombola/templates/robots.txt b/pombola/templates/robots.txt
@@ -1,4 +1,5 @@
 User-agent: *
+Crawl-delay: 120
 
 {% if staging %}
 # This is a staging server, so we ask all search engines not to index it.
@@ -8,3 +9,4 @@ Disallow: /
 Disallow: /feedback
 Disallow: /admin
 Disallow: /county-performance
+