Merge pull request #10023 from scottbarnes/help-manage-bots-hitting-a…

…uthors

Help slow down bots hitting /authors

docker/nginx.conf

@@ -63,6 +63,12 @@ http {
     # Set a more permissive limit for covers because some pages might load 20+ covers.
     limit_req_zone $rate_limit_key zone=cover_limit:10m rate=400r/m;
 
+    # For returning 200 when someone tries to randomly sort author results.
+    map $arg_sort $is_random_sort {
+      default 0;
+      ~^random_.* 1;
+    }
+
     # Things are mounted into here by the docker compose file
     include /etc/nginx/sites-enabled/*;
 }

docker/web_nginx.conf

@@ -73,6 +73,14 @@ server {
     }
 
     location / {
+        # Web rate limit.
+        limit_req zone=web_limit burst=200 nodelay;
+        limit_req_status 429;
+
+        if ($http_user_agent ~ (Bytespider) ) {
+           return 444;
+        }
+
         proxy_pass http://webnodes;
         proxy_set_header Host $http_host;
 
@@ -83,14 +91,27 @@ server {
         # app server is http only.
         proxy_set_header X-Scheme $scheme;
 
-        if ($http_user_agent ~ (Bytespider) ) {
-           return 444;
-        }
-
+    }
 
+    # Log likely bots caught in /authors random loop.
+    location ~* ^/authors/.* {
         # Web rate limit.
         limit_req zone=web_limit burst=200 nodelay;
         limit_req_status 429;
+
+        # randomly sorting will be removed. For now just return 200
+        if ($is_random_sort = 1) {
+            return 200 "";
+        }
+
+        if ($http_user_agent ~ (Bytespider) ) {
+            return 444;
+        }
+
+        proxy_pass http://webnodes;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Scheme $scheme;
     }
 
     location ^~ /.well-known/acme-challenge/ {