Improve IAM Role cache (#404)

- Allow jitter to be configurable
- Don't wait on background process to populate cache. Instead, if on
first fetch it's empty, populate.

confidant/services/iamrolemanager.py

@@ -23,11 +23,11 @@ def refresh_cache():
             exc_info=True
         )
     finally:
-        # +/- 20ish seconds for respawn, to ensure all processes do not
+        # +/- seconds for respawn, to ensure all processes do not
         # refresh at the same time
         random_refresh_rate = random.randrange(
-            refresh_rate - 20,
-            refresh_rate + 20
+            refresh_rate - settings.BACKGROUND_CACHE_IAM_ROLE_JITTER,
+            refresh_rate + settings.BACKGROUND_CACHE_IAM_ROLE_JITTER
         )
         return gevent.spawn_later(
             random_refresh_rate,
@@ -37,11 +37,12 @@ def refresh_cache():
 
 def get_iam_roles(purge=False):
     if settings.BACKGROUND_CACHE_IAM_ROLES:
-        # If the cache is empty, assume it's not populated yet, and skip cache
+        global ROLES
+        # If cache is empty, it hasn't been populated by the bg process yet
+        # Populate cache, then return
         if not ROLES:
-            return _get_iam_roles()
-        else:
-            return ROLES
+            ROLES = _get_iam_roles()
+        return ROLES
     else:
         return _get_iam_roles()
 

confidant/settings.py

@@ -484,13 +484,22 @@ def str_env(var_name, default=''):
 # gevent thread.
 BACKGROUND_CACHE_IAM_ROLES = bool_env('BACKGROUND_CACHE_IAM_ROLES', True)
 # Number of seconds between calls to refresh the IAM role cache. Calls will be
-# randomized +/- by 20s, to randomize calls across processes. Minimum value for
+# randomized +/- by BACKGROUND_CACHE_IAM_ROLE_JITTER seconds,
+# to randomize calls across processes. Minimum value for
 # this setting is 60.
 BACKGROUND_CACHE_IAM_ROLE_REFRESH_RATE = int_env(
     'BACKGROUND_CACHE_IAM_ROLE_REFRESH_RATE',
     600
 )
 
+# Seconds to add as jitter to ensure all processes do not refresh at
+# the same time which can cause AWS ratelimits to be hit.
+# Default to 20 seconds
+BACKGROUND_CACHE_IAM_ROLE_JITTER = int_env(
+    'BACKGROUND_CACHE_IAM_ROLE_JITTER',
+    20
+)
+
 # ACM Private CA configuration
 
 # ACM_PRIVATE_CAS is a comma separated list of friendly ca names. Confidant