diff --git a/lib/reindex/s3_bucketd.py b/lib/reindex/s3_bucketd.py index 469c0065..fb307c91 100644 --- a/lib/reindex/s3_bucketd.py +++ b/lib/reindex/s3_bucketd.py @@ -3,17 +3,18 @@ import itertools import json import logging -import os import re import sys import time import urllib -from collections import defaultdict, namedtuple +import uuid +from collections import namedtuple from concurrent.futures import ThreadPoolExecutor +from datetime import datetime import redis import requests -from requests import ConnectionError, HTTPError, Timeout +from requests import ConnectionError, Timeout logging.basicConfig(level=logging.INFO) _log = logging.getLogger('utapi-reindex') @@ -258,7 +259,7 @@ def _sum_objects(self, bucket, listing, only_latest_when_locked = False): total_size += size except InvalidListing: - _log.error('Invalid contents in listing. bucket:%s status_code:%s'%(bucket.name, status_code)) + _log.error('Invalid contents in listing. bucket:%s'%(bucket.name)) raise InvalidListing(bucket.name) return count, total_size @@ -384,14 +385,21 @@ def get_redis_client(options): ) def update_redis(client, resource, name, obj_count, total_size): - timestamp = int(time.time() - 15 * 60) * 1000 + now = datetime.utcnow() + # Round down to the nearest 15 minute interval + rounded_minute = now.minute - (now.minute % 15) + now = now.replace(minute=rounded_minute, second=0, microsecond=0) + # Convert to milliseconds + timestamp = int(now.timestamp()) * 1000 obj_count_key = 's3:%s:%s:numberOfObjects' % (resource, name) total_size_key = 's3:%s:%s:storageUtilized' % (resource, name) + obj_count_serialized = f"{obj_count}:{uuid.uuid4()}" + total_size_serialized = f"{total_size}:{uuid.uuid4()}" client.zremrangebyscore(obj_count_key, timestamp, timestamp) client.zremrangebyscore(total_size_key, timestamp, timestamp) - client.zadd(obj_count_key, {obj_count: timestamp}) - client.zadd(total_size_key, {total_size: timestamp}) + client.zadd(obj_count_key, {obj_count_serialized: timestamp}) + client.zadd(total_size_key, {total_size_serialized: timestamp}) client.set(obj_count_key + ':counter', obj_count) client.set(total_size_key + ':counter', total_size)