From 3d08d9d4b60bac334edeba6b3fa6c13faee38379 Mon Sep 17 00:00:00 2001 From: Taylor McKinnon Date: Wed, 9 Oct 2024 10:21:05 -0700 Subject: [PATCH 1/4] bugfix(UTAPI-109): Append uuidv4 to sorted set members during reindex --- lib/reindex/s3_bucketd.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/reindex/s3_bucketd.py b/lib/reindex/s3_bucketd.py index 469c0065..b27d103f 100644 --- a/lib/reindex/s3_bucketd.py +++ b/lib/reindex/s3_bucketd.py @@ -9,6 +9,7 @@ import time import urllib from collections import defaultdict, namedtuple +import uuid from concurrent.futures import ThreadPoolExecutor import redis @@ -387,11 +388,13 @@ def update_redis(client, resource, name, obj_count, total_size): timestamp = int(time.time() - 15 * 60) * 1000 obj_count_key = 's3:%s:%s:numberOfObjects' % (resource, name) total_size_key = 's3:%s:%s:storageUtilized' % (resource, name) + obj_count_serialized = f"{obj_count}:{uuid.uuid4()}" + total_size_serialized = f"{total_size}:{uuid.uuid4()}" client.zremrangebyscore(obj_count_key, timestamp, timestamp) client.zremrangebyscore(total_size_key, timestamp, timestamp) - client.zadd(obj_count_key, {obj_count: timestamp}) - client.zadd(total_size_key, {total_size: timestamp}) + client.zadd(obj_count_key, {obj_count_serialized: timestamp}) + client.zadd(total_size_key, {total_size_serialized: timestamp}) client.set(obj_count_key + ':counter', obj_count) client.set(total_size_key + ':counter', total_size) From 268c8b53f53fd3cc81611d2c0c22ab97f2941642 Mon Sep 17 00:00:00 2001 From: Taylor McKinnon Date: Wed, 9 Oct 2024 10:21:22 -0700 Subject: [PATCH 2/4] bugfix(UTAPI-109): Remove unused imports --- lib/reindex/s3_bucketd.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/reindex/s3_bucketd.py b/lib/reindex/s3_bucketd.py index b27d103f..38c732fd 100644 --- a/lib/reindex/s3_bucketd.py +++ b/lib/reindex/s3_bucketd.py @@ -3,18 +3,17 @@ import itertools import json import logging -import os import re import sys import time import urllib -from collections import defaultdict, namedtuple import uuid +from collections import namedtuple from concurrent.futures import ThreadPoolExecutor import redis import requests -from requests import ConnectionError, HTTPError, Timeout +from requests import ConnectionError, Timeout logging.basicConfig(level=logging.INFO) _log = logging.getLogger('utapi-reindex') From 9ccb24d44d4d8f988375264183a27f1ea0dea29b Mon Sep 17 00:00:00 2001 From: Taylor McKinnon Date: Wed, 9 Oct 2024 10:30:11 -0700 Subject: [PATCH 3/4] bugfix(UTAPI-109): Remove undefined variable reference --- lib/reindex/s3_bucketd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/reindex/s3_bucketd.py b/lib/reindex/s3_bucketd.py index 38c732fd..50c2ee70 100644 --- a/lib/reindex/s3_bucketd.py +++ b/lib/reindex/s3_bucketd.py @@ -258,7 +258,7 @@ def _sum_objects(self, bucket, listing, only_latest_when_locked = False): total_size += size except InvalidListing: - _log.error('Invalid contents in listing. bucket:%s status_code:%s'%(bucket.name, status_code)) + _log.error('Invalid contents in listing. bucket:%s'%(bucket.name)) raise InvalidListing(bucket.name) return count, total_size From 285f6ca9fa255477aa6a7ba8f68c1bb2e61440a8 Mon Sep 17 00:00:00 2001 From: Taylor McKinnon Date: Mon, 14 Oct 2024 09:50:36 -0700 Subject: [PATCH 4/4] bugfix(UTAPI-109): Round reindex timestamp correctly to 15 minute interval --- lib/reindex/s3_bucketd.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/reindex/s3_bucketd.py b/lib/reindex/s3_bucketd.py index 50c2ee70..fb307c91 100644 --- a/lib/reindex/s3_bucketd.py +++ b/lib/reindex/s3_bucketd.py @@ -10,6 +10,7 @@ import uuid from collections import namedtuple from concurrent.futures import ThreadPoolExecutor +from datetime import datetime import redis import requests @@ -384,7 +385,12 @@ def get_redis_client(options): ) def update_redis(client, resource, name, obj_count, total_size): - timestamp = int(time.time() - 15 * 60) * 1000 + now = datetime.utcnow() + # Round down to the nearest 15 minute interval + rounded_minute = now.minute - (now.minute % 15) + now = now.replace(minute=rounded_minute, second=0, microsecond=0) + # Convert to milliseconds + timestamp = int(now.timestamp()) * 1000 obj_count_key = 's3:%s:%s:numberOfObjects' % (resource, name) total_size_key = 's3:%s:%s:storageUtilized' % (resource, name) obj_count_serialized = f"{obj_count}:{uuid.uuid4()}"