Skip to content

Commit

Permalink
remove lock and rely on deque consistency
Browse files Browse the repository at this point in the history
  • Loading branch information
andrewkho committed Nov 5, 2024
1 parent 8e1f7dc commit 336859e
Showing 1 changed file with 12 additions and 16 deletions.
28 changes: 12 additions & 16 deletions torchdata/nodes/snapshot_store.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import threading
from collections import deque
from dataclasses import dataclass
from typing import Any, Optional, Protocol
Expand Down Expand Up @@ -32,23 +31,20 @@ class DequeSnapshotStore(SnapshotStore):

def __init__(self, max_size: Optional[int] = None) -> None:
self._deque: deque = deque(maxlen=max_size)
self._lock = threading.Lock()
self._max_version: int = -1

def append(self, snapshot: Any, version: int) -> None:
with self._lock:
if version <= self._max_version:
raise ValueError(f"{version=} is not strictly greater than {self._max_version=}")
self._max_version = version
self._deque.append((version, snapshot))
if version <= self._max_version:
raise ValueError(f"{version=} is not strictly greater than {self._max_version=}")
self._max_version = version
self._deque.append((version, snapshot))

def pop_version(self, version: int) -> Optional[Any]:
with self._lock:
ver, val = None, None
while self._deque and version >= self._deque[0][0]:
ver, val = self._deque.popleft()

if ver == version:
return val
else:
return None
ver, val = None, None
while self._deque and version >= self._deque[0][0]:
ver, val = self._deque.popleft()

if ver == version:
return val
else:
return None

0 comments on commit 336859e

Please sign in to comment.