Skip to content

Commit

Permalink
Introduce 'markedForEviction' state for the Item.
Browse files Browse the repository at this point in the history 
It is similar to 'moving' but requires ref count to be 0.

An item which is marked for eviction causes all incRef() calls to
that item to fail.

This will be used to ensure that once item is selected for eviction,
no one can interfere and prevent the eviction from suceeding.

'markedForEviction' relies on the same 'exlusive' bit as the 'moving'
state. To distinguish between those two states, 'moving' add 1 to the
refCount. This is hidden from the user, so getRefCount() will not
return that extra ref.
  • Loading branch information
@igchor
igchor committed Jan 9, 2023
1 parent 519f664 commit bd92995
Show file tree
Hide file tree
Showing 7 changed files with 434 additions and 176 deletions.
71 changes: 36 additions & 35 deletions cachelib/allocator/CacheAllocator-inl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -823,28 +823,29 @@ CacheAllocator<CacheTrait>::releaseBackToAllocator(Item& it,

removeFromMMContainer(*head);

// If this chained item is marked as exclusive, we will not free it.
// We must capture the exclusive state before we do the decRef when
// If this chained item is marked as moving, we will not free it.
// We must capture the moving state before we do the decRef when
// we know the item must still be valid
const bool wasExclusive = head->isExclusive();
const bool wasMoving = head->isMoving();
XDCHECK(!head->isMarkedForEviction());

// Decref and check if we were the last reference. Now if the item
// was marked exclusive, after decRef, it will be free to be released
// was marked moving, after decRef, it will be free to be released
// by slab release thread
const auto childRef = head->decRef();

// If the item is already exclusive and we already decremented the
// If the item is already moving and we already decremented the
// refcount, we don't need to free this item. We'll let the slab
// release thread take care of that
if (!wasExclusive) {
if (!wasMoving) {
if (childRef != 0) {
throw std::runtime_error(folly::sformat(
"chained item refcount is not zero. We cannot proceed! "
"Ref: {}, Chained Item: {}",
childRef, head->toString()));
}

// Item is not exclusive and refcount is 0, we can proceed to
// Item is not moving and refcount is 0, we can proceed to
// free it or recylce the memory
if (head == toRecycle) {
XDCHECK(ReleaseRes::kReleased != res);
Expand Down Expand Up @@ -1170,7 +1171,7 @@ bool CacheAllocator<CacheTrait>::moveChainedItem(ChainedItem& oldItem,

// This item has been unlinked from its parent and we're the only
// owner of it, so we're done here
if (!oldItem.isInMMContainer() || oldItem.isOnlyExclusive()) {
if (!oldItem.isInMMContainer() || oldItem.isOnlyMoving()) {
return false;
}

Expand Down Expand Up @@ -1201,7 +1202,7 @@ bool CacheAllocator<CacheTrait>::moveChainedItem(ChainedItem& oldItem,

// In case someone else had removed this chained item from its parent by now
// So we check again to see if the it has been unlinked from its parent
if (!oldItem.isInMMContainer() || oldItem.isOnlyExclusive()) {
if (!oldItem.isInMMContainer() || oldItem.isOnlyMoving()) {
return false;
}

Expand All @@ -1217,7 +1218,7 @@ bool CacheAllocator<CacheTrait>::moveChainedItem(ChainedItem& oldItem,
// parent's chain and the MMContainer.
auto oldItemHandle =
replaceChainedItemLocked(oldItem, std::move(newItemHdl), *parentHandle);
XDCHECK(oldItemHandle->isExclusive());
XDCHECK(oldItemHandle->isMoving());
XDCHECK(!oldItemHandle->isInMMContainer());

return true;
Expand Down Expand Up @@ -1246,7 +1247,7 @@ CacheAllocator<CacheTrait>::findEviction(PoolId pid, ClassId cid) {
: toRecycle;

// make sure no other thead is evicting the item
if (candidate->getRefCount() != 0 || !candidate->markExclusive()) {
if (candidate->getRefCount() != 0 || !candidate->markMoving()) {
++itr;
continue;
}
Expand All @@ -1261,11 +1262,11 @@ CacheAllocator<CacheTrait>::findEviction(PoolId pid, ClassId cid) {
? advanceIteratorAndTryEvictChainedItem(itr)
: advanceIteratorAndTryEvictRegularItem(mmContainer, itr);
evictionSuccessful = toReleaseHandle != nullptr;
// destroy toReleseHandle. The item won't be released to allocator
// since we marked it as exclusive.
// destroy toReleaseHandle. The item won't be released to allocator
// since we marked for eviction.
}

const auto ref = candidate->unmarkExclusive();
const auto ref = candidate->unmarkMoving();
if (ref == 0u) {
// Invalidate iterator since later on we may use this mmContainer
// again, which cannot be done unless we drop this iterator
Expand Down Expand Up @@ -2352,7 +2353,7 @@ void CacheAllocator<CacheTrait>::releaseSlabImpl(
// Need to mark an item for release before proceeding
// If we can't mark as moving, it means the item is already freed
const bool isAlreadyFreed =
!markExclusiveForSlabRelease(releaseContext, alloc, throttler);
!markMovingForSlabRelease(releaseContext, alloc, throttler);
if (isAlreadyFreed) {
continue;
}
Expand Down Expand Up @@ -2397,8 +2398,8 @@ bool CacheAllocator<CacheTrait>::moveForSlabRelease(
stats_.numMoveAttempts.inc();

// Nothing to move and the key is likely also bogus for chained items.
if (oldItem.isOnlyExclusive()) {
oldItem.unmarkExclusive();
if (oldItem.isOnlyMoving()) {
oldItem.unmarkMoving();
const auto res =
releaseBackToAllocator(oldItem, RemoveContext::kNormal, false);
XDCHECK(res == ReleaseRes::kReleased);
Expand Down Expand Up @@ -2437,7 +2438,7 @@ bool CacheAllocator<CacheTrait>::moveForSlabRelease(
// that's identical to this one to replace it. Here we just need to wait
// until all users have dropped the item handles before we can proceed.
startTime = util::getCurrentTimeSec();
while (!oldItem.isOnlyExclusive()) {
while (!oldItem.isOnlyMoving()) {
throttleWith(throttler, [&] {
XLOGF(WARN,
"Spent {} seconds, slab release still waiting for refcount to "
Expand Down Expand Up @@ -2491,8 +2492,8 @@ CacheAllocator<CacheTrait>::allocateNewItemForOldItem(const Item& oldItem) {
return {};
}

// Set up the destination for the move. Since oldChainedItem would have
// the exclusive bit set, it won't be picked for eviction.
// Set up the destination for the move. Since oldChainedItem would be
// marked as moving, it won't be picked for eviction.
auto newItemHdl =
allocateChainedItemInternal(parentHandle, oldChainedItem.getSize());
if (!newItemHdl) {
Expand Down Expand Up @@ -2544,7 +2545,7 @@ bool CacheAllocator<CacheTrait>::tryMovingForSlabRelease(
// item is still valid.
const std::string parentKey =
oldItem.asChainedItem().getParentItem(compressor_).getKey().str();
if (oldItem.isOnlyExclusive()) {
if (oldItem.isOnlyMoving()) {
// If chained item no longer has a refcount, its parent is already
// being released, so we abort this try to moving.
return false;
Expand Down Expand Up @@ -2574,10 +2575,10 @@ void CacheAllocator<CacheTrait>::evictForSlabRelease(
stats_.numEvictionAttempts.inc();

// if the item is already in a state where only the exclusive bit is set,
// nothing needs to be done. We simply need to unmark exclusive bit and free
// nothing needs to be done. We simply need to call unmarkMoving and free
// the item.
if (item.isOnlyExclusive()) {
item.unmarkExclusive();
if (item.isOnlyMoving()) {
item.unmarkMoving();
const auto res =
releaseBackToAllocator(item, RemoveContext::kNormal, false);
XDCHECK(ReleaseRes::kReleased == res);
Expand Down Expand Up @@ -2608,7 +2609,7 @@ void CacheAllocator<CacheTrait>::evictForSlabRelease(
stats_.numEvictionSuccesses.inc();

// we have the last handle. no longer need to hold on to the exclusive bit
item.unmarkExclusive();
item.unmarkMoving();

// manually decrement the refcount to call releaseBackToAllocator
const auto ref = decRef(*owningHandle);
Expand All @@ -2620,7 +2621,7 @@ void CacheAllocator<CacheTrait>::evictForSlabRelease(
}

if (shutDownInProgress_) {
item.unmarkExclusive();
item.unmarkMoving();
allocator_->abortSlabRelease(ctx);
throw exception::SlabReleaseAborted(
folly::sformat("Slab Release aborted while trying to evict"
Expand Down Expand Up @@ -2766,9 +2767,9 @@ CacheAllocator<CacheTrait>::advanceIteratorAndTryEvictChainedItem(
template <typename CacheTrait>
typename CacheAllocator<CacheTrait>::WriteHandle
CacheAllocator<CacheTrait>::evictNormalItemForSlabRelease(Item& item) {
XDCHECK(item.isExclusive());
XDCHECK(item.isMoving());

if (item.isOnlyExclusive()) {
if (item.isOnlyMoving()) {
return WriteHandle{};
}

Expand All @@ -2780,7 +2781,7 @@ CacheAllocator<CacheTrait>::evictNormalItemForSlabRelease(Item& item) {

// We remove the item from both access and mm containers. It doesn't matter
// if someone else calls remove on the item at this moment, the item cannot
// be freed as long as we have the exclusive bit set.
// be freed as long as it's marked for eviction.
auto handle = accessContainer_->removeIf(item, std::move(predicate));

if (!handle) {
Expand All @@ -2804,7 +2805,7 @@ CacheAllocator<CacheTrait>::evictNormalItemForSlabRelease(Item& item) {
template <typename CacheTrait>
typename CacheAllocator<CacheTrait>::WriteHandle
CacheAllocator<CacheTrait>::evictChainedItemForSlabRelease(ChainedItem& child) {
XDCHECK(child.isExclusive());
XDCHECK(child.isMoving());

// We have the child marked as moving, but dont know anything about the
// state of the parent. Unlike the case of regular eviction where we are
Expand All @@ -2826,7 +2827,7 @@ CacheAllocator<CacheTrait>::evictChainedItemForSlabRelease(ChainedItem& child) {
// check if the child is still in mmContainer and the expected parent is
// valid under the chained item lock.
if (expectedParent.getKey() != parentKey || !child.isInMMContainer() ||
child.isOnlyExclusive() ||
child.isOnlyMoving() ||
&expectedParent != &child.getParentItem(compressor_) ||
!expectedParent.isAccessible() || !expectedParent.hasChainedItem()) {
return {};
Expand Down Expand Up @@ -2881,14 +2882,14 @@ CacheAllocator<CacheTrait>::evictChainedItemForSlabRelease(ChainedItem& child) {

// In case someone else had removed this chained item from its parent by now
// So we check again to see if it has been unlinked from its parent
if (!child.isInMMContainer() || child.isOnlyExclusive()) {
if (!child.isInMMContainer() || child.isOnlyMoving()) {
return {};
}

// check after removing from the MMContainer that the parent is still not
// being marked as moving. If parent is moving, it will release the child
// item and we will wait for that.
if (parentHandle->isExclusive()) {
if (parentHandle->isMoving()) {
return {};
}

Expand Down Expand Up @@ -2921,7 +2922,7 @@ bool CacheAllocator<CacheTrait>::removeIfExpired(const ReadHandle& handle) {
}

template <typename CacheTrait>
bool CacheAllocator<CacheTrait>::markExclusiveForSlabRelease(
bool CacheAllocator<CacheTrait>::markMovingForSlabRelease(
const SlabReleaseContext& ctx, void* alloc, util::Throttler& throttler) {
// MemoryAllocator::processAllocForRelease will execute the callback
// if the item is not already free. So there are three outcomes here:
Expand All @@ -2940,7 +2941,7 @@ bool CacheAllocator<CacheTrait>::markExclusiveForSlabRelease(
// Since this callback is executed, the item is not yet freed
itemFreed = false;
Item* item = static_cast<Item*>(memory);
if (item->markExclusive()) {
if (item->markMoving()) {
markedMoving = true;
}
};
Expand Down
8 changes: 4 additions & 4 deletions cachelib/allocator/CacheAllocator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1756,9 +1756,9 @@ class CacheAllocator : public CacheBase {

// @return true when successfully marked as moving,
// fasle when this item has already been freed
bool markExclusiveForSlabRelease(const SlabReleaseContext& ctx,
void* alloc,
util::Throttler& throttler);
bool markMovingForSlabRelease(const SlabReleaseContext& ctx,
void* alloc,
util::Throttler& throttler);

// "Move" (by copying) the content in this item to another memory
// location by invoking the move callback.
Expand Down Expand Up @@ -1936,7 +1936,7 @@ class CacheAllocator : public CacheBase {
}

static bool parentEvictForSlabReleasePredicate(const Item& item) {
return item.getRefCount() == 1 && !item.isExclusive();
return item.getRefCount() == 1 && !item.isMoving();
}

std::unique_ptr<Deserializer> createDeserializer();
Expand Down
55 changes: 39 additions & 16 deletions cachelib/allocator/CacheItem-inl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,15 +148,16 @@ std::string CacheItem<CacheTrait>::toString() const {
return folly::sformat(
"item: "
"memory={}:raw-ref={}:size={}:key={}:hex-key={}:"
"isInMMContainer={}:isAccessible={}:isExclusive={}:references={}:ctime="
"isInMMContainer={}:isAccessible={}:isMarkedForEviction={}:"
"isMoving={}:references={}:ctime="
"{}:"
"expTime={}:updateTime={}:isNvmClean={}:isNvmEvicted={}:hasChainedItem="
"{}",
this, getRefCountAndFlagsRaw(), getSize(),
folly::humanify(getKey().str()), folly::hexlify(getKey()),
isInMMContainer(), isAccessible(), isExclusive(), getRefCount(),
getCreationTime(), getExpiryTime(), getLastAccessTime(), isNvmClean(),
isNvmEvicted(), hasChainedItem());
isInMMContainer(), isAccessible(), isMarkedForEviction(), isMoving(),
getRefCount(), getCreationTime(), getExpiryTime(), getLastAccessTime(),
isNvmClean(), isNvmEvicted(), hasChainedItem());
}
}

Expand Down Expand Up @@ -217,23 +218,43 @@ bool CacheItem<CacheTrait>::isInMMContainer() const noexcept {
}

template <typename CacheTrait>
bool CacheItem<CacheTrait>::markExclusive() noexcept {
return ref_.markExclusive();
bool CacheItem<CacheTrait>::markForEviction() noexcept {
return ref_.markForEviction();
}

template <typename CacheTrait>
RefcountWithFlags::Value CacheItem<CacheTrait>::unmarkExclusive() noexcept {
return ref_.unmarkExclusive();
RefcountWithFlags::Value CacheItem<CacheTrait>::unmarkForEviction() noexcept {
return ref_.unmarkForEviction();
}

template <typename CacheTrait>
bool CacheItem<CacheTrait>::isExclusive() const noexcept {
return ref_.isExclusive();
bool CacheItem<CacheTrait>::isMarkedForEviction() const noexcept {
return ref_.isMarkedForEviction();
}

template <typename CacheTrait>
bool CacheItem<CacheTrait>::isOnlyExclusive() const noexcept {
return ref_.isOnlyExclusive();
bool CacheItem<CacheTrait>::markForEvictionWhenMoving() {
return ref_.markForEvictionWhenMoving();
}

template <typename CacheTrait>
bool CacheItem<CacheTrait>::markMoving() {
return ref_.markMoving();
}

template <typename CacheTrait>
RefcountWithFlags::Value CacheItem<CacheTrait>::unmarkMoving() noexcept {
return ref_.unmarkMoving();
}

template <typename CacheTrait>
bool CacheItem<CacheTrait>::isMoving() const noexcept {
return ref_.isMoving();
}

template <typename CacheTrait>
bool CacheItem<CacheTrait>::isOnlyMoving() const noexcept {
return ref_.isOnlyMoving();
}

template <typename CacheTrait>
Expand Down Expand Up @@ -335,7 +356,7 @@ bool CacheItem<CacheTrait>::updateExpiryTime(uint32_t expiryTimeSecs) noexcept {
// check for moving to make sure we are not updating the expiry time while at
// the same time re-allocating the item with the old state of the expiry time
// in moveRegularItem(). See D6852328
if (isExclusive() || !isInMMContainer() || isChainedItem()) {
if (isMoving() || isMarkedForEviction() || !isInMMContainer() || isChainedItem()) {
return false;
}
// attempt to atomically update the value of expiryTime
Expand Down Expand Up @@ -451,12 +472,14 @@ std::string CacheChainedItem<CacheTrait>::toString() const {
return folly::sformat(
"chained item: "
"memory={}:raw-ref={}:size={}:parent-compressed-ptr={}:"
"isInMMContainer={}:isAccessible={}:isExclusive={}:references={}:ctime={}"
"isInMMContainer={}:isAccessible={}:isMarkedForEviction={}:"
"isMoving={}:references={}:ctime={}"
":"
"expTime={}:updateTime={}",
this, Item::getRefCountAndFlagsRaw(), Item::getSize(), cPtr.getRaw(),
Item::isInMMContainer(), Item::isAccessible(), Item::isExclusive(),
Item::getRefCount(), Item::getCreationTime(), Item::getExpiryTime(),
Item::isInMMContainer(), Item::isAccessible(),
Item::isMarkedForEviction(), Item::isMoving(), Item::getRefCount(),
Item::getCreationTime(), Item::getExpiryTime(),
Item::getLastAccessTime());
}

Expand Down
Loading

0 comments on commit bd92995

Please sign in to comment.