Skip to content

Commit c5976af

Browse files
guptaskguptask
committed
refactored batch promotion and added DML support
1 parent 708513f commit c5976af

File tree

6 files changed

+151
-24
lines changed

6 files changed

+151
-24
lines changed

‎cachelib/allocator/CacheAllocator-inl.h‎

Lines changed: 95 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1582,6 +1582,7 @@ void CacheAllocator<CacheTrait>::evictRegularItems(TierId tid, PoolId pid, Class
15821582
evictionData[i].candidate->toString()));
15831583
}
15841584
}
1585+
15851586
if (dmlBatchSize) {
15861587
handler = dml::submit<dml::hardware>(dml::batch, sequence);
15871588
if (!handler.valid()) {
@@ -1613,6 +1614,7 @@ void CacheAllocator<CacheTrait>::evictRegularItems(TierId tid, PoolId pid, Class
16131614
util::LatencyTracker smallItemWait{stats().evictDmlSmallItemWaitLatency_, smallBatch};
16141615
result = handler.get();
16151616
}
1617+
16161618
if (result.status != dml::status_code::ok) {
16171619
/* Re-try using CPU memmove */
16181620
for (auto i = 0U; i < dmlBatchSize; i++) {
@@ -1629,6 +1631,91 @@ void CacheAllocator<CacheTrait>::evictRegularItems(TierId tid, PoolId pid, Class
16291631
}
16301632
}
16311633

1634+
template <typename CacheTrait>
1635+
void CacheAllocator<CacheTrait>::promoteRegularItems(TierId tid, PoolId pid, ClassId cid,
1636+
std::vector<Item*>& candidates,
1637+
std::vector<WriteHandle>& newItemHdls,
1638+
bool skipAddInMMContainer,
1639+
bool fromBgThread,
1640+
std::vector<bool>& moved) {
1641+
/* Split batch for DSA-based move */
1642+
const auto& pool = allocator_[tid]->getPool(pid);
1643+
const auto& allocSizes = pool.getAllocSizes();
1644+
auto isLarge = allocSizes[cid] >= config_.largeItemMinSize;
1645+
auto dmlBatchRatio = isLarge ? config_.largeItemBatchPromoteDsaUsageFraction :
1646+
config_.smallItemBatchPromoteDsaUsageFraction;
1647+
size_t dmlBatchSize =
1648+
(config_.dsaEnabled && candidates.size() >= config_.minBatchSizeForDsaUsage) ?
1649+
static_cast<size_t>(candidates.size() * dmlBatchRatio) : 0;
1650+
auto sequence = dml::sequence<allocator_t>(dmlBatchSize);
1651+
batch_handler_t handler{};
1652+
1653+
/* Move a calculated portion of the batch using DSA (if enabled) */
1654+
for (auto i = 0U; i < dmlBatchSize; i++) {
1655+
XDCHECK(!candidates[i]->isExpired());
1656+
XDCHECK_EQ(newItemHdls[i]->getSize(), candidates[i]->getSize());
1657+
if (candidates[i]->isNvmClean()) {
1658+
newItemHdls[i]->markNvmClean();
1659+
}
1660+
dml::const_data_view srcView = dml::make_view(
1661+
reinterpret_cast<uint8_t*>(candidates[i]->getMemory()), candidates[i]->getSize());
1662+
dml::data_view dstView = dml::make_view(
1663+
reinterpret_cast<uint8_t*>(newItemHdls[i]->getMemory()), newItemHdls[i]->getSize());
1664+
if (sequence.add(dml::mem_copy, srcView, dstView) != dml::status_code::ok) {
1665+
throw std::runtime_error(folly::sformat(
1666+
"failed to add dml::mem_copy operation to the sequence for item: {}",
1667+
candidates[i]->toString()));
1668+
}
1669+
}
1670+
1671+
if (dmlBatchSize) {
1672+
handler = dml::submit<dml::hardware>(dml::batch, sequence);
1673+
if (!handler.valid()) {
1674+
auto status = handler.get();
1675+
XDCHECK(handler.valid()) << dmlErrStr(status);
1676+
throw std::runtime_error(folly::sformat(
1677+
"Failed dml sequence hw submission: {}", dmlErrStr(status)));
1678+
}
1679+
(*stats_.promoteDmlBatchSubmits)[tid][pid][cid].inc();
1680+
}
1681+
1682+
/* Move the remaining batch using CPU memmove */
1683+
for (auto i = dmlBatchSize; i < candidates.size(); i++) {
1684+
moved[i] = moveRegularItem(*candidates[i], newItemHdls[i],
1685+
skipAddInMMContainer, fromBgThread);
1686+
}
1687+
1688+
/* If DSA batch move not in use */
1689+
if (!dmlBatchSize) {
1690+
return;
1691+
}
1692+
1693+
/* Complete the DSA based batch move */
1694+
dml::batch_result result{};
1695+
{
1696+
size_t largeBatch = isLarge ? dmlBatchSize : 0;
1697+
size_t smallBatch = dmlBatchSize - largeBatch;
1698+
util::LatencyTracker largeItemWait{stats().promoteDmlLargeItemWaitLatency_, largeBatch};
1699+
util::LatencyTracker smallItemWait{stats().promoteDmlSmallItemWaitLatency_, smallBatch};
1700+
result = handler.get();
1701+
}
1702+
1703+
if (result.status != dml::status_code::ok) {
1704+
/* Re-try using CPU memmove */
1705+
for (auto i = 0U; i < dmlBatchSize; i++) {
1706+
moved[i] = moveRegularItem(*candidates[i], newItemHdls[i],
1707+
skipAddInMMContainer, fromBgThread);
1708+
}
1709+
(*stats_.promoteDmlBatchFails)[tid][pid][cid].inc();
1710+
return;
1711+
}
1712+
1713+
/* Complete book keeping for items moved successfully via DSA based batch move */
1714+
for (auto i = 0U; i < dmlBatchSize; i++) {
1715+
moved[i] = moveRegularItemBookKeeper(*candidates[i], newItemHdls[i]);
1716+
}
1717+
}
1718+
16321719
template <typename CacheTrait>
16331720
bool CacheAllocator<CacheTrait>::moveRegularItem(Item& oldItem,
16341721
WriteHandle& newItemHdl,
@@ -1921,7 +2008,7 @@ CacheAllocator<CacheTrait>::getNextCandidatesPromotion(TierId tid,
19212008
candidateHandles.push_back(std::move(candidateHandle_));
19222009
}
19232010
};
1924-
2011+
19252012
mmContainer.withPromotionIterator(iterateAndMark);
19262013

19272014
if (candidates.size() < batch) {
@@ -1934,7 +2021,7 @@ CacheAllocator<CacheTrait>::getNextCandidatesPromotion(TierId tid,
19342021
return candidates;
19352022
}
19362023
}
1937-
2024+
19382025
//1. get and item handle from a new allocation
19392026
for (int i = 0; i < candidates.size(); i++) {
19402027
Item *candidate = candidates[i];
@@ -1954,6 +2041,7 @@ CacheAllocator<CacheTrait>::getNextCandidatesPromotion(TierId tid,
19542041
folly::sformat("Was not to acquire new alloc, failed alloc {}", blankAllocs[i]));
19552042
}
19562043
}
2044+
19572045
//2. add in batch to mmContainer
19582046
auto& newMMContainer = getMMContainer(tid-1, pid, cid);
19592047
uint32_t added = newMMContainer.addBatch(newAllocs.begin(), newAllocs.end());
@@ -1963,12 +2051,15 @@ CacheAllocator<CacheTrait>::getNextCandidatesPromotion(TierId tid,
19632051
folly::sformat("Was not able to add all new items, failed item {} and handle {}",
19642052
newAllocs[added]->toString(),newHandles[added]->toString()));
19652053
}
2054+
19662055
//3. copy item data - don't need to add in mmContainer
2056+
std::vector<bool> moved(candidates.size());
2057+
promoteRegularItems(tid, pid, cid, candidates, newHandles, true, true, moved);
2058+
19672059
for (int i = 0; i < candidates.size(); i++) {
19682060
Item *candidate = candidates[i];
19692061
WriteHandle newHandle = std::move(newHandles[i]);
1970-
bool moved = moveRegularItem(*candidate,newHandle, true, true);
1971-
if (moved) {
2062+
if (moved[i]) {
19722063
XDCHECK(candidate->getKey() == newHandle->getKey());
19732064
if (markMoving) {
19742065
auto ref = candidate->unmarkMoving();
@@ -1980,7 +2071,6 @@ CacheAllocator<CacheTrait>::getNextCandidatesPromotion(TierId tid,
19802071
}
19812072
} else {
19822073
typename NvmCacheT::PutToken token{};
1983-
19842074
removeFromMMContainer(*newAllocs[i]);
19852075
auto ret = handleFailedMove(candidate,token,false,markMoving);
19862076
XDCHECK(ret);
@@ -1989,7 +2079,6 @@ CacheAllocator<CacheTrait>::getNextCandidatesPromotion(TierId tid,
19892079
releaseBackToAllocator(*candidate, RemoveContext::kNormal, false);
19902080
XDCHECK(res == ReleaseRes::kReleased);
19912081
}
1992-
19932082
}
19942083
}
19952084
return candidates;

‎cachelib/allocator/CacheAllocator.h‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1693,25 +1693,44 @@ class CacheAllocator : public CacheBase {
16931693
// not exist.
16941694
FOLLY_ALWAYS_INLINE WriteHandle findFastImpl(Key key, AccessMode mode);
16951695

1696-
// Evicts a regular item to a far memory tier.
1696+
// Evicts a batch of regular item from near to far memory tier.
16971697
//
1698-
// @param tid the id of the tier to look for evictions inside
1699-
// @param pid the id of the pool to look for evictions inside
1698+
// @param tid the id of the tier to look for evictions inside
1699+
// @param pid the id of the pool to look for evictions inside
17001700
// @param cid the id of the class to look for evictions inside
1701-
// @param evictionData Reference to the vector of items being moved
1702-
// @param newItemHdls Reference to the vector of new item handles being moved into
1703-
// @param skipAddInMMContainer
1704-
// So we can tell if we should add in mmContainer or wait
1701+
// @param evictionData reference to the vector of items being moved
1702+
// @param newItemHdls reference to the vector of new item handles being moved into
1703+
// @param skipAddInMMContainer
1704+
// so we can tell if we should add in mmContainer or wait
17051705
// to do in batch
1706-
// @param fromBgThread Use memmove instead of memcopy (for DTO testing)
1707-
// @param moved Save the status of move for each item
1706+
// @param fromBgThread use memmove instead of memcopy (for DTO testing)
1707+
// @param moved save the status of move for each item
17081708
void evictRegularItems(TierId tid, PoolId pid, ClassId cid,
17091709
std::vector<EvictionData>& evictionData,
17101710
std::vector<WriteHandle>& newItemHdls,
17111711
bool skipAddInMMContainer,
17121712
bool fromBgThread,
17131713
std::vector<bool>& moved);
17141714

1715+
// Promote a batch of regular items from far to near memory tier.
1716+
//
1717+
// @param tid the id of the tier to look for promotions inside
1718+
// @param pid the id of the pool to look for promotions inside
1719+
// @param cid the id of the class to look for promotions inside
1720+
// @param candidates reference to the vector of items being promoted
1721+
// @param newItemHdls reference to the vector of new item handles being moved into
1722+
// @param skipAddInMMContainer
1723+
// so we can tell if we should add in mmContainer or wait
1724+
// to do in batch
1725+
// @param fromBgThread use memmove instead of memcopy (for DTO testing)
1726+
// @param moved save the status of move for each item
1727+
void promoteRegularItems(TierId tid, PoolId pid, ClassId cid,
1728+
std::vector<Item*>& candidates,
1729+
std::vector<WriteHandle>& newItemHdls,
1730+
bool skipAddInMMContainer,
1731+
bool fromBgThread,
1732+
std::vector<bool>& moved);
1733+
17151734
// Moves a regular item to a different memory tier.
17161735
//
17171736
// @param oldItem Reference to the item being moved

‎cachelib/allocator/CacheAllocatorConfig.h‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -284,8 +284,10 @@ class CacheAllocatorConfig {
284284
CacheAllocatorConfig& enableDSA(bool useDsa,
285285
uint64_t minBatchSize,
286286
uint64_t largeItemSizeMin,
287-
double largeItemBatchFrac,
288-
double smallItemBatchFrac);
287+
double largeItemBatchEvictFrac,
288+
double smallItemBatchEvictFrac,
289+
double largeItemBatchPromoFrac,
290+
double smallItemBatchPromoFrac);
289291

290292
// This enables an optimization for Pool rebalancing and resizing.
291293
// The rough idea is to ensure only the least useful items are evicted when
@@ -517,12 +519,18 @@ class CacheAllocatorConfig {
517519
// Min item size (in Bytes) to get classified as Large
518520
uint64_t largeItemMinSize{8192};
519521

520-
// Large items - 80:20 split
522+
// Large items eviction - 80:20 split
521523
double largeItemBatchEvictDsaUsageFraction{0.8};
522524

523-
// Small items - 70:30 split
525+
// Small items eviction - 70:30 split
524526
double smallItemBatchEvictDsaUsageFraction{0.7};
525527

528+
// Large items promotion - 80:20 split
529+
double largeItemBatchPromoteDsaUsageFraction{0.8};
530+
531+
// Small items promotion - 70:30 split
532+
double smallItemBatchPromoteDsaUsageFraction{0.7};
533+
526534
// step size for compact cache size optimization: how many percents of the
527535
// victim to move
528536
unsigned int ccacheOptimizeStepSizePercent{1};
@@ -1047,13 +1055,18 @@ template <typename T>
10471055
CacheAllocatorConfig<T>& CacheAllocatorConfig<T>::enableDSA(bool useDsa,
10481056
uint64_t minBatchSize,
10491057
uint64_t largeItemSizeMin,
1050-
double largeItemBatchFrac,
1051-
double smallItemBatchFrac) {
1058+
double largeItemBatchEvictFrac,
1059+
double smallItemBatchEvictFrac,
1060+
double largeItemBatchPromoFrac,
1061+
double smallItemBatchPromoFrac,) {
10521062
dsaEnabled = useDsa;
10531063
minBatchSizeForDsaUsage = minBatchSize;
10541064
largeItemMinSize = largeItemSizeMin;
1055-
largeItemBatchEvictDsaUsageFraction = largeItemBatchFrac;
1056-
smallItemBatchEvictDsaUsageFraction = smallItemBatchFrac;
1065+
largeItemBatchEvictDsaUsageFraction = largeItemBatchEvictFrac;
1066+
smallItemBatchEvictDsaUsageFraction = smallItemBatchEvictFrac;
1067+
largeItemBatchPromoteDsaUsageFraction = largeItemBatchPromoFrac;
1068+
smallItemBatchPromoteDsaUsageFraction = smallItemBatchPromoFrac;
1069+
10571070
return *this;
10581071
}
10591072

‎cachelib/cachebench/cache/Cache-inl.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,9 @@ Cache<Allocator>::Cache(const CacheConfig& config,
5050
config_.minBatchSizeForDsaUsage,
5151
config_.largeItemMinSize,
5252
config_.largeItemBatchEvictDsaUsageFraction,
53-
config_.smallItemBatchEvictDsaUsageFraction);
53+
config_.smallItemBatchEvictDsaUsageFraction,
54+
config_.largeItemBatchPromoteDsaUsageFraction,
55+
config_.smallItemBatchPromoteDsaUsageFraction);
5456
allocatorConfig_.enableBackgroundEvictor(
5557
config_.getBackgroundEvictorStrategy(),
5658
std::chrono::milliseconds(config_.backgroundEvictorIntervalMilSec),

‎cachelib/cachebench/util/CacheConfig.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) {
3838
JSONSetVal(configJson, largeItemMinSize);
3939
JSONSetVal(configJson, largeItemBatchEvictDsaUsageFraction);
4040
JSONSetVal(configJson, smallItemBatchEvictDsaUsageFraction);
41+
JSONSetVal(configJson, largeItemBatchPromoteDsaUsageFraction);
42+
JSONSetVal(configJson, smallItemBatchPromoteDsaUsageFraction);
4143
JSONSetVal(configJson, moveOnSlabRelease);
4244
JSONSetVal(configJson, rebalanceStrategy);
4345
JSONSetVal(configJson, rebalanceMinSlabs);

‎cachelib/cachebench/util/CacheConfig.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ struct CacheConfig : public JSONConfig {
8181
uint64_t largeItemMinSize{8192};
8282
double largeItemBatchEvictDsaUsageFraction{0.8};
8383
double smallItemBatchEvictDsaUsageFraction{0.7};
84+
double largeItemBatchPromoteDsaUsageFraction{0.8};
85+
double smallItemBatchPromoteDsaUsageFraction{0.7};
8486
uint64_t rebalanceMinSlabs{1};
8587
double rebalanceDiffRatio{0.25};
8688
bool moveOnSlabRelease{false};

0 commit comments

Comments
 (0)