diff --git a/cache/cache.hpp b/cache/cache.hpp index e64b073..e6c4721 100644 --- a/cache/cache.hpp +++ b/cache/cache.hpp @@ -191,6 +191,10 @@ class CacheBase : public CacheMonitorSupport // access both meta and data in one function call virtual std::pair access_line(uint32_t ai, uint32_t s, uint32_t w) = 0; + virtual void replace_read(uint32_t ai, uint32_t s, uint32_t w, bool prefetch, bool genre = false) = 0; + virtual void replace_write(uint32_t ai, uint32_t s, uint32_t w, bool demand_acc, bool genre = false) = 0; + virtual void replace_manage(uint32_t ai, uint32_t s, uint32_t w, bool hit, uint32_t evict, bool genre = false) = 0; + virtual bool query_coloc(uint64_t addrA, uint64_t addrB) = 0; virtual LocInfo query_loc(uint64_t addr) { return LocInfo(id, this, addr); } virtual void query_fill_loc(LocInfo *loc, uint64_t addr) = 0; @@ -330,21 +334,30 @@ class CacheSkewed : public CacheBase data_return_buffer(buffer_data); } - virtual void hook_read(uint64_t addr, uint32_t ai, uint32_t s, uint32_t w, bool hit, bool prefetch, const CMMetadataBase * meta, const CMDataBase *data, uint64_t *delay) override { - if(ai < P) replacer[ai].access(s, w, true, prefetch); + virtual void hook_read(uint64_t addr, uint32_t ai, uint32_t s, uint32_t w, bool hit, const CMMetadataBase * meta, const CMDataBase *data, uint64_t *delay) override { if constexpr (EnMon || !C_VOID) monitors->hook_read(addr, ai, s, w, hit, meta, data, delay); } - virtual void hook_write(uint64_t addr, uint32_t ai, uint32_t s, uint32_t w, bool hit, bool demand_acc, const CMMetadataBase * meta, const CMDataBase *data, uint64_t *delay) override { - if(ai < P) replacer[ai].access(s, w, demand_acc, false); + virtual void replace_read(uint32_t ai, uint32_t s, uint32_t w, bool prefetch, bool genre = false) override { + if(ai < P) replacer[ai].access(s, w, true, prefetch); + } + + virtual void hook_write(uint64_t addr, uint32_t ai, uint32_t s, uint32_t w, bool hit, const CMMetadataBase * meta, const CMDataBase *data, uint64_t *delay) override { if constexpr (EnMon || !C_VOID) monitors->hook_write(addr, ai, s, w, hit, meta, data, delay); } + virtual void replace_write(uint32_t ai, uint32_t s, uint32_t w, bool demand_acc, bool genre = false) override { + if(ai < P) replacer[ai].access(s, w, demand_acc, false); + } + virtual void hook_manage(uint64_t addr, uint32_t ai, uint32_t s, uint32_t w, bool hit, uint32_t evict, bool writeback, const CMMetadataBase * meta, const CMDataBase *data, uint64_t *delay) override { - if(ai < P && hit && evict) replacer[ai].invalid(s, w, evict == 2); if constexpr (EnMon || !C_VOID) monitors->hook_manage(addr, ai, s, w, hit, evict, writeback, meta, data, delay); } + virtual void replace_manage(uint32_t ai, uint32_t s, uint32_t w, bool hit, uint32_t evict, bool genre = false) override { + if(ai < P && hit && evict) replacer[ai].invalid(s, w, evict == 2); + } + virtual CMDataBase *data_copy_buffer() override { if (data_buffer_pool_set.empty()) return nullptr; if constexpr (EnMT) { diff --git a/cache/coherence.hpp b/cache/coherence.hpp index 84c83ac..6a90e3a 100644 --- a/cache/coherence.hpp +++ b/cache/coherence.hpp @@ -47,7 +47,6 @@ class OuterCohPortBase virtual void acquire_req(uint64_t addr, CMMetadataBase *meta, CMDataBase *data, coh_cmd_t cmd, uint64_t *delay) = 0; virtual void writeback_req(uint64_t addr, CMMetadataBase *meta, CMDataBase *data, coh_cmd_t cmd, uint64_t *delay) = 0; - virtual void prefetch_req(uint64_t addr, coh_cmd_t cmd, uint64_t *delay) = 0; // may not implement probe_resp() and finish_req() if the port is uncached virtual std::pair probe_resp(uint64_t addr, CMMetadataBase *meta, CMDataBase *data, coh_cmd_t cmd, uint64_t *delay) { return std::make_pair(false,false); } @@ -78,7 +77,6 @@ class InnerCohPortBase virtual void acquire_resp(uint64_t addr, CMDataBase *data_inner, CMMetadataBase *meta_inner, coh_cmd_t outer_cmd, uint64_t *delay) = 0; virtual void writeback_resp(uint64_t addr, CMDataBase *data_inner, CMMetadataBase *meta_inner, coh_cmd_t outer_cmd, uint64_t *delay) = 0; - virtual void prefetch_resp(uint64_t addr, coh_cmd_t outer_cmd, uint64_t *delay) = 0; // may not implement probe_req() and finish_resp() if the port is uncached virtual std::pair probe_req(uint64_t addr, CMMetadataBase *meta, CMDataBase *data, coh_cmd_t cmd, uint64_t *delay) { return std::make_pair(false,false); } @@ -144,11 +142,6 @@ class OuterCohPortUncached : public OuterCohPortBase Policy::meta_after_writeback(outer_cmd, meta); } - virtual void prefetch_req(uint64_t addr, coh_cmd_t outer_cmd, uint64_t *delay) override { - outer_cmd.id = coh_id; - coh->prefetch_resp(addr, outer_cmd, delay); - } - virtual void query_loc_req(uint64_t addr, std::list *locs) override { coh->query_loc_resp(addr, locs); } @@ -191,18 +184,23 @@ class OuterCohPortT : public OPUC auto sync = Policy::probe_need_sync(outer_cmd, meta); if(sync.first) { auto [phit, pwb] = OuterCohPortBase::inner->probe_req(addr, meta, data, sync.second, delay); - if(pwb) cache->hook_write(addr, ai, s, w, true, false, meta, data, delay); + if(pwb) { + cache->replace_write(ai, s, w, false); + cache->hook_write(addr, ai, s, w, true, meta, data, delay); + } } // now we should be able to safely operate on the cache line if constexpr (EnMT) {assert(meta->match(addr)); meta_outer->lock(); } if((writeback = Policy::probe_need_writeback(outer_cmd, meta))) { if(data_outer) data_outer->copy(data); } // writeback if dirty Policy::meta_after_probe(outer_cmd, meta, meta_outer, coh_id, writeback); // alway update meta + cache->replace_manage(ai, s, w, hit, (coh::is_evict(outer_cmd) ? 1 : 0)); cache->hook_manage(addr, ai, s, w, hit, (coh::is_evict(outer_cmd) ? 1 : 0), writeback, meta, data, delay); if constexpr (EnMT) { meta_outer->unlock(); meta->unlock(); cache->reset_mt_state(ai, s, XactPrio::probe); } } else { if constexpr (EnMT) meta_outer->lock(); Policy::meta_after_probe(outer_cmd, meta, meta_outer, coh_id, writeback); // alway update meta + cache->replace_manage(ai, s, w, hit, (coh::is_evict(outer_cmd) ? 1 : 0)); cache->hook_manage(addr, ai, s, w, hit, (coh::is_evict(outer_cmd) ? 1 : 0), writeback, meta, data, delay); if constexpr (EnMT) meta_outer->unlock(); } @@ -225,10 +223,12 @@ class InnerCohPortUncached : public InnerCohPortBase public: virtual void acquire_resp(uint64_t addr, CMDataBase *data_inner, CMMetadataBase *meta_inner, coh_cmd_t cmd, uint64_t *delay) override { auto [meta, data, ai, s, w, hit] = access_line(addr, cmd, XactPrio::acquire, delay); + bool act_as_prefetch = coh::is_prefetch(cmd) && Policy::is_uncached(); // only tweak replace priority at the LLC accoridng to [Guo2022-MICRO] if (data_inner && data) data_inner->copy(data); Policy::meta_after_grant(cmd, meta, meta_inner); - cache->hook_read(addr, ai, s, w, hit, coh::is_prefetch(cmd), meta, data, delay); + if(!act_as_prefetch || !hit) cache->replace_read(ai, s, w, act_as_prefetch); + cache->hook_read(addr, ai, s, w, hit, meta, data, delay); finish_record(addr, coh::cmd_for_finish(cmd.id), !hit, meta, ai, s); if(cmd.id == -1) finish_resp(addr, coh::cmd_for_finish(cmd.id)); } @@ -240,10 +240,6 @@ class InnerCohPortUncached : public InnerCohPortBase write_line(addr, data_inner, meta_inner, cmd, delay); } - virtual void prefetch_resp(uint64_t addr, coh_cmd_t cmd, uint64_t *delay) override { - prefetch_line(addr, cmd, delay); - } - virtual void query_loc_resp(uint64_t addr, std::list *locs) override { outer->query_loc_req(addr, locs); locs->push_front(cache->query_loc(addr)); @@ -258,33 +254,35 @@ class InnerCohPortUncached : public InnerCohPortBase if(sync.first) { if constexpr (EnMT && Policy::sync_need_lock()) cache->set_mt_state(ai, s, XactPrio::sync); auto [phit, pwb] = probe_req(addr, meta, data, sync.second, delay); // sync if necessary - if(pwb) cache->hook_write(addr, ai, s, w, true, false, meta, data, delay); // a write occurred during the probe + if(pwb){ + cache->replace_write(ai, s, w, false); + cache->hook_write(addr, ai, s, w, true, meta, data, delay); // a write occurred during the probe + } if constexpr (EnMT && Policy::sync_need_lock()) cache->reset_mt_state(ai, s, XactPrio::sync); } auto writeback = Policy::writeback_need_writeback(meta); if(writeback.first) outer->writeback_req(addr, meta, data, writeback.second, delay); // writeback if dirty Policy::meta_after_evict(meta); + cache->replace_manage(ai, s, w, true, 1); cache->hook_manage(addr, ai, s, w, true, 1, writeback.first, meta, data, delay); } virtual std::tuple - check_hit_or_replace(uint64_t addr, uint16_t prio, bool do_replace, bool prefetch, uint64_t *delay) { // check hit or get a replacement target + check_hit_or_replace(uint64_t addr, uint16_t prio, bool do_replace, uint64_t *delay) { // check hit or get a replacement target uint32_t ai, s, w; CMMetadataBase *meta = nullptr; CMDataBase *data = nullptr; bool hit; if constexpr (EnMT) { while(true) { - hit = cache->hit(addr, &ai, &s, &w, prio, !prefetch); + hit = cache->hit(addr, &ai, &s, &w, prio, true); if(hit) { std::tie(meta, data) = cache->access_line(ai, s, w); - if(!prefetch) { // no need to lock for prefetch - meta->lock(); - if(!cache->check_mt_state(ai, s, prio) || !meta->match(addr)) { // acquire is intercepted by a probe and even invalidated - meta->unlock(); meta = nullptr; data = nullptr; - cache->reset_mt_state(ai, s, prio); - continue; // redo the hit check - } + meta->lock(); + if(!cache->check_mt_state(ai, s, prio) || !meta->match(addr)) { // acquire is intercepted by a probe and even invalidated + meta->unlock(); meta = nullptr; data = nullptr; + cache->reset_mt_state(ai, s, prio); + continue; // redo the hit check } } else if(do_replace) { // miss if(cache->replace(addr, &ai, &s, &w, prio)) { // lock the cache set and get a replacement candidate @@ -310,14 +308,16 @@ class InnerCohPortUncached : public InnerCohPortBase virtual std::tuple access_line(uint64_t addr, coh_cmd_t cmd, uint16_t prio, uint64_t *delay) { // common function for access a line in the cache - auto [hit, meta, data, ai, s, w] = check_hit_or_replace(addr, prio, true, false, delay); - + auto [hit, meta, data, ai, s, w] = check_hit_or_replace(addr, prio, true, delay); if(hit) { auto sync = Policy::access_need_sync(cmd, meta); if(sync.first) { if constexpr (EnMT && Policy::sync_need_lock()) { assert(prio < XactPrio::sync); cache->set_mt_state(ai, s, XactPrio::sync);} auto [phit, pwb] = probe_req(addr, meta, data, sync.second, delay); // sync if necessary - if(pwb) cache->hook_write(addr, ai, s, w, true, false, meta, data, delay); // a write occurred during the probe + if(pwb){ + cache->replace_write(ai, s, w, false); + cache->hook_write(addr, ai, s, w, true, meta, data, delay); // a write occurred during the probe + } if constexpr (EnMT && Policy::sync_need_lock()) cache->reset_mt_state(ai, s, XactPrio::sync); } auto [promote, promote_local, promote_cmd] = Policy::access_need_promote(cmd, meta); @@ -325,7 +325,7 @@ class InnerCohPortUncached : public InnerCohPortBase else if(promote_local) meta->to_modified(-1); } else { // miss if(meta->is_valid()) evict(meta, data, ai, s, w, delay); - outer->acquire_req(addr, meta, data, Policy::cmd_for_outer_acquire(cmd), delay); // fetch the missing block + outer->acquire_req(addr, meta, data, coh::is_prefetch(cmd) ? cmd : Policy::cmd_for_outer_acquire(cmd), delay); // fetch the missing block } return std::make_tuple(meta, data, ai, s, w, hit); } @@ -336,7 +336,8 @@ class InnerCohPortUncached : public InnerCohPortBase if(data_inner) data->copy(data_inner); Policy::meta_after_release(cmd, meta, meta_inner); assert(meta_inner); // assume meta_inner is valid for all writebacks - cache->hook_write(addr, ai, s, w, hit, false, meta, data, delay); + cache->replace_write(ai, s, w, false); + cache->hook_write(addr, ai, s, w, hit, meta, data, delay); if constexpr (EnMT) { meta->unlock(); cache->reset_mt_state(ai, s, XactPrio::release); } } @@ -344,14 +345,17 @@ class InnerCohPortUncached : public InnerCohPortBase if constexpr (!Policy::is_uncached()) outer->writeback_req(addr, nullptr, nullptr, coh::cmd_for_flush(), delay); else { - auto [hit, meta, data, ai, s, w] = check_hit_or_replace(addr, XactPrio::flush, false, false, delay); + auto [hit, meta, data, ai, s, w] = check_hit_or_replace(addr, XactPrio::flush, false, delay); auto [probe, probe_cmd] = Policy::flush_need_sync(cmd, meta); if(!hit) return; if(probe) { if constexpr (EnMT && Policy::sync_need_lock()) cache->set_mt_state(ai, s, XactPrio::sync); auto [phit, pwb] = probe_req(addr, meta, data, probe_cmd, delay); // sync if necessary - if(pwb) cache->hook_write(addr, ai, s, w, true, false, meta, data, delay); // a write occurred during the probe + if(pwb){ + cache->replace_write(ai, s, w, false); // a write occurred during the probe + cache->hook_write(addr, ai, s, w, true, meta, data, delay); // a write occurred during the probe + } if constexpr (EnMT && Policy::sync_need_lock()) cache->reset_mt_state(ai, s, XactPrio::sync); } @@ -359,26 +363,13 @@ class InnerCohPortUncached : public InnerCohPortBase if(writeback.first) outer->writeback_req(addr, meta, data, writeback.second, delay); // writeback if dirty Policy::meta_after_flush(cmd, meta, cache); + cache->replace_manage(ai, s, w, hit, (coh::is_evict(cmd) ? 2 : 0)); // identify flush to hook_manager cache->hook_manage(addr, ai, s, w, hit, (coh::is_evict(cmd) ? 2 : 0), writeback.first, meta, data, delay); // identify flush to hook_manager if constexpr (EnMT) { meta->unlock(); cache->reset_mt_state(ai, s, XactPrio::flush); } } } - virtual void prefetch_line(uint64_t addr, coh_cmd_t cmd, uint64_t *delay) { - if constexpr (!Policy::is_uncached()) - outer->prefetch_req(addr, coh::cmd_for_prefetch(), delay); - else { - auto [hit, meta, data, ai, s, w] = check_hit_or_replace(addr, XactPrio::acquire, true, true, delay); - if(!hit) { - if(meta->is_valid()) evict(meta, data, ai, s, w, delay); - outer->acquire_req(addr, meta, data, coh::cmd_for_prefetch(), delay); // fetch the missing block - cache->hook_read(addr, ai, s, w, hit, true, meta, data, delay); - finish_record(addr, coh::cmd_for_finish(-1), !hit, meta, ai, s); - finish_resp(addr, coh::cmd_for_finish(-1)); - } - } - } }; template