diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp index 78860bbe4cb..772c5a0a98b 100644 --- a/src/hotspot/share/gc/z/zGeneration.cpp +++ b/src/hotspot/share/gc/z/zGeneration.cpp @@ -948,6 +948,14 @@ void ZGenerationYoung::register_with_remset(ZPage* page) { _remembered.register_found_old(page); } +ZRemembered* ZGenerationYoung::remembered() { + return &_remembered; +} + +void ZGenerationYoung::remap_current_remset(ZRemsetTableIterator* iter) { + _remembered.remap_current(iter); +} + ZGenerationTracer* ZGenerationYoung::jfr_tracer() { return &_jfr_tracer; } @@ -1435,7 +1443,7 @@ typedef ClaimingCLDToOopClosure ZRemapCLDClosure; class ZRemapYoungRootsTask : public ZTask { private: - ZGenerationPagesParallelIterator _old_pages_parallel_iterator; + ZRemsetTableIterator _remset_table_iterator; ZRootsIteratorAllColored _roots_colored; ZRootsIteratorAllUncolored _roots_uncolored; @@ -1449,7 +1457,7 @@ private: public: ZRemapYoungRootsTask(ZPageTable* page_table, ZPageAllocator* page_allocator) : ZTask("ZRemapYoungRootsTask"), - _old_pages_parallel_iterator(page_table, ZGenerationId::old, page_allocator), + _remset_table_iterator(ZGeneration::young()->remembered(), false /* previous */), _roots_colored(ZGenerationIdOptional::old), _roots_uncolored(ZGenerationIdOptional::old), _cl_colored(), @@ -1472,11 +1480,8 @@ public: { ZStatTimerWorker timer(ZSubPhaseConcurrentRemapRememberedOld); - _old_pages_parallel_iterator.do_pages([&](ZPage* page) { - // Visit all object fields that potentially pointing into young generation - page->oops_do_current_remembered(ZBarrier::load_barrier_on_oop_field); - return true; - }); + // Visit all object fields that potentially pointing into young generation + ZGeneration::young()->remap_current_remset(&_remset_table_iterator); } } }; diff --git a/src/hotspot/share/gc/z/zGeneration.hpp b/src/hotspot/share/gc/z/zGeneration.hpp index 922efe5ef9e..13adc06b123 100644 --- a/src/hotspot/share/gc/z/zGeneration.hpp +++ b/src/hotspot/share/gc/z/zGeneration.hpp @@ -191,6 +191,7 @@ class ZGenerationYoung : public ZGeneration { friend class VM_ZMarkStartYoung; friend class VM_ZMarkStartYoungAndOld; friend class VM_ZRelocateStartYoung; + friend class ZRemapYoungRootsTask; friend class ZYoungTypeSetter; private: @@ -219,6 +220,8 @@ private: void pause_relocate_start(); void concurrent_relocate(); + ZRemembered* remembered(); + public: ZGenerationYoung(ZPageTable* page_table, const ZForwardingTable* old_forwarding_table, @@ -252,6 +255,9 @@ public: // Register old pages with remembered set void register_with_remset(ZPage* page); + // Remap the oops of the current remembered set + void remap_current_remset(ZRemsetTableIterator* iter); + // Serviceability ZGenerationTracer* jfr_tracer(); diff --git a/src/hotspot/share/gc/z/zRemembered.cpp b/src/hotspot/share/gc/z/zRemembered.cpp index b94d676242c..815b914dd8d 100644 --- a/src/hotspot/share/gc/z/zRemembered.cpp +++ b/src/hotspot/share/gc/z/zRemembered.cpp @@ -392,69 +392,71 @@ struct ZRemsetTableEntry { ZForwarding* _forwarding; }; -class ZRemsetTableIterator { -private: - ZRemembered* const _remembered; - ZPageTable* const _page_table; - const ZForwardingTable* const _old_forwarding_table; - volatile BitMap::idx_t _claimed; - -public: - ZRemsetTableIterator(ZRemembered* remembered) - : _remembered(remembered), - _page_table(remembered->_page_table), - _old_forwarding_table(remembered->_old_forwarding_table), - _claimed(0) {} +ZRemsetTableIterator::ZRemsetTableIterator(ZRemembered* remembered, bool previous) + : _remembered(remembered), + _bm(previous + ? _remembered->_found_old.previous_bitmap() + : _remembered->_found_old.current_bitmap()), + _page_table(remembered->_page_table), + _old_forwarding_table(remembered->_old_forwarding_table), + _claimed(0) {} // This iterator uses the "found old" optimization. - bool next(ZRemsetTableEntry* entry_addr) { - BitMap* const bm = _remembered->_found_old.previous_bitmap(); +bool ZRemsetTableIterator::next(ZRemsetTableEntry* entry_addr) { + BitMap::idx_t prev = Atomic::load(&_claimed); - BitMap::idx_t prev = Atomic::load(&_claimed); - - for (;;) { - if (prev == bm->size()) { - return false; - } - - const BitMap::idx_t page_index = bm->find_first_set_bit(_claimed); - if (page_index == bm->size()) { - Atomic::cmpxchg(&_claimed, prev, page_index, memory_order_relaxed); - return false; - } - - const BitMap::idx_t res = Atomic::cmpxchg(&_claimed, prev, page_index + 1, memory_order_relaxed); - if (res != prev) { - // Someone else claimed - prev = res; - continue; - } - - // Found bit - look around for page or forwarding to scan - - ZForwarding* forwarding = nullptr; - if (ZGeneration::old()->is_phase_relocate()) { - forwarding = _old_forwarding_table->at(page_index); - } - - ZPage* page = _page_table->at(page_index); - if (page != nullptr && !page->is_old()) { - page = nullptr; - } - - if (page == nullptr && forwarding == nullptr) { - // Nothing to scan - continue; - } - - // Found old page or old forwarding - entry_addr->_forwarding = forwarding; - entry_addr->_page = page; - - return true; + for (;;) { + if (prev == _bm->size()) { + return false; } + + const BitMap::idx_t page_index = _bm->find_first_set_bit(_claimed); + if (page_index == _bm->size()) { + Atomic::cmpxchg(&_claimed, prev, page_index, memory_order_relaxed); + return false; + } + + const BitMap::idx_t res = Atomic::cmpxchg(&_claimed, prev, page_index + 1, memory_order_relaxed); + if (res != prev) { + // Someone else claimed + prev = res; + continue; + } + + // Found bit - look around for page or forwarding to scan + + ZForwarding* forwarding = nullptr; + if (ZGeneration::old()->is_phase_relocate()) { + forwarding = _old_forwarding_table->at(page_index); + } + + ZPage* page = _page_table->at(page_index); + if (page != nullptr && !page->is_old()) { + page = nullptr; + } + + if (page == nullptr && forwarding == nullptr) { + // Nothing to scan + continue; + } + + // Found old page or old forwarding + entry_addr->_forwarding = forwarding; + entry_addr->_page = page; + + return true; } -}; +} + +void ZRemembered::remap_current(ZRemsetTableIterator* iter) { + for (ZRemsetTableEntry entry; iter->next(&entry);) { + assert(entry._forwarding == nullptr, "Shouldn't be looking for forwardings"); + assert(entry._page != nullptr, "Must have found a page"); + assert(entry._page->is_old(), "Should only have found old pages"); + + entry._page->oops_do_current_remembered(ZBarrier::load_barrier_on_oop_field); + } +} // This task scans the remembered set and follows pointers when possible. // Interleaving remembered set scanning with marking makes the marking times @@ -470,7 +472,7 @@ public: : ZRestartableTask("ZRememberedScanMarkFollowTask"), _remembered(remembered), _mark(mark), - _remset_table_iterator(remembered) { + _remset_table_iterator(remembered, true /* previous */) { _mark->prepare_work(); _remembered->_page_allocator->enable_safe_destroy(); } diff --git a/src/hotspot/share/gc/z/zRemembered.hpp b/src/hotspot/share/gc/z/zRemembered.hpp index 1400472f2bc..6cc76f5dc26 100644 --- a/src/hotspot/share/gc/z/zRemembered.hpp +++ b/src/hotspot/share/gc/z/zRemembered.hpp @@ -35,7 +35,9 @@ class ZMark; class ZPage; class ZPageAllocator; class ZPageTable; +class ZRemsetTableIterator; struct ZRememberedSetContaining; +struct ZRemsetTableEntry; class ZRemembered { friend class ZRememberedScanMarkFollowTask; @@ -99,6 +101,26 @@ public: // Register pages with the remembered set void register_found_old(ZPage* page); + + // Remap the current remembered set + void remap_current(ZRemsetTableIterator* iter); +}; + +// This iterator uses the "found old" optimization to skip having to iterate +// over the entire page table. Make sure to check where and how the FoundOld +// data is cycled before using this iterator. +class ZRemsetTableIterator { +private: + ZRemembered* const _remembered; + BitMap* const _bm; + ZPageTable* const _page_table; + const ZForwardingTable* const _old_forwarding_table; + volatile BitMap::idx_t _claimed; + +public: + ZRemsetTableIterator(ZRemembered* remembered, bool previous); + + bool next(ZRemsetTableEntry* entry_addr); }; #endif // SHARE_GC_Z_ZREMEMBERED_HPP