8357443: ZGC: Optimize old page iteration in remap remembered phase

Reviewed-by: aboldtch, eosterlund

src/hotspot/share/gc/z/zGeneration.cpp

@@ -948,6 +948,14 @@ void ZGenerationYoung::register_with_remset(ZPage* page) {
   _remembered.register_found_old(page);
 }
 
+ZRemembered* ZGenerationYoung::remembered() {
+  return &_remembered;
+}
+
+void ZGenerationYoung::remap_current_remset(ZRemsetTableIterator* iter) {
+  _remembered.remap_current(iter);
+}
+
 ZGenerationTracer* ZGenerationYoung::jfr_tracer() {
   return &_jfr_tracer;
 }
@@ -1435,7 +1443,7 @@ typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_none> ZRemapCLDClosure;
 
 class ZRemapYoungRootsTask : public ZTask {
 private:
-  ZGenerationPagesParallelIterator _old_pages_parallel_iterator;
+  ZRemsetTableIterator             _remset_table_iterator;
 
   ZRootsIteratorAllColored         _roots_colored;
   ZRootsIteratorAllUncolored       _roots_uncolored;
@@ -1449,7 +1457,7 @@ private:
 public:
   ZRemapYoungRootsTask(ZPageTable* page_table, ZPageAllocator* page_allocator)
     : ZTask("ZRemapYoungRootsTask"),
-      _old_pages_parallel_iterator(page_table, ZGenerationId::old, page_allocator),
+      _remset_table_iterator(ZGeneration::young()->remembered(), false /* previous */),
       _roots_colored(ZGenerationIdOptional::old),
       _roots_uncolored(ZGenerationIdOptional::old),
       _cl_colored(),
@@ -1472,11 +1480,8 @@ public:
 
     {
       ZStatTimerWorker timer(ZSubPhaseConcurrentRemapRememberedOld);
-      _old_pages_parallel_iterator.do_pages([&](ZPage* page) {
+      // Visit all object fields that potentially pointing into young generation
-        // Visit all object fields that potentially pointing into young generation
+      ZGeneration::young()->remap_current_remset(&_remset_table_iterator);
-        page->oops_do_current_remembered(ZBarrier::load_barrier_on_oop_field);
-        return true;
-      });
     }
   }
 };

src/hotspot/share/gc/z/zGeneration.hpp

@@ -191,6 +191,7 @@ class ZGenerationYoung : public ZGeneration {
   friend class VM_ZMarkStartYoung;
   friend class VM_ZMarkStartYoungAndOld;
   friend class VM_ZRelocateStartYoung;
+  friend class ZRemapYoungRootsTask;
   friend class ZYoungTypeSetter;
 
 private:
@@ -219,6 +220,8 @@ private:
   void pause_relocate_start();
   void concurrent_relocate();
 
+  ZRemembered* remembered();
+
 public:
   ZGenerationYoung(ZPageTable* page_table,
                    const ZForwardingTable* old_forwarding_table,
@@ -252,6 +255,9 @@ public:
   // Register old pages with remembered set
   void register_with_remset(ZPage* page);
 
+  // Remap the oops of the current remembered set
+  void remap_current_remset(ZRemsetTableIterator* iter);
+
   // Serviceability
   ZGenerationTracer* jfr_tracer();
 

src/hotspot/share/gc/z/zRemembered.cpp

@@ -392,69 +392,71 @@ struct ZRemsetTableEntry {
   ZForwarding* _forwarding;
 };
 
-class ZRemsetTableIterator {
+ZRemsetTableIterator::ZRemsetTableIterator(ZRemembered* remembered, bool previous)
-private:
+  : _remembered(remembered),
-  ZRemembered* const            _remembered;
+    _bm(previous
-  ZPageTable* const             _page_table;
+        ? _remembered->_found_old.previous_bitmap()
-  const ZForwardingTable* const _old_forwarding_table;
+        : _remembered->_found_old.current_bitmap()),
-  volatile BitMap::idx_t        _claimed;
+    _page_table(remembered->_page_table),
-
+    _old_forwarding_table(remembered->_old_forwarding_table),
-public:
+    _claimed(0) {}
-  ZRemsetTableIterator(ZRemembered* remembered)
-    : _remembered(remembered),
-      _page_table(remembered->_page_table),
-      _old_forwarding_table(remembered->_old_forwarding_table),
-      _claimed(0) {}
 
   // This iterator uses the "found old" optimization.
-  bool next(ZRemsetTableEntry* entry_addr)  {
+bool ZRemsetTableIterator::next(ZRemsetTableEntry* entry_addr) {
-    BitMap* const bm = _remembered->_found_old.previous_bitmap();
+  BitMap::idx_t prev = Atomic::load(&_claimed);
 
-    BitMap::idx_t prev = Atomic::load(&_claimed);
+  for (;;) {
-
+    if (prev == _bm->size()) {
-    for (;;) {
+      return false;
-      if (prev == bm->size()) {
-        return false;
-      }
-
-      const BitMap::idx_t page_index = bm->find_first_set_bit(_claimed);
-      if (page_index == bm->size()) {
-        Atomic::cmpxchg(&_claimed, prev, page_index, memory_order_relaxed);
-        return false;
-      }
-
-      const BitMap::idx_t res = Atomic::cmpxchg(&_claimed, prev, page_index + 1, memory_order_relaxed);
-      if (res != prev) {
-        // Someone else claimed
-        prev = res;
-        continue;
-      }
-
-      // Found bit - look around for page or forwarding to scan
-
-      ZForwarding* forwarding = nullptr;
-      if (ZGeneration::old()->is_phase_relocate()) {
-        forwarding = _old_forwarding_table->at(page_index);
-      }
-
-      ZPage* page = _page_table->at(page_index);
-      if (page != nullptr && !page->is_old()) {
-        page = nullptr;
-      }
-
-      if (page == nullptr && forwarding == nullptr) {
-        // Nothing to scan
-        continue;
-      }
-
-      // Found old page or old forwarding
-      entry_addr->_forwarding = forwarding;
-      entry_addr->_page = page;
-
-      return true;
     }
+
+    const BitMap::idx_t page_index = _bm->find_first_set_bit(_claimed);
+    if (page_index == _bm->size()) {
+      Atomic::cmpxchg(&_claimed, prev, page_index, memory_order_relaxed);
+      return false;
+    }
+
+    const BitMap::idx_t res = Atomic::cmpxchg(&_claimed, prev, page_index + 1, memory_order_relaxed);
+    if (res != prev) {
+      // Someone else claimed
+      prev = res;
+      continue;
+    }
+
+    // Found bit - look around for page or forwarding to scan
+
+    ZForwarding* forwarding = nullptr;
+    if (ZGeneration::old()->is_phase_relocate()) {
+      forwarding = _old_forwarding_table->at(page_index);
+    }
+
+    ZPage* page = _page_table->at(page_index);
+    if (page != nullptr && !page->is_old()) {
+      page = nullptr;
+    }
+
+    if (page == nullptr && forwarding == nullptr) {
+      // Nothing to scan
+      continue;
+    }
+
+    // Found old page or old forwarding
+    entry_addr->_forwarding = forwarding;
+    entry_addr->_page = page;
+
+    return true;
   }
-};
+}
+
+void ZRemembered::remap_current(ZRemsetTableIterator* iter) {
+  for (ZRemsetTableEntry entry; iter->next(&entry);) {
+    assert(entry._forwarding == nullptr, "Shouldn't be looking for forwardings");
+    assert(entry._page != nullptr, "Must have found a page");
+    assert(entry._page->is_old(), "Should only have found old pages");
+
+    entry._page->oops_do_current_remembered(ZBarrier::load_barrier_on_oop_field);
+  }
+}
 
 // This task scans the remembered set and follows pointers when possible.
 // Interleaving remembered set scanning with marking makes the marking times
@@ -470,7 +472,7 @@ public:
     : ZRestartableTask("ZRememberedScanMarkFollowTask"),
       _remembered(remembered),
       _mark(mark),
-      _remset_table_iterator(remembered)  {
+      _remset_table_iterator(remembered, true /* previous */) {
     _mark->prepare_work();
     _remembered->_page_allocator->enable_safe_destroy();
   }

src/hotspot/share/gc/z/zRemembered.hpp

@@ -35,7 +35,9 @@ class ZMark;
 class ZPage;
 class ZPageAllocator;
 class ZPageTable;
+class ZRemsetTableIterator;
 struct ZRememberedSetContaining;
+struct ZRemsetTableEntry;
 
 class ZRemembered {
   friend class ZRememberedScanMarkFollowTask;
@@ -99,6 +101,26 @@ public:
 
   // Register pages with the remembered set
   void register_found_old(ZPage* page);
+
+  // Remap the current remembered set
+  void remap_current(ZRemsetTableIterator* iter);
+};
+
+// This iterator uses the "found old" optimization to skip having to iterate
+// over the entire page table. Make sure to check where and how the FoundOld
+// data is cycled before using this iterator.
+class ZRemsetTableIterator {
+private:
+  ZRemembered* const            _remembered;
+  BitMap* const                 _bm;
+  ZPageTable* const             _page_table;
+  const ZForwardingTable* const _old_forwarding_table;
+  volatile BitMap::idx_t        _claimed;
+
+public:
+  ZRemsetTableIterator(ZRemembered* remembered, bool previous);
+
+  bool next(ZRemsetTableEntry* entry_addr);
 };
 
 #endif // SHARE_GC_Z_ZREMEMBERED_HPP