8256298: Shenandoah: Enable concurrent stack processing

Reviewed-by: rkennke, shade

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -24,14 +24,13 @@
 
 #include "precompiled.hpp"
 #include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahClosures.inline.hpp"
 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetClone.inline.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
 #include "gc/shenandoah/shenandoahBarrierSetNMethod.hpp"
-#include "gc/shenandoah/shenandoahCollectorPolicy.hpp"
-#include "gc/shenandoah/shenandoahConcurrentRoots.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
-#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
+#include "gc/shenandoah/shenandoahStackWatermark.hpp"
 #include "memory/iterator.inline.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #ifdef COMPILER1
@@ -112,6 +111,10 @@ void ShenandoahBarrierSet::on_thread_attach(Thread *thread) {
     ShenandoahThreadLocalData::set_gc_state(thread, _heap->gc_state());
     ShenandoahThreadLocalData::initialize_gclab(thread);
     ShenandoahThreadLocalData::set_disarmed_value(thread, ShenandoahCodeRoots::disarmed_value());
+
+    JavaThread* const jt = thread->as_Java_thread();
+    StackWatermark* const watermark = new ShenandoahStackWatermark(jt);
+    StackWatermarkSet::add_watermark(jt, watermark);
   }
 }
 
@@ -123,6 +126,16 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) {
     if (gclab != NULL) {
       gclab->retire();
     }
+
+    // SATB protocol requires to keep alive reacheable oops from roots at the beginning of GC
+    ShenandoahHeap* const heap = ShenandoahHeap::heap();
+    if (heap->is_concurrent_mark_in_progress()) {
+      ShenandoahKeepAliveClosure oops;
+      StackWatermarkSet::finish_processing(thread->as_Java_thread(), &oops, StackWatermarkKind::gc);
+    } else if (heap->is_concurrent_weak_root_in_progress() && heap->is_evacuation_in_progress()) {
+      ShenandoahContextEvacuateUpdateRootsClosure oops;
+      StackWatermarkSet::finish_processing(thread->as_Java_thread(), &oops, StackWatermarkKind::gc);
+    }
   }
 }
 
@@ -131,4 +144,3 @@ void ShenandoahBarrierSet::clone_barrier_runtime(oop src) {
     clone_barrier(src);
   }
 }
-

src/hotspot/share/gc/shenandoah/shenandoahClosures.hpp

@@ -28,6 +28,7 @@
 #include "oops/accessDecorators.hpp"
 #include "runtime/handshake.hpp"
 
+class ShenandoahBarrierSet;
 class ShenandoahHeap;
 class ShenandoahMarkingContext;
 class ShenandoahHeapRegionSet;
@@ -57,6 +58,18 @@ public:
   inline BoolObjectClosure* is_alive_closure();
 };
 
+class ShenandoahKeepAliveClosure : public OopClosure {
+private:
+  ShenandoahBarrierSet* const _bs;
+public:
+  inline ShenandoahKeepAliveClosure();
+  inline void do_oop(oop* p);
+  inline void do_oop(narrowOop* p);
+private:
+  template <typename T>
+  void do_oop_work(T* p);
+};
+
 class ShenandoahUpdateRefsClosure: public OopClosure {
 private:
   ShenandoahHeap* _heap;
@@ -70,12 +83,12 @@ private:
 };
 
 template <DecoratorSet MO = MO_UNORDERED>
-class ShenandoahEvacuateUpdateRootsClosure: public BasicOopIterateClosure {
+class ShenandoahEvacuateUpdateMetadataClosure: public BasicOopIterateClosure {
 private:
-  ShenandoahHeap* _heap;
-  Thread* _thread;
+  ShenandoahHeap* const _heap;
+  Thread* const         _thread;
 public:
-  inline ShenandoahEvacuateUpdateRootsClosure();
+  inline ShenandoahEvacuateUpdateMetadataClosure();
   inline void do_oop(oop* p);
   inline void do_oop(narrowOop* p);
 
@@ -84,12 +97,24 @@ private:
   inline void do_oop_work(T* p);
 };
 
-class ShenandoahEvacUpdateOopStorageRootsClosure : public BasicOopIterateClosure {
+// Context free version, cannot cache calling thread
+class ShenandoahEvacuateUpdateRootsClosure : public BasicOopIterateClosure {
 private:
-  ShenandoahHeap* _heap;
-  Thread* _thread;
+  ShenandoahHeap* const _heap;
 public:
-  inline ShenandoahEvacUpdateOopStorageRootsClosure();
+  inline ShenandoahEvacuateUpdateRootsClosure();
+  inline void do_oop(oop* p);
+  inline void do_oop(narrowOop* p);
+protected:
+  template <typename T>
+  inline void do_oop_work(T* p, Thread* thr);
+};
+
+class ShenandoahContextEvacuateUpdateRootsClosure : public ShenandoahEvacuateUpdateRootsClosure {
+private:
+  Thread* const _thread;
+public:
+  inline ShenandoahContextEvacuateUpdateRootsClosure();
   inline void do_oop(oop* p);
   inline void do_oop(narrowOop* p);
 };

src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp

@@ -26,7 +26,9 @@
 
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahBarrierSet.hpp"
 #include "gc/shenandoah/shenandoahClosures.hpp"
+#include "gc/shenandoah/shenandoahEvacOOMHandler.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahNMethod.inline.hpp"
 #include "oops/compressedOops.inline.hpp"
@@ -64,6 +66,30 @@ BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() {
          reinterpret_cast<BoolObjectClosure*>(&_alive_cl);
 }
 
+ShenandoahKeepAliveClosure::ShenandoahKeepAliveClosure() :
+  _bs(static_cast<ShenandoahBarrierSet*>(BarrierSet::barrier_set())) {
+}
+
+void ShenandoahKeepAliveClosure::do_oop(oop* p) {
+  do_oop_work(p);
+}
+
+void ShenandoahKeepAliveClosure::do_oop(narrowOop* p) {
+  do_oop_work(p);
+}
+
+template <typename T>
+void ShenandoahKeepAliveClosure::do_oop_work(T* p) {
+  assert(ShenandoahHeap::heap()->is_concurrent_mark_in_progress(), "Only for concurrent marking phase");
+  assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
+
+  T o = RawAccess<>::oop_load(p);
+  if (!CompressedOops::is_null(o)) {
+    oop obj = CompressedOops::decode_not_null(o);
+    _bs->enqueue(obj);
+  }
+}
+
 ShenandoahUpdateRefsClosure::ShenandoahUpdateRefsClosure() :
   _heap(ShenandoahHeap::heap()) {
 }
@@ -81,16 +107,17 @@ void ShenandoahUpdateRefsClosure::do_oop(oop* p)       { do_oop_work(p); }
 void ShenandoahUpdateRefsClosure::do_oop(narrowOop* p) { do_oop_work(p); }
 
 template <DecoratorSet MO>
-ShenandoahEvacuateUpdateRootsClosure<MO>::ShenandoahEvacuateUpdateRootsClosure() :
+ShenandoahEvacuateUpdateMetadataClosure<MO>::ShenandoahEvacuateUpdateMetadataClosure() :
   _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
 }
 
 template <DecoratorSet MO>
 template <class T>
-void ShenandoahEvacuateUpdateRootsClosure<MO>::do_oop_work(T* p) {
+void ShenandoahEvacuateUpdateMetadataClosure<MO>::do_oop_work(T* p) {
   assert(_heap->is_concurrent_weak_root_in_progress() ||
          _heap->is_concurrent_strong_root_in_progress(),
          "Only do this in root processing phase");
+  assert(_thread == Thread::current(), "Wrong thread");
 
   T o = RawAccess<>::oop_load(p);
   if (! CompressedOops::is_null(o)) {
@@ -107,41 +134,64 @@ void ShenandoahEvacuateUpdateRootsClosure<MO>::do_oop_work(T* p) {
   }
 }
 template <DecoratorSet MO>
-void ShenandoahEvacuateUpdateRootsClosure<MO>::do_oop(oop* p) {
+void ShenandoahEvacuateUpdateMetadataClosure<MO>::do_oop(oop* p) {
   do_oop_work(p);
 }
 
 template <DecoratorSet MO>
-void ShenandoahEvacuateUpdateRootsClosure<MO>::do_oop(narrowOop* p) {
+void ShenandoahEvacuateUpdateMetadataClosure<MO>::do_oop(narrowOop* p) {
   do_oop_work(p);
 }
 
-ShenandoahEvacUpdateOopStorageRootsClosure::ShenandoahEvacUpdateOopStorageRootsClosure() :
-  _heap(ShenandoahHeap::heap()), _thread(Thread::current()) {
+ShenandoahEvacuateUpdateRootsClosure::ShenandoahEvacuateUpdateRootsClosure() :
+  _heap(ShenandoahHeap::heap()) {
 }
 
-void ShenandoahEvacUpdateOopStorageRootsClosure::do_oop(oop* p) {
+template <typename T>
+void ShenandoahEvacuateUpdateRootsClosure::do_oop_work(T* p, Thread* t) {
   assert(_heap->is_concurrent_weak_root_in_progress() ||
          _heap->is_concurrent_strong_root_in_progress(),
          "Only do this in root processing phase");
+  assert(t == Thread::current(), "Wrong thread");
 
-  oop obj = RawAccess<>::oop_load(p);
-  if (! CompressedOops::is_null(obj)) {
+  T o = RawAccess<>::oop_load(p);
+  if (!CompressedOops::is_null(o)) {
+    oop obj = CompressedOops::decode_not_null(o);
     if (_heap->in_collection_set(obj)) {
       assert(_heap->is_evacuation_in_progress(), "Only do this when evacuation is in progress");
       shenandoah_assert_marked(p, obj);
       oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
       if (resolved == obj) {
-        resolved = _heap->evacuate_object(obj, _thread);
+        resolved = _heap->evacuate_object(obj, t);
       }
-
-      Atomic::cmpxchg(p, obj, resolved);
+      _heap->cas_oop(resolved, p, o);
     }
   }
 }
 
-void ShenandoahEvacUpdateOopStorageRootsClosure::do_oop(narrowOop* p) {
-  ShouldNotReachHere();
+void ShenandoahEvacuateUpdateRootsClosure::do_oop(oop* p) {
+  ShenandoahEvacOOMScope scope;
+  do_oop_work(p, Thread::current());
+}
+
+void ShenandoahEvacuateUpdateRootsClosure::do_oop(narrowOop* p) {
+  ShenandoahEvacOOMScope scope;
+  do_oop_work(p, Thread::current());
+}
+
+ShenandoahContextEvacuateUpdateRootsClosure::ShenandoahContextEvacuateUpdateRootsClosure() :
+  ShenandoahEvacuateUpdateRootsClosure(),
+  _thread(Thread::current()) {
+}
+
+void ShenandoahContextEvacuateUpdateRootsClosure::do_oop(oop* p) {
+  ShenandoahEvacOOMScope scope;
+  do_oop_work(p, _thread);
+}
+
+void ShenandoahContextEvacuateUpdateRootsClosure::do_oop(narrowOop* p) {
+  ShenandoahEvacOOMScope scope;
+  do_oop_work(p, _thread);
 }
 
 template <bool CONCURRENT, typename IsAlive, typename KeepAlive>

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -36,9 +36,10 @@
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
 #include "gc/shenandoah/shenandoahReferenceProcessor.hpp"
 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
-#include "gc/shenandoah/shenandoahVMOperations.hpp"
+#include "gc/shenandoah/shenandoahStackWatermark.hpp"
 #include "gc/shenandoah/shenandoahUtils.hpp"
 #include "gc/shenandoah/shenandoahVerifier.hpp"
+#include "gc/shenandoah/shenandoahVMOperations.hpp"
 #include "gc/shenandoah/shenandoahWorkGroup.hpp"
 #include "gc/shenandoah/shenandoahWorkerPolicy.hpp"
 #include "prims/jvmtiTagMap.hpp"
@@ -78,6 +79,11 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
   // Complete marking under STW, and start evacuation
   vmop_entry_final_mark();
 
+  // Concurrent stack processing
+  if (heap->is_evacuation_in_progress()) {
+    entry_thread_roots();
+  }
+
   // Process weak roots that might still point to regions that would be broken by cleanup
   if (heap->is_concurrent_weak_root_in_progress()) {
     entry_weak_refs();
@@ -268,6 +274,20 @@ void ShenandoahConcurrentGC::entry_mark() {
   op_mark();
 }
 
+void ShenandoahConcurrentGC::entry_thread_roots() {
+  ShenandoahHeap* const heap = ShenandoahHeap::heap();
+  static const char* msg = "Concurrent thread roots";
+  ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_thread_roots);
+  EventMark em("%s", msg);
+
+  ShenandoahWorkerScope scope(heap->workers(),
+                              ShenandoahWorkerPolicy::calc_workers_for_conc_root_processing(),
+                              msg);
+
+  heap->try_inject_alloc_failure();
+  op_thread_roots();
+}
+
 void ShenandoahConcurrentGC::entry_weak_refs() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   static const char* msg = "Concurrent weak references";
@@ -460,14 +480,6 @@ void ShenandoahConcurrentGC::op_init_mark() {
 
   heap->set_concurrent_mark_in_progress(true);
 
-  // We need to reset all TLABs because they might be below the TAMS, and we need to mark
-  // the objects in them. Do not let mutators allocate any new objects in their current TLABs.
-  // It is also a good place to resize the TLAB sizes for future allocations.
-  if (UseTLAB) {
-    ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_manage_tlabs);
-    heap->tlabs_retire(ResizeTLAB);
-  }
-
   {
     ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states);
     ShenandoahInitMarkUpdateRegionStateClosure cl;
@@ -488,8 +500,7 @@ void ShenandoahConcurrentGC::op_init_mark() {
     ShenandoahCodeRoots::arm_nmethods();
   }
 
-  _mark.mark_stw_roots();
-
+  ShenandoahStackWatermark::change_epoch_id();
   if (ShenandoahPacing) {
     heap->pacer()->setup_for_mark();
   }
@@ -533,19 +544,19 @@ void ShenandoahConcurrentGC::op_final_mark() {
       // From here on, we need to update references.
       heap->set_has_forwarded_objects(true);
 
-      // Arm nmethods for concurrent processing
-      ShenandoahCodeRoots::arm_nmethods();
-
-      // Should be gone after 8212879 and concurrent stack processing
-      heap->evacuate_and_update_roots();
-
-      // Notify JVMTI that oops are changed.
-      JvmtiTagMap::set_needs_rehashing();
-
+      // Verify before arming for concurrent processing.
+      // Otherwise, verification can trigger stack processing.
       if (ShenandoahVerify) {
         heap->verifier()->verify_during_evacuation();
       }
 
+      // Arm nmethods/stack for concurrent processing
+      ShenandoahCodeRoots::arm_nmethods();
+      ShenandoahStackWatermark::change_epoch_id();
+
+      // Notify JVMTI that oops are changed.
+      JvmtiTagMap::set_needs_rehashing();
+
       if (ShenandoahPacing) {
         heap->pacer()->setup_for_evac();
       }
@@ -561,6 +572,51 @@ void ShenandoahConcurrentGC::op_final_mark() {
   }
 }
 
+class ShenandoahConcurrentEvacThreadClosure : public ThreadClosure {
+private:
+  OopClosure* const _oops;
+
+public:
+  ShenandoahConcurrentEvacThreadClosure(OopClosure* oops);
+  void do_thread(Thread* thread);
+};
+
+ShenandoahConcurrentEvacThreadClosure::ShenandoahConcurrentEvacThreadClosure(OopClosure* oops) :
+  _oops(oops) {
+}
+
+void ShenandoahConcurrentEvacThreadClosure::do_thread(Thread* thread) {
+  JavaThread* const jt = thread->as_Java_thread();
+  StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
+}
+
+class ShenandoahConcurrentEvacUpdateThreadTask : public AbstractGangTask {
+private:
+  ShenandoahJavaThreadsIterator _java_threads;
+
+public:
+  ShenandoahConcurrentEvacUpdateThreadTask() :
+    AbstractGangTask("Shenandoah Evacuate/Update Concurrent Thread Roots"),
+    _java_threads(ShenandoahPhaseTimings::conc_thread_roots) {
+  }
+
+  void work(uint worker_id) {
+    // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure.
+    // Otherwise, may deadlock with watermark lock
+    ShenandoahContextEvacuateUpdateRootsClosure oops_cl;
+    ShenandoahConcurrentEvacThreadClosure thr_cl(&oops_cl);
+    _java_threads.threads_do(&thr_cl, worker_id);
+  }
+};
+
+void ShenandoahConcurrentGC::op_thread_roots() {
+  ShenandoahHeap* const heap = ShenandoahHeap::heap();
+  assert(heap->is_evacuation_in_progress(), "Checked by caller");
+  ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_thread_roots);
+  ShenandoahConcurrentEvacUpdateThreadTask task;
+  heap->workers()->run_task(&task);
+}
+
 void ShenandoahConcurrentGC::op_weak_refs() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase");
@@ -680,7 +736,7 @@ public:
 
       // String dedup weak roots
       ShenandoahForwardedIsAliveClosure is_alive;
-      ShenandoahEvacuateUpdateRootsClosure<MO_RELEASE> keep_alive;
+      ShenandoahEvacuateUpdateMetadataClosure<MO_RELEASE> keep_alive;
       _dedup_roots.oops_do(&is_alive, &keep_alive, worker_id);
     }
 
@@ -741,8 +797,8 @@ void ShenandoahConcurrentGC::op_class_unloading() {
 
 class ShenandoahEvacUpdateCodeCacheClosure : public NMethodClosure {
 private:
-  BarrierSetNMethod* const               _bs;
-  ShenandoahEvacuateUpdateRootsClosure<> _cl;
+  BarrierSetNMethod* const                  _bs;
+  ShenandoahEvacuateUpdateMetadataClosure<> _cl;
 
 public:
   ShenandoahEvacUpdateCodeCacheClosure() :
@@ -797,12 +853,12 @@ public:
       {
         // vm_roots and weak_roots are OopStorage backed roots, concurrent iteration
         // may race against OopStorage::release() calls.
-        ShenandoahEvacUpdateOopStorageRootsClosure cl;
-        _vm_roots.oops_do<ShenandoahEvacUpdateOopStorageRootsClosure>(&cl, worker_id);
+        ShenandoahContextEvacuateUpdateRootsClosure cl;
+        _vm_roots.oops_do<ShenandoahContextEvacuateUpdateRootsClosure>(&cl, worker_id);
       }
 
       {
-        ShenandoahEvacuateUpdateRootsClosure<> cl;
+        ShenandoahEvacuateUpdateMetadataClosure<> cl;
         CLDToOopClosure clds(&cl, ClassLoaderData::_claim_strong);
         _cld_roots.cld_do(&clds, worker_id);
       }

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp

@@ -72,6 +72,7 @@ private:
   void entry_reset();
   void entry_mark_roots();
   void entry_mark();
+  void entry_thread_roots();
   void entry_weak_refs();
   void entry_weak_roots();
   void entry_class_unloading();
@@ -89,6 +90,7 @@ private:
   void op_mark_roots();
   void op_mark();
   void op_final_mark();
+  void op_thread_roots();
   void op_weak_refs();
   void op_weak_roots();
   void op_class_unloading();

src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp

@@ -181,50 +181,9 @@ public:
   }
 };
 
-class ShenandoahInitMarkRootsTask : public AbstractGangTask {
-private:
-  ShenandoahRootScanner              _root_scanner;
-  ShenandoahObjToScanQueueSet* const _task_queues;
-public:
-  ShenandoahInitMarkRootsTask(uint n_workers, ShenandoahObjToScanQueueSet* task_queues) :
-    AbstractGangTask("Shenandoah Init Mark Roots"),
-    _root_scanner(n_workers, ShenandoahPhaseTimings::scan_roots),
-    _task_queues(task_queues) {
-    assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
-  }
-
-  void work(uint worker_id) {
-    ShenandoahParallelWorkerSession worker_session(worker_id);
-    assert(_task_queues->get_reserved() > worker_id, "Queue has not been reserved for worker id: %d", worker_id);
-
-    ShenandoahObjToScanQueue* q = _task_queues->queue(worker_id);
-    ShenandoahInitMarkRootsClosure mark_cl(q);
-    _root_scanner.roots_do(worker_id, &mark_cl);
-  }
-};
-
 ShenandoahConcurrentMark::ShenandoahConcurrentMark() :
   ShenandoahMark() {}
 
-void ShenandoahConcurrentMark::mark_stw_roots() {
-  assert(Thread::current()->is_VM_thread(), "Can only do this in VMThread");
-  assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint");
-  assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected");
-
-  ShenandoahGCPhase phase(ShenandoahPhaseTimings::scan_roots);
-
-  WorkGang* workers = ShenandoahHeap::heap()->workers();
-  uint nworkers = workers->active_workers();
-
-  assert(nworkers <= task_queues()->size(), "Just check");
-
-  TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
-  task_queues()->reserve(nworkers);
-
-  ShenandoahInitMarkRootsTask mark_roots(nworkers, task_queues());
-  workers->run_task(&mark_roots);
-}
-
 // Mark concurrent roots during concurrent phases
 class ShenandoahMarkConcurrentRootsTask : public AbstractGangTask {
 private:
@@ -263,6 +222,8 @@ void ShenandoahConcurrentMark::mark_concurrent_roots() {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   assert(!heap->has_forwarded_objects(), "Not expected");
 
+  TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats());
+
   WorkGang* workers = heap->workers();
   ShenandoahReferenceProcessor* rp = heap->ref_processor();
   task_queues()->reserve(workers->active_workers());

src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp

@@ -40,17 +40,13 @@ class ShenandoahConcurrentMark: public ShenandoahMark {
 
 public:
   ShenandoahConcurrentMark();
-
-  // When concurrent stack processing is not supported
-  void mark_stw_roots();
+  // Concurrent mark roots
   void mark_concurrent_roots();
-
   // Concurrent mark
   void concurrent_mark();
   // Finish mark at a safepoint
   void finish_mark();
 
-
   static void cancel();
 
 private:

src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp

@@ -241,6 +241,16 @@ void ShenandoahDegenGC::op_prepare_evacuation() {
   // Prepare regions and collection set
   heap->prepare_regions_and_collection_set(false /*concurrent*/);
 
+  // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
+  // This is needed for two reasons. Strong one: new allocations would be with new freeset,
+  // which would be outside the collection set, so no cset writes would happen there.
+  // Weaker one: new allocations would happen past update watermark, and so less work would
+  // be needed for reference updates (would update the large filler instead).
+  if (UseTLAB) {
+    ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_final_manage_labs);
+    heap->tlabs_retire(false);
+  }
+
   if (!heap->collection_set()->is_empty()) {
     heap->set_evacuation_in_progress(true);
     heap->set_has_forwarded_objects(true);

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -610,6 +610,10 @@ void ShenandoahHeap::post_initialize() {
   // gclab can not be initialized early during VM startup, as it can not determinate its max_size.
   // Now, we will let WorkGang to initialize gclab when new worker is created.
   _workers->set_initialize_gclab();
+  if (_safepoint_workers != NULL) {
+    _safepoint_workers->threads_do(&init_gclabs);
+    _safepoint_workers->set_initialize_gclab();
+  }
 
   _heuristics->initialize();
 
@@ -1117,42 +1121,10 @@ void ShenandoahHeap::gclabs_retire(bool resize) {
     cl.do_thread(t);
   }
   workers()->threads_do(&cl);
-}
 
-class ShenandoahEvacuateUpdateRootsTask : public AbstractGangTask {
-private:
-  ShenandoahRootEvacuator* _rp;
-
-public:
-  ShenandoahEvacuateUpdateRootsTask(ShenandoahRootEvacuator* rp) :
-    AbstractGangTask("Shenandoah Evacuate/Update Roots"),
-    _rp(rp) {}
-
-  void work(uint worker_id) {
-    ShenandoahParallelWorkerSession worker_session(worker_id);
-    ShenandoahEvacOOMScope oom_evac_scope;
-    ShenandoahEvacuateUpdateRootsClosure<> cl;
-    MarkingCodeBlobClosure blobsCl(&cl, CodeBlobToOopClosure::FixRelocations);
-    _rp->roots_do(worker_id, &cl);
+  if (safepoint_workers() != NULL) {
+    safepoint_workers()->threads_do(&cl);
   }
-};
-
-void ShenandoahHeap::evacuate_and_update_roots() {
-#if COMPILER2_OR_JVMCI
-  DerivedPointerTable::clear();
-#endif
-  assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Only iterate roots while world is stopped");
-  {
-    // Include concurrent roots if current cycle can not process those roots concurrently
-    ShenandoahRootEvacuator rp(workers()->active_workers(),
-                               ShenandoahPhaseTimings::init_evac);
-    ShenandoahEvacuateUpdateRootsTask roots_task(&rp);
-    workers()->run_task(&roots_task);
-  }
-
-#if COMPILER2_OR_JVMCI
-  DerivedPointerTable::update_pointers();
-#endif
 }
 
 // Returns size in bytes
@@ -1666,17 +1638,6 @@ void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) {
     assert_pinned_region_status();
   }
 
-  // Retire the TLABs, which will force threads to reacquire their TLABs after the pause.
-  // This is needed for two reasons. Strong one: new allocations would be with new freeset,
-  // which would be outside the collection set, so no cset writes would happen there.
-  // Weaker one: new allocations would happen past update watermark, and so less work would
-  // be needed for reference updates (would update the large filler instead).
-  if (UseTLAB) {
-    ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_manage_labs :
-                                         ShenandoahPhaseTimings::degen_gc_final_manage_labs);
-    tlabs_retire(false);
-  }
-
   {
     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset :
                                          ShenandoahPhaseTimings::degen_gc_choose_cset);

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -506,6 +506,11 @@ public:
   void sync_pinned_region_status();
   void assert_pinned_region_status() NOT_DEBUG_RETURN;
 
+// ---------- Concurrent Stack Processing support
+//
+public:
+  bool uses_stack_watermark_barrier() const { return true; }
+
 // ---------- Allocation support
 //
 private:
@@ -594,8 +599,6 @@ private:
   ShenandoahCollectionSet* _collection_set;
   ShenandoahEvacOOMHandler _oom_evac_handler;
 
-  void evacuate_and_update_roots();
-
 public:
   static address in_cset_fast_test_addr();
 

src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp

@@ -150,30 +150,6 @@ ShenandoahNMethod* ShenandoahNMethod::for_nmethod(nmethod* nm) {
   return new ShenandoahNMethod(nm, oops, non_immediate_oops);
 }
 
-template <bool HAS_FWD>
-class ShenandoahKeepNMethodMetadataAliveClosure : public OopClosure {
-private:
-  ShenandoahBarrierSet* const _bs;
-public:
-  ShenandoahKeepNMethodMetadataAliveClosure() :
-    _bs(static_cast<ShenandoahBarrierSet*>(BarrierSet::barrier_set())) {
-  }
-
-  virtual void do_oop(oop* p) {
-    oop obj = RawAccess<>::oop_load(p);
-    if (!CompressedOops::is_null(obj)) {
-      if (HAS_FWD) {
-        obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj);
-      }
-      _bs->enqueue(obj);
-    }
-  }
-
-  virtual void do_oop(narrowOop* p) {
-    ShouldNotReachHere();
-  }
-};
-
 void ShenandoahNMethod::heal_nmethod(nmethod* nm) {
   ShenandoahNMethod* data = gc_data(nm);
   assert(data != NULL, "Sanity");
@@ -181,13 +157,8 @@ void ShenandoahNMethod::heal_nmethod(nmethod* nm) {
 
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   if (heap->is_concurrent_mark_in_progress()) {
-    if (heap->has_forwarded_objects()) {
-      ShenandoahKeepNMethodMetadataAliveClosure<true> cl;
-      data->oops_do(&cl);
-    } else {
-      ShenandoahKeepNMethodMetadataAliveClosure<false> cl;
-      data->oops_do(&cl);
-    }
+    ShenandoahKeepAliveClosure cl;
+    data->oops_do(&cl);
   } else if (heap->is_concurrent_weak_root_in_progress() ||
              heap->is_concurrent_strong_root_in_progress()) {
     ShenandoahEvacOOMScope evac_scope;

src/hotspot/share/gc/shenandoah/shenandoahNMethod.inline.hpp

@@ -73,7 +73,7 @@ void ShenandoahNMethod::oops_do(OopClosure* oops, bool fix_relocations) {
 }
 
 void ShenandoahNMethod::heal_nmethod_metadata(ShenandoahNMethod* nmethod_data) {
-  ShenandoahEvacuateUpdateRootsClosure<> cl;
+  ShenandoahEvacuateUpdateMetadataClosure<> cl;
   nmethod_data->oops_do(&cl, true /*fix relocation*/);
 }
 

src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp

@@ -97,7 +97,6 @@ bool ShenandoahPhaseTimings::is_worker_phase(Phase phase) {
   assert(phase >= 0 && phase < _num_phases, "Out of bounds");
   switch (phase) {
     case init_evac:
-    case scan_roots:
     case finish_mark:
     case purge_weak_par:
     case full_gc_mark:
@@ -114,6 +113,7 @@ bool ShenandoahPhaseTimings::is_worker_phase(Phase phase) {
     case degen_gc_purge_weak_par:
     case heap_iteration_roots:
     case conc_mark_roots:
+    case conc_thread_roots:
     case conc_weak_roots_work:
     case conc_weak_refs_work:
     case conc_strong_roots:
@@ -125,7 +125,6 @@ bool ShenandoahPhaseTimings::is_worker_phase(Phase phase) {
 
 bool ShenandoahPhaseTimings::is_root_work_phase(Phase phase) {
   switch (phase) {
-    case scan_roots:
     case finish_mark:
     case init_evac:
     case degen_gc_update_roots:

src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp

@@ -53,8 +53,6 @@ class outputStream;
   f(init_mark,                                      "Pause Init Mark (N)")             \
   f(init_manage_tlabs,                              "  Manage TLABs")                  \
   f(init_update_region_states,                      "  Update Region States")          \
-  f(scan_roots,                                     "  Scan Roots")                    \
-  SHENANDOAH_PAR_PHASE_DO(scan_,                    "    S: ", f)                      \
                                                                                        \
   f(conc_mark_roots,                                "  Roots ")                        \
   SHENANDOAH_PAR_PHASE_DO(conc_mark_roots,          "    CM: ", f)                     \
@@ -75,6 +73,9 @@ class outputStream;
   f(init_evac,                                      "  Initial Evacuation")            \
   SHENANDOAH_PAR_PHASE_DO(evac_,                    "    E: ", f)                      \
                                                                                        \
+  f(conc_thread_roots,                              "Concurrent Stack Processing")     \
+  f(conc_thread_roots_work,                           "  Threads")                     \
+  SHENANDOAH_PAR_PHASE_DO(conc_thread_roots_work_,    "    CT: ", f)                   \
   f(conc_weak_refs,                                 "Concurrent Weak References")      \
   f(conc_weak_refs_work,                            "  Process")                       \
   SHENANDOAH_PAR_PHASE_DO(conc_weak_refs_work_,     "    CWRF: ", f)                   \

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.cpp

@@ -31,12 +31,30 @@
 #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp"
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahPhaseTimings.hpp"
+#include "gc/shenandoah/shenandoahStackWatermark.hpp"
 #include "gc/shenandoah/shenandoahStringDedup.hpp"
 #include "memory/iterator.hpp"
 #include "memory/resourceArea.hpp"
-#include "prims/jvmtiExport.hpp"
+#include "runtime/stackWatermarkSet.inline.hpp"
 #include "runtime/thread.hpp"
 
+ShenandoahJavaThreadsIterator::ShenandoahJavaThreadsIterator(ShenandoahPhaseTimings::Phase phase) :
+  _threads(),
+  _claimed(0),
+  _phase(phase) {
+}
+
+uint ShenandoahJavaThreadsIterator::claim() {
+  return Atomic::fetch_and_add(&_claimed, 1u);
+}
+
+void ShenandoahJavaThreadsIterator::threads_do(ThreadClosure* cl, uint worker_id) {
+  ShenandoahWorkerTimingsTracker timer(_phase, ShenandoahPhaseTimings::ThreadRoots, worker_id);
+  for (uint i = claim(); i < _threads.length(); i = claim()) {
+    cl->do_thread(_threads.thread_at(i));
+  }
+}
+
 ShenandoahThreadRoots::ShenandoahThreadRoots(ShenandoahPhaseTimings::Phase phase, bool is_par) :
   _phase(phase), _is_par(is_par) {
   Threads::change_thread_claim_token();
@@ -166,9 +184,30 @@ ShenandoahSTWRootScanner::ShenandoahSTWRootScanner(ShenandoahPhaseTimings::Phase
    _unload_classes(ShenandoahHeap::heap()->unload_classes()) {
 }
 
+class ShenandoahConcurrentMarkThreadClosure : public ThreadClosure {
+private:
+  OopClosure* const _oops;
+
+public:
+  ShenandoahConcurrentMarkThreadClosure(OopClosure* oops);
+  void do_thread(Thread* thread);
+};
+
+ShenandoahConcurrentMarkThreadClosure::ShenandoahConcurrentMarkThreadClosure(OopClosure* oops) :
+  _oops(oops) {
+}
+
+void ShenandoahConcurrentMarkThreadClosure::do_thread(Thread* thread) {
+  assert(thread->is_Java_thread(), "Must be");
+  JavaThread* const jt = thread->as_Java_thread();
+
+  StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc);
+}
+
 ShenandoahConcurrentRootScanner::ShenandoahConcurrentRootScanner(uint n_workers,
                                                                  ShenandoahPhaseTimings::Phase phase) :
    ShenandoahRootProcessor(phase),
+   _java_threads(phase),
   _vm_roots(phase),
   _cld_roots(phase, n_workers),
   _codecache_snapshot(NULL),
@@ -177,6 +216,7 @@ ShenandoahConcurrentRootScanner::ShenandoahConcurrentRootScanner(uint n_workers,
     CodeCache_lock->lock_without_safepoint_check();
     _codecache_snapshot = ShenandoahCodeRoots::table()->snapshot_for_iteration();
   }
+  update_tlab_stats();
   assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expecting forwarded pointers during concurrent marking");
 }
 
@@ -190,6 +230,10 @@ ShenandoahConcurrentRootScanner::~ShenandoahConcurrentRootScanner() {
 void ShenandoahConcurrentRootScanner::roots_do(OopClosure* oops, uint worker_id) {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   CLDToOopClosure clds_cl(oops, ClassLoaderData::_claim_strong);
+
+  ShenandoahConcurrentMarkThreadClosure thr_cl(oops);
+  _java_threads.threads_do(&thr_cl, worker_id);
+
   _vm_roots.oops_do(oops, worker_id);
 
   if (!heap->unload_classes()) {
@@ -202,22 +246,18 @@ void ShenandoahConcurrentRootScanner::roots_do(OopClosure* oops, uint worker_id)
   }
 }
 
-ShenandoahRootEvacuator::ShenandoahRootEvacuator(uint n_workers,
-                                                 ShenandoahPhaseTimings::Phase phase) :
-  ShenandoahRootProcessor(phase),
-  _thread_roots(phase, n_workers > 1) {
-  nmethod::oops_do_marking_prologue();
-}
-
-ShenandoahRootEvacuator::~ShenandoahRootEvacuator() {
-  nmethod::oops_do_marking_epilogue();
-}
-
-void ShenandoahRootEvacuator::roots_do(uint worker_id, OopClosure* oops) {
-  // Always disarm on-stack nmethods, because we are evacuating/updating them
-  // here
-  ShenandoahCodeBlobAndDisarmClosure codeblob_cl(oops);
-  _thread_roots.oops_do(oops, &codeblob_cl, worker_id);
+void ShenandoahConcurrentRootScanner::update_tlab_stats() {
+  if (UseTLAB) {
+    ThreadLocalAllocStats total;
+    for (uint i = 0; i < _java_threads.length(); i ++) {
+      Thread* thr = _java_threads.thread_at(i);
+      if (thr->is_Java_thread()) {
+        ShenandoahStackWatermark* wm = StackWatermarkSet::get<ShenandoahStackWatermark>(thr->as_Java_thread(), StackWatermarkKind::gc);
+        total.update(wm->stats());
+      }
+    }
+    total.publish();
+  }
 }
 
 ShenandoahRootUpdater::ShenandoahRootUpdater(uint n_workers, ShenandoahPhaseTimings::Phase phase) :

src/hotspot/share/gc/shenandoah/shenandoahRootProcessor.hpp

@@ -65,6 +65,21 @@ public:
   void oops_do(T* cl, uint worker_id);
 };
 
+class ShenandoahJavaThreadsIterator {
+private:
+  ThreadsListHandle             _threads;
+  volatile uint                 _claimed;
+  ShenandoahPhaseTimings::Phase _phase;
+
+  uint claim();
+public:
+  ShenandoahJavaThreadsIterator(ShenandoahPhaseTimings::Phase phase);
+  void threads_do(ThreadClosure* cl, uint worker_id);
+
+  uint length() const { return _threads.length(); }
+  Thread* thread_at(uint index) const { return _threads.thread_at(index); }
+};
+
 class ShenandoahThreadRoots {
 private:
   ShenandoahPhaseTimings::Phase _phase;
@@ -183,6 +198,7 @@ public:
 
 class ShenandoahConcurrentRootScanner : public ShenandoahRootProcessor {
 private:
+  ShenandoahJavaThreadsIterator             _java_threads;
   ShenandoahVMRoots<true /*concurrent*/>    _vm_roots;
   ShenandoahClassLoaderDataRoots<true /*concurrent*/, false /* single-threaded*/>
                                            _cld_roots;
@@ -194,6 +210,9 @@ public:
   ~ShenandoahConcurrentRootScanner();
 
   void roots_do(OopClosure* oops, uint worker_id);
+
+private:
+  void update_tlab_stats();
 };
 
 // This scanner is only for SH::object_iteration() and only supports single-threaded
@@ -215,17 +234,6 @@ public:
   void roots_do(OopClosure* cl);
 };
 
-// Evacuate all roots at a safepoint
-class ShenandoahRootEvacuator : public ShenandoahRootProcessor {
-private:
-  ShenandoahThreadRoots                                     _thread_roots;
-public:
-  ShenandoahRootEvacuator(uint n_workers, ShenandoahPhaseTimings::Phase phase);
-  ~ShenandoahRootEvacuator();
-
-  void roots_do(uint worker_id, OopClosure* oops);
-};
-
 // Update all roots at a safepoint
 class ShenandoahRootUpdater : public ShenandoahRootProcessor {
 private:

src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.cpp

@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+
+#include "gc/shenandoah/shenandoahAsserts.hpp"
+#include "gc/shenandoah/shenandoahClosures.inline.hpp"
+#include "gc/shenandoah/shenandoahCodeRoots.hpp"
+#include "gc/shenandoah/shenandoahHeap.inline.hpp"
+#include "gc/shenandoah/shenandoahStackWatermark.hpp"
+#include "gc/shenandoah/shenandoahUtils.hpp"
+#include "runtime/safepointVerifiers.hpp"
+
+uint32_t ShenandoahStackWatermark::_epoch_id = 1;
+
+ShenandoahOnStackCodeBlobClosure::ShenandoahOnStackCodeBlobClosure() :
+    _bs_nm(BarrierSet::barrier_set()->barrier_set_nmethod()) {}
+
+void ShenandoahOnStackCodeBlobClosure::do_code_blob(CodeBlob* cb) {
+  nmethod* const nm = cb->as_nmethod_or_null();
+  if (nm != NULL) {
+    const bool result = _bs_nm->nmethod_entry_barrier(nm);
+    assert(result, "NMethod on-stack must be alive");
+  }
+}
+
+ThreadLocalAllocStats& ShenandoahStackWatermark::stats() {
+  return _stats;
+}
+
+uint32_t ShenandoahStackWatermark::epoch_id() const {
+  return _epoch_id;
+}
+
+void ShenandoahStackWatermark::change_epoch_id() {
+  shenandoah_assert_safepoint();
+  _epoch_id++;
+}
+
+ShenandoahStackWatermark::ShenandoahStackWatermark(JavaThread* jt) :
+  StackWatermark(jt, StackWatermarkKind::gc, _epoch_id),
+  _heap(ShenandoahHeap::heap()),
+  _stats(),
+  _keep_alive_cl(),
+  _evac_update_oop_cl(),
+  _cb_cl() {}
+
+OopClosure* ShenandoahStackWatermark::closure_from_context(void* context) {
+  if (context != NULL) {
+    assert(_heap->is_concurrent_weak_root_in_progress() ||
+           _heap->is_concurrent_mark_in_progress(),
+           "Only these two phases");
+    assert(Thread::current()->is_Worker_thread(), "Unexpected thread passing in context: " PTR_FORMAT, p2i(context));
+    return reinterpret_cast<OopClosure*>(context);
+  } else {
+    if (_heap->is_concurrent_mark_in_progress()) {
+      return &_keep_alive_cl;
+    } else if (_heap->is_concurrent_weak_root_in_progress()) {
+      assert(_heap->is_evacuation_in_progress(), "Nothing to evacuate");
+      return &_evac_update_oop_cl;
+    } else {
+      ShouldNotReachHere();
+      return NULL;
+    }
+  }
+}
+
+void ShenandoahStackWatermark::start_processing_impl(void* context) {
+  NoSafepointVerifier nsv;
+  ShenandoahHeap* const heap = ShenandoahHeap::heap();
+
+  // Process the non-frame part of the thread
+  if (heap->is_concurrent_mark_in_progress()) {
+    // We need to reset all TLABs because they might be below the TAMS, and we need to mark
+    // the objects in them. Do not let mutators allocate any new objects in their current TLABs.
+    // It is also a good place to resize the TLAB sizes for future allocations.
+    retire_tlab();
+
+    _jt->oops_do_no_frames(closure_from_context(context), &_cb_cl);
+  } else if (heap->is_concurrent_weak_root_in_progress()) {
+    assert(heap->is_evacuation_in_progress(), "Should not be armed");
+    // Retire the TLABs, which will force threads to reacquire their TLABs.
+    // This is needed for two reasons. Strong one: new allocations would be with new freeset,
+    // which would be outside the collection set, so no cset writes would happen there.
+    // Weaker one: new allocations would happen past update watermark, and so less work would
+    // be needed for reference updates (would update the large filler instead).
+    retire_tlab();
+
+    _jt->oops_do_no_frames(closure_from_context(context), &_cb_cl);
+  } else {
+    ShouldNotReachHere();
+  }
+
+  // Publishes the processing start to concurrent threads
+  StackWatermark::start_processing_impl(context);
+}
+
+void ShenandoahStackWatermark::retire_tlab() {
+  // Retire TLAB
+  if (UseTLAB) {
+    _stats.reset();
+    _jt->tlab().retire(&_stats);
+    if (ResizeTLAB) {
+      _jt->tlab().resize();
+    }
+  }
+}
+
+void ShenandoahStackWatermark::process(const frame& fr, RegisterMap& register_map, void* context) {
+  OopClosure* oops = closure_from_context(context);
+  assert(oops != NULL, "Should not get to here");
+  ShenandoahHeap* const heap = ShenandoahHeap::heap();
+  assert((heap->is_concurrent_weak_root_in_progress() && heap->is_evacuation_in_progress()) ||
+         heap->is_concurrent_mark_in_progress(),
+         "Only these two phases");
+  fr.oops_do(oops, &_cb_cl, &register_map, DerivedPointerIterationMode::_directly);
+}

src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.hpp

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2021, Red Hat, Inc. All rights reserved.
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSTACKWATERMARK_HPP
+#define SHARE_GC_SHENANDOAH_SHENANDOAHSTACKWATERMARK_HPP
+
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/barrierSetNMethod.hpp"
+#include "gc/shenandoah/shenandoahClosures.hpp"
+#include "memory/allocation.hpp"
+#include "memory/iterator.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "runtime/stackWatermark.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class frame;
+class JavaThread;
+
+class ShenandoahOnStackCodeBlobClosure : public CodeBlobClosure {
+private:
+  BarrierSetNMethod* _bs_nm;
+
+  void do_code_blob(CodeBlob* cb);
+public:
+  ShenandoahOnStackCodeBlobClosure();
+};
+
+class ShenandoahStackWatermark : public StackWatermark {
+private:
+  static uint32_t                      _epoch_id;
+  ShenandoahHeap* const                _heap;
+  ThreadLocalAllocStats                _stats;
+
+  // Closures
+  ShenandoahKeepAliveClosure           _keep_alive_cl;
+  ShenandoahEvacuateUpdateRootsClosure _evac_update_oop_cl;
+  ShenandoahOnStackCodeBlobClosure     _cb_cl;
+public:
+  ShenandoahStackWatermark(JavaThread* jt);
+  ThreadLocalAllocStats& stats();
+
+  static void change_epoch_id();
+private:
+  OopClosure* closure_from_context(void* context);
+  uint32_t epoch_id() const;
+  void start_processing_impl(void* context);
+  void process(const frame& fr, RegisterMap& register_map, void* context);
+
+  void retire_tlab();
+};
+
+#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSTACKWATERMARK_HPP

src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp

@@ -46,6 +46,7 @@ protected:
   uint         _gc_id;
 public:
   VM_ShenandoahOperation() : _gc_id(GCId::current()) {};
+  virtual bool skip_thread_oop_barriers() const { return true; }
 };
 
 class VM_ShenandoahReferenceOperation : public VM_ShenandoahOperation {