8348595: GenShen: Fix generational free-memory no-progress check

Reviewed-by: phh, xpeng

src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp

@@ -303,7 +303,7 @@ void ShenandoahDegenGC::op_degenerated() {
 
   // Check for futility and fail. There is no reason to do several back-to-back Degenerated cycles,
   // because that probably means the heap is overloaded and/or fragmented.
-  if (!metrics.is_good_progress()) {
+  if (!metrics.is_good_progress(_generation)) {
     heap->cancel_gc(GCCause::_shenandoah_upgrade_to_full_gc);
     op_degenerated_futile();
   } else {

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp

@@ -435,6 +435,9 @@ public:
   // Acquire heap lock and log status, assuming heap lock is not acquired by the caller.
   void log_status_under_lock();
 
+  // Note that capacity is the number of regions that had available memory at most recent rebuild.  It is not the
+  // entire size of the young or global generation.  (Regions within the generation that were fully utilized at time of
+  // rebuild are not counted as part of capacity.)
   inline size_t capacity()  const { return _partitions.capacity_of(ShenandoahFreeSetPartitionId::Mutator); }
   inline size_t used()      const { return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator);     }
 

src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp

@@ -119,7 +119,7 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) {
 
   metrics.snap_after();
 
-  if (metrics.is_good_progress()) {
+  if (metrics.is_good_progress(heap->global_generation())) {
     heap->notify_gc_progress();
   } else {
     // Nothing to do. Tell the allocation path that we have failed to make

src/hotspot/share/gc/shenandoah/shenandoahMetrics.cpp

@@ -43,10 +43,28 @@ void ShenandoahMetricsSnapshot::snap_after() {
   _ef_after = _heap->free_set()->external_fragmentation();
 }
 
-bool ShenandoahMetricsSnapshot::is_good_progress() {
+// For degenerated GC, generation is Young in generational mode, Global in non-generational mode.
+// For full GC, generation is always Global.
+//
+// Note that the size of the chosen collection set is proportional to the relevant generation's collection set.
+// Note also that the generation size may change following selection of the collection set, as a side effect
+// of evacuation.  Evacuation may promote objects, causing old to grow and young to shrink.  Or this may be a
+// mixed evacuation.  When old regions are evacuated, this typically allows young to expand.  In all of these
+// various scenarios, the purpose of asking is_good_progress() is to determine if there is enough memory available
+// within young generation to justify making an attempt to perform a concurrent collection.  For this reason, we'll
+// use the current size of the generation (which may not be different than when the collection set was chosen) to
+// assess how much free memory we require in order to consider the most recent GC to have had good progress.
+
+bool ShenandoahMetricsSnapshot::is_good_progress(ShenandoahGeneration* generation) {
   // Under the critical threshold?
-  size_t free_actual   = _heap->free_set()->available();
-  size_t free_expected = _heap->max_capacity() / 100 * ShenandoahCriticalFreeThreshold;
+  ShenandoahFreeSet* free_set = _heap->free_set();
+  size_t free_actual   = free_set->available();
+
+  // ShenandoahCriticalFreeThreshold is expressed as a percentage.  We multiple this percentage by 1/100th
+  // of the generation capacity to determine whether the available memory within the generation exceeds the
+  // critical threshold.
+  size_t free_expected = (generation->max_capacity() / 100) * ShenandoahCriticalFreeThreshold;
+
   bool prog_free = free_actual >= free_expected;
   log_info(gc, ergo)("%s progress for free space: %zu%s, need %zu%s",
                      prog_free ? "Good" : "Bad",

src/hotspot/share/gc/shenandoah/shenandoahMetrics.hpp

@@ -40,7 +40,7 @@ public:
   void snap_before();
   void snap_after();
 
-  bool is_good_progress();
+  bool is_good_progress(ShenandoahGeneration *generation);
 };
 
 #endif // SHARE_GC_SHENANDOAH_SHENANDOAHMETRICS_HPP