Align plabs on card boundaries

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -198,21 +198,64 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
   HeapWord* result = NULL;
   size_t size = req.size();
 
+  // req.size() is in words, free() is in bytes.
   if (ShenandoahElasticTLAB && req.is_lab_alloc()) {
-    size_t free = align_down(r->free() >> LogHeapWordSize, MinObjAlignment);
-    if (size > free) {
-      size = free;
+    if (req.type() == ShenandoahAllocRequest::_alloc_plab) {
+      // Need to assure that plabs are aligned on multiple of card region.
+      size_t free = r->free();
+      size_t usable_free = (free / CardTable::card_size) << CardTable::card_shift;
+      free /= HeapWordSize;
+      usable_free /= HeapWordSize;
+      if (size > usable_free) {
+        size = usable_free;
+      }
+      if (size >= req.min_size()) {
+        result = r->allocate_aligned(size, req, CardTable::card_size);
+        assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, usable_free, size);
+        if (free > usable_free) {
+          // Account for the alignment padding
+          size_t padding = (free - usable_free) * HeapWordSize;
+          increase_used(padding);
+          assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "All PLABs reside in old-gen");
+          _heap->old_generation()->increase_used(padding);
+          // For verification consistency, we need to report this padding to _heap
+          _heap->increase_used(padding);
+        }
+      }
+    } else {
+      size_t free = align_down(r->free() >> LogHeapWordSize, MinObjAlignment);
+      if (size > free) {
+        size = free;
+      }
+      if (size >= req.min_size()) {
+        result = r->allocate(size, req);
+        assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size);
+      }
     }
-    if (size >= req.min_size()) {
-      result = r->allocate(size, req);
-      assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size);
+  } else if (req.is_lab_alloc() && req.type() == ShenandoahAllocRequest::_alloc_plab) {
+    size_t free = r->free();
+    size_t usable_free = (free / CardTable::card_size) << CardTable::card_shift;
+    free /= HeapWordSize;
+    usable_free /= HeapWordSize;
+    if (size <= usable_free) {
+      assert(size % CardTable::card_size_in_words == 0, "PLAB size must be multiple of remembered set card size");
+
+      result = r->allocate_aligned(size, req, CardTable::card_size);
+      assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, usable_free, size);
+
+      // Account for the alignment padding
+      size_t padding = (free - usable_free) * HeapWordSize;
+      increase_used(padding);
+      assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "All PLABs reside in old-gen");
+      _heap->old_generation()->increase_used(padding);
+      // For verification consistency, we need to report this padding to _heap
+      _heap->increase_used(padding);
     }
   } else {
     result = r->allocate(size, req);
   }
 
   if (result != NULL) {
-
     // Record actual allocation size
     req.set_actual_size(size);
 

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -894,9 +894,15 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) {
   new_size = MIN2(new_size, PLAB::max_size());
   new_size = MAX2(new_size, PLAB::min_size());
 
+  size_t unalignment = new_size % CardTable::card_size_in_words;
+  if (unalignment != 0) {
+    new_size = new_size - unalignment + CardTable::card_size_in_words;
+  }
+
   // Record new heuristic value even if we take any shortcut. This captures
   // the case when moderately-sized objects always take a shortcut. At some point,
-  // heuristics should catch up with them.
+  // heuristics should catch up with them.  Note that the requested new_size may
+  // not be honored, but we remember that this is the preferred size.
   ShenandoahThreadLocalData::set_plab_size(thread, new_size);
 
   if (new_size < size) {

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -347,6 +347,9 @@ class ShenandoahHeapRegion {
     return _index;
   }
 
+  // Allocation (return NULL if full)
+  inline HeapWord* allocate_aligned(size_t word_size, ShenandoahAllocRequest req, size_t alignment_in_words);
+
   // Allocation (return NULL if full)
   inline HeapWord* allocate(size_t word_size, ShenandoahAllocRequest req);
 

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp

@@ -31,6 +31,40 @@
 #include "gc/shenandoah/shenandoahPacer.inline.hpp"
 #include "runtime/atomic.hpp"
 
+// If next available memory is not aligned on address that is multiple of alignment, fill the empty space
+// so that returned object is aligned on an address that is a multiple of alignment_in_words.  Requested
+// size is in words.
+HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocRequest req, size_t alignment_in_bytes) {
+  shenandoah_assert_heaplocked_or_safepoint();
+  assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size);
+
+  HeapWord* obj = top();
+  uintptr_t addr_as_int = (uintptr_t) obj;
+
+  size_t unalignment_bytes = addr_as_int % alignment_in_bytes;
+  size_t unalignment_words = unalignment_bytes / HeapWordSize;
+  if (pointer_delta(end(), obj + unalignment_words) >= size) {
+    if (unalignment_words > 0) {
+      size_t pad_words = (alignment_in_bytes / HeapWordSize) - unalignment_words;
+      ShenandoahHeap::fill_with_object(obj, pad_words);
+      ShenandoahHeap::heap()->card_scan()->register_object(obj);
+      obj += pad_words;
+    }
+
+    make_regular_allocation(req.affiliation());
+    adjust_alloc_metadata(req.type(), size);
+
+    HeapWord* new_top = obj + size;
+    set_top(new_top);
+    assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top));
+    assert(((uintptr_t) obj) % (alignment_in_bytes) == 0, "obj is not aligned: " PTR_FORMAT, p2i(obj));
+
+    return obj;
+  } else {
+    return NULL;
+  }
+}
+
 HeapWord* ShenandoahHeapRegion::allocate(size_t size, ShenandoahAllocRequest req) {
   shenandoah_assert_heaplocked_or_safepoint();
   assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size);