diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp
index 2761b9513949b..3bb42433cd61c 100644
--- a/src/hotspot/share/gc/z/zMark.cpp
+++ b/src/hotspot/share/gc/z/zMark.cpp
@@ -66,6 +66,8 @@ static const ZStatSubPhase ZSubPhaseConcurrentMarkTryFlush("Concurrent Mark Try
static const ZStatSubPhase ZSubPhaseConcurrentMarkTryTerminate("Concurrent Mark Try Terminate");
static const ZStatSubPhase ZSubPhaseMarkTryComplete("Pause Mark Try Complete");
+volatile bool ZMark::_push_local_stripe = false;
+
ZMark::ZMark(ZWorkers* workers, ZPageTable* page_table) :
_workers(workers),
_page_table(page_table),
@@ -117,6 +119,8 @@ void ZMark::start() {
const size_t nstripes = calculate_nstripes(_nworkers);
_stripes.set_nstripes(nstripes);
+ _push_local_stripe = false;
+
// Update statistics
ZStatMark::set_at_mark_start(nstripes);
@@ -157,7 +161,9 @@ bool ZMark::is_array(uintptr_t addr) const {
void ZMark::push_partial_array(uintptr_t addr, size_t size, bool finalizable) {
assert(is_aligned(addr, ZMarkPartialArrayMinSize), "Address misaligned");
ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current());
- ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr);
+ ZMarkStripe* const stripe = push_local_stripe() ?
+ _stripes.stripe_for_worker(_nworkers, ZThread::worker_id()) :
+ _stripes.stripe_for_addr(addr);
const uintptr_t offset = ZAddress::offset(addr) >> ZMarkPartialArrayMinSizeShift;
const uintptr_t length = size / oopSize;
const ZMarkStackEntry entry(offset, length, finalizable);
diff --git a/src/hotspot/share/gc/z/zMark.hpp b/src/hotspot/share/gc/z/zMark.hpp
index 9711899e89bfe..58955b6aa917c 100644
--- a/src/hotspot/share/gc/z/zMark.hpp
+++ b/src/hotspot/share/gc/z/zMark.hpp
@@ -53,6 +53,7 @@ class ZMark {
size_t _ntrycomplete;
size_t _ncontinue;
uint _nworkers;
+ static volatile bool _push_local_stripe;
size_t calculate_nstripes(uint nworkers) const;
@@ -109,6 +110,13 @@ class ZMark {
void mark(bool initial);
bool end();
+ static void set_push_local_stripe(bool mode) {
+ Atomic::store(&_push_local_stripe, mode);
+ }
+ static bool push_local_stripe() {
+ return Atomic::load(&_push_local_stripe);
+ }
+
void flush_and_free();
bool flush_and_free(Thread* thread);
};
diff --git a/src/hotspot/share/gc/z/zMark.inline.hpp b/src/hotspot/share/gc/z/zMark.inline.hpp
index 95f1e336a378b..17cf017945f00 100644
--- a/src/hotspot/share/gc/z/zMark.inline.hpp
+++ b/src/hotspot/share/gc/z/zMark.inline.hpp
@@ -28,6 +28,7 @@
#include "gc/z/zMark.hpp"
#include "gc/z/zMarkStack.inline.hpp"
#include "gc/z/zThreadLocalData.hpp"
+#include "gc/z/zThread.inline.hpp"
#include "runtime/thread.hpp"
#include "utilities/debug.hpp"
@@ -35,7 +36,9 @@ template <bool follow, bool finalizable, bool publish>
inline void ZMark::mark_object(uintptr_t addr) {
assert(ZAddress::is_marked(addr), "Should be marked");
ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current());
- ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr);
+ ZMarkStripe* const stripe = push_local_stripe() && ZThread::is_worker() ?
+ _stripes.stripe_for_worker(_nworkers, ZThread::worker_id()) :
+ _stripes.stripe_for_addr(addr);
ZMarkStackEntry entry(addr, follow, finalizable);
stacks->push(&_allocator, &_stripes, stripe, entry, publish);
diff --git a/src/hotspot/share/gc/z/zMarkStackAllocator.cpp b/src/hotspot/share/gc/z/zMarkStackAllocator.cpp
index 2c3feabe105f1..3bdaa9910bed0 100644
--- a/src/hotspot/share/gc/z/zMarkStackAllocator.cpp
+++ b/src/hotspot/share/gc/z/zMarkStackAllocator.cpp
@@ -27,6 +27,7 @@
#include "gc/z/zLock.inline.hpp"
#include "gc/z/zMarkStack.inline.hpp"
#include "gc/z/zMarkStackAllocator.hpp"
+#include "gc/z/zMark.hpp"
#include "logging/log.hpp"
#include "runtime/atomic.hpp"
#include "runtime/os.hpp"
@@ -128,7 +129,8 @@ uintptr_t ZMarkStackSpace::alloc(size_t size) {
ZMarkStackAllocator::ZMarkStackAllocator() :
_freelist(),
- _space() {
+ _space(),
+ _nmagazine(0) {
// Prime free list to avoid an immediate space
// expansion when marking starts.
if (_space.is_initialized()) {
@@ -159,13 +161,45 @@ ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t ad
assert(success, "Magazine should never get full");
}
+ inc_nmagazine();
+
return magazine;
}
+void ZMarkStackAllocator::inc_nmagazine() {
+ int cur = Atomic::add(&_nmagazine, 1);
+ if (!ZMark::push_local_stripe() &&
+ cur * ZMarkStackMagazineSize >= ZMarkStackSpaceLimit * ZStackModeChangeRatio) {
+ ZMark::set_push_local_stripe(true);
+ }
+}
+
+void ZMarkStackAllocator::dec_nmagazine() {
+ int nmagazine = Atomic::load(&_nmagazine);
+
+ for (;;) {
+ if (nmagazine == 0) {
+ break;
+ }
+ const int new_nmagazine = nmagazine - 1;
+ const int prev_nmagazine = Atomic::cmpxchg(&_nmagazine, nmagazine, new_nmagazine);
+ if (prev_nmagazine == nmagazine) {
+ break;
+ }
+ nmagazine = prev_nmagazine;
+ }
+
+ if (ZMark::push_local_stripe() &&
+ nmagazine * ZMarkStackMagazineSize * 2.0 < ZMarkStackSpaceLimit * ZStackModeChangeRatio) {
+ ZMark::set_push_local_stripe(false);
+ }
+}
+
ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() {
// Try allocating from the free list first
ZMarkStackMagazine* const magazine = _freelist.pop();
if (magazine != NULL) {
+ inc_nmagazine();
return magazine;
}
@@ -180,4 +214,5 @@ ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() {
void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) {
_freelist.push(magazine);
+ dec_nmagazine();
}
diff --git a/src/hotspot/share/gc/z/zMarkStackAllocator.hpp b/src/hotspot/share/gc/z/zMarkStackAllocator.hpp
index 55ce6e2e58186..123195c0c8b1b 100644
--- a/src/hotspot/share/gc/z/zMarkStackAllocator.hpp
+++ b/src/hotspot/share/gc/z/zMarkStackAllocator.hpp
@@ -52,8 +52,11 @@ class ZMarkStackAllocator {
private:
ZCACHE_ALIGNED ZMarkStackMagazineList _freelist;
ZCACHE_ALIGNED ZMarkStackSpace _space;
+ ZCACHE_ALIGNED volatile int _nmagazine;
void prime_freelist();
+ void inc_nmagazine();
+ void dec_nmagazine();
ZMarkStackMagazine* create_magazine_from_space(uintptr_t addr, size_t size);
public:
diff --git a/src/hotspot/share/gc/z/z_globals.hpp b/src/hotspot/share/gc/z/z_globals.hpp
index 4a49d775c8cb3..52641d60e3dcf 100644
--- a/src/hotspot/share/gc/z/z_globals.hpp
+++ b/src/hotspot/share/gc/z/z_globals.hpp
@@ -42,6 +42,9 @@
"Maximum number of bytes allocated for mark stacks") \
range(32*M, 1024*G) \
\
+ product(double, ZStackModeChangeRatio, 0.25, \
+ "Mark push mode is changed when usage is above the ratio") \
+ \
product(double, ZCollectionInterval, 0, \
"Force GC at a fixed time interval (in seconds)") \
\