/*
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/
#include "precompiled.hpp"
#include "gc_implementation/parNew/parNewGeneration.hpp"
#include "gc_implementation/parNew/parOopClosures.inline.hpp"
#include "gc_implementation/shared/adaptiveSizePolicy.hpp"
#include "gc_implementation/shared/ageTable.hpp"
#include "gc_implementation/shared/parGCAllocBuffer.hpp"
#include "gc_implementation/shared/gcHeapSummary.hpp"
#include "gc_implementation/shared/gcTimer.hpp"
#include "gc_implementation/shared/gcTrace.hpp"
#include "gc_implementation/shared/gcTraceTime.hpp"
#include "gc_implementation/shared/copyFailedInfo.hpp"
#include "gc_implementation/shared/spaceDecorator.hpp"
#include "memory/defNewGeneration.inline.hpp"
#include "memory/genCollectedHeap.hpp"
#include "memory/genOopClosures.inline.hpp"
#include "memory/generation.hpp"
#include "memory/generation.inline.hpp"
#include "memory/referencePolicy.hpp"
#include "memory/resourceArea.hpp"
#include "memory/sharedHeap.hpp"
#include "oops/objArrayOop.hpp"
#include "oops/oop.inline.hpp"
#include "oops/oop.pcgc.inline.hpp"
#include "runtime/handles.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/thread.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/workgroup.hpp"
#ifdef _MSC_VER
#endif
int thread_num_,
_ageTable(false), // false ==> not the global age table, no perf data.
_older_gen_closure(gen_, this),
work_queue_set_, &term_),
{
#if TASKQUEUE_STATS
_term_attempts = 0;
_overflow_refills = 0;
#endif // TASKQUEUE_STATS
}
#ifdef _MSC_VER
#endif
// A non-null SCA implies that we want the PLAB data recorded.
}
}
return new_obj->is_objArray() &&
}
// Process ParGCArrayScanChunk elements now
// and push the remainder back onto queue
// Test above combines last partial chunk with a full chunk
// Push remainder.
} else {
// Restore length so that it can be used if there
// is a promotion failure and forwarding pointers
// must be removed.
}
// process our set of indices (include header in first chunk)
// should make sure end is even (aligned to HeapWord in case of compressed oops)
// object is in to_space
} else {
// object is in old generation
}
}
do {
if (obj_to_scan->is_objArray() &&
obj_to_scan->is_forwarded() &&
} else {
// object is in to_space
}
} else {
// object is in old generation
}
}
}
// For the case of compressed oops, we have a private, non-shared
// overflow stack, so we eagerly drain it so as to more evenly
// distribute load early. Note: this may be good to do in
// general rather than delay for the final stealing phase.
// If applicable, we'll transfer a set of objects over to our
// work queue, allowing them to be stolen and draining our
// private overflow stack.
}
// Transfer the most recent num_take_elems from the overflow
// stack to our work queue.
for (size_t i = 0; i != num_take_elems; i++) {
obj_to_push = cur;
}
}
return num_take_elems > 0; // was something transferred?
}
overflow_stack()->push(p);
}
// Otherwise, if the object is small enough, try to reallocate the
// buffer.
if (!_to_space_full) {
if (word_sz * 100 <
// Is small enough; abandon this buffer and start a new one.
"Invariant");
}
}
// Note that we cannot compare buf_size < word_sz below
// because of AlignmentReserve (see ParGCAllocBuffer::allocate()).
"Else should have been able to allocate");
// It's conceivable that we may be able to use the
// buffer we just grabbed for subsequent small requests
// even if not for this one.
} else {
// We're used up.
_to_space_full = true;
}
} else {
// Too large; allocate the object individually.
}
}
return obj;
}
// Is the alloc in the current alloc buffer?
"Should contain whole object.");
} else {
}
}
}
}
public:
// Initializes states for the specified number of threads;
inline ParScanThreadState& thread_state(int i);
void flush();
#if TASKQUEUE_STATS
static void
static void
void reset_stats();
#endif // TASKQUEUE_STATS
private:
public:
};
{
"overflow_stack allocation mismatch");
// Initialize states.
for (int i = 0; i < num_threads; ++i) {
new ((ParScanThreadState*)_data + i)
}
}
{
return ((ParScanThreadState*)_data)[i];
}
for (int i = 0; i < length(); ++i) {
if (thread_state(i).promotion_failed()) {
}
}
}
{
if (promotion_failed) {
for (int i = 0; i < length(); ++i) {
}
}
}
#if TASKQUEUE_STATS
void
{
taskqueue_stats().reset();
_term_attempts = 0;
_overflow_refills = 0;
}
{
for (int i = 0; i < length(); ++i) {
thread_state(i).reset_stats();
}
}
void
{
"-------termination-------");
" ms % attempts");
"--------- ------ --------");
}
{
for (int i = 0; i < length(); ++i) {
}
}
// Print stats related to work queue activity.
{
}
{
for (int i = 0; i < length(); ++i) {
if (pss.overflow_refills() > 0) {
}
}
}
#endif // TASKQUEUE_STATS
{
// Work in this loop should be kept as lightweight as
// possible since this might otherwise become a bottleneck
// to scaling. Should we add heavy-weight work into this
// loop, consider parallelizing the loop into the worker threads.
for (int i = 0; i < length(); ++i) {
// Flush stats related to To-space PLAB activity and
// retire the last buffer.
true /* end_of_gc */,
false /* retain */);
// Every thread has its own age table. We need to merge
// them all into one.
// Inform old gen that we're done.
}
if (UseConcMarkSweepGC && ParallelGCThreads > 0) {
// We need to call this even when ResizeOldPLAB is disabled
// so as to avoid breaking some asserts. While we may be able
// to avoid this by reorganizing the code a bit, I am loathe
// to do that unless we find cases where ergo leads to bad
// performance.
}
}
{
}
void ParScanWithBarrierClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, true, false); }
void ParScanWithoutBarrierClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, false, false); }
void ParRootScanWithBarrierTwoGensClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, true, true); }
void ParRootScanWithBarrierTwoGensClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, true, true); }
void ParRootScanWithoutBarrierClosure::do_oop(oop* p) { ParScanClosure::do_oop_work(p, false, true); }
void ParRootScanWithoutBarrierClosure::do_oop(narrowOop* p) { ParScanClosure::do_oop_work(p, false, true); }
{}
#ifdef WIN32
#pragma warning(disable: 4786) /* identifier was truncated to '255' characters in the browser information */
#endif
{}
while (true) {
// Scan to-space and old-gen objs until we run out of both.
par_scan_state()->trim_queues(0);
// We have no local work, attempt to steal from other threads.
// attempt to steal work from promoted.
par_scan_state()->hash_seed(),
obj_to_scan)) {
// if successful, goto Start.
continue;
// try global overflow list.
continue;
}
// Otherwise, offer termination.
if (terminator()->offer_termination()) break;
par_scan_state()->end_term_time();
}
"Broken overflow list?");
// Finish the last termination pause.
par_scan_state()->end_term_time();
}
AbstractGangTask("ParNewGeneration collection"),
{}
// Reset the terminator for the given number of
// active threads.
// Should the heap be passed in? There's only 1 for now so
// grab it instead.
}
// The "i" passed to this method is the part of the work for
// this thread. It is not the worker ID. The "i" is derived
// from _started_workers which is incremented in internal_note_start()
// called in GangWorker loop() and which is called under the
// which is called under the protection of the gang monitor and is
// called after a task is started. So "i" is based on
// first-come-first-served.
// Since this is being done in a separate thread, need new resource
// and handle marks.
// We would need multiple old-gen queues otherwise.
true, // Process younger gens, if any,
// as strong roots.
false, // no scope; this is parallel code
false, // not collecting perm generation.
true, // walk *all* scavengable nmethods
// "evacuate followers".
}
#ifdef _MSC_VER
#endif
_is_alive_closure(this),
{
NOT_PRODUCT(_num_par_pushes = 0;)
ObjToScanQueue *q = new ObjToScanQueue();
}
if (ParGCUseLocalOverflow) {
// typedef to workaround NEW_C_HEAP_ARRAY macro, which can not deal
// with ','
for (size_t i = 0; i < ParallelGCThreads; ++i) {
}
}
if (UsePerfData) {
const char* cname =
}
}
#ifdef _MSC_VER
#endif
// ParNewGeneration::
template <class T>
#ifdef ASSERT
{
// We never expect to see a null reference being processed
// as a weak reference.
}
#endif // ASSERT
}
}
void /*ParNewGeneration::*/ParKeepAliveClosure::do_oop(oop* p) { ParKeepAliveClosure::do_oop_work(p); }
void /*ParNewGeneration::*/ParKeepAliveClosure::do_oop(narrowOop* p) { ParKeepAliveClosure::do_oop_work(p); }
// ParNewGeneration::
template <class T>
#ifdef ASSERT
{
// We never expect to see a null reference being processed
// as a weak reference.
}
#endif // ASSERT
}
}
void /*ParNewGeneration::*/KeepAliveClosure::do_oop(narrowOop* p) { KeepAliveClosure::do_oop_work(p); }
}
if (_gc_barrier) {
// If p points to a younger generation, mark the card.
}
}
}
}
public:
private:
}
private:
};
: AbstractGangTask("ParNewGeneration parallel reference processing"),
{
}
{
}
public:
: AbstractGangTask("ParNewGeneration parallel reference enqueue"),
{ }
{
}
};
{
"not a generational heap");
}
{
}
{
_state_set.flush();
gch->save_marks();
}
ScanClosure(g, gc_barrier) {}
{}
do {
// Beware: this call will lead to closure applications via virtual
// calls.
}
// A Generation that does parallel young-gen collection.
// Set the desired survivor size to half the real survivor space
}
void ParNewGeneration::handle_promotion_failed(GenCollectedHeap* gch, ParScanThreadStateSet& thread_state_set, ParNewTracer& gc_tracer) {
if (PrintGCDetails) {
}
// All the spaces are in play for mark-sweep.
swap_spaces(); // Make life simpler for CMS || rescan; see 6483690.
// Inform the next generation that a promotion failure occurred.
// Trace promotion failure in the parallel GC threads
// Single threaded code may have reported promotion failure to the global state
if (_promotion_failed_info.has_failed()) {
}
// Reset the PromotionFailureALot counters.
}
bool clear_all_soft_refs,
bool is_tlab) {
"not a CMS generational heap");
int active_workers =
"This must be the youngest gen, and not the only gen");
"Par collection currently only works with single older gen.");
// Do we have to avoid promotion_undo?
set_avoid_promotion_undo(true);
}
// If the next generation is too full to accommodate worst-case promotion
// from this generation, pass on collection; let the next generation
// do it.
if (!collection_attempt_is_safe()) {
return;
}
if (UseAdaptiveSizePolicy) {
set_survivor_overflow(false);
}
// Capture heap used before collection (for printing).
gch->save_marks();
// Set the correct parallelism (number of queues) in the reference processor
// Always set the terminator for the active number of workers
// because only those workers go through the termination protocol.
// It turns out that even when we're using 1 thread, doing the work in a
// separate thread causes wide variance in run times. We can't help this
// in the multi-threaded case, but we special-case n=1 here to get
// repeatable measurements of the 1-thread overhead of the parallel code.
if (n_workers > 1) {
} else {
}
promotion_failed());
// Process (weak) reference objects found during scavenge.
IsAliveClosure is_alive(this);
ScanWeakRefClosure scan_weak_ref(this);
ScanClosure scan_without_gc_barrier(this, false);
ScanClosureWithParBarrier scan_with_gc_barrier(this, true);
// Can the mt_degree be set later (at run_task() time would be best)?
if (rp->processing_is_mt()) {
} else {
gch->save_marks();
}
if (!promotion_failed()) {
// Swap the survivor spaces.
if (ZapUnusedHeapArea) {
// This is now done here because of the piece-meal mangling which
// can check for valid mangling at intermediate points in the
// collection(s). When a minor collection fails to collect
// sufficient space resizing of the young generation can occur
// an redistribute the spaces in the young generation. Mangle
// here so that unzapped regions don't get distributed to
// other spaces.
to()->mangle_unused_area();
}
swap_spaces();
// A successful scavenge should restart the GC time limit count which is
// for full GC's.
} else {
}
// set new iteration safe limit for the survivor spaces
if (ResizePLAB) {
}
if (PrintGC && !PrintGCDetails) {
}
if (PrintGCDetails && ParallelGCVerbose) {
}
if (UseAdaptiveSizePolicy) {
}
// We need to use a monotonically non-deccreasing time in ms
// or we will see time-warp warnings and os::javaTimeMillis()
// does not guarantee monotonicity.
rp->set_enqueuing_is_done(true);
if (rp->processing_is_mt()) {
} else {
}
}
static int sum;
for (int i = 0; i < 100; i++) {
sum += i;
}
}
// Because of concurrency, there are times where an object for which
// "is_forwarded()" is true contains an "interim" forwarding pointer
// value. Such a value will soon be overwritten with a real value.
// This method requires "obj" to have a forwarding pointer, and waits, if
// necessary for a real one to be inserted, and returns it.
if (forward_ptr != ClaimedForwardPtr) {
return forward_ptr;
} else {
return real_forwardee_slow(obj);
}
}
// Spin-read if it is claimed but not yet written by another thread.
while (forward_ptr == ClaimedForwardPtr) {
}
return forward_ptr;
}
#ifdef ASSERT
return
(_avoid_promotion_undo && p == ClaimedForwardPtr)
}
#endif
if (m->must_be_preserved_for_promotion_failure(obj)) {
// We should really have separate per-worker stacks, rather
// than use locking of a common pair of stacks.
preserve_mark(obj, m);
}
}
// Multiple GC threads may try to promote an object. If the object
// is successfully promoted, a forwarding pointer will be installed in
// the object in the young generation. This method claims the right
// to install the forwarding pointer before it copies the object,
// thus avoiding the need to undo the copy as in
// copy_to_survivor_space_avoiding_with_undo.
// In the sequential version, this assert also says that the object is
// not forwarded. That might not be the case here. It is the case that
// the caller observed it to be not forwarded at some time in the past.
// The sequential code read "old->age()" below. That doesn't work here,
// since the age is in the mark word, and that might be overwritten with
// a forwarding pointer by a parallel thread. So we must save the mark
// word in a local and then analyze it.
"should not be called with forwarding pointer mark word.");
// Try allocating obj in to-space (unless too old)
set_survivor_overflow(true);
}
}
// Either to-space is full or we decided to promote
// try allocating obj tenured
// Attempt to install a null forwarding pointer (atomically),
// to claim the right to install the real forwarding pointer.
if (forward_ptr != NULL) {
// someone else beat us to it.
return real_forwardee(old);
}
// promotion failed, forward to self
_promotion_failed = true;
}
forward_ptr = NULL;
} else {
// Is in to-space; do copying ourselves.
// Restore the mark word copied above.
// Increment age if obj still in new generation
}
if (forward_ptr == NULL) {
// Length field used as index of next element to be scanned.
// Real length can be obtained from real_forwardee()
obj_to_push = old;
"push forwarded object");
}
// Push it on one of the queues of to-be-scanned objects.
bool simulate_overflow = false;
if (ParGCWorkQueueOverflowALot && should_simulate_overflow()) {
// simulate a stack overflow
simulate_overflow = true;
}
)
// Add stats for overflow pushes.
if (Verbose && PrintGCDetails) {
}
}
return new_obj;
}
// Oops. Someone beat us to it. Undo the allocation. Where did we
// allocate it?
if (is_in_reserved(new_obj)) {
// Must be in to_space.
if (forward_ptr == ClaimedForwardPtr) {
// Wait to get the real forwarding pointer value.
}
}
return forward_ptr;
}
// Multiple GC threads may try to promote the same object. If two
// or more GC threads copy the object, only one wins the race to install
// the forwarding pointer. The other threads have to undo their copy.
// In the sequential version, this assert also says that the object is
// not forwarded. That might not be the case here. It is the case that
// the caller observed it to be not forwarded at some time in the past.
// The sequential code read "old->age()" below. That doesn't work here,
// since the age is in the mark word, and that might be overwritten with
// a forwarding pointer by a parallel thread. So we must save the mark
// word here, install it in a local oopDesc, and then analyze it.
"should not be called with forwarding pointer mark word.");
bool failed_to_promote = false;
// Try allocating obj in to-space (unless too old)
set_survivor_overflow(true);
}
}
// Either to-space is full or we decided to promote
// try allocating obj tenured
// promotion failed, forward to self
if (forward_ptr != NULL) {
return forward_ptr; // someone else succeeded
}
_promotion_failed = true;
failed_to_promote = true;
}
} else {
// Is in to-space; do copying ourselves.
// Restore the mark word copied above.
// Increment age if new_obj still in new generation
}
// Now attempt to install the forwarding pointer (atomically).
// We have to copy the mark word before overwriting with forwarding
// ptr, so we can restore it below in the copy.
if (!failed_to_promote) {
}
if (forward_ptr == NULL) {
// Length field used as index of next element to be scanned.
// Real length can be obtained from real_forwardee()
obj_to_push = old;
"push forwarded object");
}
// Push it on one of the queues of to-be-scanned objects.
bool simulate_overflow = false;
if (ParGCWorkQueueOverflowALot && should_simulate_overflow()) {
// simulate a stack overflow
simulate_overflow = true;
}
)
// Add stats for overflow pushes.
}
return new_obj;
}
// Oops. Someone beat us to it. Undo the allocation. Where did we
// allocate it?
if (is_in_reserved(new_obj)) {
// Must be in to_space.
} else {
}
return forward_ptr;
}
#ifndef PRODUCT
// It's OK to call this multi-threaded; the worst thing
// that can happen is that we'll get a bunch of closely
// spaced simulated oveflows, but that's OK, in fact
// probably good as it would exercise the overflow code
// under contention.
if (_overflow_counter-- <= 0) { // just being defensive
return true;
} else {
return false;
}
}
#endif
// In case we are using compressed oops, we need to be careful.
// If the object being pushed is an object array, then its length
// field keeps track of the "grey boundary" at which the next
// incremental scan will be done (see ParGCArrayScanChunk).
// When using compressed oops, this length field is kept in the
// lower 32 bits of the erstwhile klass word and cannot be used
// for the overflow chaining pointer (OCP below). As such the OCP
// would itself need to be compressed into the top 32-bits in this
// case. Unfortunately, see below, in the event that we have a
// promotion failure, the node to be pushed on the list can be
// outside of the Java heap, so the heap-based pointer compression
// would not work (we would have potential aliasing between C-heap
// and Java-heap pointers). For this reason, when using compressed
// oops, we simply use a worker-thread-local, non-shared overflow
// list in the form of a growable array, with a slightly different
// stacks here, we can go back to using (fat) pointer chains
// (although some performance comparisons would be useful since
// single global lists have their own performance disadvantages
// as we were made painfully aware not long ago, see 6786503).
void ParNewGeneration::push_on_overflow_list(oop from_space_obj, ParScanThreadState* par_scan_state) {
if (ParGCUseLocalOverflow) {
// In the case of compressed oops, we use a private, not-shared
// overflow stack.
} else {
// if the object has been forwarded to itself, then we cannot
// use the klass pointer for the linked list. Instead we have
// to allocate an oopDesc in the C-Heap and use that for the linked list.
// XXX This is horribly inefficient when a promotion failure occurs
// and should be fixed. XXX FIX ME !!!
#ifndef PRODUCT
#endif
}
do {
if (cur_overflow_list != BUSY) {
} else {
}
} while (cur_overflow_list != observed_overflow_list);
}
}
bool res;
if (ParGCUseLocalOverflow) {
} else {
}
return res;
}
// *NOTE*: The overflow list manipulation code here and
// in CMSCollector:: are very similar in shape,
// except that in the CMS case we thread the objects
// directly into the list via their mark word, and do
// not need to deal with special cases below related
// to chunking of object arrays and promotion failure
// handling.
// CR 6797058 has been filed to attempt consolidation of
// the common code.
// Because of the common code, if you make any changes in
// the code below, please check the CMS version to see if
// similar changes might be needed.
// See CMSCollector::par_take_from_overflow_list() for
// more extensive documentation comments.
// How many to take?
if (_overflow_list == NULL) return false;
// Otherwise, there was something there; try claiming the list.
// Trim off a prefix of at most objsFromOverflow items
// someone grabbed it before we did ...
// ... we spin for a short while...
if (_overflow_list == NULL) {
// nothing left to take
return false;
} else if (_overflow_list != BUSY) {
// try and grab the prefix
}
}
// Nothing to take or waited long enough
// Write back the NULL in case we overwrote it with BUSY above
// and it is still the same value.
}
return false;
}
size_t i = 1;
}
// Reattach remaining (suffix) to overflow list
// Write back the NULL in lieu of the BUSY we wrote
// above and it is still the same value.
if (_overflow_list == BUSY) {
}
} else {
// It's possible that the list is still in the empty(busy) state
// we left it in a short while ago; in that case we may be
// able to place back the suffix.
bool attached = false;
if (cur_overflow_list == observed_overflow_list) {
attached = true;
break;
} else cur_overflow_list = observed_overflow_list;
}
if (!attached) {
// Too bad, someone else got in in between; we'll need to do a splice.
// Find the last item of suffix list
}
// Atomically prepend suffix to current overflow list
do {
if (cur_overflow_list != BUSY) {
// Do the splice ...
} else { // cur_overflow_list == BUSY
}
} while (cur_overflow_list != observed_overflow_list);
}
}
// Push objects on prefix list onto this thread's work queue
ssize_t n = 0;
// This may be an array object that is self-forwarded. In that case, the list pointer
// space, cur, is not in the Java heap, but rather in the C-heap and should be freed.
if (!is_in_reserved(cur)) {
// This can become a scaling bottleneck when there is work queue overflow coincident
// with promotion failure.
obj_to_push = cur;
}
n++;
}
#ifndef PRODUCT
#endif
return true;
}
if (_ref_processor == NULL) {
// Allocate and initialize a reference processor
(int) ParallelGCThreads, // mt processing degree
refs_discovery_is_mt(), // mt discovery
(int) ParallelGCThreads, // mt discovery degree
refs_discovery_is_atomic(), // atomic_discovery
NULL, // is_alive_non_header
false); // write barrier for next field updates
}
}
return "par new generation";
}
return UseParNewGC && ParallelGCThreads > 0;
}