1berry/openjdk
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

8204331: AArch64: fix CAS not embedded in normal graph error

Browse Source 
JDK fails with assert on AArch64 after changes made by JDK-8202377

Reviewed-by: roland, rkennke
This commit is contained in:
Andrew Dinn 2018-06-22 11:21:34 +01:00
parent 51c3a9d4d1
commit 3ada65c7c2
1 changed files with 301 additions and 298 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

599
src/hotspot/cpu/aarch64/aarch64.ad
View File

@ -1193,21 +1193,28 @@ source %{
// MemBarRelease // MemBarRelease
// MemBarCPUOrder // MemBarCPUOrder
// StoreX[mo_release] {CardMark}-optional // StoreX[mo_release] {CardMark}-optional
// MemBarCPUOrder
// MemBarVolatile // MemBarVolatile
// //
// n.b. as an aside, the cpuorder membar is not itself subject to // n.b. as an aside, a cpuorder membar is not itself subject to
// matching and translation by adlc rules. However, the rule // matching and translation by adlc rules. However, the rule
// predicates need to detect its presence in order to correctly // predicates need to detect its presence in order to correctly
// select the desired adlc rules. // select the desired adlc rules.
// //
// Inlined unsafe volatile gets manifest as a somewhat different // Inlined unsafe volatile gets manifest as a slightly different
// node sequence to a normal volatile get // node sequence to a normal volatile get because of the
// introduction of some CPUOrder memory barriers to bracket the
// Load. However, but the same basic skeleton of a LoadX feeding a
// MemBarAcquire, possibly thorugh an optional DecodeN, is still
// present
// //
// MemBarCPUOrder // MemBarCPUOrder
// || \\ // || \\
// MemBarAcquire LoadX[mo_acquire] // MemBarCPUOrder LoadX[mo_acquire]
// || // || |
// MemBarCPUOrder // || {DecodeN} optional
// || /
// MemBarAcquire
// //
// In this case the acquire membar does not directly depend on the // In this case the acquire membar does not directly depend on the
// load. However, we can be sure that the load is generated from an // load. However, we can be sure that the load is generated from an
@ -1314,8 +1321,8 @@ source %{
MemBarNode *child_membar(const MemBarNode *n) MemBarNode *child_membar(const MemBarNode *n)
{ {
ProjNode *ctl = n->proj_out(TypeFunc::Control); ProjNode *ctl = n->proj_out_or_null(TypeFunc::Control);
ProjNode *mem = n->proj_out(TypeFunc::Memory); ProjNode *mem = n->proj_out_or_null(TypeFunc::Memory);
// MemBar needs to have both a Ctl and Mem projection // MemBar needs to have both a Ctl and Mem projection
if (! ctl || ! mem) if (! ctl || ! mem)
@ -1432,6 +1439,8 @@ source %{
// | \ / // | \ /
// | MergeMem // | MergeMem
// | / // | /
// {MemBarCPUOrder} -- optional
// { || }
// MemBarVolatile // MemBarVolatile
// //
// where // where
@ -1453,6 +1462,8 @@ source %{
// | MergeMem // | MergeMem
// | / // | /
// || / // || /
// {MemBarCPUOrder} -- optional
// { || }
// MemBarVolatile // MemBarVolatile
// //
// i.e. the leading membar feeds Ctl to a CastP2X (which converts // i.e. the leading membar feeds Ctl to a CastP2X (which converts
@ -1505,6 +1516,7 @@ source %{
// | / // | /
// MergeMem // MergeMem
// | // |
// {MemBarCPUOrder}
// MemBarVolatile // MemBarVolatile
// //
// This is referred to as a *normal* subgraph. It can easily be // This is referred to as a *normal* subgraph. It can easily be
@ -1567,7 +1579,7 @@ source %{
// object put and the corresponding conditional card mark. CMS // object put and the corresponding conditional card mark. CMS
// employs a post-write GC barrier while G1 employs both a pre- and // employs a post-write GC barrier while G1 employs both a pre- and
// post-write GC barrier. Of course the extra nodes may be absent -- // post-write GC barrier. Of course the extra nodes may be absent --
// they are only inserted for object puts. This significantly // they are only inserted for object puts/swaps. This significantly
// complicates the task of identifying whether a MemBarRelease, // complicates the task of identifying whether a MemBarRelease,
// StoreX[mo_release] or MemBarVolatile forms part of a volatile put // StoreX[mo_release] or MemBarVolatile forms part of a volatile put
// when using these GC configurations (see below). It adds similar // when using these GC configurations (see below). It adds similar
@ -1575,8 +1587,8 @@ source %{
// CompareAndSwapX or MemBarAcquire forms part of a CAS. // CompareAndSwapX or MemBarAcquire forms part of a CAS.
// //
// In both cases the post-write subtree includes an auxiliary // In both cases the post-write subtree includes an auxiliary
// MemBarVolatile (StoreLoad barrier) separating the object put and // MemBarVolatile (StoreLoad barrier) separating the object put/swap
// the read of the corresponding card. This poses two additional // and the read of the corresponding card. This poses two additional
// problems. // problems.
// //
// Firstly, a card mark MemBarVolatile needs to be distinguished // Firstly, a card mark MemBarVolatile needs to be distinguished
@ -1638,6 +1650,7 @@ source %{
// | . . . \ / Bot // | . . . \ / Bot
// | MergeMem // | MergeMem
// | | // | |
// {MemBarCPUOrder}
// MemBarVolatile (trailing) // MemBarVolatile (trailing)
// //
// The first MergeMem merges the AliasIdxBot Mem slice from the // The first MergeMem merges the AliasIdxBot Mem slice from the
@ -1647,53 +1660,39 @@ source %{
// from the StoreCM into the trailing membar (n.b. the latter // from the StoreCM into the trailing membar (n.b. the latter
// proceeds via a Phi associated with the If region). // proceeds via a Phi associated with the If region).
// //
// The graph for a CAS varies slightly, the obvious difference being // The graph for a CAS varies slightly, the difference being
// that the StoreN/P node is replaced by a CompareAndSwapP/N node // that the StoreN/P node is replaced by a CompareAndSwapP/N node
// and the trailing MemBarVolatile by a MemBarCPUOrder + // and the trailing MemBarVolatile by a MemBarCPUOrder +
// MemBarAcquire pair. The other important difference is that the // MemBarAcquire pair.
// CompareAndSwap node's SCMemProj is not merged into the card mark
// membar - it still feeds the trailing MergeMem. This also means
// that the card mark membar receives its Mem feed directly from the
// leading membar rather than via a MergeMem.
// //
// MemBarRelease // MemBarRelease
// MemBarCPUOrder__(leading)_________________________ // MemBarCPUOrder_(leading)_______________
// || \\ C \ // C | M \ \\ C \
// MemBarVolatile (card mark) CompareAndSwapN/P CastP2X // | \ CompareAndSwapN/P CastP2X
// C | || M | | // | \ |
// | LoadB | ______/| // | \ SCMemProj
// | | | / | // | Bot \ /
// | Cmp | / SCMemProj // | MergeMem
// | / | / | // | /
// If | / / // MemBarVolatile (card mark)
// | \ | / / // C | || M |
// IfFalse IfTrue | / / // | LoadB |
// \ / \ |/ prec / // | | |
// \ / StoreCM / // | Cmp |\
// \ / | / // | / | \
// Region . . . / // If | \
// | \ / // | \ | \
// | . . . \ / Bot // IfFalse IfTrue | \
// \ / \ | \
// \ / StoreCM |
// \ / | |
// Region . . . |
// | \ /
// | . . . \ / Bot
// | MergeMem // | MergeMem
// | | // | |
// MemBarCPUOrder // {MemBarCPUOrder}
// MemBarAcquire (trailing) // MemBarVolatile (trailing)
//
// This has a slightly different memory subgraph to the one seen
// previously but the core of it is the same as for the CAS normal
// sungraph
//
// MemBarRelease
// MemBarCPUOrder____
// || \ . . .
// MemBarVolatile CompareAndSwapX . . .
// | \ |
// . . . SCMemProj
// | / . . .
// MergeMem
// |
// MemBarCPUOrder
// MemBarAcquire
// //
// //
// G1 is quite a lot more complicated. The nodes inserted on behalf // G1 is quite a lot more complicated. The nodes inserted on behalf
@ -1742,15 +1741,13 @@ source %{
// (post write subtree elided) // (post write subtree elided)
// . . . // . . .
// C \ M / // C \ M /
// MemBarVolatile (trailing) // \ /
// {MemBarCPUOrder}
// MemBarVolatile (trailing)
// //
// n.b. the LoadB in this subgraph is not the card read -- it's a // n.b. the LoadB in this subgraph is not the card read -- it's a
// read of the SATB queue active flag. // read of the SATB queue active flag.
// //
// Once again the CAS graph is a minor variant on the above with the
// expected substitutions of CompareAndSawpX for StoreN/P and
// MemBarCPUOrder + MemBarAcquire for trailing MemBarVolatile.
//
// The G1 post-write subtree is also optional, this time when the // The G1 post-write subtree is also optional, this time when the
// new value being written is either null or can be identified as a // new value being written is either null or can be identified as a
// newly allocated (young gen) object with no intervening control // newly allocated (young gen) object with no intervening control
@ -1773,7 +1770,8 @@ source %{
// checking if card_val != young). n.b. although this test requires // checking if card_val != young). n.b. although this test requires
// a pre-read of the card it can safely be done before the StoreLoad // a pre-read of the card it can safely be done before the StoreLoad
// barrier. However that does not bypass the need to reread the card // barrier. However that does not bypass the need to reread the card
// after the barrier. // after the barrier. A final, 4th If tests if the card is already
// marked.
// //
// (pre-write subtree elided) // (pre-write subtree elided)
// . . . . . . . . . . . . // . . . . . . . . . . . .
@ -1826,6 +1824,7 @@ source %{
// | | | / Bot // | | | / Bot
// \ MergeMem // \ MergeMem
// \ / // \ /
// {MemBarCPUOrder}
// MemBarVolatile // MemBarVolatile
// //
// As with CMS the initial MergeMem merges the AliasIdxBot Mem slice // As with CMS the initial MergeMem merges the AliasIdxBot Mem slice
@ -1845,26 +1844,29 @@ source %{
// otherwise it is 3. // otherwise it is 3.
// //
// The CAS graph when using G1GC also includes a pre-write subgraph // The CAS graph when using G1GC also includes a pre-write subgraph
// and an optional post-write subgraph. Teh sam evarioations are // and an optional post-write subgraph. The same variations are
// introduced as for CMS with conditional card marking i.e. the // introduced as for CMS with conditional card marking i.e. the
// StoreP/N is swapped for a CompareAndSwapP/N, the tariling // StoreP/N is swapped for a CompareAndSwapP/N with a following
// MemBarVolatile for a MemBarCPUOrder + MemBarAcquire pair and the // SCMemProj, the trailing MemBarVolatile for a MemBarCPUOrder +
// Mem feed from the CompareAndSwapP/N includes a precedence // MemBarAcquire pair. There may be an extra If test introduced in
// dependency feed to the StoreCM and a feed via an SCMemProj to the // the CAS case, when the boolean result of the CAS is tested by the
// trailing membar. So, as before the configuration includes the // caller. In that case an extra Region and AliasIdxBot Phi may be
// normal CAS graph as a subgraph of the memory flow. // introduced before the MergeMem
// //
// So, the upshot is that in all cases the volatile put graph will // So, the upshot is that in all cases the subgraph will include a
// include a *normal* memory subgraph betwen the leading membar and // *normal* memory subgraph betwen the leading membar and its child
// its child membar, either a volatile put graph (including a // membar: either a normal volatile put graph including a releasing
// releasing StoreX) or a CAS graph (including a CompareAndSwapX). // StoreX and terminating with a trailing volatile membar or card
// When that child is not a card mark membar then it marks the end // mark volatile membar; or a normal CAS graph including a
// of the volatile put or CAS subgraph. If the child is a card mark // CompareAndSwapX + SCMemProj pair and terminating with a card mark
// membar then the normal subgraph will form part of a volatile put // volatile membar or a trailing cpu order and acquire membar
// subgraph if and only if the child feeds an AliasIdxBot Mem feed // pair. If the child membar is not a (volatile) card mark membar
// to a trailing barrier via a MergeMem. That feed is either direct // then it marks the end of the volatile put or CAS subgraph. If the
// (for CMS) or via 2 or 3 Phi nodes merging the leading barrier // child is a card mark membar then the normal subgraph will form
// memory flow (for G1). // part of a larger volatile put or CAS subgraph if and only if the
// child feeds an AliasIdxBot Mem feed to a trailing barrier via a
// MergeMem. That feed is either direct (for CMS) or via 2, 3 or 4
// Phi nodes merging the leading barrier memory flow (for G1).
// //
// The predicates controlling generation of instructions for store // The predicates controlling generation of instructions for store
// and barrier nodes employ a few simple helper functions (described // and barrier nodes employ a few simple helper functions (described
@ -1907,13 +1909,27 @@ source %{
} }
} }
// helper to determine the maximum number of Phi nodes we may need to
// traverse when searching from a card mark membar for the merge mem
// feeding a trailing membar or vice versa
int max_phis()
{
if (UseG1GC) {
return 4;
} else if (UseConcMarkSweepGC && UseCondCardMark) {
return 1;
} else {
return 0;
}
}
// leading_to_normal // leading_to_normal
// //
//graph traversal helper which detects the normal case Mem feed from // graph traversal helper which detects the normal case Mem feed
// a release membar (or, optionally, its cpuorder child) to a // from a release membar (or, optionally, its cpuorder child) to a
// dependent volatile membar i.e. it ensures that one or other of // dependent volatile or acquire membar i.e. it ensures that one of
// the following Mem flow subgraph is present. // the following 3 Mem flow subgraphs is present.
// //
// MemBarRelease // MemBarRelease
// MemBarCPUOrder {leading} // MemBarCPUOrder {leading}
@ -1922,19 +1938,27 @@ source %{
// | / // | /
// MergeMem // MergeMem
// | // |
// {MemBarCPUOrder}
// MemBarVolatile {trailing or card mark} // MemBarVolatile {trailing or card mark}
// //
// MemBarRelease // MemBarRelease
// MemBarCPUOrder {leading} // MemBarCPUOrder {leading}
// | \ . . . // | \ . . .
// | CompareAndSwapX . . . // | CompareAndSwapX . . .
// | // | /
// . . . SCMemProj // MergeMem
// \ | // |
// | MergeMem // MemBarVolatile {card mark}
// | / //
// MemBarCPUOrder // MemBarRelease
// MemBarAcquire {trailing} // MemBarCPUOrder {leading}
// | \ . . .
// | CompareAndSwapX . . .
// | /
// MergeMem
// |
// MemBarCPUOrder
// MemBarAcquire {trailing}
// //
// if the correct configuration is present returns the trailing // if the correct configuration is present returns the trailing
// membar otherwise NULL. // membar otherwise NULL.
@ -1991,61 +2015,65 @@ source %{
return NULL; return NULL;
} }
// must have a merge if we also have st // must have a merge
if (st && !mm) { if (!mm) {
return NULL; return NULL;
} }
Node *y = NULL; Node *feed = NULL;
if (cas) { if (cas) {
// look for an SCMemProj // look for an SCMemProj
for (DUIterator_Fast imax, i = cas->fast_outs(imax); i < imax; i++) { for (DUIterator_Fast imax, i = cas->fast_outs(imax); i < imax; i++) {
x = cas->fast_out(i); x = cas->fast_out(i);
if (x->is_Proj()) { if (x->Opcode() == Op_SCMemProj) {
y = x; feed = x;
break; break;
} }
} }
if (y == NULL) { if (feed == NULL) {
return NULL; return NULL;
} }
// the proj must feed a MergeMem
for (DUIterator_Fast imax, i = y->fast_outs(imax); i < imax; i++) {
x = y->fast_out(i);
if (x->is_MergeMem()) {
mm = x->as_MergeMem();
break;
}
}
if (mm == NULL)
return NULL;
} else { } else {
// ensure the store feeds the existing mergemem; feed = st;
for (DUIterator_Fast imax, i = st->fast_outs(imax); i < imax; i++) { }
if (st->fast_out(i) == mm) { // ensure the feed node feeds the existing mergemem;
y = st; for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) {
break; x = feed->fast_out(i);
} if (x == mm) {
} break;
if (y == NULL) {
return NULL;
} }
} }
if (x != mm) {
return NULL;
}
MemBarNode *mbar = NULL; MemBarNode *mbar = NULL;
// ensure the merge feeds to the expected type of membar // ensure the merge feeds to the expected type of membar
for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) { for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
x = mm->fast_out(i); x = mm->fast_out(i);
if (x->is_MemBar()) { if (x->is_MemBar()) {
int opcode = x->Opcode(); if (x->Opcode() == Op_MemBarCPUOrder) {
if (opcode == Op_MemBarVolatile && st) { // with a store any cpu order membar should precede a
mbar = x->as_MemBar(); // trailing volatile membar. with a cas it should precede a
} else if (cas && opcode == Op_MemBarCPUOrder) { // trailing acquire membar. in either case try to skip to
// that next membar
MemBarNode *y = x->as_MemBar(); MemBarNode *y = x->as_MemBar();
y = child_membar(y); y = child_membar(y);
if (y != NULL && y->Opcode() == Op_MemBarAcquire) { if (y != NULL) {
mbar = y; // skip to this new membar to do the check
x = y;
} }
}
if (x->Opcode() == Op_MemBarVolatile) {
mbar = x->as_MemBar();
// for a volatile store this can be either a trailing membar
// or a card mark membar. for a cas it must be a card mark
// membar
assert(cas == NULL || is_card_mark_membar(mbar),
"in CAS graph volatile membar must be a card mark");
} else if (cas != NULL && x->Opcode() == Op_MemBarAcquire) {
mbar = x->as_MemBar();
} }
break; break;
} }
@ -2059,28 +2087,36 @@ source %{
// graph traversal helper which detects the normal case Mem feed // graph traversal helper which detects the normal case Mem feed
// from either a card mark or a trailing membar to a preceding // from either a card mark or a trailing membar to a preceding
// release membar (optionally its cpuorder child) i.e. it ensures // release membar (optionally its cpuorder child) i.e. it ensures
// that one or other of the following Mem flow subgraphs is present. // that one of the following 3 Mem flow subgraphs is present.
// //
// MemBarRelease // MemBarRelease
// MemBarCPUOrder {leading} // {MemBarCPUOrder} {leading}
// | \ . . . // | \ . . .
// | StoreN/P[mo_release] . . . // | StoreN/P[mo_release] . . .
// | / // | /
// MergeMem // MergeMem
// | // |
// MemBarVolatile {card mark or trailing} // {MemBarCPUOrder}
// MemBarVolatile {trailing or card mark}
// //
// MemBarRelease // MemBarRelease
// MemBarCPUOrder {leading} // MemBarCPUOrder {leading}
// | \ . . . // | \ . . .
// | CompareAndSwapX . . . // | CompareAndSwapX . . .
// | // | /
// . . . SCMemProj // MergeMem
// \ | // |
// | MergeMem // MemBarVolatile {card mark}
// | / //
// MemBarCPUOrder // MemBarRelease
// MemBarAcquire {trailing} // MemBarCPUOrder {leading}
// | \ . . .
// | CompareAndSwapX . . .
// | /
// MergeMem
// |
// MemBarCPUOrder
// MemBarAcquire {trailing}
// //
// this predicate checks for the same flow as the previous predicate // this predicate checks for the same flow as the previous predicate
// but starting from the bottom rather than the top. // but starting from the bottom rather than the top.
@ -2097,20 +2133,19 @@ source %{
assert((barrier->Opcode() == Op_MemBarVolatile || assert((barrier->Opcode() == Op_MemBarVolatile ||
barrier->Opcode() == Op_MemBarAcquire), barrier->Opcode() == Op_MemBarAcquire),
"expecting a volatile or an acquire membar"); "expecting a volatile or an acquire membar");
Node *x; bool barrier_is_acquire = barrier->Opcode() == Op_MemBarAcquire;
bool is_cas = barrier->Opcode() == Op_MemBarAcquire;
// if we have an acquire membar then it must be fed via a CPUOrder // if we have an intervening cpu order membar then start the
// membar // search from it
Node *x = parent_membar(barrier);
if (is_cas) { if (x == NULL) {
// skip to parent barrier which must be a cpuorder // stick with the original barrier
x = parent_membar(barrier);
if (x->Opcode() != Op_MemBarCPUOrder)
return NULL;
} else {
// start from the supplied barrier
x = (Node *)barrier; x = (Node *)barrier;
} else if (x->Opcode() != Op_MemBarCPUOrder) {
// any other barrier means this is not the graph we want
return NULL;
} }
// the Mem feed to the membar should be a merge // the Mem feed to the membar should be a merge
@ -2120,30 +2155,8 @@ source %{
MergeMemNode *mm = x->as_MergeMem(); MergeMemNode *mm = x->as_MergeMem();
if (is_cas) { // the merge should get its Bottom mem feed from the leading membar
// the merge should be fed from the CAS via an SCMemProj node x = mm->in(Compile::AliasIdxBot);
x = NULL;
for (uint idx = 1; idx < mm->req(); idx++) {
if (mm->in(idx)->Opcode() == Op_SCMemProj) {
x = mm->in(idx);
break;
}
}
if (x == NULL) {
return NULL;
}
// check for a CAS feeding this proj
x = x->in(0);
int opcode = x->Opcode();
if (!is_CAS(opcode)) {
return NULL;
}
// the CAS should get its mem feed from the leading membar
x = x->in(MemNode::Memory);
} else {
// the merge should get its Bottom mem feed from the leading membar
x = mm->in(Compile::AliasIdxBot);
}
// ensure this is a non control projection // ensure this is a non control projection
if (!x->is_Proj() || x->is_CFG()) { if (!x->is_Proj() || x->is_CFG()) {
@ -2188,15 +2201,34 @@ source %{
if (st == NULL & cas == NULL) { if (st == NULL & cas == NULL) {
return NULL; return NULL;
} }
if (st == NULL) { if (st == NULL) {
// nothing more to check // if we started from a volatile membar and found a CAS then the
return leading; // original membar ought to be for a card mark
} else { assert((barrier_is_acquire || is_card_mark_membar(barrier)),
// we should not have a store if we started from an acquire "unexpected volatile barrier (i.e. not card mark) in CAS graph");
if (is_cas) { // check that the CAS feeds the merge we used to get here via an
return NULL; // intermediary SCMemProj
Node *scmemproj = NULL;
for (DUIterator_Fast imax, i = cas->fast_outs(imax); i < imax; i++) {
x = cas->fast_out(i);
if (x->Opcode() == Op_SCMemProj) {
scmemproj = x;
break;
}
} }
if (scmemproj == NULL) {
return NULL;
}
for (DUIterator_Fast imax, i = scmemproj->fast_outs(imax); i < imax; i++) {
x = scmemproj->fast_out(i);
if (x == mm) {
return leading;
}
}
} else {
// we should not have found a store if we started from an acquire
assert(!barrier_is_acquire,
"unexpected trailing acquire barrier in volatile store graph");
// the store should feed the merge we used to get here // the store should feed the merge we used to get here
for (DUIterator_Fast imax, i = st->fast_outs(imax); i < imax; i++) { for (DUIterator_Fast imax, i = st->fast_outs(imax); i < imax; i++) {
@ -2227,8 +2259,9 @@ source %{
// Bot | / // Bot | /
// MergeMem // MergeMem
// | // |
// | // {MemBarCPUOrder} OR MemBarCPUOrder
// MemBarVolatile {trailing} // MemBarVolatile {trailing} MemBarAcquire {trailing}
//
// //
// 2) // 2)
// MemBarRelease/CPUOrder (leading) // MemBarRelease/CPUOrder (leading)
@ -2246,8 +2279,8 @@ source %{
// Bot | / // Bot | /
// MergeMem // MergeMem
// | // |
// MemBarVolatile {trailing} // {MemBarCPUOrder} OR MemBarCPUOrder
// // MemBarVolatile {trailing} MemBarAcquire {trailing}
// //
// 3) // 3)
// MemBarRelease/CPUOrder (leading) // MemBarRelease/CPUOrder (leading)
@ -2269,12 +2302,44 @@ source %{
// MergeMem // MergeMem
// | // |
// | // |
// MemBarVolatile {trailing} // {MemBarCPUOrder} OR MemBarCPUOrder
// MemBarVolatile {trailing} MemBarAcquire {trailing}
//
// 4)
// MemBarRelease/CPUOrder (leading)
// |
// |\
// | \
// | \
// | \
// |\ \
// | \ \
// | \ \ . . .
// | \ \ |
// |\ \ \ MemBarVolatile (card mark)
// | \ \ \ / |
// | \ \ \ / StoreCM . . .
// | \ \ Phi
// \ \ \ /
// \ \ Phi
// \ \ /
// \ Phi
// \ /
// Phi . . .
// Bot | /
// MergeMem
// |
// |
// MemBarCPUOrder
// MemBarAcquire {trailing}
// //
// configuration 1 is only valid if UseConcMarkSweepGC && // configuration 1 is only valid if UseConcMarkSweepGC &&
// UseCondCardMark // UseCondCardMark
// //
// configurations 2 and 3 are only valid if UseG1GC. // configuration 2, is only valid if UseConcMarkSweepGC &&
// UseCondCardMark or if UseG1GC
//
// configurations 3 and 4 are only valid if UseG1GC.
// //
// if a valid configuration is present returns the trailing membar // if a valid configuration is present returns the trailing membar
// otherwise NULL. // otherwise NULL.
@ -2292,8 +2357,8 @@ source %{
Node *x; Node *x;
MergeMemNode *mm = NULL; MergeMemNode *mm = NULL;
const int MAX_PHIS = 3; // max phis we will search through const int MAX_PHIS = max_phis(); // max phis we will search through
int phicount = 0; // current search count int phicount = 0; // current search count
bool retry_feed = true; bool retry_feed = true;
while (retry_feed) { while (retry_feed) {
@ -2308,7 +2373,7 @@ source %{
} }
if (mm) { if (mm) {
retry_feed = false; retry_feed = false;
} else if (UseG1GC & phicount++ < MAX_PHIS) { } else if (phicount++ < MAX_PHIS) {
// the barrier may feed indirectly via one or two Phi nodes // the barrier may feed indirectly via one or two Phi nodes
PhiNode *phi = NULL; PhiNode *phi = NULL;
for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) { for (DUIterator_Fast imax, i = feed->fast_outs(imax); i < imax; i++) {
@ -2334,12 +2399,24 @@ source %{
assert(mm->as_MergeMem()->in(Compile::AliasIdxBot) == feed, "expecting membar to feed AliasIdxBot slice to Merge"); assert(mm->as_MergeMem()->in(Compile::AliasIdxBot) == feed, "expecting membar to feed AliasIdxBot slice to Merge");
MemBarNode *trailing = NULL; MemBarNode *trailing = NULL;
// be sure we have a trailing membar the merge // be sure we have a trailing membar fed by the merge
for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) { for (DUIterator_Fast imax, i = mm->fast_outs(imax); i < imax; i++) {
x = mm->fast_out(i); x = mm->fast_out(i);
if (x->is_MemBar() && x->Opcode() == Op_MemBarVolatile) { if (x->is_MemBar()) {
trailing = x->as_MemBar(); // if this is an intervening cpu order membar skip to the
break; // following membar
if (x->Opcode() == Op_MemBarCPUOrder) {
MemBarNode *y = x->as_MemBar();
y = child_membar(y);
if (y != NULL) {
x = y;
}
}
if (x->Opcode() == Op_MemBarVolatile ||
x->Opcode() == Op_MemBarAcquire) {
trailing = x->as_MemBar();
}
break;
} }
} }
@ -2360,18 +2437,33 @@ source %{
// otherwise NULL // otherwise NULL
// //
// n.b. the supplied membar is expected to be a trailing // n.b. the supplied membar is expected to be a trailing
// MemBarVolatile i.e. the caller must ensure the input node has the // MemBarVolatile or MemBarAcquire i.e. the caller must ensure the
// correct opcode // input node has the correct opcode
MemBarNode *trailing_to_card_mark(const MemBarNode *trailing) MemBarNode *trailing_to_card_mark(const MemBarNode *trailing)
{ {
assert(trailing->Opcode() == Op_MemBarVolatile, assert(trailing->Opcode() == Op_MemBarVolatile ||
"expecting a volatile membar"); trailing->Opcode() == Op_MemBarAcquire,
"expecting a volatile or acquire membar");
assert(!is_card_mark_membar(trailing), assert(!is_card_mark_membar(trailing),
"not expecting a card mark membar"); "not expecting a card mark membar");
Node *x = (Node *)trailing;
// look for a preceding cpu order membar
MemBarNode *y = parent_membar(x->as_MemBar());
if (y != NULL) {
// make sure it is a cpu order membar
if (y->Opcode() != Op_MemBarCPUOrder) {
// this is nto the graph we were looking for
return NULL;
}
// start the search from here
x = y;
}
// the Mem feed to the membar should be a merge // the Mem feed to the membar should be a merge
Node *x = trailing->in(TypeFunc::Memory); x = x->in(TypeFunc::Memory);
if (!x->is_MergeMem()) { if (!x->is_MergeMem()) {
return NULL; return NULL;
} }
@ -2382,20 +2474,20 @@ source %{
// with G1 we may possibly see a Phi or two before we see a Memory // with G1 we may possibly see a Phi or two before we see a Memory
// Proj from the card mark membar // Proj from the card mark membar
const int MAX_PHIS = 3; // max phis we will search through const int MAX_PHIS = max_phis(); // max phis we will search through
int phicount = 0; // current search count int phicount = 0; // current search count
bool retry_feed = !x->is_Proj(); bool retry_feed = !x->is_Proj();
while (retry_feed) { while (retry_feed) {
if (UseG1GC && x->is_Phi() && phicount++ < MAX_PHIS) { if (x->is_Phi() && phicount++ < MAX_PHIS) {
PhiNode *phi = x->as_Phi(); PhiNode *phi = x->as_Phi();
ProjNode *proj = NULL; ProjNode *proj = NULL;
PhiNode *nextphi = NULL; PhiNode *nextphi = NULL;
bool found_leading = false; bool found_leading = false;
for (uint i = 1; i < phi->req(); i++) { for (uint i = 1; i < phi->req(); i++) {
x = phi->in(i); x = phi->in(i);
if (x->is_Phi()) { if (x->is_Phi() && x->adr_type() == TypePtr::BOTTOM) {
nextphi = x->as_Phi(); nextphi = x->as_Phi();
} else if (x->is_Proj()) { } else if (x->is_Proj()) {
int opcode = x->in(0)->Opcode(); int opcode = x->in(0)->Opcode();
@ -2475,10 +2567,8 @@ source %{
return leading; return leading;
} }
// nothing more to do if this is an acquire // there is no normal path from trailing to leading membar. see if
if (trailing->Opcode() == Op_MemBarAcquire) { // we can arrive via a card mark membar
return NULL;
}
MemBarNode *card_mark_membar = trailing_to_card_mark(trailing); MemBarNode *card_mark_membar = trailing_to_card_mark(trailing);
@ -2506,15 +2596,6 @@ bool unnecessary_acquire(const Node *barrier)
// with a bogus read dependency on it's preceding load. so in those // with a bogus read dependency on it's preceding load. so in those
// cases we will find the load node at the PARMS offset of the // cases we will find the load node at the PARMS offset of the
// acquire membar. n.b. there may be an intervening DecodeN node. // acquire membar. n.b. there may be an intervening DecodeN node.
//
// a volatile load derived from an inlined unsafe field access
// manifests as a cpuorder membar with Ctl and Mem projections
// feeding both an acquire membar and a LoadX[mo_acquire]. The
// acquire then feeds another cpuorder membar via Ctl and Mem
// projections. The load has no output dependency on these trailing
// membars because subsequent nodes inserted into the graph take
// their control feed from the final membar cpuorder meaning they
// are all ordered after the load.
Node *x = barrier->lookup(TypeFunc::Parms); Node *x = barrier->lookup(TypeFunc::Parms);
if (x) { if (x) {
@ -2537,61 +2618,7 @@ bool unnecessary_acquire(const Node *barrier)
return (x->is_Load() && x->as_Load()->is_acquire()); return (x->is_Load() && x->as_Load()->is_acquire());
} }
// now check for an unsafe volatile get // other option for unnecessary membar is that it is a trailing node
// need to check for
//
// MemBarCPUOrder
// || \\
// MemBarAcquire* LoadX[mo_acquire]
// ||
// MemBarCPUOrder
//
// where * tags node we were passed
// and || or \\ are Ctl+Mem feeds via intermediate Proj Nodes
// check for a parent MemBarCPUOrder
ProjNode *ctl;
ProjNode *mem;
MemBarNode *parent = parent_membar(barrier);
if (!parent || parent->Opcode() != Op_MemBarCPUOrder)
return false;
ctl = parent->proj_out(TypeFunc::Control);
mem = parent->proj_out(TypeFunc::Memory);
if (!ctl || !mem) {
return false;
}
// ensure the proj nodes both feed a LoadX[mo_acquire]
LoadNode *ld = NULL;
for (DUIterator_Fast imax, i = ctl->fast_outs(imax); i < imax; i++) {
x = ctl->fast_out(i);
// if we see a load we keep hold of it and stop searching
if (x->is_Load()) {
ld = x->as_Load();
break;
}
}
// it must be an acquiring load
if (ld && ld->is_acquire()) {
for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) {
x = mem->fast_out(i);
// if we see the same load we drop it and stop searching
if (x == ld) {
ld = NULL;
break;
}
}
// we must have dropped the load
if (ld == NULL) {
// check for a child cpuorder membar
MemBarNode *child = child_membar(barrier->as_MemBar());
if (child && child->Opcode() == Op_MemBarCPUOrder)
return true;
}
}
// final option for unnecessary mebar is that it is a trailing node
// belonging to a CAS // belonging to a CAS
MemBarNode *leading = trailing_to_leading(barrier->as_MemBar()); MemBarNode *leading = trailing_to_leading(barrier->as_MemBar());
@ -2647,39 +2674,7 @@ bool needs_acquiring_load(const Node *n)
return true; return true;
} }
// now check for an unsafe volatile get return false;
// check if Ctl and Proj feed comes from a MemBarCPUOrder
//
// MemBarCPUOrder
// || \\
// MemBarAcquire* LoadX[mo_acquire]
// ||
// MemBarCPUOrder
MemBarNode *membar;
membar = parent_membar(ld);
if (!membar || !membar->Opcode() == Op_MemBarCPUOrder) {
return false;
}
// ensure that there is a CPUOrder->Acquire->CPUOrder membar chain
membar = child_membar(membar);
if (!membar || !membar->Opcode() == Op_MemBarAcquire) {
return false;
}
membar = child_membar(membar);
if (!membar || !membar->Opcode() == Op_MemBarCPUOrder) {
return false;
}
return true;
} }
bool unnecessary_release(const Node *n) bool unnecessary_release(const Node *n)
@ -2739,7 +2734,7 @@ bool unnecessary_volatile(const Node *n)
} }
// ok, if it's not a card mark then we still need to check if it is // ok, if it's not a card mark then we still need to check if it is
// a trailing membar of a volatile put hgraph. // a trailing membar of a volatile put graph.
return (trailing_to_leading(mbvol) != NULL); return (trailing_to_leading(mbvol) != NULL);
} }
@ -2848,6 +2843,14 @@ bool needs_acquiring_load_exclusive(const Node *n)
assert(mbar != NULL, "CAS not embedded in normal graph!"); assert(mbar != NULL, "CAS not embedded in normal graph!");
// if this is a card mark membar check we have a trailing acquire
if (is_card_mark_membar(mbar)) {
mbar = card_mark_to_trailing(mbar);
}
assert(mbar != NULL, "card mark membar for CAS not embedded in normal graph!");
assert(mbar->Opcode() == Op_MemBarAcquire, "trailing membar should be an acquire"); assert(mbar->Opcode() == Op_MemBarAcquire, "trailing membar should be an acquire");
#endif // ASSERT #endif // ASSERT
// so we can just return true here // so we can just return true here