Skip to content

Commit

Permalink
Merge pull request #20333 from midronij/offheap-unsafe-getAndAdd
Browse files Browse the repository at this point in the history 
Adjust arguments to transformed UnsafeAPI calls when offheap is enabled
  • Loading branch information
@zl-wang
zl-wang authored Nov 26, 2024
2 parents c44f0de + cdfb2f5 commit 395aab5
Show file tree
Hide file tree
Showing 2 changed files with 96 additions and 5 deletions.
68 changes: 68 additions & 0 deletions runtime/compiler/optimizer/J9RecognizedCallTransformer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -638,6 +638,8 @@ void J9::RecognizedCallTransformer::processUnsafeAtomicCall(TR::TreeTop* treetop
TR::CFG* cfg = comp()->getMethodSymbol()->getFlowGraph();
TR::Node* isObjectNullNode = NULL;
TR::TreeTop* isObjectNullTreeTop = NULL;
TR::Node* isObjectArrayNode = NULL;
TR::TreeTop* isObjectArrayTreeTop = NULL;
TR::Node* isNotLowTaggedNode = NULL;
TR::TreeTop* isNotLowTaggedTreeTop = NULL;

Expand Down Expand Up @@ -692,6 +694,28 @@ void J9::RecognizedCallTransformer::processUnsafeAtomicCall(TR::TreeTop* treetop
if (enableTrace)
traceMsg(comp(), "Created isObjectNull test node n%dn, non-null object will fall through to Block_%d\n", isObjectNullNode->getGlobalIndex(), treetop->getEnclosingBlock()->getNumber());

// Test if object is array - offheap only
#if defined (J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION)
if (TR::Compiler->om.isOffHeapAllocationEnabled() && comp()->target().is64Bit())
{
//generate array check treetop
TR::Node *vftLoad = TR::Node::createWithSymRef(TR::aloadi, 1, 1,
objectNode->duplicateTree(),
comp()->getSymRefTab()->findOrCreateVftSymbolRef());

isObjectArrayNode = TR::Node::createif(TR::ificmpne,
comp()->fej9()->testIsClassArrayType(vftLoad),
TR::Node::create(TR::iconst, 0),
NULL);
isObjectArrayTreeTop = TR::TreeTop::create(comp(), isObjectArrayNode, NULL, NULL);
treetop->insertBefore(isObjectArrayTreeTop);
treetop->getEnclosingBlock()->split(treetop, cfg, fixupCommoning);

if (enableTrace)
traceMsg(comp(), "Created isObjectArray test node n%dn, array will branch to array access block\n", isObjectArrayNode->getGlobalIndex());
}
#endif /* J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION */

// Test if low tag is set
isNotLowTaggedNode = TR::Node::createif(TR::iflcmpne,
TR::Node::create(TR::land, 2, offsetNode->duplicateTree(), TR::Node::lconst(J9_SUN_STATIC_FIELD_OFFSET_TAG)),
Expand Down Expand Up @@ -829,6 +853,50 @@ void J9::RecognizedCallTransformer::processUnsafeAtomicCall(TR::TreeTop* treetop
// Split the block that contains the original helper call into a separate block
treetop->getEnclosingBlock()->split(treetop, cfg, fixupCommoning);

// Create another helper call for array access (offheap only)
#if defined (J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION)
if (isObjectArrayTreeTop != NULL)
{
TR::TreeTop *arrayAccessTreeTop = treetop->duplicateTree();
TR::Node *addrToAccessNode = arrayAccessTreeTop->getNode()->getChild(0)->getChild(0);
TR::Block *arrayAccessBlock = TR::Block::createEmptyBlock(arrayAccessTreeTop->getNode(), comp(),
treetop->getEnclosingBlock()->getFrequency());
arrayAccessBlock->append(arrayAccessTreeTop);
arrayAccessBlock->append(TR::TreeTop::create(comp(), TR::Node::create(arrayAccessTreeTop->getNode(),
TR::Goto, 0,
returnBlock->getEntry())));

//load dataAddr
TR::Node *objBaseAddrNode = addrToAccessNode->getChild(0);
TR::Node *dataAddrNode = TR::TransformUtil::generateDataAddrLoadTrees(comp(), objBaseAddrNode);
addrToAccessNode->setChild(0, dataAddrNode);

//correct refcounts
objBaseAddrNode->decReferenceCount();
dataAddrNode->incReferenceCount();

//set as array test destination and insert array access into IL trees
// - if object is array, goto array access block
// - else, fall through to lowtag test
unsafeCallRamStaticsTT->getEnclosingBlock()->getExit()->insertTreeTopsAfterMe(arrayAccessBlock->getEntry(), arrayAccessBlock->getExit());
isObjectArrayNode->setBranchDestination(arrayAccessTreeTop->getEnclosingBlock()->getEntry());

cfg->addNode(arrayAccessBlock);
cfg->addEdge(TR::CFGEdge::createEdge(isObjectArrayTreeTop->getEnclosingBlock(), arrayAccessBlock, comp()->trMemory()));
cfg->addEdge(TR::CFGEdge::createEdge(arrayAccessBlock, returnBlock, comp()->trMemory()));

// Store the return value from the helper call for array access
if (storeReturnNode)
{
TR::Node *storeNode = TR::Node::createStore(unsafeCall, storeReturnNode->getSymbolReference(), arrayAccessTreeTop->getNode()->getFirstChild());
arrayAccessTreeTop->insertTreeTopsAfterMe(TR::TreeTop::create(comp(), storeNode));
}

if (enableTrace)
traceMsg(comp(), "Created array access helper block_%d that loads dataAddr pointer from array object address\n", arrayAccessBlock->getNumber());
}
#endif /* J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION */

// Setup CFG edges
cfg->addEdge(unsafeCallRamStaticsTT->getEnclosingBlock(), returnBlock);

Expand Down
33 changes: 28 additions & 5 deletions runtime/compiler/optimizer/UnsafeFastPath.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ static bool isVarHandleOperationMethodOnNonStaticField(TR::RecognizedMethod rm)
bool TR_UnsafeFastPath::tryTransformUnsafeAtomicCallInVarHandleAccessMethod(TR::TreeTop* callTree, TR::RecognizedMethod callerMethod, TR::RecognizedMethod calleeMethod)
{
TR::Node* node = callTree->getNode()->getFirstChild();
TR::Node* unsafeAddress = NULL;

// Give up on arraylet
//
Expand All @@ -279,6 +280,20 @@ bool TR_UnsafeFastPath::tryTransformUnsafeAtomicCallInVarHandleAccessMethod(TR::
if (isVarHandleOperationMethodOnNonStaticField(callerMethod) &&
performTransformation(comp(), "%s transforming Unsafe.CAS [" POINTER_PRINTF_FORMAT "] into codegen inlineable\n", optDetailString(), node))
{
#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION)
if (isUnsafeCallerAccessingArrayElement(callerMethod) && TR::Compiler->om.isOffHeapAllocationEnabled() && comp()->target().is64Bit())
{
TR::Node *object = node->getChild(1);

TR::Node *baseAddr = TR::TransformUtil::generateDataAddrLoadTrees(comp(), object);
node->setChild(1, baseAddr);

//correct refcounts
object->decReferenceCount();
baseAddr->incReferenceCount();
}
#endif /* J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION */

node->setIsSafeForCGToFastPathUnsafeCall(true);
if (!isVarHandleOperationMethodOnArray(callerMethod))
{
Expand Down Expand Up @@ -310,7 +325,6 @@ bool TR_UnsafeFastPath::tryTransformUnsafeAtomicCallInVarHandleAccessMethod(TR::
if (!performTransformation(comp(), "%s turning the call [" POINTER_PRINTF_FORMAT "] into atomic intrinsic\n", optDetailString(), node))
return false;

TR::Node* unsafeAddress = NULL;
if (isUnsafeCallerAccessingStaticField(callerMethod))
{
TR::Node *jlClass = node->getChild(1);
Expand All @@ -326,10 +340,19 @@ bool TR_UnsafeFastPath::tryTransformUnsafeAtomicCallInVarHandleAccessMethod(TR::
}
else
{
TR::Node* object = node->getChild(1);
TR::Node* offset = node->getChild(2);
unsafeAddress = comp()->target().is32Bit() ? TR::Node::create(node, TR::aiadd, 2, object, TR::Node::create(node, TR::l2i, 1, offset)) :
TR::Node::create(node, TR::aladd, 2, object, offset);
TR::Node *object = node->getChild(1);
TR::Node *offset = node->getChild(2);

TR::Node *baseAddr = object;

//load dataAddr only if offheap is enabled and object is array
#if defined(J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION)
if (isUnsafeCallerAccessingArrayElement(callerMethod) && TR::Compiler->om.isOffHeapAllocationEnabled() && comp()->target().is64Bit())
baseAddr = TR::TransformUtil::generateDataAddrLoadTrees(comp(), object);
#endif /* J9VM_GC_ENABLE_SPARSE_HEAP_ALLOCATION */

unsafeAddress = comp()->target().is32Bit() ? TR::Node::create(node, TR::aiadd, 2, baseAddr, TR::Node::create(node, TR::l2i, 1, offset)) :
TR::Node::create(node, TR::aladd, 2, baseAddr, offset);
unsafeAddress->setIsInternalPointer(true);
}

Expand Down

0 comments on commit 395aab5

Please sign in to comment.