From 9a6d5feb58f806f382c385c7895b7d854a44e3bc Mon Sep 17 00:00:00 2001 From: Gita Koblents Date: Fri, 17 May 2024 22:04:06 -0400 Subject: [PATCH 1/2] Move out-of-line instructions to warm cache - during code cache disclaim, it's beneficial to move OOL code into warm code cache --- compiler/x/codegen/OMRCodeGenerator.cpp | 62 +++++++++++++++++++++++++ compiler/x/codegen/OMRCodeGenerator.hpp | 6 +++ 2 files changed, 68 insertions(+) diff --git a/compiler/x/codegen/OMRCodeGenerator.cpp b/compiler/x/codegen/OMRCodeGenerator.cpp index 76edcbe0a7d..76d7479a2d3 100644 --- a/compiler/x/codegen/OMRCodeGenerator.cpp +++ b/compiler/x/codegen/OMRCodeGenerator.cpp @@ -1867,6 +1867,9 @@ void OMR::X86::CodeGenerator::doRegisterAssignment(TR_RegisterKinds kindsToAssig self()->doBackwardsRegisterAssignment(kindsToAssign, self()->getAppendInstruction()); } + + if (TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming)) + moveOutOfLineInstructionsToWarm(); } bool OMR::X86::CodeGenerator::isReturnInstruction(TR::Instruction *instr) @@ -3500,3 +3503,62 @@ OMR::X86::CodeGenerator::getOutOfLineCodeSize() return totalSize; } + +void +OMR::X86::CodeGenerator::moveOutOfLineInstructionsToWarm() + { + // OOL instructions are already attached at the end of the IL (after cold instructions) + // and register allocated. + // Move them to immediately after the last warm instruction, unless they already happend to be + // there (e.g. there are no cold instructions) + // + if (self()->comp()->getOption(TR_TraceCG)) + traceMsg(self()->comp(), "Moving OutOfLine instructions\n"); + + auto oiIterator = self()->getOutlinedInstructionsList().begin(); + + while (oiIterator != self()->getOutlinedInstructionsList().end()) + { + TR::Instruction *firstInstruction = (*oiIterator)->getFirstInstruction(); + TR::Instruction *lastInstruction = (*oiIterator)->getAppendInstruction(); + + TR_ASSERT_FATAL(firstInstruction, "VFPRestore instruciton should preceeed any OOL section\n"); + TR_ASSERT_FATAL(self()->getLastWarmInstruction() != self()->getAppendInstruction(), + "Last warm instruction can't be append instruction since OOL code was attached already\n"); + + if (self()->getLastWarmInstruction() && + self()->getLastWarmInstruction() != firstInstruction->getPrev()) + { + TR::Instruction *appendInstruction; + + // remove from prrevious location + if (firstInstruction->getPrev()) + firstInstruction->getPrev()->setNext(lastInstruction->getNext()); + + if (lastInstruction->getNext()) + lastInstruction->getNext()->setPrev(firstInstruction->getPrev()); + + // update codegen append instruction + if (lastInstruction == self()->getAppendInstruction()) + self()->setAppendInstruction(firstInstruction->getPrev()); + + // insert after last warm instruction + appendInstruction = self()->getLastWarmInstruction(); + appendInstruction->setLastWarmInstruction(false); + lastInstruction->setLastWarmInstruction(true); + self()->setLastWarmInstruction(lastInstruction); + + TR::Instruction *followInstruction = appendInstruction->getNext(); + + appendInstruction->setNext(firstInstruction); + firstInstruction->setPrev(appendInstruction); + + lastInstruction->setNext(followInstruction); + + if (followInstruction) + followInstruction->setPrev(lastInstruction); + } + + ++oiIterator; + } + } diff --git a/compiler/x/codegen/OMRCodeGenerator.hpp b/compiler/x/codegen/OMRCodeGenerator.hpp index 29bbe6b002c..b815c0fc906 100644 --- a/compiler/x/codegen/OMRCodeGenerator.hpp +++ b/compiler/x/codegen/OMRCodeGenerator.hpp @@ -654,6 +654,12 @@ class OMR_EXTENSIBLE CodeGenerator : public OMR::CodeGenerator bool considerTypeForGRA(TR::DataType dt); bool considerTypeForGRA(TR::SymbolReference *symRef); + /* + * \brief move out-of-line instructions from cold code to warm + * + */ + void moveOutOfLineInstructionsToWarm(); + uint32_t getOutOfLineCodeSize(); /* From f93c09fd7d3207a41b1022c9092eba74e0fa53ef Mon Sep 17 00:00:00 2001 From: Gita Koblents Date: Wed, 7 Aug 2024 13:30:58 -0400 Subject: [PATCH 2/2] ool_in_warm_pr: PR comments 1 --- compiler/control/OMROptions.cpp | 1 + compiler/control/OMROptions.hpp | 2 +- compiler/x/codegen/OMRCodeGenerator.cpp | 58 ++++++++++++------------- compiler/x/codegen/OMRCodeGenerator.hpp | 2 +- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/compiler/control/OMROptions.cpp b/compiler/control/OMROptions.cpp index 9370c140d97..e0a086c4db7 100644 --- a/compiler/control/OMROptions.cpp +++ b/compiler/control/OMROptions.cpp @@ -1002,6 +1002,7 @@ TR::OptionTable OMR::Options::_jitOptions[] = { TR::Options::setStaticNumeric, (intptr_t)&OMR::Options::_minProfiledCheckcastFrequency, 0, "F%d", NOT_IN_SUBSET}, {"minSleepTimeMsForCompThrottling=", "M\tLower bound for sleep time during compilation throttling (ms)", TR::Options::setStaticNumeric, (intptr_t)&OMR::Options::_minSleepTimeMsForCompThrottling, 0, "F%d", NOT_IN_SUBSET }, + {"moveOOLInstructionsToWarmCode", "M\tmove out-of-line instructions to after last warm instruction", SET_OPTION_BIT(TR_MoveOOLInstructionsToWarmCode), "F"}, {"noAotSecondRunDetection", "M\tdo not do second run detection for AOT", SET_OPTION_BIT(TR_NoAotSecondRunDetection), "F", NOT_IN_SUBSET }, #ifdef DEBUG {"noExceptions", "C\tfail compilation for methods with exceptions", diff --git a/compiler/control/OMROptions.hpp b/compiler/control/OMROptions.hpp index a124a6c7447..d6c5056c88e 100644 --- a/compiler/control/OMROptions.hpp +++ b/compiler/control/OMROptions.hpp @@ -381,7 +381,7 @@ enum TR_CompilationOptions TR_BreakOnNew = 0x08000000 + 9, TR_DisableInliningUnrecognizedIntrinsics = 0x10000000 + 9, TR_EnableVectorAPIExpansion = 0x20000000 + 9, - // Available = 0x40000000 + 9, + TR_MoveOOLInstructionsToWarmCode = 0x40000000 + 9, // Available = 0x80000000 + 9, // Option word 10 diff --git a/compiler/x/codegen/OMRCodeGenerator.cpp b/compiler/x/codegen/OMRCodeGenerator.cpp index 76d7479a2d3..d7a5499e029 100644 --- a/compiler/x/codegen/OMRCodeGenerator.cpp +++ b/compiler/x/codegen/OMRCodeGenerator.cpp @@ -1868,8 +1868,8 @@ void OMR::X86::CodeGenerator::doRegisterAssignment(TR_RegisterKinds kindsToAssig self()->doBackwardsRegisterAssignment(kindsToAssign, self()->getAppendInstruction()); } - if (TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming)) - moveOutOfLineInstructionsToWarm(); + if (TR::Options::getCmdLineOptions()->getOption(TR_MoveOOLInstructionsToWarmCode)) + moveOutOfLineInstructionsToWarmCode(); } bool OMR::X86::CodeGenerator::isReturnInstruction(TR::Instruction *instr) @@ -3505,58 +3505,58 @@ OMR::X86::CodeGenerator::getOutOfLineCodeSize() } void -OMR::X86::CodeGenerator::moveOutOfLineInstructionsToWarm() +OMR::X86::CodeGenerator::moveOutOfLineInstructionsToWarmCode() { // OOL instructions are already attached at the end of the IL (after cold instructions) // and register allocated. - // Move them to immediately after the last warm instruction, unless they already happend to be + // Move them to immediately after the last warm instruction, unless they already happened to be // there (e.g. there are no cold instructions) // + if (!self()->getLastWarmInstruction()) + return; + if (self()->comp()->getOption(TR_TraceCG)) - traceMsg(self()->comp(), "Moving OutOfLine instructions\n"); + traceMsg(self()->comp(), "Moving OutOfLine instructions to after %p\n", self()->getLastWarmInstruction()); auto oiIterator = self()->getOutlinedInstructionsList().begin(); while (oiIterator != self()->getOutlinedInstructionsList().end()) { - TR::Instruction *firstInstruction = (*oiIterator)->getFirstInstruction(); - TR::Instruction *lastInstruction = (*oiIterator)->getAppendInstruction(); + TR::Instruction *firstOLInstruction = (*oiIterator)->getFirstInstruction(); + TR::Instruction *lastOLInstruction = (*oiIterator)->getAppendInstruction(); - TR_ASSERT_FATAL(firstInstruction, "VFPRestore instruciton should preceeed any OOL section\n"); + TR_ASSERT_FATAL(firstOLInstruction, "VFPRestore instruction should preceeed any OOL section\n"); TR_ASSERT_FATAL(self()->getLastWarmInstruction() != self()->getAppendInstruction(), "Last warm instruction can't be append instruction since OOL code was attached already\n"); - if (self()->getLastWarmInstruction() && - self()->getLastWarmInstruction() != firstInstruction->getPrev()) + if (self()->getLastWarmInstruction() != firstOLInstruction->getPrev()) { - TR::Instruction *appendInstruction; - - // remove from prrevious location - if (firstInstruction->getPrev()) - firstInstruction->getPrev()->setNext(lastInstruction->getNext()); + // remove from previous location + if (firstOLInstruction->getPrev()) + firstOLInstruction->getPrev()->setNext(lastOLInstruction->getNext()); - if (lastInstruction->getNext()) - lastInstruction->getNext()->setPrev(firstInstruction->getPrev()); + if (lastOLInstruction->getNext()) + lastOLInstruction->getNext()->setPrev(firstOLInstruction->getPrev()); // update codegen append instruction - if (lastInstruction == self()->getAppendInstruction()) - self()->setAppendInstruction(firstInstruction->getPrev()); + if (lastOLInstruction == self()->getAppendInstruction()) + self()->setAppendInstruction(firstOLInstruction->getPrev()); // insert after last warm instruction - appendInstruction = self()->getLastWarmInstruction(); - appendInstruction->setLastWarmInstruction(false); - lastInstruction->setLastWarmInstruction(true); - self()->setLastWarmInstruction(lastInstruction); + TR::Instruction *mainlineAppendInstruction = self()->getLastWarmInstruction(); + mainlineAppendInstruction->setLastWarmInstruction(false); + lastOLInstruction->setLastWarmInstruction(true); + self()->setLastWarmInstruction(lastOLInstruction); - TR::Instruction *followInstruction = appendInstruction->getNext(); + TR::Instruction *mainlineFollowInstruction = mainlineAppendInstruction->getNext(); - appendInstruction->setNext(firstInstruction); - firstInstruction->setPrev(appendInstruction); + mainlineAppendInstruction->setNext(firstOLInstruction); + firstOLInstruction->setPrev(mainlineAppendInstruction); - lastInstruction->setNext(followInstruction); + lastOLInstruction->setNext(mainlineFollowInstruction); - if (followInstruction) - followInstruction->setPrev(lastInstruction); + if (mainlineFollowInstruction) + mainlineFollowInstruction->setPrev(lastOLInstruction); } ++oiIterator; diff --git a/compiler/x/codegen/OMRCodeGenerator.hpp b/compiler/x/codegen/OMRCodeGenerator.hpp index b815c0fc906..8a8a803849a 100644 --- a/compiler/x/codegen/OMRCodeGenerator.hpp +++ b/compiler/x/codegen/OMRCodeGenerator.hpp @@ -658,7 +658,7 @@ class OMR_EXTENSIBLE CodeGenerator : public OMR::CodeGenerator * \brief move out-of-line instructions from cold code to warm * */ - void moveOutOfLineInstructionsToWarm(); + void moveOutOfLineInstructionsToWarmCode(); uint32_t getOutOfLineCodeSize();