diff --git a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h b/llvm/include/llvm/CodeGen/RegAllocEvictionAdvisor.h similarity index 71% rename from llvm/lib/CodeGen/RegAllocEvictionAdvisor.h rename to llvm/include/llvm/CodeGen/RegAllocEvictionAdvisor.h index 52dd946a68540..a14548ff6959e 100644 --- a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.h +++ b/llvm/include/llvm/CodeGen/RegAllocEvictionAdvisor.h @@ -9,13 +9,18 @@ #ifndef LLVM_CODEGEN_REGALLOCEVICTIONADVISOR_H #define LLVM_CODEGEN_REGALLOCEVICTIONADVISOR_H +#include "llvm/ADT/Any.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" +#include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/Register.h" #include "llvm/Config/llvm-config.h" +#include "llvm/IR/PassManager.h" #include "llvm/MC/MCRegister.h" #include "llvm/Pass.h" +#include "llvm/Support/Compiler.h" namespace llvm { class AllocationOrder; @@ -149,6 +154,35 @@ class RegAllocEvictionAdvisor { const bool EnableLocalReassign; }; +/// Common provider for legacy and new pass managers. +/// This keeps the state for logging, and sets up and holds the provider. +/// The legacy pass itself used to keep the logging state and provider, +/// so this extraction helps the NPM analysis to reuse the logic. +/// TODO: Coalesce this with the NPM analysis when legacy PM is removed. +class RegAllocEvictionAdvisorProvider { +public: + enum class AdvisorMode : int { Default, Release, Development }; + RegAllocEvictionAdvisorProvider(AdvisorMode Mode, LLVMContext &Ctx) + : Ctx(Ctx), Mode(Mode) {} + + virtual ~RegAllocEvictionAdvisorProvider() = default; + + virtual void logRewardIfNeeded(const MachineFunction &MF, + llvm::function_ref GetReward) {} + + virtual std::unique_ptr + getAdvisor(const MachineFunction &MF, const RAGreedy &RA, + MachineBlockFrequencyInfo *MBFI, MachineLoopInfo *Loops) = 0; + + AdvisorMode getAdvisorMode() const { return Mode; } + +protected: + LLVMContext &Ctx; + +private: + const AdvisorMode Mode; +}; + /// ImmutableAnalysis abstraction for fetching the Eviction Advisor. We model it /// as an analysis to decouple the user from the implementation insofar as /// dependencies on other analyses goes. The motivation for it being an @@ -164,20 +198,20 @@ class RegAllocEvictionAdvisor { /// /// Because we need to offer additional services in 'development' mode, the /// implementations of this analysis need to implement RTTI support. -class RegAllocEvictionAdvisorAnalysis : public ImmutablePass { +class RegAllocEvictionAdvisorAnalysisLegacy : public ImmutablePass { public: enum class AdvisorMode : int { Default, Release, Development }; - RegAllocEvictionAdvisorAnalysis(AdvisorMode Mode) - : ImmutablePass(ID), Mode(Mode){}; + RegAllocEvictionAdvisorAnalysisLegacy(AdvisorMode Mode) + : ImmutablePass(ID), Mode(Mode) {}; static char ID; /// Get an advisor for the given context (i.e. machine function, etc) - virtual std::unique_ptr - getAdvisor(const MachineFunction &MF, const RAGreedy &RA) = 0; + RegAllocEvictionAdvisorProvider &getProvider() { return *Provider; } + AdvisorMode getAdvisorMode() const { return Mode; } virtual void logRewardIfNeeded(const MachineFunction &MF, - llvm::function_ref GetReward){}; + function_ref GetReward) {}; protected: // This analysis preserves everything, and subclasses may have additional @@ -185,19 +219,65 @@ class RegAllocEvictionAdvisorAnalysis : public ImmutablePass { void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); } + std::unique_ptr Provider; private: StringRef getPassName() const override; const AdvisorMode Mode; }; +/// A MachineFunction analysis for fetching the Eviction Advisor. +/// This sets up the Provider lazily and caches it. +/// - in the ML implementation case, the evaluator is stateless but (especially +/// in the development mode) expensive to set up. With a Module Analysis, we +/// `require` it and set it up once. +/// - in the 'development' mode ML case, we want to capture the training log +/// during allocation (this is a log of features encountered and decisions +/// made), and then measure a score, potentially a few steps after allocation +/// completes. So we need a Module analysis to keep the logger state around +/// until we can make that measurement. +class RegAllocEvictionAdvisorAnalysis + : public AnalysisInfoMixin { + static AnalysisKey Key; + friend AnalysisInfoMixin; + +public: + struct Result { + // owned by this analysis + RegAllocEvictionAdvisorProvider *Provider; + + bool invalidate(MachineFunction &MF, const PreservedAnalyses &PA, + MachineFunctionAnalysisManager::Invalidator &Inv) { + // Provider is stateless and constructed only once. Do not get + // invalidated. + return false; + } + }; + + Result run(MachineFunction &MF, MachineFunctionAnalysisManager &MAM); + +private: + void + initializeProvider(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode Mode, + LLVMContext &Ctx); + + std::unique_ptr Provider; +}; + /// Specialization for the API used by the analysis infrastructure to create /// an instance of the eviction advisor. -template <> Pass *callDefaultCtor(); +template <> Pass *callDefaultCtor(); + +RegAllocEvictionAdvisorAnalysisLegacy *createReleaseModeAdvisorAnalysisLegacy(); + +RegAllocEvictionAdvisorAnalysisLegacy * +createDevelopmentModeAdvisorAnalysisLegacy(); -RegAllocEvictionAdvisorAnalysis *createReleaseModeAdvisor(); +LLVM_ATTRIBUTE_RETURNS_NONNULL RegAllocEvictionAdvisorProvider * +createReleaseModeAdvisorProvider(LLVMContext &Ctx); -RegAllocEvictionAdvisorAnalysis *createDevelopmentModeAdvisor(); +RegAllocEvictionAdvisorProvider * +createDevelopmentModeAdvisorProvider(LLVMContext &Ctx); // TODO: move to RegAllocEvictionAdvisor.cpp when we move implementation // out of RegAllocGreedy.cpp diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h index da4ffcd83213a..81a602c8889d8 100644 --- a/llvm/include/llvm/InitializePasses.h +++ b/llvm/include/llvm/InitializePasses.h @@ -251,7 +251,7 @@ void initializePseudoProbeInserterPass(PassRegistry &); void initializeRAGreedyPass(PassRegistry &); void initializeReachingDefAnalysisPass(PassRegistry &); void initializeReassociateLegacyPassPass(PassRegistry &); -void initializeRegAllocEvictionAdvisorAnalysisPass(PassRegistry &); +void initializeRegAllocEvictionAdvisorAnalysisLegacyPass(PassRegistry &); void initializeRegAllocFastPass(PassRegistry &); void initializeRegAllocPriorityAdvisorAnalysisPass(PassRegistry &); void initializeRegAllocScoringPass(PassRegistry &); diff --git a/llvm/include/llvm/Passes/CodeGenPassBuilder.h b/llvm/include/llvm/Passes/CodeGenPassBuilder.h index 1458318ff021a..12781e2b84623 100644 --- a/llvm/include/llvm/Passes/CodeGenPassBuilder.h +++ b/llvm/include/llvm/Passes/CodeGenPassBuilder.h @@ -57,6 +57,7 @@ #include "llvm/CodeGen/PeepholeOptimizer.h" #include "llvm/CodeGen/PostRASchedulerList.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" +#include "llvm/CodeGen/RegAllocEvictionAdvisor.h" #include "llvm/CodeGen/RegAllocFast.h" #include "llvm/CodeGen/RegUsageInfoCollector.h" #include "llvm/CodeGen/RegUsageInfoPropagate.h" diff --git a/llvm/include/llvm/Passes/MachinePassRegistry.def b/llvm/include/llvm/Passes/MachinePassRegistry.def index 075ebcb829553..2b5e258682585 100644 --- a/llvm/include/llvm/Passes/MachinePassRegistry.def +++ b/llvm/include/llvm/Passes/MachinePassRegistry.def @@ -114,6 +114,7 @@ MACHINE_FUNCTION_ANALYSIS("machine-post-dom-tree", MachinePostDominatorTreeAnalysis()) MACHINE_FUNCTION_ANALYSIS("machine-trace-metrics", MachineTraceMetricsAnalysis()) MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis(PIC)) +MACHINE_FUNCTION_ANALYSIS("regalloc-evict", RegAllocEvictionAdvisorAnalysis()) MACHINE_FUNCTION_ANALYSIS("slot-indexes", SlotIndexesAnalysis()) MACHINE_FUNCTION_ANALYSIS("spill-code-placement", SpillPlacementAnalysis()) MACHINE_FUNCTION_ANALYSIS("virtregmap", VirtRegMapAnalysis()) diff --git a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp index 9656774c6eaae..1a8e11de909e8 100644 --- a/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp +++ b/llvm/lib/CodeGen/MLRegAllocEvictAdvisor.cpp @@ -11,11 +11,11 @@ //===----------------------------------------------------------------------===// #include "AllocationOrder.h" -#include "RegAllocEvictionAdvisor.h" #include "RegAllocGreedy.h" #include "llvm/Analysis/InteractiveModelRunner.h" #include "llvm/Analysis/MLModelRunner.h" #include "llvm/Analysis/TensorSpec.h" +#include "llvm/CodeGen/RegAllocEvictionAdvisor.h" #if defined(LLVM_HAVE_TF_AOT_REGALLOCEVICTMODEL) || defined(LLVM_HAVE_TFLITE) #include "llvm/Analysis/ModelUnderTrainingRunner.h" #include "llvm/Analysis/NoInferenceModelRunner.h" @@ -115,7 +115,7 @@ class RegAllocScoring : public MachineFunctionPass { /// RegAllocReward analysis usage. void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); - AU.addRequired(); + AU.addRequired(); AU.addRequired(); AU.addRequired(); MachineFunctionPass::getAnalysisUsage(AU); @@ -389,11 +389,12 @@ class MLEvictAdvisor : public RegAllocEvictionAdvisor { // =================================== // Release (AOT) - specifics // =================================== -class ReleaseModeEvictionAdvisorAnalysis final - : public RegAllocEvictionAdvisorAnalysis { +/// Common provider for legacy and new pass managers. +class ReleaseModeEvictionAdvisorProvider final + : public RegAllocEvictionAdvisorProvider { public: - ReleaseModeEvictionAdvisorAnalysis() - : RegAllocEvictionAdvisorAnalysis(AdvisorMode::Release) { + ReleaseModeEvictionAdvisorProvider(LLVMContext &Ctx) + : RegAllocEvictionAdvisorProvider(AdvisorMode::Release, Ctx) { if (EnableDevelopmentFeatures) { InputFeatures = {RA_EVICT_FEATURES_LIST( _DECL_FEATURES) RA_EVICT_FIRST_DEVELOPMENT_FEATURE(_DECL_FEATURES) @@ -403,21 +404,13 @@ class ReleaseModeEvictionAdvisorAnalysis final } } // support for isa<> and dyn_cast. - static bool classof(const RegAllocEvictionAdvisorAnalysis *R) { + static bool classof(const RegAllocEvictionAdvisorProvider *R) { return R->getAdvisorMode() == AdvisorMode::Release; } -private: - std::vector InputFeatures; - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); - AU.addRequired(); - RegAllocEvictionAdvisorAnalysis::getAnalysisUsage(AU); - } - std::unique_ptr - getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override { + getAdvisor(const MachineFunction &MF, const RAGreedy &RA, + MachineBlockFrequencyInfo *MBFI, MachineLoopInfo *Loops) override { if (!Runner) { if (InteractiveChannelBaseName.empty()) Runner = std::make_unique>( @@ -428,14 +421,45 @@ class ReleaseModeEvictionAdvisorAnalysis final InteractiveChannelBaseName + ".out", InteractiveChannelBaseName + ".in"); } - return std::make_unique( - MF, RA, Runner.get(), - getAnalysis().getMBFI(), - getAnalysis().getLI()); + assert(MBFI && Loops && + "Invalid provider state: must have analysis available"); + return std::make_unique(MF, RA, Runner.get(), *MBFI, + *Loops); } + +private: + std::vector InputFeatures; std::unique_ptr Runner; }; +class ReleaseModeEvictionAdvisorAnalysisLegacy final + : public RegAllocEvictionAdvisorAnalysisLegacy { +public: + ReleaseModeEvictionAdvisorAnalysisLegacy() + : RegAllocEvictionAdvisorAnalysisLegacy(AdvisorMode::Release) {} + + void logRewardIfNeeded(const MachineFunction &MF, + llvm::function_ref GetReward) override { + // No-op in release mode + } + + bool doInitialization(Module &M) override { + Provider = + std::make_unique(M.getContext()); + return false; + } + + static bool classof(const RegAllocEvictionAdvisorAnalysisLegacy *R) { + return R->getAdvisorMode() == AdvisorMode::Release; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); + RegAllocEvictionAdvisorAnalysisLegacy::getAnalysisUsage(AU); + } +}; + // =================================== // Development mode-specifics // =================================== @@ -468,11 +492,11 @@ class DevelopmentModeEvictAdvisor : public MLEvictAdvisor { Logger *const Log; }; -class DevelopmentModeEvictionAdvisorAnalysis final - : public RegAllocEvictionAdvisorAnalysis { +class DevelopmentModeEvictionAdvisorProvider final + : public RegAllocEvictionAdvisorProvider { public: - DevelopmentModeEvictionAdvisorAnalysis() - : RegAllocEvictionAdvisorAnalysis(AdvisorMode::Development) { + DevelopmentModeEvictionAdvisorProvider(LLVMContext &Ctx) + : RegAllocEvictionAdvisorProvider(AdvisorMode::Development, Ctx) { if (EnableDevelopmentFeatures) { InputFeatures = {RA_EVICT_FEATURES_LIST( _DECL_FEATURES) RA_EVICT_FIRST_DEVELOPMENT_FEATURE(_DECL_FEATURES) @@ -492,44 +516,10 @@ class DevelopmentModeEvictionAdvisorAnalysis final TensorSpec::createSpec("action_step_type", {1}), TensorSpec::createSpec("action_reward", {1})}; } - } - // support for isa<> and dyn_cast. - static bool classof(const RegAllocEvictionAdvisorAnalysis *R) { - return R->getAdvisorMode() == AdvisorMode::Development; - } - - void logRewardIfNeeded(const MachineFunction &MF, - llvm::function_ref GetReward) override { - if (!Log || !Log->hasAnyObservationForContext(MF.getName())) - return; - // The function pass manager would run all the function passes for a - // function, so we assume the last context belongs to this function. If - // this invariant ever changes, we can implement at that time switching - // contexts. At this point, it'd be an error - if (Log->currentContext() != MF.getName()) { - MF.getFunction().getContext().emitError( - "The training log context shouldn't have had changed."); - } - if (Log->hasObservationInProgress()) - Log->logReward(GetReward()); - } - -private: - std::vector InputFeatures; - std::vector TrainingInputFeatures; - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); - AU.addRequired(); - RegAllocEvictionAdvisorAnalysis::getAnalysisUsage(AU); - } - - bool doInitialization(Module &M) override { - LLVMContext &Ctx = M.getContext(); if (ModelUnderTraining.empty() && TrainingLog.empty()) { Ctx.emitError("Regalloc development mode should be requested with at " "least logging enabled and/or a training model"); - return false; + return; } if (ModelUnderTraining.empty()) Runner = std::make_unique(Ctx, InputFeatures); @@ -538,15 +528,15 @@ class DevelopmentModeEvictionAdvisorAnalysis final Ctx, ModelUnderTraining, DecisionName, TrainingInputFeatures); if (!Runner) { Ctx.emitError("Regalloc: could not set up the model runner"); - return false; + return; } if (TrainingLog.empty()) - return false; + return; std::error_code EC; auto OS = std::make_unique(TrainingLog, EC); if (EC) { - M.getContext().emitError(EC.message() + ":" + TrainingLog); - return false; + Ctx.emitError(EC.message() + ":" + TrainingLog); + return; } std::vector LFS = InputFeatures; if (auto *MUTR = dyn_cast(Runner.get())) @@ -558,25 +548,80 @@ class DevelopmentModeEvictionAdvisorAnalysis final Log = std::make_unique(std::move(OS), LFS, Reward, /*IncludeReward*/ true); - return false; + return; + } + + // support for isa<> and dyn_cast. + static bool classof(const RegAllocEvictionAdvisorProvider *R) { + return R->getAdvisorMode() == AdvisorMode::Development; + } + + void logRewardIfNeeded(const MachineFunction &MF, + llvm::function_ref GetReward) override { + if (!Log || !Log->hasAnyObservationForContext(MF.getName())) + return; + // The function pass manager would run all the function passes for a + // function, so we assume the last context belongs to this function. If + // this invariant ever changes, we can implement at that time switching + // contexts. At this point, it'd be an error + if (Log->currentContext() != MF.getName()) { + MF.getFunction().getContext().emitError( + "The training log context shouldn't have had changed."); + } + if (Log->hasObservationInProgress()) + Log->logReward(GetReward()); } std::unique_ptr - getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override { + getAdvisor(const MachineFunction &MF, const RAGreedy &RA, + MachineBlockFrequencyInfo *MBFI, MachineLoopInfo *Loops) override { if (!Runner) return nullptr; if (Log) Log->switchContext(MF.getName()); + assert(MBFI && Loops && + "Invalid provider state: must have analysis available"); return std::make_unique( - MF, RA, Runner.get(), - getAnalysis().getMBFI(), - getAnalysis().getLI(), Log.get()); + MF, RA, Runner.get(), *MBFI, *Loops, Log.get()); } +private: + std::vector InputFeatures; + std::vector TrainingInputFeatures; + std::unique_ptr Runner; std::unique_ptr Log; }; +class DevelopmentModeEvictionAdvisorAnalysisLegacy final + : public RegAllocEvictionAdvisorAnalysisLegacy { +public: + DevelopmentModeEvictionAdvisorAnalysisLegacy() + : RegAllocEvictionAdvisorAnalysisLegacy(AdvisorMode::Development) {} + + bool doInitialization(Module &M) override { + Provider = std::make_unique( + M.getContext()); + return false; + } + + void logRewardIfNeeded(const MachineFunction &MF, + llvm::function_ref GetReward) override { + Provider->logRewardIfNeeded(MF, GetReward); + } + + // support for isa<> and dyn_cast. + static bool classof(const RegAllocEvictionAdvisorAnalysisLegacy *R) { + return R->getAdvisorMode() == AdvisorMode::Development; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addRequired(); + RegAllocEvictionAdvisorAnalysisLegacy::getAnalysisUsage(AU); + } +}; + #endif // #ifdef LLVM_HAVE_TFLITE } // namespace @@ -1127,8 +1172,9 @@ void llvm::extractMBBFrequency( // Development mode-specific implementations #ifdef LLVM_HAVE_TFLITE -RegAllocEvictionAdvisorAnalysis *llvm::createDevelopmentModeAdvisor() { - return new DevelopmentModeEvictionAdvisorAnalysis(); +RegAllocEvictionAdvisorAnalysisLegacy * +llvm::createDevelopmentModeAdvisorAnalysisLegacy() { + return new DevelopmentModeEvictionAdvisorAnalysisLegacy(); } int64_t DevelopmentModeEvictAdvisor::tryFindEvictionCandidatePosition( @@ -1194,18 +1240,32 @@ bool RegAllocScoring::runOnMachineFunction(MachineFunction &MF) { return *CachedReward; }; - getAnalysis().logRewardIfNeeded(MF, - GetReward); + getAnalysis().logRewardIfNeeded( + MF, GetReward); getAnalysis().logRewardIfNeeded(MF, GetReward); return false; } #endif // #ifdef LLVM_HAVE_TFLITE -RegAllocEvictionAdvisorAnalysis *llvm::createReleaseModeAdvisor() { +RegAllocEvictionAdvisorProvider * +llvm::createReleaseModeAdvisorProvider(LLVMContext &Ctx) { + return new ReleaseModeEvictionAdvisorProvider(Ctx); +} + +RegAllocEvictionAdvisorProvider * +llvm::createDevelopmentModeAdvisorProvider(LLVMContext &Ctx) { +#if defined(LLVM_HAVE_TFLITE) + return new DevelopmentModeEvictionAdvisorProvider(Ctx); +#endif + return nullptr; +} + +RegAllocEvictionAdvisorAnalysisLegacy * +llvm::createReleaseModeAdvisorAnalysisLegacy() { return llvm::isEmbeddedModelEvaluatorValid() || !InteractiveChannelBaseName.empty() - ? new ReleaseModeEvictionAdvisorAnalysis() + ? new ReleaseModeEvictionAdvisorAnalysisLegacy() : nullptr; } diff --git a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp index a1f441ebd0d5e..2369615ef0fb6 100644 --- a/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp +++ b/llvm/lib/CodeGen/RegAllocEvictionAdvisor.cpp @@ -9,12 +9,14 @@ // Implementation of the default eviction advisor and of the Analysis pass. // //===----------------------------------------------------------------------===// - -#include "RegAllocEvictionAdvisor.h" +#include "llvm/CodeGen/RegAllocEvictionAdvisor.h" #include "AllocationOrder.h" #include "RegAllocGreedy.h" +#include "RegAllocPriorityAdvisor.h" #include "llvm/CodeGen/LiveRegMatrix.h" +#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" #include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/VirtRegMap.h" #include "llvm/IR/Module.h" @@ -26,17 +28,18 @@ using namespace llvm; -static cl::opt Mode( +static cl::opt Mode( "regalloc-enable-advisor", cl::Hidden, - cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default), + cl::init(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values( - clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default, + clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default, "default", "Default"), - clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release, + clEnumValN(RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release, "release", "precompiled"), - clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development, - "development", "for training"))); + clEnumValN( + RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development, + "development", "for training"))); static cl::opt EnableLocalReassignment( "enable-local-reassign", cl::Hidden, @@ -59,59 +62,112 @@ cl::opt EvictInterferenceCutoff( #define LLVM_HAVE_TF_AOT #endif -char RegAllocEvictionAdvisorAnalysis::ID = 0; -INITIALIZE_PASS(RegAllocEvictionAdvisorAnalysis, "regalloc-evict", +char RegAllocEvictionAdvisorAnalysisLegacy::ID = 0; +INITIALIZE_PASS(RegAllocEvictionAdvisorAnalysisLegacy, "regalloc-evict", "Regalloc eviction policy", false, true) namespace { -class DefaultEvictionAdvisorAnalysis final - : public RegAllocEvictionAdvisorAnalysis { +class DefaultEvictionAdvisorProvider final + : public RegAllocEvictionAdvisorProvider { public: - DefaultEvictionAdvisorAnalysis(bool NotAsRequested) - : RegAllocEvictionAdvisorAnalysis(AdvisorMode::Default), - NotAsRequested(NotAsRequested) {} + DefaultEvictionAdvisorProvider(bool NotAsRequested, LLVMContext &Ctx) + : RegAllocEvictionAdvisorProvider(AdvisorMode::Default, Ctx) { + if (NotAsRequested) + Ctx.emitError("Requested regalloc eviction advisor analysis " + "could not be created. Using default"); + } // support for isa<> and dyn_cast. - static bool classof(const RegAllocEvictionAdvisorAnalysis *R) { + static bool classof(const RegAllocEvictionAdvisorProvider *R) { return R->getAdvisorMode() == AdvisorMode::Default; } -private: std::unique_ptr - getAdvisor(const MachineFunction &MF, const RAGreedy &RA) override { + getAdvisor(const MachineFunction &MF, const RAGreedy &RA, + MachineBlockFrequencyInfo *, MachineLoopInfo *) override { return std::make_unique(MF, RA); } +}; + +class DefaultEvictionAdvisorAnalysisLegacy final + : public RegAllocEvictionAdvisorAnalysisLegacy { +public: + DefaultEvictionAdvisorAnalysisLegacy(bool NotAsRequested) + : RegAllocEvictionAdvisorAnalysisLegacy(AdvisorMode::Default), + NotAsRequested(NotAsRequested) {} + bool doInitialization(Module &M) override { - if (NotAsRequested) - M.getContext().emitError("Requested regalloc eviction advisor analysis " - "could not be created. Using default"); - return RegAllocEvictionAdvisorAnalysis::doInitialization(M); + Provider.reset( + new DefaultEvictionAdvisorProvider(NotAsRequested, M.getContext())); + return false; + } + + // support for isa<> and dyn_cast. + static bool classof(const RegAllocEvictionAdvisorAnalysisLegacy *R) { + return R->getAdvisorMode() == AdvisorMode::Default; } + +private: const bool NotAsRequested; }; } // namespace -template <> Pass *llvm::callDefaultCtor() { - Pass *Ret = nullptr; +AnalysisKey RegAllocEvictionAdvisorAnalysis::Key; + +void RegAllocEvictionAdvisorAnalysis::initializeProvider( + RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode Mode, LLVMContext &Ctx) { + if (Provider) + return; + switch (Mode) { + case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default: + Provider.reset( + new DefaultEvictionAdvisorProvider(/*NotAsRequested=*/false, Ctx)); + return; + case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development: +#if defined(LLVM_HAVE_TFLITE) + Provider.reset(createDevelopmentModeAdvisorProvider(Ctx)); +#else + Provider.reset( + new DefaultEvictionAdvisorProvider(/*NotAsRequested=*/true, Ctx)); +#endif + return; + case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release: + Provider.reset(createReleaseModeAdvisorProvider(Ctx)); + return; + } +} + +RegAllocEvictionAdvisorAnalysis::Result +RegAllocEvictionAdvisorAnalysis::run(MachineFunction &MF, + MachineFunctionAnalysisManager &MFAM) { + // Lazy initialization of the provider. + initializeProvider(::Mode, MF.getFunction().getContext()); + return Result{Provider.get()}; +} + +template <> +Pass *llvm::callDefaultCtor() { switch (Mode) { - case RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default: - Ret = new DefaultEvictionAdvisorAnalysis(/*NotAsRequested*/ false); - break; - case RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development: + case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Default: + return new DefaultEvictionAdvisorAnalysisLegacy(/*NotAsRequested=*/false); + case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Release: { + Pass *Ret = createReleaseModeAdvisorAnalysisLegacy(); + // release mode advisor may not be supported + if (Ret) + return Ret; + return new DefaultEvictionAdvisorAnalysisLegacy(/*NotAsRequested=*/true); + } + case RegAllocEvictionAdvisorAnalysisLegacy::AdvisorMode::Development: #if defined(LLVM_HAVE_TFLITE) - Ret = createDevelopmentModeAdvisor(); + return createDevelopmentModeAdvisorAnalysisLegacy(); +#else + return new DefaultEvictionAdvisorAnalysisLegacy(/*NotAsRequested=*/true); #endif - break; - case RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release: - Ret = createReleaseModeAdvisor(); - break; } - if (Ret) - return Ret; - return new DefaultEvictionAdvisorAnalysis(/*NotAsRequested*/ true); + llvm_unreachable("unexpected advisor mode"); } -StringRef RegAllocEvictionAdvisorAnalysis::getPassName() const { +StringRef RegAllocEvictionAdvisorAnalysisLegacy::getPassName() const { switch (getAdvisorMode()) { case AdvisorMode::Default: return "Default Regalloc Eviction Advisor"; diff --git a/llvm/lib/CodeGen/RegAllocGreedy.cpp b/llvm/lib/CodeGen/RegAllocGreedy.cpp index 7c7eb2ad52b41..9318c1df0b5e2 100644 --- a/llvm/lib/CodeGen/RegAllocGreedy.cpp +++ b/llvm/lib/CodeGen/RegAllocGreedy.cpp @@ -15,7 +15,6 @@ #include "AllocationOrder.h" #include "InterferenceCache.h" #include "RegAllocBase.h" -#include "RegAllocEvictionAdvisor.h" #include "RegAllocPriorityAdvisor.h" #include "SplitKit.h" #include "llvm/ADT/ArrayRef.h" @@ -46,6 +45,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RegAllocEvictionAdvisor.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/CodeGen/SlotIndexes.h" @@ -164,7 +164,7 @@ INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy) INITIALIZE_PASS_DEPENDENCY(EdgeBundlesWrapperLegacy) INITIALIZE_PASS_DEPENDENCY(SpillPlacementWrapperLegacy) INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass) -INITIALIZE_PASS_DEPENDENCY(RegAllocEvictionAdvisorAnalysis) +INITIALIZE_PASS_DEPENDENCY(RegAllocEvictionAdvisorAnalysisLegacy) INITIALIZE_PASS_DEPENDENCY(RegAllocPriorityAdvisorAnalysis) INITIALIZE_PASS_END(RAGreedy, "greedy", "Greedy Register Allocator", false, false) @@ -219,7 +219,7 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); AU.addRequired(); AU.addRequired(); - AU.addRequired(); + AU.addRequired(); AU.addRequired(); MachineFunctionPass::getAnalysisUsage(AU); } @@ -2765,8 +2765,11 @@ bool RAGreedy::runOnMachineFunction(MachineFunction &mf) { : TRI->reverseLocalAssignment(); ExtraInfo.emplace(); - EvictAdvisor = - getAnalysis().getAdvisor(*MF, *this); + + auto &EvictAdvisorProvider = + getAnalysis().getProvider(); + EvictAdvisor = EvictAdvisorProvider.getAdvisor(*MF, *this, MBFI, Loops); + PriorityAdvisor = getAnalysis().getAdvisor(*MF, *this); diff --git a/llvm/lib/CodeGen/RegAllocGreedy.h b/llvm/lib/CodeGen/RegAllocGreedy.h index e1ec63b4a5296..1d55a8241d760 100644 --- a/llvm/lib/CodeGen/RegAllocGreedy.h +++ b/llvm/lib/CodeGen/RegAllocGreedy.h @@ -14,7 +14,6 @@ #include "InterferenceCache.h" #include "RegAllocBase.h" -#include "RegAllocEvictionAdvisor.h" #include "RegAllocPriorityAdvisor.h" #include "SplitKit.h" #include "llvm/ADT/ArrayRef.h" diff --git a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h index 32e4598b71539..0758743c2b140 100644 --- a/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h +++ b/llvm/lib/CodeGen/RegAllocPriorityAdvisor.h @@ -9,7 +9,7 @@ #ifndef LLVM_CODEGEN_REGALLOCPRIORITYADVISOR_H #define LLVM_CODEGEN_REGALLOCPRIORITYADVISOR_H -#include "RegAllocEvictionAdvisor.h" +#include "llvm/CodeGen/RegAllocEvictionAdvisor.h" #include "llvm/CodeGen/SlotIndexes.h" #include "llvm/Pass.h" diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 8d5c0b3c13e01..96939f89279c6 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -127,6 +127,7 @@ #include "llvm/CodeGen/PeepholeOptimizer.h" #include "llvm/CodeGen/PostRASchedulerList.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" +#include "llvm/CodeGen/RegAllocEvictionAdvisor.h" #include "llvm/CodeGen/RegAllocFast.h" #include "llvm/CodeGen/RegUsageInfoCollector.h" #include "llvm/CodeGen/RegUsageInfoPropagate.h"