vasilito
cb424d7448
verify-patch-sanity.py validates every active recipe .patch has internally- consistent hunk line counts — catching the 'malformed patch at line N' failure at commit/CI/preflight time instead of hours into a cook. This cycle hit that class three times (qtwaylandscanner, sddm, xwayland), each only discovered when cookbook tried to apply the patch. Running it across the repo found 29 latent malformed patches (validated against GNU patch: e.g. relibc/P3-sysv-ipc reproduces 'malformed patch at line 22'). They were harmless only because they sit in vendored recipes (baked, not re- applied) — but would fail on any version-bump re-derivation. --fix recounts the hunk headers (body untouched) and repaired all 29. Wired into build-preflight.sh (Phase 1.0D) and redbear-ci.yml, with a unit test (test-patch-sanity.sh). Skips archived/legacy trees and unvalidatable formats (empty placeholders, bare-@@ git hunks).
396 lines
15 KiB
C++
396 lines
15 KiB
C++
//===- bolt/Passes/FrameOptimizer.cpp -------------------------------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file implements the FrameOptimizerPass class.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "bolt/Passes/FrameOptimizer.h"
|
|
#include "bolt/Core/BinaryFunctionCallGraph.h"
|
|
#include "bolt/Core/ParallelUtilities.h"
|
|
#include "bolt/Passes/DataflowInfoManager.h"
|
|
#include "bolt/Passes/ShrinkWrapping.h"
|
|
#include "bolt/Passes/StackAvailableExpressions.h"
|
|
#include "bolt/Passes/StackReachingUses.h"
|
|
#include "bolt/Utils/CommandLineOpts.h"
|
|
#include "llvm/Support/Timer.h"
|
|
#include <deque>
|
|
|
|
#define DEBUG_TYPE "fop"
|
|
|
|
using namespace llvm;
|
|
|
|
namespace opts {
|
|
extern cl::opt<unsigned> Verbosity;
|
|
extern cl::opt<bool> TimeOpts;
|
|
extern cl::OptionCategory BoltOptCategory;
|
|
|
|
using namespace bolt;
|
|
|
|
cl::opt<FrameOptimizationType>
|
|
FrameOptimization("frame-opt",
|
|
cl::init(FOP_NONE),
|
|
cl::desc("optimize stack frame accesses"),
|
|
cl::values(
|
|
clEnumValN(FOP_NONE, "none", "do not perform frame optimization"),
|
|
clEnumValN(FOP_HOT, "hot", "perform FOP on hot functions"),
|
|
clEnumValN(FOP_ALL, "all", "perform FOP on all functions")),
|
|
cl::ZeroOrMore,
|
|
cl::cat(BoltOptCategory));
|
|
|
|
static cl::opt<bool> RemoveStores(
|
|
"frame-opt-rm-stores", cl::init(FOP_NONE),
|
|
cl::desc("apply additional analysis to remove stores (experimental)"),
|
|
cl::cat(BoltOptCategory));
|
|
|
|
} // namespace opts
|
|
|
|
namespace llvm {
|
|
namespace bolt {
|
|
|
|
void FrameOptimizerPass::removeUnnecessaryLoads(const RegAnalysis &RA,
|
|
const FrameAnalysis &FA,
|
|
BinaryFunction &BF) {
|
|
StackAvailableExpressions SAE(RA, FA, BF);
|
|
SAE.run();
|
|
|
|
LLVM_DEBUG(dbgs() << "Performing unnecessary loads removal\n");
|
|
std::deque<std::pair<BinaryBasicBlock *, MCInst *>> ToErase;
|
|
bool Changed = false;
|
|
const auto ExprEnd = SAE.expr_end();
|
|
MCPlusBuilder *MIB = BF.getBinaryContext().MIB.get();
|
|
for (BinaryBasicBlock &BB : BF) {
|
|
LLVM_DEBUG(dbgs() << "\tNow at BB " << BB.getName() << "\n");
|
|
const MCInst *Prev = nullptr;
|
|
for (MCInst &Inst : BB) {
|
|
LLVM_DEBUG({
|
|
dbgs() << "\t\tNow at ";
|
|
Inst.dump();
|
|
for (auto I = Prev ? SAE.expr_begin(*Prev) : SAE.expr_begin(BB);
|
|
I != ExprEnd; ++I) {
|
|
dbgs() << "\t\t\tReached by: ";
|
|
(*I)->dump();
|
|
}
|
|
});
|
|
// if Inst is a load from stack and the current available expressions show
|
|
// this value is available in a register or immediate, replace this load
|
|
// with move from register or from immediate.
|
|
ErrorOr<const FrameIndexEntry &> FIEX = FA.getFIEFor(Inst);
|
|
if (!FIEX) {
|
|
Prev = &Inst;
|
|
continue;
|
|
}
|
|
// FIXME: Change to remove IsSimple == 0. We're being conservative here,
|
|
// but once replaceMemOperandWithReg is ready, we should feed it with all
|
|
// sorts of complex instructions.
|
|
if (FIEX->IsLoad == false || FIEX->IsSimple == false ||
|
|
FIEX->StackOffset >= 0) {
|
|
Prev = &Inst;
|
|
continue;
|
|
}
|
|
|
|
for (auto I = Prev ? SAE.expr_begin(*Prev) : SAE.expr_begin(BB);
|
|
I != ExprEnd; ++I) {
|
|
const MCInst *AvailableInst = *I;
|
|
ErrorOr<const FrameIndexEntry &> FIEY = FA.getFIEFor(*AvailableInst);
|
|
if (!FIEY)
|
|
continue;
|
|
assert(FIEY->IsStore && FIEY->IsSimple);
|
|
if (FIEX->StackOffset != FIEY->StackOffset || FIEX->Size != FIEY->Size)
|
|
continue;
|
|
// TODO: Change push/pops to stack adjustment instruction
|
|
if (MIB->isPop(Inst))
|
|
continue;
|
|
|
|
++NumRedundantLoads;
|
|
FreqRedundantLoads += BB.getKnownExecutionCount();
|
|
Changed = true;
|
|
LLVM_DEBUG(dbgs() << "Redundant load instruction: ");
|
|
LLVM_DEBUG(Inst.dump());
|
|
LLVM_DEBUG(dbgs() << "Related store instruction: ");
|
|
LLVM_DEBUG(AvailableInst->dump());
|
|
LLVM_DEBUG(dbgs() << "@BB: " << BB.getName() << "\n");
|
|
// Replace load
|
|
if (FIEY->IsStoreFromReg) {
|
|
if (!MIB->replaceMemOperandWithReg(Inst, FIEY->RegOrImm)) {
|
|
LLVM_DEBUG(dbgs() << "FAILED to change operand to a reg\n");
|
|
break;
|
|
}
|
|
FreqLoadsChangedToReg += BB.getKnownExecutionCount();
|
|
MIB->removeAnnotation(Inst, "FrameAccessEntry");
|
|
LLVM_DEBUG(dbgs() << "Changed operand to a reg\n");
|
|
if (MIB->isRedundantMove(Inst)) {
|
|
++NumLoadsDeleted;
|
|
FreqLoadsDeleted += BB.getKnownExecutionCount();
|
|
LLVM_DEBUG(dbgs() << "Created a redundant move\n");
|
|
// Delete it!
|
|
ToErase.push_front(std::make_pair(&BB, &Inst));
|
|
}
|
|
} else {
|
|
char Buf[8] = {0, 0, 0, 0, 0, 0, 0, 0};
|
|
support::ulittle64_t::ref(Buf + 0) = FIEY->RegOrImm;
|
|
LLVM_DEBUG(dbgs() << "Changing operand to an imm... ");
|
|
if (!MIB->replaceMemOperandWithImm(Inst, StringRef(Buf, 8), 0)) {
|
|
LLVM_DEBUG(dbgs() << "FAILED\n");
|
|
} else {
|
|
FreqLoadsChangedToImm += BB.getKnownExecutionCount();
|
|
MIB->removeAnnotation(Inst, "FrameAccessEntry");
|
|
LLVM_DEBUG(dbgs() << "Ok\n");
|
|
}
|
|
}
|
|
LLVM_DEBUG(dbgs() << "Changed to: ");
|
|
LLVM_DEBUG(Inst.dump());
|
|
break;
|
|
}
|
|
Prev = &Inst;
|
|
}
|
|
}
|
|
if (Changed)
|
|
LLVM_DEBUG(dbgs() << "FOP modified \"" << BF.getPrintName() << "\"\n");
|
|
|
|
// TODO: Implement an interface of eraseInstruction that works out the
|
|
// complete list of elements to remove.
|
|
for (std::pair<BinaryBasicBlock *, MCInst *> I : ToErase)
|
|
I.first->eraseInstruction(I.first->findInstruction(I.second));
|
|
}
|
|
|
|
void FrameOptimizerPass::removeUnusedStores(const FrameAnalysis &FA,
|
|
BinaryFunction &BF) {
|
|
StackReachingUses SRU(FA, BF);
|
|
SRU.run();
|
|
|
|
LLVM_DEBUG(dbgs() << "Performing unused stores removal\n");
|
|
std::vector<std::pair<BinaryBasicBlock *, MCInst *>> ToErase;
|
|
bool Changed = false;
|
|
for (BinaryBasicBlock &BB : BF) {
|
|
LLVM_DEBUG(dbgs() << "\tNow at BB " << BB.getName() << "\n");
|
|
const MCInst *Prev = nullptr;
|
|
for (MCInst &Inst : llvm::reverse(BB)) {
|
|
LLVM_DEBUG({
|
|
dbgs() << "\t\tNow at ";
|
|
Inst.dump();
|
|
for (auto I = Prev ? SRU.expr_begin(*Prev) : SRU.expr_begin(BB);
|
|
I != SRU.expr_end(); ++I) {
|
|
dbgs() << "\t\t\tReached by: ";
|
|
(*I)->dump();
|
|
}
|
|
});
|
|
ErrorOr<const FrameIndexEntry &> FIEX = FA.getFIEFor(Inst);
|
|
if (!FIEX) {
|
|
Prev = &Inst;
|
|
continue;
|
|
}
|
|
if (FIEX->IsLoad || !FIEX->IsSimple || FIEX->StackOffset >= 0) {
|
|
Prev = &Inst;
|
|
continue;
|
|
}
|
|
|
|
if (SRU.isStoreUsed(*FIEX,
|
|
Prev ? SRU.expr_begin(*Prev) : SRU.expr_begin(BB))) {
|
|
Prev = &Inst;
|
|
continue;
|
|
}
|
|
// TODO: Change push/pops to stack adjustment instruction
|
|
if (BF.getBinaryContext().MIB->isPush(Inst))
|
|
continue;
|
|
|
|
++NumRedundantStores;
|
|
FreqRedundantStores += BB.getKnownExecutionCount();
|
|
Changed = true;
|
|
LLVM_DEBUG(dbgs() << "Unused store instruction: ");
|
|
LLVM_DEBUG(Inst.dump());
|
|
LLVM_DEBUG(dbgs() << "@BB: " << BB.getName() << "\n");
|
|
LLVM_DEBUG(dbgs() << "FIE offset = " << FIEX->StackOffset
|
|
<< " size = " << (int)FIEX->Size << "\n");
|
|
// Delete it!
|
|
ToErase.emplace_back(&BB, &Inst);
|
|
Prev = &Inst;
|
|
}
|
|
}
|
|
|
|
for (std::pair<BinaryBasicBlock *, MCInst *> I : ToErase)
|
|
I.first->eraseInstruction(I.first->findInstruction(I.second));
|
|
|
|
if (Changed)
|
|
LLVM_DEBUG(dbgs() << "FOP modified \"" << BF.getPrintName() << "\"\n");
|
|
}
|
|
|
|
Error FrameOptimizerPass::runOnFunctions(BinaryContext &BC) {
|
|
if (opts::FrameOptimization == FOP_NONE)
|
|
return Error::success();
|
|
|
|
std::unique_ptr<BinaryFunctionCallGraph> CG;
|
|
std::unique_ptr<FrameAnalysis> FA;
|
|
std::unique_ptr<RegAnalysis> RA;
|
|
|
|
{
|
|
NamedRegionTimer T1("callgraph", "create call graph", "FOP",
|
|
"FOP breakdown", opts::TimeOpts);
|
|
CG = std::make_unique<BinaryFunctionCallGraph>(buildCallGraph(BC));
|
|
}
|
|
|
|
{
|
|
NamedRegionTimer T1("frameanalysis", "frame analysis", "FOP",
|
|
"FOP breakdown", opts::TimeOpts);
|
|
FA = std::make_unique<FrameAnalysis>(BC, *CG);
|
|
}
|
|
|
|
{
|
|
NamedRegionTimer T1("reganalysis", "reg analysis", "FOP", "FOP breakdown",
|
|
opts::TimeOpts);
|
|
RA = std::make_unique<RegAnalysis>(BC, &BC.getBinaryFunctions(), CG.get());
|
|
}
|
|
|
|
// Perform caller-saved register optimizations, then callee-saved register
|
|
// optimizations (shrink wrapping)
|
|
for (auto &I : BC.getBinaryFunctions()) {
|
|
if (!FA->hasFrameInfo(I.second))
|
|
continue;
|
|
// Restrict pass execution if user asked to only run on hot functions
|
|
if (opts::FrameOptimization == FOP_HOT) {
|
|
if (I.second.getKnownExecutionCount() < BC.getHotThreshold())
|
|
continue;
|
|
LLVM_DEBUG(
|
|
dbgs() << "Considering " << I.second.getPrintName()
|
|
<< " for frame optimizations because its execution count ( "
|
|
<< I.second.getKnownExecutionCount()
|
|
<< " ) exceeds our hotness threshold ( "
|
|
<< BC.getHotThreshold() << " )\n");
|
|
}
|
|
|
|
{
|
|
NamedRegionTimer T1("removeloads", "remove loads", "FOP", "FOP breakdown",
|
|
opts::TimeOpts);
|
|
if (!FA->hasStackArithmetic(I.second))
|
|
removeUnnecessaryLoads(*RA, *FA, I.second);
|
|
}
|
|
|
|
if (opts::RemoveStores) {
|
|
NamedRegionTimer T1("removestores", "remove stores", "FOP",
|
|
"FOP breakdown", opts::TimeOpts);
|
|
if (!FA->hasStackArithmetic(I.second))
|
|
removeUnusedStores(*FA, I.second);
|
|
}
|
|
// Don't even start shrink wrapping if no profiling info is available
|
|
if (I.second.getKnownExecutionCount() == 0)
|
|
continue;
|
|
}
|
|
|
|
{
|
|
NamedRegionTimer T1("shrinkwrapping", "shrink wrapping", "FOP",
|
|
"FOP breakdown", opts::TimeOpts);
|
|
if (Error E = performShrinkWrapping(*RA, *FA, BC))
|
|
return Error(std::move(E));
|
|
}
|
|
|
|
BC.outs() << "BOLT-INFO: FOP optimized " << NumRedundantLoads
|
|
<< " redundant load(s) and " << NumRedundantStores
|
|
<< " unused store(s)\n";
|
|
BC.outs() << "BOLT-INFO: Frequency of redundant loads is "
|
|
<< FreqRedundantLoads << " and frequency of unused stores is "
|
|
<< FreqRedundantStores << "\n";
|
|
BC.outs() << "BOLT-INFO: Frequency of loads changed to use a register is "
|
|
<< FreqLoadsChangedToReg
|
|
<< " and frequency of loads changed to use an immediate is "
|
|
<< FreqLoadsChangedToImm << "\n";
|
|
BC.outs() << "BOLT-INFO: FOP deleted " << NumLoadsDeleted
|
|
<< " load(s) (dyn count: " << FreqLoadsDeleted << ") and "
|
|
<< NumRedundantStores << " store(s)\n";
|
|
FA->printStats();
|
|
ShrinkWrapping::printStats(BC);
|
|
return Error::success();
|
|
}
|
|
|
|
Error FrameOptimizerPass::performShrinkWrapping(const RegAnalysis &RA,
|
|
const FrameAnalysis &FA,
|
|
BinaryContext &BC) {
|
|
// Initialize necessary annotations to allow safe parallel accesses to
|
|
// annotation index in MIB
|
|
BC.MIB->getOrCreateAnnotationIndex(CalleeSavedAnalysis::getSaveTagName());
|
|
BC.MIB->getOrCreateAnnotationIndex(CalleeSavedAnalysis::getRestoreTagName());
|
|
BC.MIB->getOrCreateAnnotationIndex(StackLayoutModifier::getTodoTagName());
|
|
BC.MIB->getOrCreateAnnotationIndex(StackLayoutModifier::getSlotTagName());
|
|
BC.MIB->getOrCreateAnnotationIndex(
|
|
StackLayoutModifier::getOffsetCFIRegTagName());
|
|
BC.MIB->getOrCreateAnnotationIndex("ReachingDefs");
|
|
BC.MIB->getOrCreateAnnotationIndex("ReachingUses");
|
|
BC.MIB->getOrCreateAnnotationIndex("LivenessAnalysis");
|
|
BC.MIB->getOrCreateAnnotationIndex("StackReachingUses");
|
|
BC.MIB->getOrCreateAnnotationIndex("PostDominatorAnalysis");
|
|
BC.MIB->getOrCreateAnnotationIndex("DominatorAnalysis");
|
|
BC.MIB->getOrCreateAnnotationIndex("StackPointerTracking");
|
|
BC.MIB->getOrCreateAnnotationIndex("StackPointerTrackingForInternalCalls");
|
|
BC.MIB->getOrCreateAnnotationIndex("StackAvailableExpressions");
|
|
BC.MIB->getOrCreateAnnotationIndex("StackAllocationAnalysis");
|
|
BC.MIB->getOrCreateAnnotationIndex("ShrinkWrap-Todo");
|
|
BC.MIB->getOrCreateAnnotationIndex("PredictiveStackPointerTracking");
|
|
BC.MIB->getOrCreateAnnotationIndex("ReachingInsnsBackward");
|
|
BC.MIB->getOrCreateAnnotationIndex("ReachingInsns");
|
|
BC.MIB->getOrCreateAnnotationIndex("AccessesDeletedPos");
|
|
BC.MIB->getOrCreateAnnotationIndex("DeleteMe");
|
|
|
|
std::vector<std::pair<uint64_t, const BinaryFunction *>> Top10Funcs;
|
|
auto LogFunc = [&](BinaryFunction &BF) {
|
|
auto Lower = llvm::lower_bound(
|
|
Top10Funcs, BF.getKnownExecutionCount(),
|
|
[](const std::pair<uint64_t, const BinaryFunction *> &Elmt,
|
|
uint64_t Value) { return Elmt.first > Value; });
|
|
if (Lower == Top10Funcs.end() && Top10Funcs.size() >= 10)
|
|
return;
|
|
Top10Funcs.insert(Lower,
|
|
std::make_pair<>(BF.getKnownExecutionCount(), &BF));
|
|
if (Top10Funcs.size() > 10)
|
|
Top10Funcs.resize(10);
|
|
};
|
|
(void)LogFunc;
|
|
|
|
ParallelUtilities::PredicateTy SkipPredicate = [&](const BinaryFunction &BF) {
|
|
if (BF.getFunctionScore() == 0)
|
|
return true;
|
|
|
|
return false;
|
|
};
|
|
|
|
const bool HotOnly = opts::FrameOptimization == FOP_HOT;
|
|
|
|
Error SWError = Error::success();
|
|
|
|
ParallelUtilities::WorkFuncWithAllocTy WorkFunction =
|
|
[&](BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocatorId) {
|
|
DataflowInfoManager Info(BF, &RA, &FA, AllocatorId);
|
|
ShrinkWrapping SW(FA, BF, Info, AllocatorId);
|
|
|
|
auto ChangedOrErr = SW.perform(HotOnly);
|
|
if (auto E = ChangedOrErr.takeError()) {
|
|
std::lock_guard<std::mutex> Lock(FuncsChangedMutex);
|
|
SWError = joinErrors(std::move(SWError), Error(std::move(E)));
|
|
return;
|
|
}
|
|
const bool Changed = *ChangedOrErr;
|
|
if (Changed) {
|
|
std::lock_guard<std::mutex> Lock(FuncsChangedMutex);
|
|
FuncsChanged.insert(&BF);
|
|
LLVM_DEBUG(LogFunc(BF));
|
|
}
|
|
};
|
|
|
|
ParallelUtilities::runOnEachFunctionWithUniqueAllocId(
|
|
BC, ParallelUtilities::SchedulingPolicy::SP_INST_QUADRATIC, WorkFunction,
|
|
SkipPredicate, "shrink-wrapping");
|
|
|
|
if (!Top10Funcs.empty()) {
|
|
BC.outs() << "BOLT-INFO: top 10 functions changed by shrink wrapping:\n";
|
|
for (const auto &Elmt : Top10Funcs)
|
|
BC.outs() << Elmt.first << " : " << Elmt.second->getPrintName() << "\n";
|
|
}
|
|
return SWError;
|
|
}
|
|
|
|
} // namespace bolt
|
|
} // namespace llvm
|