Skip to content

Commit aca01bf

Browse files
mtrofinmtrofin
authored
[ctx_prof] CtxProfAnalysis: populate module data (#102930)
Continuing from #102084, which introduced the analysis, we now populate it with info about functions contained in the module. When we will update the profile due to e.g. inlined callsites, we'll ingest the callee's counters and callsites to the caller. We'll move those to the caller's respective index space (counter and callers), so we need to know and maintain where those currently end. We also don't need to keep profiles not pertinent to this module. This patch also introduces an arguably much simpler way to track the GUID of a function from the frontend compilation, through ThinLTO, and into the post-thinlink compilation step, which doesn't rely on keeping names around. A separate RFC and patches will discuss extending this to the current PGO (instrumented and sampled) and other consumers as an infrastructural component.
1 parent 4411d1e commit aca01bf

File tree

9 files changed

+385
-41
lines changed

9 files changed

+385
-41
lines changed

‎llvm/include/llvm/Analysis/CtxProfAnalysis.h

Lines changed: 56 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,23 +9,39 @@
99
#ifndef LLVM_ANALYSIS_CTXPROFANALYSIS_H
1010
#define LLVM_ANALYSIS_CTXPROFANALYSIS_H
1111

12+
#include "llvm/ADT/DenseMap.h"
1213
#include "llvm/IR/GlobalValue.h"
1314
#include "llvm/IR/PassManager.h"
1415
#include "llvm/ProfileData/PGOCtxProfReader.h"
15-
#include <map>
1616

1717
namespace llvm {
1818

1919
class CtxProfAnalysis;
2020

2121
/// The instrumented contextual profile, produced by the CtxProfAnalysis.
2222
class PGOContextualProfile {
23+
friend class CtxProfAnalysis;
24+
friend class CtxProfAnalysisPrinterPass;
25+
struct FunctionInfo {
26+
uint32_t NextCounterIndex = 0;
27+
uint32_t NextCallsiteIndex = 0;
28+
const std::string Name;
29+
30+
FunctionInfo(StringRef Name) : Name(Name) {}
31+
};
2332
std::optional<PGOCtxProfContext::CallTargetMapTy> Profiles;
33+
// For the GUIDs in this module, associate metadata about each function which
34+
// we'll need when we maintain the profiles during IPO transformations.
35+
DenseMap<GlobalValue::GUID, FunctionInfo> FuncInfo;
2436

25-
public:
26-
explicit PGOContextualProfile(PGOCtxProfContext::CallTargetMapTy &&Profiles)
27-
: Profiles(std::move(Profiles)) {}
37+
/// Get the GUID of this Function if it's defined in this module.
38+
GlobalValue::GUID getDefinedFunctionGUID(const Function &F) const;
39+
40+
// This is meant to be constructed from CtxProfAnalysis, which will also set
41+
// its state piecemeal.
2842
PGOContextualProfile() = default;
43+
44+
public:
2945
PGOContextualProfile(const PGOContextualProfile &) = delete;
3046
PGOContextualProfile(PGOContextualProfile &&) = default;
3147

@@ -35,6 +51,20 @@ class PGOContextualProfile {
3551
return *Profiles;
3652
}
3753

54+
bool isFunctionKnown(const Function &F) const {
55+
return getDefinedFunctionGUID(F) != 0;
56+
}
57+
58+
uint32_t allocateNextCounterIndex(const Function &F) {
59+
assert(isFunctionKnown(F));
60+
return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCounterIndex++;
61+
}
62+
63+
uint32_t allocateNextCallsiteIndex(const Function &F) {
64+
assert(isFunctionKnown(F));
65+
return FuncInfo.find(getDefinedFunctionGUID(F))->second.NextCallsiteIndex++;
66+
}
67+
3868
bool invalidate(Module &, const PreservedAnalyses &PA,
3969
ModuleAnalysisManager::Invalidator &) {
4070
// Check whether the analysis has been explicitly invalidated. Otherwise,
@@ -66,5 +96,27 @@ class CtxProfAnalysisPrinterPass
6696
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
6797
static bool isRequired() { return true; }
6898
};
99+
100+
/// Assign a GUID to functions as metadata. GUID calculation takes linkage into
101+
/// account, which may change especially through and after thinlto. By
102+
/// pre-computing and assigning as metadata, this mechanism is resilient to such
103+
/// changes (as well as name changes e.g. suffix ".llvm." additions).
104+
105+
// FIXME(mtrofin): we can generalize this mechanism to calculate a GUID early in
106+
// the pass pipeline, associate it with any Global Value, and then use it for
107+
// PGO and ThinLTO.
108+
// At that point, this should be moved elsewhere.
109+
class AssignGUIDPass : public PassInfoMixin<AssignGUIDPass> {
110+
public:
111+
explicit AssignGUIDPass() = default;
112+
113+
/// Assign a GUID *if* one is not already assign, as a function metadata named
114+
/// `GUIDMetadataName`.
115+
PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM);
116+
static const char *GUIDMetadataName;
117+
// This should become GlobalValue::getGUID
118+
static uint64_t getGUID(const Function &F);
119+
};
120+
69121
} // namespace llvm
70122
#endif // LLVM_ANALYSIS_CTXPROFANALYSIS_H

‎llvm/lib/Analysis/CtxProfAnalysis.cpp

Lines changed: 90 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,14 @@
1414
#include "llvm/Analysis/CtxProfAnalysis.h"
1515
#include "llvm/ADT/STLExtras.h"
1616
#include "llvm/IR/Analysis.h"
17+
#include "llvm/IR/IntrinsicInst.h"
1718
#include "llvm/IR/Module.h"
1819
#include "llvm/IR/PassManager.h"
1920
#include "llvm/ProfileData/PGOCtxProfReader.h"
2021
#include "llvm/Support/CommandLine.h"
2122
#include "llvm/Support/JSON.h"
2223
#include "llvm/Support/MemoryBuffer.h"
24+
#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
2325

2426
#define DEBUG_TYPE "ctx_prof"
2527

@@ -64,10 +66,39 @@ Value toJSON(const PGOCtxProfContext::CallTargetMapTy &P) {
6466
} // namespace json
6567
} // namespace llvm
6668

69+
const char *AssignGUIDPass::GUIDMetadataName = "guid";
70+
71+
PreservedAnalyses AssignGUIDPass::run(Module &M, ModuleAnalysisManager &MAM) {
72+
for (auto &F : M.functions()) {
73+
if (F.isDeclaration())
74+
continue;
75+
if (F.getMetadata(GUIDMetadataName))
76+
continue;
77+
const GlobalValue::GUID GUID = F.getGUID();
78+
F.setMetadata(GUIDMetadataName,
79+
MDNode::get(M.getContext(),
80+
{ConstantAsMetadata::get(ConstantInt::get(
81+
Type::getInt64Ty(M.getContext()), GUID))}));
82+
}
83+
return PreservedAnalyses::none();
84+
}
85+
86+
GlobalValue::GUID AssignGUIDPass::getGUID(const Function &F) {
87+
if (F.isDeclaration()) {
88+
assert(GlobalValue::isExternalLinkage(F.getLinkage()));
89+
return GlobalValue::getGUID(F.getGlobalIdentifier());
90+
}
91+
auto *MD = F.getMetadata(GUIDMetadataName);
92+
assert(MD && "guid not found for defined function");
93+
return cast<ConstantInt>(cast<ConstantAsMetadata>(MD->getOperand(0))
94+
->getValue()
95+
->stripPointerCasts())
96+
->getZExtValue();
97+
}
6798
AnalysisKey CtxProfAnalysis::Key;
6899

69-
CtxProfAnalysis::Result CtxProfAnalysis::run(Module &M,
70-
ModuleAnalysisManager &MAM) {
100+
PGOContextualProfile CtxProfAnalysis::run(Module &M,
101+
ModuleAnalysisManager &MAM) {
71102
ErrorOr<std::unique_ptr<MemoryBuffer>> MB = MemoryBuffer::getFile(Profile);
72103
if (auto EC = MB.getError()) {
73104
M.getContext().emitError("could not open contextual profile file: " +
@@ -81,7 +112,55 @@ CtxProfAnalysis::Result CtxProfAnalysis::run(Module &M,
81112
toString(MaybeCtx.takeError()));
82113
return {};
83114
}
84-
return Result(std::move(*MaybeCtx));
115+
116+
PGOContextualProfile Result;
117+
118+
for (const auto &F : M) {
119+
if (F.isDeclaration())
120+
continue;
121+
auto GUID = AssignGUIDPass::getGUID(F);
122+
assert(GUID && "guid not found for defined function");
123+
const auto &Entry = F.begin();
124+
uint32_t MaxCounters = 0; // we expect at least a counter.
125+
for (const auto &I : *Entry)
126+
if (auto *C = dyn_cast<InstrProfIncrementInst>(&I)) {
127+
MaxCounters =
128+
static_cast<uint32_t>(C->getNumCounters()->getZExtValue());
129+
break;
130+
}
131+
if (!MaxCounters)
132+
continue;
133+
uint32_t MaxCallsites = 0;
134+
for (const auto &BB : F)
135+
for (const auto &I : BB)
136+
if (auto *C = dyn_cast<InstrProfCallsite>(&I)) {
137+
MaxCallsites =
138+
static_cast<uint32_t>(C->getNumCounters()->getZExtValue());
139+
break;
140+
}
141+
auto [It, Ins] = Result.FuncInfo.insert(
142+
{GUID, PGOContextualProfile::FunctionInfo(F.getName())});
143+
(void)Ins;
144+
assert(Ins);
145+
It->second.NextCallsiteIndex = MaxCallsites;
146+
It->second.NextCounterIndex = MaxCounters;
147+
}
148+
// If we made it this far, the Result is valid - which we mark by setting
149+
// .Profiles.
150+
// Trim first the roots that aren't in this module.
151+
DenseSet<GlobalValue::GUID> ProfiledGUIDs;
152+
for (auto &[RootGuid, _] : llvm::make_early_inc_range(*MaybeCtx))
153+
if (!Result.FuncInfo.contains(RootGuid))
154+
MaybeCtx->erase(RootGuid);
155+
Result.Profiles = std::move(*MaybeCtx);
156+
return Result;
157+
}
158+
159+
GlobalValue::GUID
160+
PGOContextualProfile::getDefinedFunctionGUID(const Function &F) const {
161+
if (auto It = FuncInfo.find(AssignGUIDPass::getGUID(F)); It != FuncInfo.end())
162+
return It->first;
163+
return 0;
85164
}
86165

87166
PreservedAnalyses CtxProfAnalysisPrinterPass::run(Module &M,
@@ -91,8 +170,16 @@ PreservedAnalyses CtxProfAnalysisPrinterPass::run(Module &M,
91170
M.getContext().emitError("Invalid CtxProfAnalysis");
92171
return PreservedAnalyses::all();
93172
}
173+
174+
OS << "Function Info:\n";
175+
for (const auto &[Guid, FuncInfo] : C.FuncInfo)
176+
OS << Guid << " : " << FuncInfo.Name
177+
<< ". MaxCounterID: " << FuncInfo.NextCounterIndex
178+
<< ". MaxCallsiteID: " << FuncInfo.NextCallsiteIndex << "\n";
179+
94180
const auto JSONed = ::llvm::json::toJSON(C.profiles());
95181

182+
OS << "\nCurrent Profile:\n";
96183
OS << formatv("{0:2}", JSONed);
97184
OS << "\n";
98185
return PreservedAnalyses::all();

‎llvm/lib/Passes/PassBuilderPipelines.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "llvm/Analysis/AliasAnalysis.h"
1919
#include "llvm/Analysis/BasicAliasAnalysis.h"
2020
#include "llvm/Analysis/CGSCCPassManager.h"
21+
#include "llvm/Analysis/CtxProfAnalysis.h"
2122
#include "llvm/Analysis/GlobalsModRef.h"
2223
#include "llvm/Analysis/InlineAdvisor.h"
2324
#include "llvm/Analysis/ProfileSummaryInfo.h"
@@ -1196,6 +1197,9 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
11961197
// In pre-link, we just want the instrumented IR. We use the contextual
11971198
// profile in the post-thinlink phase.
11981199
// The instrumentation will be removed in post-thinlink after IPO.
1200+
// FIXME(mtrofin): move AssignGUIDPass if there is agreement to use this
1201+
// mechanism for GUIDs.
1202+
MPM.addPass(AssignGUIDPass());
11991203
if (IsCtxProfUse)
12001204
return MPM;
12011205
addPostPGOLoopRotation(MPM, Level);

‎llvm/lib/Passes/PassRegistry.def

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ MODULE_ALIAS_ANALYSIS("globals-aa", GlobalsAA())
4646
#endif
4747
MODULE_PASS("always-inline", AlwaysInlinerPass())
4848
MODULE_PASS("annotation2metadata", Annotation2MetadataPass())
49+
MODULE_PASS("assign-guid", AssignGUIDPass())
4950
MODULE_PASS("attributor", AttributorPass())
5051
MODULE_PASS("attributor-light", AttributorLightPass())
5152
MODULE_PASS("called-value-propagation", CalledValuePropagationPass())

‎llvm/lib/Transforms/Instrumentation/PGOCtxProfLowering.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
//
99

1010
#include "llvm/Transforms/Instrumentation/PGOCtxProfLowering.h"
11+
#include "llvm/Analysis/CtxProfAnalysis.h"
1112
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
1213
#include "llvm/IR/Analysis.h"
1314
#include "llvm/IR/DiagnosticInfo.h"
@@ -16,6 +17,7 @@
1617
#include "llvm/IR/IntrinsicInst.h"
1718
#include "llvm/IR/Module.h"
1819
#include "llvm/IR/PassManager.h"
20+
#include "llvm/ProfileData/InstrProf.h"
1921
#include "llvm/Support/CommandLine.h"
2022
#include <utility>
2123

@@ -223,8 +225,8 @@ bool CtxInstrumentationLowerer::lowerFunction(Function &F) {
223225
assert(Mark->getIndex()->isZero());
224226

225227
IRBuilder<> Builder(Mark);
226-
// FIXME(mtrofin): use InstrProfSymtab::getCanonicalName
227-
Guid = Builder.getInt64(F.getGUID());
228+
229+
Guid = Builder.getInt64(AssignGUIDPass::getGUID(F));
228230
// The type of the context of this function is now knowable since we have
229231
// NrCallsites and NrCounters. We delcare it here because it's more
230232
// convenient - we have the Builder.

‎llvm/test/Analysis/CtxProfAnalysis/full-cycle.ll

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
; REQUIRES: x86_64-linux
2+
;
3+
; RUN: rm -rf %t
4+
; RUN: split-file %s %t
5+
;
6+
; Test that the GUID metadata survives through thinlink.
7+
;
8+
; RUN: llvm-ctxprof-util fromJSON --input=%t/profile.json --output=%t/profile.ctxprofdata
9+
;
10+
; RUN: opt -module-summary -passes='thinlto-pre-link<O2>' -use-ctx-profile=%t/profile.ctxprofdata -o %t/m1.bc %t/m1.ll
11+
; RUN: opt -module-summary -passes='thinlto-pre-link<O2>' -use-ctx-profile=%t/profile.ctxprofdata -o %t/m2.bc %t/m2.ll
12+
;
13+
; RUN: rm -rf %t/postlink
14+
; RUN: mkdir %t/postlink
15+
;
16+
;
17+
; RUN: llvm-lto2 run %t/m1.bc %t/m2.bc -o %t/ -thinlto-distributed-indexes \
18+
; RUN: -use-ctx-profile=%t/profile.ctxprofdata \
19+
; RUN: -r %t/m1.bc,f1,plx \
20+
; RUN: -r %t/m2.bc,f1 \
21+
; RUN: -r %t/m2.bc,entrypoint,plx
22+
; RUN: opt --passes='function-import,require<ctx-prof-analysis>,print<ctx-prof-analysis>' \
23+
; RUN: -summary-file=%t/m2.bc.thinlto.bc -use-ctx-profile=%t/profile.ctxprofdata %t/m2.bc \
24+
; RUN: -S -o %t/m2.post.ll 2> %t/profile.txt
25+
; RUN: diff %t/expected.txt %t/profile.txt
26+
;--- m1.ll
27+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
28+
target triple = "x86_64-pc-linux-gnu"
29+
30+
source_filename = "random_path/m1.cc"
31+
32+
define private void @f2() #0 !guid !0 {
33+
ret void
34+
}
35+
36+
define void @f1() #0 {
37+
call void @f2()
38+
ret void
39+
}
40+
41+
attributes #0 = { noinline }
42+
!0 = !{ i64 3087265239403591524 }
43+
44+
;--- m2.ll
45+
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
46+
target triple = "x86_64-pc-linux-gnu"
47+
48+
source_filename = "random_path/m2.cc"
49+
50+
declare void @f1()
51+
52+
define void @entrypoint() {
53+
call void @f1()
54+
ret void
55+
}
56+
;--- profile.json
57+
[
58+
{
59+
"Callsites": [
60+
[
61+
{
62+
"Callsites": [
63+
[
64+
{
65+
"Counters": [
66+
10
67+
],
68+
"Guid": 3087265239403591524
69+
}
70+
]
71+
],
72+
"Counters": [
73+
7
74+
],
75+
"Guid": 2072045998141807037
76+
}
77+
]
78+
],
79+
"Counters": [
80+
1
81+
],
82+
"Guid": 10507721908651011566
83+
}
84+
]
85+
;--- expected.txt
86+
Function Info:
87+
10507721908651011566 : entrypoint. MaxCounterID: 1. MaxCallsiteID: 1
88+
3087265239403591524 : f2.llvm.0. MaxCounterID: 1. MaxCallsiteID: 0
89+
2072045998141807037 : f1. MaxCounterID: 1. MaxCallsiteID: 1
90+
91+
Current Profile:
92+
[
93+
{
94+
"Callsites": [
95+
[
96+
{
97+
"Callsites": [
98+
[
99+
{
100+
"Counters": [
101+
10
102+
],
103+
"Guid": 3087265239403591524
104+
}
105+
]
106+
],
107+
"Counters": [
108+
7
109+
],
110+
"Guid": 2072045998141807037
111+
}
112+
]
113+
],
114+
"Counters": [
115+
1
116+
],
117+
"Guid": 10507721908651011566
118+
}
119+
]

0 commit comments

Comments
 (0)