Skip to content

Commit 49dec3e

Browse files
lukel97lukel97
committed
[LV] Expand VPWidenIntOrFpInductionRecipe into separate recipes
1 parent 2f18b5e commit 49dec3e

19 files changed

+427
-183
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1032,29 +1032,19 @@ void VPlan::execute(VPTransformState *State) {
10321032
if (isa<VPWidenPHIRecipe>(&R))
10331033
continue;
10341034

1035-
if (isa<VPWidenPointerInductionRecipe>(&R) ||
1036-
isa<VPWidenIntOrFpInductionRecipe>(&R)) {
1037-
PHINode *Phi = nullptr;
1038-
if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
1039-
Phi = cast<PHINode>(State->get(R.getVPSingleValue()));
1040-
} else {
1041-
auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
1042-
assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) &&
1043-
"recipe generating only scalars should have been replaced");
1044-
auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi));
1045-
Phi = cast<PHINode>(GEP->getPointerOperand());
1046-
}
1035+
if (isa<VPWidenPointerInductionRecipe>(&R)) {
1036+
auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
1037+
assert(!WidenPhi->onlyScalarsGenerated(State->VF.isScalable()) &&
1038+
"recipe generating only scalars should have been replaced");
1039+
auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi));
1040+
PHINode *Phi = cast<PHINode>(GEP->getPointerOperand());
10471041

10481042
Phi->setIncomingBlock(1, VectorLatchBB);
10491043

10501044
// Move the last step to the end of the latch block. This ensures
10511045
// consistent placement of all induction updates.
10521046
Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
10531047
Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
1054-
1055-
// Use the steps for the last part as backedge value for the induction.
1056-
if (auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&R))
1057-
Inc->setOperand(0, State->get(IV->getLastUnrolledPartOperand()));
10581048
continue;
10591049
}
10601050

‎llvm/lib/Transforms/Vectorize/VPlan.h

Lines changed: 151 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2089,7 +2089,9 @@ class VPHeaderPHIRecipe : public VPSingleDefRecipe {
20892089
};
20902090

20912091
/// A recipe for handling phi nodes of integer and floating-point inductions,
2092-
/// producing their vector values.
2092+
/// producing their vector values. This won't execute any LLVM IR and will get
2093+
/// expanded later into VPWidenIntOrFpInitialRecipe, VPWidenIntOrFpPHIRecipe and
2094+
/// VPWidenIntOrFpBackedgeRecipe.
20932095
class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe {
20942096
PHINode *IV;
20952097
TruncInst *Trunc;
@@ -2122,9 +2124,10 @@ class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe {
21222124

21232125
VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC)
21242126

2125-
/// Generate the vectorized and scalarized versions of the phi node as
2126-
/// needed by their users.
2127-
void execute(VPTransformState &State) override;
2127+
void execute(VPTransformState &State) override {
2128+
llvm_unreachable("cannot execute this recipe, should be expanded via "
2129+
"expandVPWidenIntOrFpInductionRecipe");
2130+
}
21282131

21292132
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
21302133
/// Print the recipe.
@@ -2180,10 +2183,152 @@ class VPWidenIntOrFpInductionRecipe : public VPHeaderPHIRecipe {
21802183
}
21812184

21822185
/// Returns the VPValue representing the value of this induction at
2183-
/// the last unrolled part, if it exists. Returns itself if unrolling did not
2186+
/// the last unrolled part, if it exists. Returns nullptr if unrolling did not
21842187
/// take place.
21852188
VPValue *getLastUnrolledPartOperand() {
2186-
return getNumOperands() == 5 ? getOperand(4) : this;
2189+
return getNumOperands() == 5 ? getOperand(4) : nullptr;
2190+
}
2191+
};
2192+
2193+
/// A recipe to compute the initial value for a widened IV, expanded from
2194+
/// VPWidenIntOrFpInductionRecipe.
2195+
class VPWidenIntOrFpInductionInitialRecipe : public VPSingleDefRecipe {
2196+
Instruction *IV;
2197+
const InductionDescriptor &ID;
2198+
2199+
public:
2200+
VPWidenIntOrFpInductionInitialRecipe(Instruction *IV, VPValue *Start,
2201+
VPValue *Step,
2202+
const InductionDescriptor &ID)
2203+
: VPSingleDefRecipe(VPDef::VPWidenIntOrFpInductionStartSC, {Start, Step}),
2204+
IV(IV), ID(ID) {
2205+
assert((isa<PHINode>(IV) || isa<TruncInst>(IV)) &&
2206+
"Expected either an induction phi-node or a truncate of it!");
2207+
}
2208+
2209+
~VPWidenIntOrFpInductionInitialRecipe() override = default;
2210+
2211+
VPWidenIntOrFpInductionInitialRecipe *clone() override {
2212+
return new VPWidenIntOrFpInductionInitialRecipe(IV, getOperand(0),
2213+
getOperand(1), ID);
2214+
}
2215+
2216+
VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionStartSC)
2217+
2218+
void execute(VPTransformState &State) override;
2219+
2220+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2221+
/// Print the recipe.
2222+
void print(raw_ostream &O, const Twine &Indent,
2223+
VPSlotTracker &SlotTracker) const override;
2224+
#endif
2225+
2226+
VPValue *getStartValue() { return getOperand(0); }
2227+
const VPValue *getStartValue() const { return getOperand(0); }
2228+
2229+
VPValue *getStepValue() { return getOperand(1); }
2230+
const VPValue *getStepValue() const { return getOperand(1); }
2231+
2232+
/// Returns the scalar type of the induction.
2233+
Type *getScalarType() const { return IV->getType(); }
2234+
2235+
bool onlyFirstLaneUsed(const VPValue *Op) const override {
2236+
assert(is_contained(operands(), Op) &&
2237+
"Op must be an operand of the recipe");
2238+
return true;
2239+
}
2240+
};
2241+
2242+
/// A recipe to generate the PHI of a widened IV, expanded from
2243+
/// VPWidenIntOrFpInductionRecipe.
2244+
class VPWidenIntOrFpInductionPHIRecipe : public VPHeaderPHIRecipe {
2245+
Instruction *IV;
2246+
2247+
public:
2248+
VPWidenIntOrFpInductionPHIRecipe(Instruction *IV, VPValue *Start)
2249+
: VPHeaderPHIRecipe(VPDef::VPWidenIntOrFpInductionPHISC, IV, Start),
2250+
IV(IV) {
2251+
assert((isa<PHINode>(IV) || isa<TruncInst>(IV)) &&
2252+
"Expected either an induction phi-node or a truncate of it!");
2253+
}
2254+
2255+
~VPWidenIntOrFpInductionPHIRecipe() override = default;
2256+
2257+
VPWidenIntOrFpInductionPHIRecipe *clone() override {
2258+
auto *R = new VPWidenIntOrFpInductionPHIRecipe(IV, getOperand(0));
2259+
R->addOperand(getBackedgeValue());
2260+
return R;
2261+
}
2262+
2263+
VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionPHISC)
2264+
2265+
void execute(VPTransformState &State) override;
2266+
2267+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2268+
/// Print the recipe.
2269+
void print(raw_ostream &O, const Twine &Indent,
2270+
VPSlotTracker &SlotTracker) const override;
2271+
#endif
2272+
};
2273+
2274+
/// A recipe to compute the backedge value for a widened IV, expanded from
2275+
/// VPWidenIntOrFpInductionRecipe.
2276+
class VPWidenIntOrFpInductionBackedgeRecipe : public VPSingleDefRecipe {
2277+
Instruction *IV;
2278+
const InductionDescriptor &ID;
2279+
2280+
public:
2281+
VPWidenIntOrFpInductionBackedgeRecipe(Instruction *IV, VPValue *Step,
2282+
VPValue *VF, VPValue *Prev,
2283+
VPValue *SplatVF,
2284+
const InductionDescriptor &ID)
2285+
: VPSingleDefRecipe(VPDef::VPWidenIntOrFpInductionSC, {Step, VF, Prev}),
2286+
IV(IV), ID(ID) {
2287+
assert((isa<PHINode>(IV) || isa<TruncInst>(IV)) &&
2288+
"Expected either an induction phi-node or a truncate of it!");
2289+
if (SplatVF)
2290+
addOperand(SplatVF);
2291+
}
2292+
2293+
~VPWidenIntOrFpInductionBackedgeRecipe() override = default;
2294+
2295+
VPWidenIntOrFpInductionBackedgeRecipe *clone() override {
2296+
return new VPWidenIntOrFpInductionBackedgeRecipe(
2297+
IV, getOperand(0), getOperand(1), getOperand(2), getOperand(3), ID);
2298+
}
2299+
2300+
VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionIncSC)
2301+
2302+
void execute(VPTransformState &State) override;
2303+
2304+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2305+
/// Print the recipe.
2306+
void print(raw_ostream &O, const Twine &Indent,
2307+
VPSlotTracker &SlotTracker) const override;
2308+
#endif
2309+
2310+
VPValue *getStepValue() { return getOperand(0); }
2311+
const VPValue *getStepValue() const { return getOperand(0); }
2312+
2313+
VPValue *getVFValue() { return getOperand(1); }
2314+
const VPValue *getVFValue() const { return getOperand(1); }
2315+
2316+
VPValue *getPrevValue() { return getOperand(2); }
2317+
const VPValue *getPrevValue() const { return getOperand(2); }
2318+
2319+
VPValue *getSplatVFValue() {
2320+
// If the recipe has been unrolled (4 operands), return the VPValue for the
2321+
// induction increment.
2322+
return getNumOperands() == 4 ? getOperand(3) : nullptr;
2323+
}
2324+
2325+
/// Returns the scalar type of the induction.
2326+
Type *getScalarType() const { return IV->getType(); }
2327+
2328+
bool onlyFirstLaneUsed(const VPValue *Op) const override {
2329+
assert(is_contained(operands(), Op) &&
2330+
"Op must be an operand of the recipe");
2331+
return Op == getOperand(0) || Op == getOperand(1);
21872332
}
21882333
};
21892334

‎llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -214,14 +214,17 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
214214
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
215215
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
216216
VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe,
217-
VPScalarPHIRecipe>([this](const auto *R) {
218-
// Handle header phi recipes, except VPWidenIntOrFpInduction
219-
// which needs special handling due it being possibly truncated.
220-
// TODO: consider inferring/caching type of siblings, e.g.,
221-
// backedge value, here and in cases below.
222-
return inferScalarType(R->getStartValue());
223-
})
224-
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
217+
VPScalarPHIRecipe, VPWidenIntOrFpInductionPHIRecipe>(
218+
[this](const auto *R) {
219+
// Handle header phi recipes, except VPWidenIntOrFpInduction
220+
// which needs special handling due it being possibly truncated.
221+
// TODO: consider inferring/caching type of siblings, e.g.,
222+
// backedge value, here and in cases below.
223+
return inferScalarType(R->getStartValue());
224+
})
225+
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe,
226+
VPWidenIntOrFpInductionInitialRecipe,
227+
VPWidenIntOrFpInductionBackedgeRecipe>(
225228
[](const auto *R) { return R->getScalarType(); })
226229
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,
227230
VPScalarIVStepsRecipe, VPWidenGEPRecipe, VPVectorPointerRecipe,

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Lines changed: 67 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1635,47 +1635,73 @@ static Constant *getSignedIntOrFpConstant(Type *Ty, int64_t C) {
16351635
: ConstantFP::get(Ty, C);
16361636
}
16371637

1638-
void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
1638+
void VPWidenIntOrFpInductionInitialRecipe::execute(VPTransformState &State) {
16391639
assert(!State.Lane && "Int or FP induction being replicated.");
16401640

1641-
Value *Start = getStartValue()->getLiveInIRValue();
1642-
const InductionDescriptor &ID = getInductionDescriptor();
1643-
TruncInst *Trunc = getTruncInst();
1641+
Value *Start = State.get(getStartValue(), true);
16441642
IRBuilderBase &Builder = State.Builder;
1645-
assert(IV->getType() == ID.getStartValue()->getType() && "Types must match");
16461643
assert(State.VF.isVector() && "must have vector VF");
16471644

1648-
// The value from the original loop to which we are mapping the new induction
1649-
// variable.
1650-
Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
1651-
16521645
// Fast-math-flags propagate from the original induction instruction.
16531646
IRBuilder<>::FastMathFlagGuard FMFG(Builder);
1654-
if (ID.getInductionBinOp() && isa<FPMathOperator>(ID.getInductionBinOp()))
1647+
if (isa_and_nonnull<FPMathOperator>(ID.getInductionBinOp()))
16551648
Builder.setFastMathFlags(ID.getInductionBinOp()->getFastMathFlags());
16561649

16571650
// Now do the actual transformations, and start with fetching the step value.
16581651
Value *Step = State.get(getStepValue(), VPLane(0));
16591652

1660-
assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
1661-
"Expected either an induction phi-node or a truncate of it!");
1662-
1663-
// Construct the initial value of the vector IV in the vector loop preheader
1664-
auto CurrIP = Builder.saveIP();
1665-
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1666-
Builder.SetInsertPoint(VectorPH->getTerminator());
1667-
if (isa<TruncInst>(EntryVal)) {
1668-
assert(Start->getType()->isIntegerTy() &&
1669-
"Truncation requires an integer type");
1670-
auto *TruncType = cast<IntegerType>(EntryVal->getType());
1671-
Step = Builder.CreateTrunc(Step, TruncType);
1672-
Start = Builder.CreateCast(Instruction::Trunc, Start, TruncType);
1673-
}
1674-
1653+
// Construct the initial value of the vector IV
16751654
Value *Zero = getSignedIntOrFpConstant(Start->getType(), 0);
16761655
Value *SplatStart = Builder.CreateVectorSplat(State.VF, Start);
16771656
Value *SteppedStart = getStepVector(
16781657
SplatStart, Zero, Step, ID.getInductionOpcode(), State.VF, State.Builder);
1658+
State.set(this, SteppedStart);
1659+
}
1660+
1661+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1662+
void VPWidenIntOrFpInductionInitialRecipe::print(
1663+
raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const {
1664+
O << Indent;
1665+
printAsOperand(O, SlotTracker);
1666+
O << " = WIDEN-INDUCTION-START ";
1667+
printOperands(O, SlotTracker);
1668+
}
1669+
#endif
1670+
1671+
void VPWidenIntOrFpInductionPHIRecipe::execute(VPTransformState &State) {
1672+
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1673+
1674+
Value *Start = State.get(getOperand(0));
1675+
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, "vec.ind");
1676+
Phi->addIncoming(Start, VectorPH);
1677+
Phi->setDebugLoc(IV->getDebugLoc());
1678+
State.set(this, Phi);
1679+
}
1680+
1681+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1682+
void VPWidenIntOrFpInductionPHIRecipe::print(raw_ostream &O,
1683+
const Twine &Indent,
1684+
VPSlotTracker &SlotTracker) const {
1685+
O << Indent;
1686+
printAsOperand(O, SlotTracker);
1687+
O << " = WIDEN-INDUCTION-PHI ";
1688+
printOperands(O, SlotTracker);
1689+
}
1690+
#endif
1691+
1692+
void VPWidenIntOrFpInductionBackedgeRecipe::execute(VPTransformState &State) {
1693+
IRBuilderBase &Builder = State.Builder;
1694+
1695+
// Fast-math-flags propagate from the original induction instruction.
1696+
IRBuilder<>::FastMathFlagGuard FMFG(Builder);
1697+
if (isa_and_nonnull<FPMathOperator>(ID.getInductionBinOp()))
1698+
Builder.setFastMathFlags(ID.getInductionBinOp()->getFastMathFlags());
1699+
1700+
Value *Step = State.get(getStepValue(), VPLane(0));
1701+
1702+
auto CurrIP = Builder.saveIP();
1703+
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1704+
Builder.SetInsertPoint(VectorPH->getTerminator());
16791705

16801706
// We create vector phi nodes for both integer and floating-point induction
16811707
// variables. Here, we determine the kind of arithmetic we will perform.
@@ -1710,29 +1736,27 @@ void VPWidenIntOrFpInductionRecipe::execute(VPTransformState &State) {
17101736
}
17111737

17121738
Builder.restoreIP(CurrIP);
1713-
1714-
// We may need to add the step a number of times, depending on the unroll
1715-
// factor. The last of those goes into the PHI.
1716-
PHINode *VecInd = PHINode::Create(SteppedStart->getType(), 2, "vec.ind");
1717-
VecInd->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
1718-
VecInd->setDebugLoc(EntryVal->getDebugLoc());
1719-
State.set(this, VecInd);
1739+
Value *PrevVal = State.get(getPrevValue());
17201740

17211741
Instruction *LastInduction = cast<Instruction>(
1722-
Builder.CreateBinOp(AddOp, VecInd, SplatVF, "vec.ind.next"));
1723-
if (isa<TruncInst>(EntryVal))
1724-
State.addMetadata(LastInduction, EntryVal);
1725-
LastInduction->setDebugLoc(EntryVal->getDebugLoc());
1742+
Builder.CreateBinOp(AddOp, PrevVal, SplatVF, "vec.ind.next"));
1743+
if (isa<TruncInst>(IV))
1744+
State.addMetadata(LastInduction, IV);
1745+
LastInduction->setDebugLoc(IV->getDebugLoc());
17261746

1727-
VecInd->addIncoming(SteppedStart, VectorPH);
1728-
// Add induction update using an incorrect block temporarily. The phi node
1729-
// will be fixed after VPlan execution. Note that at this point the latch
1730-
// block cannot be used, as it does not exist yet.
1731-
// TODO: Model increment value in VPlan, by turning the recipe into a
1732-
// multi-def and a subclass of VPHeaderPHIRecipe.
1733-
VecInd->addIncoming(LastInduction, VectorPH);
1747+
State.set(this, LastInduction);
17341748
}
17351749

1750+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1751+
void VPWidenIntOrFpInductionBackedgeRecipe::print(
1752+
raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const {
1753+
O << Indent;
1754+
printAsOperand(O, SlotTracker);
1755+
O << " = WIDEN-INDUCTION-INC ";
1756+
printOperands(O, SlotTracker);
1757+
}
1758+
#endif
1759+
17361760
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
17371761
void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O, const Twine &Indent,
17381762
VPSlotTracker &SlotTracker) const {

0 commit comments

Comments
 (0)