@@ -1744,7 +1744,7 @@ namespace slpvectorizer {
1744
1744
1745
1745
/// Bottom Up SLP Vectorizer.
1746
1746
class BoUpSLP {
1747
- struct TreeEntry;
1747
+ class TreeEntry;
1748
1748
class ScheduleEntity;
1749
1749
class ScheduleData;
1750
1750
class ScheduleBundle;
@@ -3607,7 +3607,8 @@ class BoUpSLP {
3607
3607
/// opportunities.
3608
3608
void reorderGatherNode(TreeEntry &TE);
3609
3609
3610
- struct TreeEntry {
3610
+ class TreeEntry {
3611
+ public:
3611
3612
using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>;
3612
3613
TreeEntry(VecTreeTy &Container) : Container(Container) {}
3613
3614
@@ -3774,6 +3775,9 @@ class BoUpSLP {
3774
3775
/// Interleaving factor for interleaved loads Vectorize nodes.
3775
3776
unsigned InterleaveFactor = 0;
3776
3777
3778
+ /// True if the node does not require scheduling.
3779
+ bool DoesNotNeedToSchedule = false;
3780
+
3777
3781
/// Set this bundle's \p OpIdx'th operand to \p OpVL.
3778
3782
void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
3779
3783
if (Operands.size() < OpIdx + 1)
@@ -3791,6 +3795,16 @@ class BoUpSLP {
3791
3795
/// Sets interleaving factor for the interleaving nodes.
3792
3796
void setInterleave(unsigned Factor) { InterleaveFactor = Factor; }
3793
3797
3798
+ /// Marks the node as one that does not require scheduling.
3799
+ void setDoesNotNeedToSchedule() {
3800
+ assert(::doesNotNeedToSchedule(Scalars) &&
3801
+ "Expected to not need scheduling");
3802
+ DoesNotNeedToSchedule = true;
3803
+ }
3804
+ /// Returns true if the node is marked as one that does not require
3805
+ /// scheduling.
3806
+ bool doesNotNeedToSchedule() const { return DoesNotNeedToSchedule; }
3807
+
3794
3808
/// Set this bundle's operands from \p Operands.
3795
3809
void setOperands(ArrayRef<ValueList> Operands) {
3796
3810
for (unsigned I : seq<unsigned>(Operands.size()))
@@ -4107,6 +4121,8 @@ class BoUpSLP {
4107
4121
}
4108
4122
}
4109
4123
} else if (!Last->isGather()) {
4124
+ if (doesNotNeedToSchedule(VL))
4125
+ Last->setDoesNotNeedToSchedule();
4110
4126
SmallPtrSet<Value *, 4> Processed;
4111
4127
for (Value *V : VL) {
4112
4128
if (isa<PoisonValue>(V))
@@ -4124,7 +4140,7 @@ class BoUpSLP {
4124
4140
// Update the scheduler bundle to point to this TreeEntry.
4125
4141
assert((!Bundle.getBundle().empty() || isa<PHINode>(S.getMainOp()) ||
4126
4142
isVectorLikeInstWithConstOps(S.getMainOp()) ||
4127
- doesNotNeedToSchedule(VL )) &&
4143
+ Last-> doesNotNeedToSchedule()) &&
4128
4144
"Bundle and VL out of sync");
4129
4145
if (!Bundle.getBundle().empty()) {
4130
4146
#if !defined(NDEBUG) || defined(EXPENSIVE_CHECKS)
@@ -15331,8 +15347,8 @@ BoUpSLP::isGatherShuffledSingleRegisterEntry(
15331
15347
}
15332
15348
15333
15349
if (!TEUseEI.UserTE->isGather() && !UserPHI &&
15334
- doesNotNeedToSchedule( TEUseEI.UserTE->Scalars ) !=
15335
- doesNotNeedToSchedule( UseEI.UserTE->Scalars ) &&
15350
+ TEUseEI.UserTE->doesNotNeedToSchedule( ) !=
15351
+ UseEI.UserTE->doesNotNeedToSchedule( ) &&
15336
15352
is_contained(UseEI.UserTE->Scalars, TEInsertPt))
15337
15353
continue;
15338
15354
// Check if the user node of the TE comes after user node of TEPtr,
@@ -16127,7 +16143,7 @@ void BoUpSLP::setInsertPointAfterBundle(const TreeEntry *E) {
16127
16143
}
16128
16144
if (IsPHI ||
16129
16145
(!E->isGather() && E->State != TreeEntry::SplitVectorize &&
16130
- doesNotNeedToSchedule( E->Scalars )) ||
16146
+ E->doesNotNeedToSchedule( )) ||
16131
16147
(GatheredLoadsEntriesFirst.has_value() &&
16132
16148
E->Idx >= *GatheredLoadsEntriesFirst && !E->isGather() &&
16133
16149
E->getOpcode() == Instruction::Load)) {
@@ -19803,7 +19819,7 @@ void BoUpSLP::scheduleBlock(BlockScheduling *BS) {
19803
19819
if (ScheduleData *SD = BS->getScheduleData(I)) {
19804
19820
[[maybe_unused]] ArrayRef<TreeEntry *> SDTEs = getTreeEntries(I);
19805
19821
assert((isVectorLikeInstWithConstOps(SD->getInst()) || SDTEs.empty() ||
19806
- doesNotNeedToSchedule( SDTEs.front()->Scalars )) &&
19822
+ SDTEs.front()->doesNotNeedToSchedule( )) &&
19807
19823
"scheduler and vectorizer bundle mismatch");
19808
19824
SD->setSchedulingPriority(Idx++);
19809
19825
continue;
0 commit comments