diff --git a/llvm/include/llvm/Analysis/IVDescriptors.h b/llvm/include/llvm/Analysis/IVDescriptors.h index d12780cde71d7..15eb21a4a60dc 100644 --- a/llvm/include/llvm/Analysis/IVDescriptors.h +++ b/llvm/include/llvm/Analysis/IVDescriptors.h @@ -82,11 +82,12 @@ class RecurrenceDescriptor { RecurKind K, FastMathFlags FMF, Instruction *ExactFP, Type *RT, bool Signed, bool Ordered, SmallPtrSetImpl &CI, - unsigned MinWidthCastToRecurTy) + unsigned MinWidthCastToRecurTy, Value *Sentinel) : IntermediateStore(Store), StartValue(Start), LoopExitInstr(Exit), Kind(K), FMF(FMF), ExactFPMathInst(ExactFP), RecurrenceType(RT), IsSigned(Signed), IsOrdered(Ordered), - MinWidthCastToRecurrenceType(MinWidthCastToRecurTy) { + MinWidthCastToRecurrenceType(MinWidthCastToRecurTy), + SentinelValue(Sentinel) { CastInsts.insert_range(CI); } @@ -97,9 +98,10 @@ class RecurrenceDescriptor { : IsRecurrence(IsRecur), PatternLastInst(I), RecKind(RecurKind::None), ExactFPMathInst(ExactFP) {} - InstDesc(Instruction *I, RecurKind K, Instruction *ExactFP = nullptr) + InstDesc(Instruction *I, RecurKind K, Value *Sentinel = nullptr, + Instruction *ExactFP = nullptr) : IsRecurrence(true), PatternLastInst(I), RecKind(K), - ExactFPMathInst(ExactFP) {} + ExactFPMathInst(ExactFP), SentinelValue(Sentinel) {} bool isRecurrence() const { return IsRecurrence; } @@ -111,6 +113,8 @@ class RecurrenceDescriptor { Instruction *getPatternInst() const { return PatternLastInst; } + Value *getSentinelValue() const { return SentinelValue; } + private: // Is this instruction a recurrence candidate. bool IsRecurrence; @@ -121,6 +125,9 @@ class RecurrenceDescriptor { RecurKind RecKind; // Recurrence does not allow floating-point reassociation. Instruction *ExactFPMathInst; + // The value is used to replace the start value if required by the + // recurrence. + Value *SentinelValue = nullptr; }; /// Returns a struct describing if the instruction 'I' can be a recurrence @@ -271,9 +278,7 @@ class RecurrenceDescriptor { /// value. Value *getSentinelValue() const { assert(isFindLastIVRecurrenceKind(Kind) && "Unexpected recurrence kind"); - Type *Ty = StartValue->getType(); - return ConstantInt::get(Ty, - APInt::getSignedMinValue(Ty->getIntegerBitWidth())); + return SentinelValue; } /// Returns a reference to the instructions used for type-promoting the @@ -332,6 +337,8 @@ class RecurrenceDescriptor { SmallPtrSet CastInsts; // The minimum width used by the recurrence. unsigned MinWidthCastToRecurrenceType; + /// The value used to represent the starting value. + Value *SentinelValue = nullptr; }; /// A struct for saving information about induction variables. diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index b7c7bcab168cc..f558a1c359cef 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -288,6 +288,8 @@ bool RecurrenceDescriptor::AddReductionVar( // The first instruction in the use-def chain of the Phi node that requires // exact floating point operations. Instruction *ExactFPMathInst = nullptr; + // Record the sentinel value on demand. + Value *SentinelValue = nullptr; // A value in the reduction can be used: // - By the reduction: @@ -376,6 +378,10 @@ bool RecurrenceDescriptor::AddReductionVar( ExactFPMathInst = ExactFPMathInst == nullptr ? ReduxDesc.getExactFPMathInst() : ExactFPMathInst; + if (auto *Sentinel = ReduxDesc.getSentinelValue()) { + assert(!SentinelValue && "Sentinel value can only be assigned once"); + SentinelValue = Sentinel; + } if (!ReduxDesc.isRecurrence()) return false; // FIXME: FMF is allowed on phi, but propagation is not handled correctly. @@ -596,7 +602,8 @@ bool RecurrenceDescriptor::AddReductionVar( // Save the description of this reduction variable. RecurrenceDescriptor RD(RdxStart, ExitInstruction, IntermediateStore, Kind, FMF, ExactFPMathInst, RecurrenceType, IsSigned, - IsOrdered, CastInsts, MinWidthCastToRecurrenceType); + IsOrdered, CastInsts, MinWidthCastToRecurrenceType, + SentinelValue); RedDes = RD; return true; @@ -700,18 +707,18 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi, m_Value(NonRdxPhi))))) return InstDesc(false, I); - auto IsIncreasingLoopInduction = [&](Value *V) { + auto IsIncreasingLoopInduction = [&](Value *V) -> std::pair { Type *Ty = V->getType(); if (!SE.isSCEVable(Ty)) - return false; + return {false, nullptr}; auto *AR = dyn_cast(SE.getSCEV(V)); if (!AR || AR->getLoop() != TheLoop) - return false; + return {false, nullptr}; const SCEV *Step = AR->getStepRecurrence(SE); if (!SE.isKnownPositive(Step)) - return false; + return {false, nullptr}; const ConstantRange IVRange = SE.getSignedRange(AR); unsigned NumBits = Ty->getIntegerBitWidth(); @@ -730,17 +737,32 @@ RecurrenceDescriptor::isFindLastIVPattern(Loop *TheLoop, PHINode *OrigPhi, << IVRange << "\n"); // Ensure the induction variable does not wrap around by verifying that its // range is fully contained within the valid range. - return ValidRange.contains(IVRange); + if (!ValidRange.contains(IVRange)) + return {false, nullptr}; + + // No sentinel is needed if it can be proven that the start value of + // reduction is strictly less than the start value of increasing induction + // variable. + if (auto *ConstIVStart = dyn_cast(AR->getStart())) { + Value *RdxStart = + OrigPhi->getIncomingValueForBlock(TheLoop->getLoopPreheader()); + if (auto *ConstRdxStart = dyn_cast(RdxStart)) + if (ConstRdxStart->getValue().slt(ConstIVStart->getAPInt())) + return {true, nullptr}; + } + + return {true, ConstantInt::get(Ty, Sentinel)}; }; // We are looking for selects of the form: // select(cmp(), phi, increasing_loop_induction) or // select(cmp(), increasing_loop_induction, phi) // TODO: Support for monotonically decreasing induction variable - if (!IsIncreasingLoopInduction(NonRdxPhi)) + auto [IsRecurrence, Sentinel] = IsIncreasingLoopInduction(NonRdxPhi); + if (!IsRecurrence) return InstDesc(false, I); - return InstDesc(I, RecurKind::FindLastIV); + return InstDesc(I, RecurKind::FindLastIV, Sentinel); } RecurrenceDescriptor::InstDesc diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index 2fff9521017ff..a08c15b670876 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -1247,15 +1247,17 @@ Value *llvm::createFindLastIVReduction(IRBuilderBase &Builder, Value *Src, assert(RecurrenceDescriptor::isFindLastIVRecurrenceKind( Desc.getRecurrenceKind()) && "Unexpected reduction kind"); - Value *Sentinel = Desc.getSentinelValue(); - Value *MaxRdx = Src->getType()->isVectorTy() + Value *Result = Src->getType()->isVectorTy() ? Builder.CreateIntMaxReduce(Src, true) : Src; // Correct the final reduction result back to the start value if the maximum // reduction is sentinel value. - Value *Cmp = - Builder.CreateCmp(CmpInst::ICMP_NE, MaxRdx, Sentinel, "rdx.select.cmp"); - return Builder.CreateSelect(Cmp, MaxRdx, Start, "rdx.select"); + if (Value *Sentinel = Desc.getSentinelValue()) { + Value *Cmp = + Builder.CreateCmp(CmpInst::ICMP_NE, Result, Sentinel, "rdx.select.cmp"); + Result = Builder.CreateSelect(Cmp, Result, Start, "rdx.select"); + } + return Result; } Value *llvm::getReductionIdentity(Intrinsic::ID RdxID, Type *Ty, diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 2fe59a464457f..a8b2fd9e89512 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7568,7 +7568,8 @@ static void fixReductionScalarResumeWhenVectorizingEpilog( "start value"); MainResumeValue = Cmp->getOperand(0); } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind( - RdxDesc.getRecurrenceKind())) { + RdxDesc.getRecurrenceKind()) && + RdxDesc.getSentinelValue()) { using namespace llvm::PatternMatch; Value *Cmp, *OrigResumeV, *CmpOp; bool IsExpectedPattern = @@ -9625,7 +9626,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions( // Adjust the start value for FindLastIV recurrences to use the sentinel // value after generating the ResumePhi recipe, which uses the original // start value. - PhiR->setOperand(0, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue())); + if (auto *Sentinel = RdxDesc.getSentinelValue()) + PhiR->setOperand(0, Plan->getOrAddLiveIn(Sentinel)); } } for (VPRecipeBase *R : ToDelete) @@ -10114,7 +10116,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L, IRBuilder<> Builder(PBB, PBB->getFirstNonPHIIt()); ResumeV = Builder.CreateICmpNE(ResumeV, RdxDesc.getRecurrenceStartValue()); - } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) { + } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK) && + RdxDesc.getSentinelValue()) { ToFrozen[RdxDesc.getRecurrenceStartValue()] = cast(ResumeV)->getIncomingValueForBlock( EPI.MainLoopIterationCountCheck); diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 72aa771b0b98e..a1e5e0c9af69d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -3848,15 +3848,11 @@ void VPReductionPHIRecipe::execute(VPTransformState &State) { StartV = Iden = State.get(StartVPV); } } else if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(RK)) { - // [I|F]FindLastIV will use a sentinel value to initialize the reduction - // phi or the resume value from the main vector loop when vectorizing the - // epilogue loop. In the exit block, ComputeReductionResult will generate - // checks to verify if the reduction result is the sentinel value. If the - // result is the sentinel value, it will be corrected back to the start - // value. - // TODO: The sentinel value is not always necessary. When the start value is - // a constant, and smaller than the start value of the induction variable, - // the start value can be directly used to initialize the reduction phi. + // FindLastIV may use a sentinel value to initialize the reduction phi or + // the resume value from the main vector loop when vectorizing the epilogue + // loop. In the exit block, ComputeFindLastIVResult will generate checks to + // verify if the reduction result is the sentinel value. If the result is + // the sentinel value, it will be corrected back to the start value. Iden = StartV; if (!ScalarPHI) { IRBuilderBase::InsertPointGuard IPBuilder(Builder); diff --git a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll index e4597ebfe7dc8..28495d96b4245 100644 --- a/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll +++ b/llvm/test/Transforms/LoopVectorize/iv-select-cmp-trunc.ll @@ -441,7 +441,7 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) { ; CHECK-VF4IC1-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK-VF4IC1: [[VECTOR_BODY]]: ; CHECK-VF4IC1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-VF4IC1-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ] +; CHECK-VF4IC1-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP4:%.*]], %[[VECTOR_BODY]] ] ; CHECK-VF4IC1-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ , %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-VF4IC1-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] ; CHECK-VF4IC1-NEXT: [[TMP2:%.*]] = getelementptr inbounds float, ptr [[TMP1]], i32 0 @@ -454,12 +454,10 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) { ; CHECK-VF4IC1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK-VF4IC1: [[MIDDLE_BLOCK]]: ; CHECK-VF4IC1-NEXT: [[TMP6:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[TMP4]]) -; CHECK-VF4IC1-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i32 [[TMP6]], -2147483648 -; CHECK-VF4IC1-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[TMP6]], i32 -1 ; CHECK-VF4IC1-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] ; CHECK-VF4IC1: [[SCALAR_PH]]: ; CHECK-VF4IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 2147483648, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] -; CHECK-VF4IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] +; CHECK-VF4IC1-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] ; CHECK-VF4IC1-NEXT: br label %[[FOR_BODY:.*]] ; CHECK-VF4IC1: [[FOR_BODY]]: ; CHECK-VF4IC1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] @@ -473,7 +471,7 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) { ; CHECK-VF4IC1-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 2147483648 ; CHECK-VF4IC1-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK-VF4IC1: [[EXIT]]: -; CHECK-VF4IC1-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ] +; CHECK-VF4IC1-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[TMP6]], %[[MIDDLE_BLOCK]] ] ; CHECK-VF4IC1-NEXT: ret i32 [[SPEC_SELECT_LCSSA]] ; ; CHECK-VF4IC4-LABEL: define i32 @select_fcmp_max_valid_const_ub( @@ -484,10 +482,10 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) { ; CHECK-VF4IC4-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK-VF4IC4: [[VECTOR_BODY]]: ; CHECK-VF4IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ] -; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ] -; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ] -; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ splat (i32 -2147483648), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ] +; CHECK-VF4IC4-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP10:%.*]], %[[VECTOR_BODY]] ] +; CHECK-VF4IC4-NEXT: [[VEC_PHI1:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP11:%.*]], %[[VECTOR_BODY]] ] +; CHECK-VF4IC4-NEXT: [[VEC_PHI2:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP12:%.*]], %[[VECTOR_BODY]] ] +; CHECK-VF4IC4-NEXT: [[VEC_PHI3:%.*]] = phi <4 x i32> [ splat (i32 -1), %[[VECTOR_PH]] ], [ [[TMP13:%.*]], %[[VECTOR_BODY]] ] ; CHECK-VF4IC4-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ , %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-VF4IC4-NEXT: [[STEP_ADD:%.*]] = add <4 x i32> [[VEC_IND]], splat (i32 4) ; CHECK-VF4IC4-NEXT: [[STEP_ADD_2:%.*]] = add <4 x i32> [[STEP_ADD]], splat (i32 4) @@ -518,12 +516,10 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) { ; CHECK-VF4IC4-NEXT: [[RDX_MINMAX7:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[RDX_MINMAX]], <4 x i32> [[TMP12]]) ; CHECK-VF4IC4-NEXT: [[RDX_MINMAX8:%.*]] = call <4 x i32> @llvm.smax.v4i32(<4 x i32> [[RDX_MINMAX7]], <4 x i32> [[TMP13]]) ; CHECK-VF4IC4-NEXT: [[TMP15:%.*]] = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> [[RDX_MINMAX8]]) -; CHECK-VF4IC4-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i32 [[TMP15]], -2147483648 -; CHECK-VF4IC4-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[TMP15]], i32 -1 ; CHECK-VF4IC4-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] ; CHECK-VF4IC4: [[SCALAR_PH]]: ; CHECK-VF4IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 2147483648, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] -; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] +; CHECK-VF4IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP15]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] ; CHECK-VF4IC4-NEXT: br label %[[FOR_BODY:.*]] ; CHECK-VF4IC4: [[FOR_BODY]]: ; CHECK-VF4IC4-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] @@ -537,7 +533,7 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) { ; CHECK-VF4IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 2147483648 ; CHECK-VF4IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK-VF4IC4: [[EXIT]]: -; CHECK-VF4IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ] +; CHECK-VF4IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[TMP15]], %[[MIDDLE_BLOCK]] ] ; CHECK-VF4IC4-NEXT: ret i32 [[SPEC_SELECT_LCSSA]] ; ; CHECK-VF1IC4-LABEL: define i32 @select_fcmp_max_valid_const_ub( @@ -548,10 +544,10 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) { ; CHECK-VF1IC4-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK-VF1IC4: [[VECTOR_BODY]]: ; CHECK-VF1IC4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ -2147483648, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ] -; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ -2147483648, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ] -; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ -2147483648, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ] -; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ -2147483648, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ] +; CHECK-VF1IC4-NEXT: [[VEC_PHI:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP21:%.*]], %[[VECTOR_BODY]] ] +; CHECK-VF1IC4-NEXT: [[VEC_PHI1:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP22:%.*]], %[[VECTOR_BODY]] ] +; CHECK-VF1IC4-NEXT: [[VEC_PHI2:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP23:%.*]], %[[VECTOR_BODY]] ] +; CHECK-VF1IC4-NEXT: [[VEC_PHI3:%.*]] = phi i32 [ -1, %[[VECTOR_PH]] ], [ [[TMP24:%.*]], %[[VECTOR_BODY]] ] ; CHECK-VF1IC4-NEXT: [[TMP1:%.*]] = add i64 [[INDEX]], 1 ; CHECK-VF1IC4-NEXT: [[TMP2:%.*]] = add i64 [[INDEX]], 2 ; CHECK-VF1IC4-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 3 @@ -582,12 +578,10 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) { ; CHECK-VF1IC4-NEXT: [[RDX_MINMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP21]], i32 [[TMP22]]) ; CHECK-VF1IC4-NEXT: [[RDX_MINMAX4:%.*]] = call i32 @llvm.smax.i32(i32 [[RDX_MINMAX]], i32 [[TMP23]]) ; CHECK-VF1IC4-NEXT: [[RDX_MINMAX5:%.*]] = call i32 @llvm.smax.i32(i32 [[RDX_MINMAX4]], i32 [[TMP24]]) -; CHECK-VF1IC4-NEXT: [[RDX_SELECT_CMP:%.*]] = icmp ne i32 [[RDX_MINMAX5]], -2147483648 -; CHECK-VF1IC4-NEXT: [[RDX_SELECT:%.*]] = select i1 [[RDX_SELECT_CMP]], i32 [[RDX_MINMAX5]], i32 -1 ; CHECK-VF1IC4-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] ; CHECK-VF1IC4: [[SCALAR_PH]]: ; CHECK-VF1IC4-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 2147483648, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] -; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] +; CHECK-VF1IC4-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[RDX_MINMAX5]], %[[MIDDLE_BLOCK]] ], [ -1, %[[ENTRY]] ] ; CHECK-VF1IC4-NEXT: br label %[[FOR_BODY:.*]] ; CHECK-VF1IC4: [[FOR_BODY]]: ; CHECK-VF1IC4-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[INC:%.*]], %[[FOR_BODY]] ] @@ -601,7 +595,7 @@ define i32 @select_fcmp_max_valid_const_ub(ptr %a) { ; CHECK-VF1IC4-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], 2147483648 ; CHECK-VF1IC4-NEXT: br i1 [[EXITCOND_NOT]], label %[[EXIT]], label %[[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK-VF1IC4: [[EXIT]]: -; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[RDX_SELECT]], %[[MIDDLE_BLOCK]] ] +; CHECK-VF1IC4-NEXT: [[SPEC_SELECT_LCSSA:%.*]] = phi i32 [ [[SPEC_SELECT]], %[[FOR_BODY]] ], [ [[RDX_MINMAX5]], %[[MIDDLE_BLOCK]] ] ; CHECK-VF1IC4-NEXT: ret i32 [[SPEC_SELECT_LCSSA]] ; entry: