diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 5b8725178521e..277e43a38018e 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8532,7 +8532,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes( // failures. VPlanTransforms::addExitUsersForFirstOrderRecurrences(*Plan, Range); DenseMap IVEndValues; - VPlanTransforms::addScalarResumePhis(*Plan, RecipeBuilder, IVEndValues); + VPlanTransforms::updateScalarResumePhis(*Plan, IVEndValues); // --------------------------------------------------------------------------- // Transform initial VPlan: Apply previously taken decisions, in order, to @@ -8630,23 +8630,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) { *TLI)) return nullptr; - // Collect mapping of IR header phis to header phi recipes, to be used in - // addScalarResumePhis. - DenseMap BlockMaskCache; - VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE, - Builder, BlockMaskCache); - for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) { - if (isa(&R)) - continue; - auto *HeaderR = cast(&R); - RecipeBuilder.setRecipe(HeaderR->getUnderlyingInstr(), HeaderR); - } - DenseMap IVEndValues; // TODO: IVEndValues are not used yet in the native path, to optimize exit // values. // TODO: We can't call runPass on the transform yet, due to verifier // failures. - VPlanTransforms::addScalarResumePhis(*Plan, RecipeBuilder, IVEndValues); + DenseMap IVEndValues; + VPlanTransforms::updateScalarResumePhis(*Plan, IVEndValues); assert(verifyVPlanIsValid(*Plan) && "VPlan is invalid"); return Plan; diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 0fc527e07e634..405f83a6ce8e5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1134,7 +1134,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, OpcodeTy Opcode; /// An optional name that can be used for the generated IR instruction. - const std::string Name; + std::string Name; /// Returns true if we can generate a scalar for the first lane only if /// needed. @@ -1225,6 +1225,9 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, /// Returns the symbolic name assigned to the VPInstruction. StringRef getName() const { return Name; } + /// Set the symbolic name for the VPInstruction. + void setName(StringRef NewName) { Name = NewName.str(); } + protected: #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) /// Print the VPInstruction to \p O. diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index dbbde1cafa9f2..5fbd61a929fe2 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -554,6 +554,15 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL, Plan.getEntry()->swapSuccessors(); createExtractsForLiveOuts(Plan, MiddleVPBB); + + VPBuilder ScalarPHBuilder(ScalarPH); + for (const auto &[PhiR, ScalarPhiR] : zip_equal( + drop_begin(HeaderVPBB->phis()), Plan.getScalarHeader()->phis())) { + auto *VectorPhiR = cast(&PhiR); + auto *ResumePhiR = ScalarPHBuilder.createScalarPhi( + {VectorPhiR, VectorPhiR->getOperand(0)}, VectorPhiR->getDebugLoc()); + cast(&ScalarPhiR)->addOperand(ResumePhiR); + } } std::unique_ptr diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a6a03c0d805fd..dc2d013a24029 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -4526,12 +4526,13 @@ void VPlanTransforms::addBranchWeightToMiddleTerminator( MiddleTerm->setMetadata(LLVMContext::MD_prof, BranchWeights); } -/// Create and return a ResumePhi for \p WideIV, unless it is truncated. If the -/// induction recipe is not canonical, creates a VPDerivedIVRecipe to compute -/// the end value of the induction. -static VPInstruction *addResumePhiRecipeForInduction( - VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, - VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC) { +/// Compute and return the end value for \p WideIV, unless it is truncated. If +/// the induction recipe is not canonical, creates a VPDerivedIVRecipe to +/// compute the end value of the induction. +static VPValue *tryToComputeEndValueForInduction(VPWidenInductionRecipe *WideIV, + VPBuilder &VectorPHBuilder, + VPTypeAnalysis &TypeInfo, + VPValue *VectorTC) { auto *WideIntOrFp = dyn_cast(WideIV); // Truncated wide inductions resume from the last lane of their vector value // in the last vector iteration which is handled elsewhere. @@ -4557,14 +4558,11 @@ static VPInstruction *addResumePhiRecipeForInduction( WideIV->getDebugLoc()); } - auto *ResumePhiRecipe = ScalarPHBuilder.createScalarPhi( - {EndValue, Start}, WideIV->getDebugLoc(), "bc.resume.val"); - return ResumePhiRecipe; + return EndValue; } -void VPlanTransforms::addScalarResumePhis( - VPlan &Plan, VPRecipeBuilder &Builder, - DenseMap &IVEndValues) { +void VPlanTransforms::updateScalarResumePhis( + VPlan &Plan, DenseMap &IVEndValues) { VPTypeAnalysis TypeInfo(Plan); auto *ScalarPH = Plan.getScalarPreheader(); auto *MiddleVPBB = cast(ScalarPH->getPredecessors()[0]); @@ -4572,21 +4570,18 @@ void VPlanTransforms::addScalarResumePhis( VPBuilder VectorPHBuilder( cast(VectorRegion->getSinglePredecessor())); VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi()); - VPBuilder ScalarPHBuilder(ScalarPH); - for (VPRecipeBase &ScalarPhiR : Plan.getScalarHeader()->phis()) { - auto *ScalarPhiIRI = cast(&ScalarPhiR); + for (VPRecipeBase &PhiR : Plan.getScalarPreheader()->phis()) { + auto *ResumePhiR = cast(&PhiR); // TODO: Extract final value from induction recipe initially, optimize to // pre-computed end value together in optimizeInductionExitUsers. - auto *VectorPhiR = - cast(Builder.getRecipe(&ScalarPhiIRI->getIRPhi())); + auto *VectorPhiR = cast(ResumePhiR->getOperand(0)); if (auto *WideIVR = dyn_cast(VectorPhiR)) { - if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction( - WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo, - &Plan.getVectorTripCount())) { - assert(isa(ResumePhi) && "Expected a phi"); - IVEndValues[WideIVR] = ResumePhi->getOperand(0); - ScalarPhiIRI->addOperand(ResumePhi); + if (VPValue *EndValue = tryToComputeEndValueForInduction( + WideIVR, VectorPHBuilder, TypeInfo, &Plan.getVectorTripCount())) { + IVEndValues[WideIVR] = EndValue; + ResumePhiR->setOperand(0, EndValue); + ResumePhiR->setName("bc.resume.val"); continue; } // TODO: Also handle truncated inductions here. Computing end-values @@ -4608,10 +4603,8 @@ void VPlanTransforms::addScalarResumePhis( ResumeFromVectorLoop = MiddleBuilder.createNaryOp( VPInstruction::ExtractLastElement, {ResumeFromVectorLoop}, {}, "vector.recur.extract"); - StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx"; - auto *ResumePhiR = ScalarPHBuilder.createScalarPhi( - {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name); - ScalarPhiIRI->addOperand(ResumePhiR); + ResumePhiR->setName(IsFOR ? "scalar.recur.init" : "bc.merge.rdx"); + ResumePhiR->setOperand(0, ResumeFromVectorLoop); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 708ea4185e1cb..5fd3f756c55e3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -372,12 +372,12 @@ struct VPlanTransforms { addBranchWeightToMiddleTerminator(VPlan &Plan, ElementCount VF, std::optional VScaleForTuning); - /// Create resume phis in the scalar preheader for first-order recurrences, - /// reductions and inductions, and update the VPIRInstructions wrapping the - /// original phis in the scalar header. End values for inductions are added to - /// \p IVEndValues. - static void addScalarResumePhis(VPlan &Plan, VPRecipeBuilder &Builder, - DenseMap &IVEndValues); + /// Update the resume phis in the scalar preheader after creating wide recipes + /// for first-order recurrences, reductions and inductions. End values for + /// inductions are added to \p IVEndValues. + static void + updateScalarResumePhis(VPlan &Plan, + DenseMap &IVEndValues); /// Handle users in the exit block for first order reductions in the original /// exit block. The penultimate value of recurrences is fed to their LCSSA phi diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp index 5742df2aa3c53..127e830b1bc14 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp @@ -113,12 +113,13 @@ compound=true N0 -> N2 [ label="F"] N1 [label = "scalar.ph:\l" + + " EMIT-SCALAR vp\<%6\> = phi [ ir\<%indvars.iv\>, middle.block ], [ ir\<0\>, ir-bb\ ]\l" + "Successor(s): ir-bb\\l" ] N1 -> N3 [ label=""] N3 [label = "ir-bb\:\l" + - " IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\l" + + " IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] (extra operand: vp\<%6\> from scalar.ph)\l" + " IR %arr.idx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv\l" + " IR %l1 = load i32, ptr %arr.idx, align 4\l" + " IR %res = add i32 %l1, 10\l" + @@ -282,12 +283,13 @@ compound=true N0 -> N2 [ label="F"] N1 [label = "scalar.ph:\l" + + " EMIT-SCALAR vp\<%6\> = phi [ ir\<%iv\>, middle.block ], [ ir\<0\>, ir-bb\ ]\l" + "Successor(s): ir-bb\\l" ] N1 -> N3 [ label=""] N3 [label = "ir-bb\:\l" + - " IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]\l" + + " IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp\<%6\> from scalar.ph)\l" + " IR %arr.idx = getelementptr inbounds i32, ptr %A, i64 %iv\l" + " IR %l1 = load i32, ptr %arr.idx, align 4\l" + " IR %c = icmp eq i32 %l1, 0\l" + diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index 1b0cd1074c1b5..ec8f06a1eb755 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -346,6 +346,8 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInScalarHeaderVPIRBB) { Function *F = M.getFunction("f"); BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); auto Plan = buildVPlan(LoopHeader); + VPValue *Zero = Plan->getConstantInt(32, 0); + Plan->getScalarHeader()->front().addOperand(Zero); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -387,8 +389,6 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInExitVPIRBB) { {HeaderBlock->front().getVPSingleValue()}); DefI->insertBefore(Plan->getMiddleBlock()->getTerminator()); Plan->getExitBlocks()[0]->front().addOperand(DefI); - VPValue *Zero = Plan->getConstantInt(32, 0); - Plan->getScalarHeader()->front().addOperand(Zero); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr();