Skip to content

Commit

Permalink
[VPlan] Limit stride replacement to vector region and middle VPBB (NFC).
Browse files Browse the repository at this point in the history
At the moment this in NFC, but ensures we only replace uses that are
dominated by runtime checks as we model more of the skeleton in VPlan.
  • Loading branch information
fhahn committed Oct 24, 2024
1 parent 0558fe3 commit 7b9f988
Showing 1 changed file with 10 additions and 2 deletions.
12 changes: 10 additions & 2 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9189,6 +9189,14 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {

// Replace VPValues for known constant strides guaranteed by predicate scalar
// evolution.
auto CanUseVersionedStride = [&Plan](VPUser &U, unsigned) {
auto *R = dyn_cast<VPRecipeBase>(&U);
if (!R)
return false;
return R->getParent()->getParent() ||
R->getParent() ==
Plan->getVectorLoopRegion()->getSinglePredecessor();
};
for (auto [_, Stride] : Legal->getLAI()->getSymbolicStrides()) {
auto *StrideV = cast<SCEVUnknown>(Stride)->getValue();
auto *ScevStride = dyn_cast<SCEVConstant>(PSE.getSCEV(StrideV));
Expand All @@ -9199,7 +9207,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
auto *CI = Plan->getOrAddLiveIn(
ConstantInt::get(Stride->getType(), ScevStride->getAPInt()));
if (VPValue *StrideVPV = Plan->getLiveIn(StrideV))
StrideVPV->replaceAllUsesWith(CI);
StrideVPV->replaceUsesWithIf(CI, CanUseVersionedStride);

// The versioned value may not be used in the loop directly but through a
// sext/zext. Add new live-ins in those cases.
Expand All @@ -9213,7 +9221,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) {
APInt C = isa<SExtInst>(U) ? ScevStride->getAPInt().sext(BW)
: ScevStride->getAPInt().zext(BW);
VPValue *CI = Plan->getOrAddLiveIn(ConstantInt::get(U->getType(), C));
StrideVPV->replaceAllUsesWith(CI);
StrideVPV->replaceUsesWithIf(CI, CanUseVersionedStride);
}
}

Expand Down

0 comments on commit 7b9f988

Please sign in to comment.