carbon-language/carbon-lang
1
0
Fork 0
mirror of https://github.com/carbon-language/carbon-lang.git synced 2026-10-05 22:02:55 +01:00
Code Issues Packages Projects Releases 10 Wiki Activity

[SVE] Remove invalid calls to VectorType::getNumElements from BasicTTIImpl

Browse Source 
Summary:
Most of these operations are reasonable for scalable vectors. Due to
this, we have decided not to change the interface to specifically take
FixedVectorType despite the fact that the current implementations make
fixed width assumptions. Instead, we cast to FixedVectorType and assert
in the body. If a developer makes some change in the future that causes
one of these asserts to fire, they should either change their code or
make the function they are trying to call handle scalable vectors.

Reviewers: efriedma, samparker, RKSimon, craig.topper, sdesmalen, c-rhodes

Reviewed By: efriedma

Subscribers: tschuett, rkruppe, psnobl, llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D81495
This commit is contained in:
Christopher Tetreault
2020-06-16 14:16:15 -07:00
parent 369d00df60
commit b3e77c6d55
1 changed files with 57 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
llvm/include/llvm/CodeGen/BasicTTIImpl.h
+57 -40
View File
@@ -80,7 +80,7 @@ private:
/// Estimate a cost of Broadcast as an extract and sequence of insert
/// operations.
unsigned getBroadcastShuffleOverhead(VectorType *VTy) {
unsigned getBroadcastShuffleOverhead(FixedVectorType *VTy) {
unsigned Cost = 0;
// Broadcast cost is equal to the cost of extracting the zero'th element
// plus the cost of inserting it into every element of the result vector.
@@ -96,7 +96,7 @@ private:
/// Estimate a cost of shuffle as a sequence of extract and insert
/// operations.
unsigned getPermuteShuffleOverhead(VectorType *VTy) {
unsigned getPermuteShuffleOverhead(FixedVectorType *VTy) {
unsigned Cost = 0;
// Shuffle cost is equal to the cost of extracting element from its argument
// plus the cost of inserting them onto the result vector.
@@ -116,8 +116,8 @@ private:
/// Estimate a cost of subvector extraction as a sequence of extract and
/// insert operations.
unsigned getExtractSubvectorOverhead(VectorType *VTy, int Index,
VectorType *SubVTy) {
unsigned getExtractSubvectorOverhead(FixedVectorType *VTy, int Index,
FixedVectorType *SubVTy) {
assert(VTy && SubVTy &&
"Can only extract subvectors from vectors");
int NumSubElts = SubVTy->getNumElements();
@@ -139,8 +139,8 @@ private:
/// Estimate a cost of subvector insertion as a sequence of extract and
/// insert operations.
unsigned getInsertSubvectorOverhead(VectorType *VTy, int Index,
VectorType *SubVTy) {
unsigned getInsertSubvectorOverhead(FixedVectorType *VTy, int Index,
FixedVectorType *SubVTy) {
assert(VTy && SubVTy &&
"Can only insert subvectors into vectors");
int NumSubElts = SubVTy->getNumElements();
@@ -525,8 +525,12 @@ public:
/// Estimate the overhead of scalarizing an instruction. Insert and Extract
/// are set if the demanded result elements need to be inserted and/or
/// extracted from vectors.
unsigned getScalarizationOverhead(VectorType *Ty, const APInt &DemandedElts,
unsigned getScalarizationOverhead(VectorType *InTy, const APInt &DemandedElts,
bool Insert, bool Extract) {
/// FIXME: a bitfield is not a reasonable abstraction for talking about
/// which elements are needed from a scalable vector
auto *Ty = cast<FixedVectorType>(InTy);
assert(DemandedElts.getBitWidth() == Ty->getNumElements() &&
"Vector size mismatch");
@@ -547,7 +551,10 @@ public:
}
/// Helper wrapper for the DemandedElts variant of getScalarizationOverhead.
unsigned getScalarizationOverhead(VectorType *Ty, bool Insert, bool Extract) {
unsigned getScalarizationOverhead(VectorType *InTy, bool Insert,
bool Extract) {
auto *Ty = cast<FixedVectorType>(InTy);
APInt DemandedElts = APInt::getAllOnesValue(Ty->getNumElements());
return static_cast<T *>(this)->getScalarizationOverhead(Ty, DemandedElts,
Insert, Extract);
@@ -565,11 +572,12 @@ public:
auto *VecTy = dyn_cast<VectorType>(A->getType());
if (VecTy) {
// If A is a vector operand, VF should be 1 or correspond to A.
assert((VF == 1 || VF == VecTy->getNumElements()) &&
assert((VF == 1 ||
VF == cast<FixedVectorType>(VecTy)->getNumElements()) &&
"Vector argument does not match VF");
}
else
VecTy = VectorType::get(A->getType(), VF);
VecTy = FixedVectorType::get(A->getType(), VF);
Cost += getScalarizationOverhead(VecTy, false, true);
}
@@ -578,7 +586,10 @@ public:
return Cost;
}
unsigned getScalarizationOverhead(VectorType *Ty, ArrayRef<const Value *> Args) {
unsigned getScalarizationOverhead(VectorType *InTy,
ArrayRef<const Value *> Args) {
auto *Ty = cast<FixedVectorType>(InTy);
unsigned Cost = 0;
Cost += getScalarizationOverhead(Ty, true, false);
@@ -638,7 +649,7 @@ public:
// TODO: If one of the types get legalized by splitting, handle this
// similarly to what getCastInstrCost() does.
if (auto *VTy = dyn_cast<VectorType>(Ty)) {
unsigned Num = VTy->getNumElements();
unsigned Num = cast<FixedVectorType>(VTy)->getNumElements();
unsigned Cost = static_cast<T *>(this)->getArithmeticInstrCost(
Opcode, VTy->getScalarType(), CostKind);
// Return the cost of multiple scalar invocation plus the cost of
@@ -652,19 +663,22 @@ public:
unsigned getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp, int Index,
VectorType *SubTp) {
switch (Kind) {
case TTI::SK_Broadcast:
return getBroadcastShuffleOverhead(Tp);
return getBroadcastShuffleOverhead(cast<FixedVectorType>(Tp));
case TTI::SK_Select:
case TTI::SK_Reverse:
case TTI::SK_Transpose:
case TTI::SK_PermuteSingleSrc:
case TTI::SK_PermuteTwoSrc:
return getPermuteShuffleOverhead(Tp);
return getPermuteShuffleOverhead(cast<FixedVectorType>(Tp));
case TTI::SK_ExtractSubvector:
return getExtractSubvectorOverhead(Tp, Index, SubTp);
return getExtractSubvectorOverhead(cast<FixedVectorType>(Tp), Index,
cast<FixedVectorType>(SubTp));
case TTI::SK_InsertSubvector:
return getInsertSubvectorOverhead(Tp, Index, SubTp);
return getInsertSubvectorOverhead(cast<FixedVectorType>(Tp), Index,
cast<FixedVectorType>(SubTp));
}
llvm_unreachable("Unknown TTI::ShuffleKind");
}
@@ -784,12 +798,11 @@ public:
bool SplitDst =
TLI->getTypeAction(Dst->getContext(), TLI->getValueType(DL, Dst)) ==
TargetLowering::TypeSplitVector;
if ((SplitSrc || SplitDst) && SrcVTy->getNumElements() > 1 &&
DstVTy->getNumElements() > 1) {
Type *SplitDstTy = VectorType::get(DstVTy->getElementType(),
DstVTy->getNumElements() / 2);
Type *SplitSrcTy = VectorType::get(SrcVTy->getElementType(),
SrcVTy->getNumElements() / 2);
if ((SplitSrc || SplitDst) &&
cast<FixedVectorType>(SrcVTy)->getNumElements() > 1 &&
cast<FixedVectorType>(DstVTy)->getNumElements() > 1) {
Type *SplitDstTy = VectorType::getHalfElementsVectorType(DstVTy);
Type *SplitSrcTy = VectorType::getHalfElementsVectorType(SrcVTy);
T *TTI = static_cast<T *>(this);
// If both types need to be split then the split is free.
unsigned SplitCost =
@@ -801,7 +814,7 @@ public:
// In other cases where the source or destination are illegal, assume
// the operation will get scalarized.
unsigned Num = DstVTy->getNumElements();
unsigned Num = cast<FixedVectorType>(DstVTy)->getNumElements();
unsigned Cost = static_cast<T *>(this)->getCastInstrCost(
Opcode, Dst->getScalarType(), Src->getScalarType(),
CostKind, I);
@@ -867,7 +880,7 @@ public:
// TODO: If one of the types get legalized by splitting, handle this
// similarly to what getCastInstrCost() does.
if (auto *ValVTy = dyn_cast<VectorType>(ValTy)) {
unsigned Num = ValVTy->getNumElements();
unsigned Num = cast<FixedVectorType>(ValVTy)->getNumElements();
if (CondTy)
CondTy = CondTy->getScalarType();
unsigned Cost = static_cast<T *>(this)->getCmpSelInstrCost(
@@ -935,13 +948,13 @@ public:
TTI::TargetCostKind CostKind,
bool UseMaskForCond = false,
bool UseMaskForGaps = false) {
auto *VT = cast<VectorType>(VecTy);
auto *VT = cast<FixedVectorType>(VecTy);
unsigned NumElts = VT->getNumElements();
assert(Factor > 1 && NumElts % Factor == 0 && "Invalid interleave factor");
unsigned NumSubElts = NumElts / Factor;
VectorType *SubVT = VectorType::get(VT->getElementType(), NumSubElts);
auto *SubVT = FixedVectorType::get(VT->getElementType(), NumSubElts);
// Firstly, the cost of load/store operation.
unsigned Cost;
@@ -1050,8 +1063,8 @@ public:
return Cost;
Type *I8Type = Type::getInt8Ty(VT->getContext());
VectorType *MaskVT = VectorType::get(I8Type, NumElts);
SubVT = VectorType::get(I8Type, NumSubElts);
auto *MaskVT = FixedVectorType::get(I8Type, NumElts);
SubVT = FixedVectorType::get(I8Type, NumSubElts);
// The Mask shuffling cost is extract all the elements of the Mask
// and insert each of them Factor times into the wide vector:
@@ -1119,7 +1132,8 @@ public:
Type *RetTy = ICA.getReturnType();
unsigned VF = ICA.getVectorFactor();
unsigned RetVF =
(RetTy->isVectorTy() ? cast<VectorType>(RetTy)->getNumElements() : 1);
(RetTy->isVectorTy() ? cast<FixedVectorType>(RetTy)->getNumElements()
: 1);
assert((RetVF == 1 || VF == 1) && "VF > 1 and RetVF is a vector type");
const IntrinsicInst *I = ICA.getInst();
const SmallVectorImpl<Value *> &Args = ICA.getArgs();
@@ -1132,11 +1146,11 @@ public:
for (Value *Op : Args) {
Type *OpTy = Op->getType();
assert(VF == 1 || !OpTy->isVectorTy());
Types.push_back(VF == 1 ? OpTy : VectorType::get(OpTy, VF));
Types.push_back(VF == 1 ? OpTy : FixedVectorType::get(OpTy, VF));
}
if (VF > 1 && !RetTy->isVoidTy())
RetTy = VectorType::get(RetTy, VF);
RetTy = FixedVectorType::get(RetTy, VF);
// Compute the scalarization overhead based on Args for a vector
// intrinsic. A vectorizer will pass a scalar RetTy and VF > 1, while
@@ -1262,7 +1276,8 @@ public:
if (auto *RetVTy = dyn_cast<VectorType>(RetTy)) {
if (!SkipScalarizationCost)
ScalarizationCost = getScalarizationOverhead(RetVTy, true, false);
ScalarCalls = std::max(ScalarCalls, RetVTy->getNumElements());
ScalarCalls = std::max(ScalarCalls,
cast<FixedVectorType>(RetVTy)->getNumElements());
ScalarRetTy = RetTy->getScalarType();
}
SmallVector<Type *, 4> ScalarTys;
@@ -1271,7 +1286,8 @@ public:
if (auto *VTy = dyn_cast<VectorType>(Ty)) {
if (!SkipScalarizationCost)
ScalarizationCost += getScalarizationOverhead(VTy, false, true);
ScalarCalls = std::max(ScalarCalls, VTy->getNumElements());
ScalarCalls = std::max(ScalarCalls,
cast<FixedVectorType>(VTy)->getNumElements());
Ty = Ty->getScalarType();
}
ScalarTys.push_back(Ty);
@@ -1629,7 +1645,7 @@ public:
unsigned ScalarizationCost = SkipScalarizationCost ?
ScalarizationCostPassed : getScalarizationOverhead(RetVTy, true, false);
unsigned ScalarCalls = RetVTy->getNumElements();
unsigned ScalarCalls = cast<FixedVectorType>(RetVTy)->getNumElements();
SmallVector<Type *, 4> ScalarTys;
for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
Type *Ty = Tys[i];
@@ -1643,7 +1659,8 @@ public:
if (auto *VTy = dyn_cast<VectorType>(Tys[i])) {
if (!ICA.skipScalarizationCost())
ScalarizationCost += getScalarizationOverhead(VTy, false, true);
ScalarCalls = std::max(ScalarCalls, VTy->getNumElements());
ScalarCalls = std::max(ScalarCalls,
cast<FixedVectorType>(VTy)->getNumElements());
}
}
return ScalarCalls * ScalarCost + ScalarizationCost;
@@ -1718,7 +1735,7 @@ public:
bool IsPairwise,
TTI::TargetCostKind CostKind) {
Type *ScalarTy = Ty->getElementType();
unsigned NumVecElts = Ty->getNumElements();
unsigned NumVecElts = cast<FixedVectorType>(Ty)->getNumElements();
unsigned NumReduxLevels = Log2_32(NumVecElts);
unsigned ArithCost = 0;
unsigned ShuffleCost = 0;
@@ -1730,7 +1747,7 @@ public:
LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
while (NumVecElts > MVTLen) {
NumVecElts /= 2;
VectorType *SubTy = VectorType::get(ScalarTy, NumVecElts);
VectorType *SubTy = FixedVectorType::get(ScalarTy, NumVecElts);
// Assume the pairwise shuffles add a cost.
ShuffleCost += (IsPairwise + 1) *
ConcreteTTI->getShuffleCost(TTI::SK_ExtractSubvector, Ty,
@@ -1769,7 +1786,7 @@ public:
TTI::TargetCostKind CostKind) {
Type *ScalarTy = Ty->getElementType();
Type *ScalarCondTy = CondTy->getElementType();
unsigned NumVecElts = Ty->getNumElements();
unsigned NumVecElts = cast<FixedVectorType>(Ty)->getNumElements();
unsigned NumReduxLevels = Log2_32(NumVecElts);
unsigned CmpOpcode;
if (Ty->isFPOrFPVectorTy()) {
@@ -1789,8 +1806,8 @@ public:
LT.second.isVector() ? LT.second.getVectorNumElements() : 1;
while (NumVecElts > MVTLen) {
NumVecElts /= 2;
VectorType *SubTy = VectorType::get(ScalarTy, NumVecElts);
CondTy = VectorType::get(ScalarCondTy, NumVecElts);
auto *SubTy = FixedVectorType::get(ScalarTy, NumVecElts);
CondTy = FixedVectorType::get(ScalarCondTy, NumVecElts);
// Assume the pairwise shuffles add a cost.
ShuffleCost += (IsPairwise + 1) *