Skip to content

Commit c57e522

Browse files
petar-avramovicpetar-avramovic
committed
AMDGPU/GlobalISel: add RegBankLegalize rules for extends and trunc
Uniform S1: Truncs to uniform S1 and AnyExts from S1 are left as is as they are meant to be combined away. Uniform S1 ZExt and SExt are lowered using select. Divergent S1: Trunc of VGPR to VCC is lowered as compare. Extends of VCC are lowered using select. For remaining types: S32 to S64 ZExt and SExt are lowered using merge values, AnyExt and Trunc are again left as is to be combined away. Notably uniform S16 for SExt and Zext is not lowered to S32 and left as is for instruction select to deal with them. This is because there are patterns that check for S16 type.
1 parent ba48a1a commit c57e522

10 files changed

+357
-183
lines changed

‎llvm/lib/Target/AMDGPU/AMDGPURegBankLegalize.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,13 @@ class AMDGPURegBankLegalizeCombiner {
214214
return;
215215
}
216216

217+
if (DstTy == S64 && TruncSrcTy == S32) {
218+
B.buildMergeLikeInstr(MI.getOperand(0).getReg(),
219+
{TruncSrc, B.buildUndef({SgprRB, S32})});
220+
cleanUpAfterCombine(MI, Trunc);
221+
return;
222+
}
223+
217224
if (DstTy == S32 && TruncSrcTy == S16) {
218225
B.buildAnyExt(Dst, TruncSrc);
219226
cleanUpAfterCombine(MI, Trunc);

‎llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.cpp

Lines changed: 81 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,40 @@ void RegBankLegalizeHelper::widenLoad(MachineInstr &MI, LLT WideTy,
131131
MI.eraseFromParent();
132132
}
133133

134+
void RegBankLegalizeHelper::lowerVccExtToSel(MachineInstr &MI) {
135+
Register Dst = MI.getOperand(0).getReg();
136+
LLT Ty = MRI.getType(Dst);
137+
Register Src = MI.getOperand(1).getReg();
138+
unsigned Opc = MI.getOpcode();
139+
if (Ty == S32 || Ty == S16) {
140+
auto True = B.buildConstant({VgprRB, Ty}, Opc == G_SEXT ? -1 : 1);
141+
auto False = B.buildConstant({VgprRB, Ty}, 0);
142+
B.buildSelect(Dst, Src, True, False);
143+
}
144+
if (Ty == S64) {
145+
auto True = B.buildConstant({VgprRB, S32}, Opc == G_SEXT ? -1 : 1);
146+
auto False = B.buildConstant({VgprRB, S32}, 0);
147+
auto Lo = B.buildSelect({VgprRB, S32}, Src, True, False);
148+
MachineInstrBuilder Hi;
149+
switch (Opc) {
150+
case G_SEXT:
151+
Hi = Lo;
152+
break;
153+
case G_ZEXT:
154+
Hi = False;
155+
break;
156+
case G_ANYEXT:
157+
Hi = B.buildUndef({VgprRB_S32});
158+
break;
159+
default:
160+
llvm_unreachable("Opcode not supported");
161+
}
162+
163+
B.buildMergeValues(Dst, {Lo.getReg(0), Hi.getReg(0)});
164+
}
165+
MI.eraseFromParent();
166+
}
167+
134168
bool isSignedBFE(MachineInstr &MI) {
135169
if (isa<GIntrinsic>(MI)) {
136170
switch (MI.getOperand(1).getIntrinsicID()) {
@@ -267,26 +301,8 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
267301
switch (Mapping.LoweringMethod) {
268302
case DoNotLower:
269303
return;
270-
case VccExtToSel: {
271-
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
272-
Register Src = MI.getOperand(1).getReg();
273-
unsigned Opc = MI.getOpcode();
274-
if (Ty == S32 || Ty == S16) {
275-
auto True = B.buildConstant({VgprRB, Ty}, Opc == G_SEXT ? -1 : 1);
276-
auto False = B.buildConstant({VgprRB, Ty}, 0);
277-
B.buildSelect(MI.getOperand(0).getReg(), Src, True, False);
278-
}
279-
if (Ty == S64) {
280-
auto True = B.buildConstant({VgprRB, S32}, Opc == G_SEXT ? -1 : 1);
281-
auto False = B.buildConstant({VgprRB, S32}, 0);
282-
auto Sel = B.buildSelect({VgprRB, S32}, Src, True, False);
283-
B.buildMergeValues(
284-
MI.getOperand(0).getReg(),
285-
{Sel.getReg(0), Opc == G_SEXT ? Sel.getReg(0) : False.getReg(0)});
286-
}
287-
MI.eraseFromParent();
288-
return;
289-
}
304+
case VccExtToSel:
305+
return lowerVccExtToSel(MI);
290306
case UniExtToSel: {
291307
LLT Ty = MRI.getType(MI.getOperand(0).getReg());
292308
auto True = B.buildConstant({SgprRB, Ty},
@@ -303,13 +319,23 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
303319
case Ext32To64: {
304320
const RegisterBank *RB = MRI.getRegBank(MI.getOperand(0).getReg());
305321
MachineInstrBuilder Hi;
306-
307-
if (MI.getOpcode() == AMDGPU::G_ZEXT) {
322+
switch (MI.getOpcode()) {
323+
case AMDGPU::G_ZEXT: {
308324
Hi = B.buildConstant({RB, S32}, 0);
309-
} else {
325+
break;
326+
}
327+
case AMDGPU::G_SEXT: {
310328
// Replicate sign bit from 32-bit extended part.
311329
auto ShiftAmt = B.buildConstant({RB, S32}, 31);
312330
Hi = B.buildAShr({RB, S32}, MI.getOperand(1).getReg(), ShiftAmt);
331+
break;
332+
}
333+
case AMDGPU::G_ANYEXT: {
334+
Hi = B.buildUndef({RB, S32});
335+
break;
336+
}
337+
default:
338+
llvm_unreachable("Unsuported Opcode in Ext32To64");
313339
}
314340

315341
B.buildMergeLikeInstr(MI.getOperand(0).getReg(),
@@ -332,7 +358,7 @@ void RegBankLegalizeHelper::lower(MachineInstr &MI,
332358
// compares all bits in register.
333359
Register BoolSrc = MRI.createVirtualRegister({VgprRB, Ty});
334360
if (Ty == S64) {
335-
auto Src64 = B.buildUnmerge({VgprRB, Ty}, Src);
361+
auto Src64 = B.buildUnmerge(VgprRB_S32, Src);
336362
auto One = B.buildConstant(VgprRB_S32, 1);
337363
auto AndLo = B.buildAnd(VgprRB_S32, Src64.getReg(0), One);
338364
auto Zero = B.buildConstant(VgprRB_S32, 0);
@@ -420,8 +446,11 @@ LLT RegBankLegalizeHelper::getTyFromID(RegBankLLTMappingApplyID ID) {
420446
case Sgpr32AExt:
421447
case Sgpr32AExtBoolInReg:
422448
case Sgpr32SExt:
449+
case Sgpr32ZExt:
423450
case UniInVgprS32:
424451
case Vgpr32:
452+
case Vgpr32SExt:
453+
case Vgpr32ZExt:
425454
return LLT::scalar(32);
426455
case Sgpr64:
427456
case Vgpr64:
@@ -532,6 +561,7 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
532561
case Sgpr32AExt:
533562
case Sgpr32AExtBoolInReg:
534563
case Sgpr32SExt:
564+
case Sgpr32ZExt:
535565
return SgprRB;
536566
case Vgpr16:
537567
case Vgpr32:
@@ -548,6 +578,8 @@ RegBankLegalizeHelper::getRegBankFromID(RegBankLLTMappingApplyID ID) {
548578
case VgprB128:
549579
case VgprB256:
550580
case VgprB512:
581+
case Vgpr32SExt:
582+
case Vgpr32ZExt:
551583
return VgprRB;
552584
default:
553585
return nullptr;
@@ -753,8 +785,8 @@ void RegBankLegalizeHelper::applyMappingSrc(
753785
assert(Ty.getSizeInBits() == 1);
754786
assert(RB == SgprRB);
755787
auto Aext = B.buildAnyExt(SgprRB_S32, Reg);
756-
// Zext SgprS1 is not legal, this instruction is most of times meant to be
757-
// combined away in RB combiner, so do not make AND with 1.
788+
// Zext SgprS1 is not legal, make AND with 1 instead. This instruction is
789+
// most of times meant to be combined away in AMDGPURegBankCombiner.
758790
auto Cst1 = B.buildConstant(SgprRB_S32, 1);
759791
auto BoolInReg = B.buildAnd(SgprRB_S32, Aext, Cst1);
760792
Op.setReg(BoolInReg.getReg(0));
@@ -767,6 +799,29 @@ void RegBankLegalizeHelper::applyMappingSrc(
767799
Op.setReg(Sext.getReg(0));
768800
break;
769801
}
802+
case Sgpr32ZExt: {
803+
assert(1 < Ty.getSizeInBits() && Ty.getSizeInBits() < 32);
804+
assert(RB == SgprRB);
805+
auto Zext = B.buildZExt({SgprRB, S32}, Reg);
806+
Op.setReg(Zext.getReg(0));
807+
break;
808+
}
809+
case Vgpr32SExt: {
810+
// Note this ext allows S1, and it is meant to be combined away.
811+
assert(Ty.getSizeInBits() < 32);
812+
assert(RB == VgprRB);
813+
auto Sext = B.buildSExt({VgprRB, S32}, Reg);
814+
Op.setReg(Sext.getReg(0));
815+
break;
816+
}
817+
case Vgpr32ZExt: {
818+
// Note this ext allows S1, and it is meant to be combined away.
819+
assert(Ty.getSizeInBits() < 32);
820+
assert(RB == VgprRB);
821+
auto Zext = B.buildZExt({VgprRB, S32}, Reg);
822+
Op.setReg(Zext.getReg(0));
823+
break;
824+
}
770825
default:
771826
llvm_unreachable("ID not supported");
772827
}

‎llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeHelper.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ class RegBankLegalizeHelper {
110110
void lower(MachineInstr &MI, const RegBankLLTMapping &Mapping,
111111
SmallSet<Register, 4> &SgprWaterfallOperandRegs);
112112

113+
void lowerVccExtToSel(MachineInstr &MI);
113114
void lowerDiv_BFE(MachineInstr &MI);
114115
void lowerUni_BFE(MachineInstr &MI);
115116
void lowerSplitTo32(MachineInstr &MI);

‎llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.cpp

Lines changed: 43 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -489,22 +489,61 @@ RegBankLegalizeRules::RegBankLegalizeRules(const GCNSubtarget &_ST,
489489
.Uni(B32, {{SgprB32}, {Sgpr32AExtBoolInReg, SgprB32, SgprB32}});
490490

491491
addRulesForGOpcs({G_ANYEXT})
492+
.Any({{UniS16, S1}, {{None}, {None}}}) // should be combined away
492493
.Any({{UniS32, S1}, {{None}, {None}}}) // should be combined away
493-
.Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}});
494+
.Any({{UniS64, S1}, {{None}, {None}}}) // should be combined away
495+
.Any({{DivS16, S1}, {{Vgpr16}, {Vcc}, VccExtToSel}})
496+
.Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})
497+
.Any({{DivS64, S1}, {{Vgpr64}, {Vcc}, VccExtToSel}})
498+
.Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})
499+
.Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}})
500+
.Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}})
501+
.Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}});
494502

495503
// In global-isel G_TRUNC in-reg is treated as no-op, inst selected into COPY.
496504
// It is up to user to deal with truncated bits.
497505
addRulesForGOpcs({G_TRUNC})
506+
.Any({{UniS1, UniS16}, {{None}, {None}}}) // should be combined away
498507
.Any({{UniS1, UniS32}, {{None}, {None}}}) // should be combined away
508+
.Any({{UniS1, UniS64}, {{None}, {None}}}) // should be combined away
499509
.Any({{UniS16, S32}, {{Sgpr16}, {Sgpr32}}})
510+
.Any({{DivS16, S32}, {{Vgpr16}, {Vgpr32}}})
511+
.Any({{UniS32, S64}, {{Sgpr32}, {Sgpr64}}})
512+
.Any({{DivS32, S64}, {{Vgpr32}, {Vgpr64}}})
500513
// This is non-trivial. VgprToVccCopy is done using compare instruction.
501-
.Any({{DivS1, DivS32}, {{Vcc}, {Vgpr32}, VgprToVccCopy}});
514+
.Any({{DivS1, DivS16}, {{Vcc}, {Vgpr16}, VgprToVccCopy}})
515+
.Any({{DivS1, DivS32}, {{Vcc}, {Vgpr32}, VgprToVccCopy}})
516+
.Any({{DivS1, DivS64}, {{Vcc}, {Vgpr64}, VgprToVccCopy}});
502517

503-
addRulesForGOpcs({G_ZEXT, G_SEXT})
518+
addRulesForGOpcs({G_ZEXT})
519+
.Any({{UniS16, S1}, {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
520+
.Any({{UniS32, S1}, {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
521+
.Any({{UniS64, S1}, {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
522+
.Any({{DivS16, S1}, {{Vgpr16}, {Vcc}, VccExtToSel}})
523+
.Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})
524+
.Any({{DivS64, S1}, {{Vgpr64}, {Vcc}, VccExtToSel}})
525+
.Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})
526+
.Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}})
527+
// not extending S16 to S32 is questionable.
528+
.Any({{UniS64, S16}, {{Sgpr64}, {Sgpr32ZExt}, Ext32To64}})
529+
.Any({{DivS64, S16}, {{Vgpr64}, {Vgpr32ZExt}, Ext32To64}})
530+
.Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}})
531+
.Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}});
532+
533+
addRulesForGOpcs({G_SEXT})
534+
.Any({{UniS16, S1}, {{Sgpr32Trunc}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
504535
.Any({{UniS32, S1}, {{Sgpr32}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
536+
.Any({{UniS64, S1}, {{Sgpr64}, {Sgpr32AExtBoolInReg}, UniExtToSel}})
537+
.Any({{DivS16, S1}, {{Vgpr16}, {Vcc}, VccExtToSel}})
505538
.Any({{DivS32, S1}, {{Vgpr32}, {Vcc}, VccExtToSel}})
539+
.Any({{DivS64, S1}, {{Vgpr64}, {Vcc}, VccExtToSel}})
506540
.Any({{UniS64, S32}, {{Sgpr64}, {Sgpr32}, Ext32To64}})
507-
.Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}});
541+
.Any({{DivS64, S32}, {{Vgpr64}, {Vgpr32}, Ext32To64}})
542+
// not extending S16 to S32 is questionable.
543+
.Any({{UniS64, S16}, {{Sgpr64}, {Sgpr32SExt}, Ext32To64}})
544+
.Any({{DivS64, S16}, {{Vgpr64}, {Vgpr32SExt}, Ext32To64}})
545+
.Any({{UniS32, S16}, {{Sgpr32}, {Sgpr16}}})
546+
.Any({{DivS32, S16}, {{Vgpr32}, {Vgpr16}}});
508547

509548
bool hasUnalignedLoads = ST->getGeneration() >= AMDGPUSubtarget::GFX12;
510549
bool hasSMRDSmall = ST->hasScalarSubwordLoads();

‎llvm/lib/Target/AMDGPU/AMDGPURegBankLegalizeRules.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,9 @@ enum RegBankLLTMappingApplyID {
159159
Sgpr32AExt,
160160
Sgpr32AExtBoolInReg,
161161
Sgpr32SExt,
162+
Sgpr32ZExt,
163+
Vgpr32SExt,
164+
Vgpr32ZExt,
162165
};
163166

164167
// Instruction needs to be replaced with sequence of instructions. Lowering was

0 commit comments

Comments
 (0)