Skip to content

Commit ce61b0e

Browse files
RoboTuxRoboTux
authored
Add out-of-line-atomics support to GlobalISel (#74588)
This patch implement the GlobalISel counterpart to 4d7df43.
1 parent 5ed11e7 commit ce61b0e

File tree

12 files changed

+5594
-2945
lines changed

12 files changed

+5594
-2945
lines changed

‎llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,11 @@ struct TypePairAndMemDesc {
224224
}
225225
};
226226

227+
/// True iff P is false.
228+
template <typename Predicate> Predicate predNot(Predicate P) {
229+
return [=](const LegalityQuery &Query) { return !P(Query); };
230+
}
231+
227232
/// True iff P0 and P1 are true.
228233
template<typename Predicate>
229234
Predicate all(Predicate P0, Predicate P1) {

‎llvm/include/llvm/CodeGen/RuntimeLibcalls.h

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,12 @@ namespace RTLIB {
8282
/// UNKNOWN_LIBCALL if there is none.
8383
Libcall getSYNC(unsigned Opc, MVT VT);
8484

85+
/// Return the outline atomics value for the given atomic ordering, access
86+
/// size and set of libcalls for a given atomic, or UNKNOWN_LIBCALL if there
87+
/// is none.
88+
Libcall getOutlineAtomicHelper(const Libcall (&LC)[5][4],
89+
AtomicOrdering Order, uint64_t MemSize);
90+
8591
/// Return the outline atomics value for the given opcode, atomic ordering
8692
/// and type, or UNKNOWN_LIBCALL if there is none.
8793
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT);

‎llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include "llvm/CodeGen/MachineConstantPool.h"
2626
#include "llvm/CodeGen/MachineFrameInfo.h"
2727
#include "llvm/CodeGen/MachineRegisterInfo.h"
28+
#include "llvm/CodeGen/RuntimeLibcalls.h"
2829
#include "llvm/CodeGen/TargetFrameLowering.h"
2930
#include "llvm/CodeGen/TargetInstrInfo.h"
3031
#include "llvm/CodeGen/TargetLowering.h"
@@ -796,6 +797,132 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
796797
return LegalizerHelper::Legalized;
797798
}
798799

800+
static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI) {
801+
unsigned Opc = MI.getOpcode();
802+
auto &AtomicMI = cast<GMemOperation>(MI);
803+
auto &MMO = AtomicMI.getMMO();
804+
auto Ordering = MMO.getMergedOrdering();
805+
LLT MemType = MMO.getMemoryType();
806+
uint64_t MemSize = MemType.getSizeInBytes();
807+
if (MemType.isVector())
808+
return RTLIB::UNKNOWN_LIBCALL;
809+
810+
#define LCALLS(A, B) \
811+
{ A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
812+
#define LCALL5(A) \
813+
LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
814+
switch (Opc) {
815+
case TargetOpcode::G_ATOMIC_CMPXCHG:
816+
case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
817+
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_CAS)};
818+
return getOutlineAtomicHelper(LC, Ordering, MemSize);
819+
}
820+
case TargetOpcode::G_ATOMICRMW_XCHG: {
821+
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_SWP)};
822+
return getOutlineAtomicHelper(LC, Ordering, MemSize);
823+
}
824+
case TargetOpcode::G_ATOMICRMW_ADD:
825+
case TargetOpcode::G_ATOMICRMW_SUB: {
826+
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDADD)};
827+
return getOutlineAtomicHelper(LC, Ordering, MemSize);
828+
}
829+
case TargetOpcode::G_ATOMICRMW_AND: {
830+
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDCLR)};
831+
return getOutlineAtomicHelper(LC, Ordering, MemSize);
832+
}
833+
case TargetOpcode::G_ATOMICRMW_OR: {
834+
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDSET)};
835+
return getOutlineAtomicHelper(LC, Ordering, MemSize);
836+
}
837+
case TargetOpcode::G_ATOMICRMW_XOR: {
838+
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDEOR)};
839+
return getOutlineAtomicHelper(LC, Ordering, MemSize);
840+
}
841+
default:
842+
return RTLIB::UNKNOWN_LIBCALL;
843+
}
844+
#undef LCALLS
845+
#undef LCALL5
846+
}
847+
848+
static LegalizerHelper::LegalizeResult
849+
createAtomicLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI) {
850+
auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
851+
852+
Type *RetTy;
853+
SmallVector<Register> RetRegs;
854+
SmallVector<CallLowering::ArgInfo, 3> Args;
855+
unsigned Opc = MI.getOpcode();
856+
switch (Opc) {
857+
case TargetOpcode::G_ATOMIC_CMPXCHG:
858+
case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
859+
Register Success;
860+
LLT SuccessLLT;
861+
auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
862+
MI.getFirst4RegLLTs();
863+
RetRegs.push_back(Ret);
864+
RetTy = IntegerType::get(Ctx, RetLLT.getSizeInBits());
865+
if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
866+
std::tie(Ret, RetLLT, Success, SuccessLLT, Mem, MemLLT, Cmp, CmpLLT, New,
867+
NewLLT) = MI.getFirst5RegLLTs();
868+
RetRegs.push_back(Success);
869+
RetTy = StructType::get(
870+
Ctx, {RetTy, IntegerType::get(Ctx, SuccessLLT.getSizeInBits())});
871+
}
872+
Args.push_back({Cmp, IntegerType::get(Ctx, CmpLLT.getSizeInBits()), 0});
873+
Args.push_back({New, IntegerType::get(Ctx, NewLLT.getSizeInBits()), 0});
874+
Args.push_back({Mem, PointerType::get(Ctx, MemLLT.getAddressSpace()), 0});
875+
break;
876+
}
877+
case TargetOpcode::G_ATOMICRMW_XCHG:
878+
case TargetOpcode::G_ATOMICRMW_ADD:
879+
case TargetOpcode::G_ATOMICRMW_SUB:
880+
case TargetOpcode::G_ATOMICRMW_AND:
881+
case TargetOpcode::G_ATOMICRMW_OR:
882+
case TargetOpcode::G_ATOMICRMW_XOR: {
883+
auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] = MI.getFirst3RegLLTs();
884+
RetRegs.push_back(Ret);
885+
RetTy = IntegerType::get(Ctx, RetLLT.getSizeInBits());
886+
if (Opc == TargetOpcode::G_ATOMICRMW_AND)
887+
Val =
888+
MIRBuilder.buildXor(ValLLT, MIRBuilder.buildConstant(ValLLT, -1), Val)
889+
.getReg(0);
890+
else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
891+
Val =
892+
MIRBuilder.buildSub(ValLLT, MIRBuilder.buildConstant(ValLLT, 0), Val)
893+
.getReg(0);
894+
Args.push_back({Val, IntegerType::get(Ctx, ValLLT.getSizeInBits()), 0});
895+
Args.push_back({Mem, PointerType::get(Ctx, MemLLT.getAddressSpace()), 0});
896+
break;
897+
}
898+
default:
899+
llvm_unreachable("unsupported opcode");
900+
}
901+
902+
auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
903+
auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
904+
RTLIB::Libcall RTLibcall = getOutlineAtomicLibcall(MI);
905+
const char *Name = TLI.getLibcallName(RTLibcall);
906+
907+
// Unsupported libcall on the target.
908+
if (!Name) {
909+
LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
910+
<< MIRBuilder.getTII().getName(Opc) << "\n");
911+
return LegalizerHelper::UnableToLegalize;
912+
}
913+
914+
CallLowering::CallLoweringInfo Info;
915+
Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
916+
Info.Callee = MachineOperand::CreateES(Name);
917+
Info.OrigRet = CallLowering::ArgInfo(RetRegs, RetTy, 0);
918+
919+
std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
920+
if (!CLI.lowerCall(MIRBuilder, Info))
921+
return LegalizerHelper::UnableToLegalize;
922+
923+
return LegalizerHelper::Legalized;
924+
}
925+
799926
static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
800927
Type *FromType) {
801928
auto ToMVT = MVT::getVT(ToType);
@@ -1081,6 +1208,19 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
10811208
return Status;
10821209
break;
10831210
}
1211+
case TargetOpcode::G_ATOMICRMW_XCHG:
1212+
case TargetOpcode::G_ATOMICRMW_ADD:
1213+
case TargetOpcode::G_ATOMICRMW_SUB:
1214+
case TargetOpcode::G_ATOMICRMW_AND:
1215+
case TargetOpcode::G_ATOMICRMW_OR:
1216+
case TargetOpcode::G_ATOMICRMW_XOR:
1217+
case TargetOpcode::G_ATOMIC_CMPXCHG:
1218+
case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1219+
auto Status = createAtomicLibcall(MIRBuilder, MI);
1220+
if (Status != Legalized)
1221+
return Status;
1222+
break;
1223+
}
10841224
case TargetOpcode::G_BZERO:
10851225
case TargetOpcode::G_MEMCPY:
10861226
case TargetOpcode::G_MEMMOVE:

‎llvm/lib/CodeGen/TargetLoweringBase.cpp

Lines changed: 26 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -520,27 +520,28 @@ RTLIB::Libcall RTLIB::getFREXP(EVT RetVT) {
520520
FREXP_PPCF128);
521521
}
522522

523-
RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order,
524-
MVT VT) {
523+
RTLIB::Libcall RTLIB::getOutlineAtomicHelper(const Libcall (&LC)[5][4],
524+
AtomicOrdering Order,
525+
uint64_t MemSize) {
525526
unsigned ModeN, ModelN;
526-
switch (VT.SimpleTy) {
527-
case MVT::i8:
527+
switch (MemSize) {
528+
case 1:
528529
ModeN = 0;
529530
break;
530-
case MVT::i16:
531+
case 2:
531532
ModeN = 1;
532533
break;
533-
case MVT::i32:
534+
case 4:
534535
ModeN = 2;
535536
break;
536-
case MVT::i64:
537+
case 8:
537538
ModeN = 3;
538539
break;
539-
case MVT::i128:
540+
case 16:
540541
ModeN = 4;
541542
break;
542543
default:
543-
return UNKNOWN_LIBCALL;
544+
return RTLIB::UNKNOWN_LIBCALL;
544545
}
545546

546547
switch (Order) {
@@ -561,34 +562,44 @@ RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order,
561562
return UNKNOWN_LIBCALL;
562563
}
563564

565+
return LC[ModeN][ModelN];
566+
}
567+
568+
RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order,
569+
MVT VT) {
570+
unsigned ModeN, ModelN;
571+
if (!VT.isScalarInteger())
572+
return UNKNOWN_LIBCALL;
573+
uint64_t MemSize = VT.getScalarSizeInBits() / 8;
574+
564575
#define LCALLS(A, B) \
565576
{ A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
566577
#define LCALL5(A) \
567578
LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
568579
switch (Opc) {
569580
case ISD::ATOMIC_CMP_SWAP: {
570581
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)};
571-
return LC[ModeN][ModelN];
582+
return getOutlineAtomicHelper(LC, Order, MemSize);
572583
}
573584
case ISD::ATOMIC_SWAP: {
574585
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)};
575-
return LC[ModeN][ModelN];
586+
return getOutlineAtomicHelper(LC, Order, MemSize);
576587
}
577588
case ISD::ATOMIC_LOAD_ADD: {
578589
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)};
579-
return LC[ModeN][ModelN];
590+
return getOutlineAtomicHelper(LC, Order, MemSize);
580591
}
581592
case ISD::ATOMIC_LOAD_OR: {
582593
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)};
583-
return LC[ModeN][ModelN];
594+
return getOutlineAtomicHelper(LC, Order, MemSize);
584595
}
585596
case ISD::ATOMIC_LOAD_CLR: {
586597
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)};
587-
return LC[ModeN][ModelN];
598+
return getOutlineAtomicHelper(LC, Order, MemSize);
588599
}
589600
case ISD::ATOMIC_LOAD_XOR: {
590601
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)};
591-
return LC[ModeN][ModelN];
602+
return getOutlineAtomicHelper(LC, Order, MemSize);
592603
}
593604
default:
594605
return UNKNOWN_LIBCALL;

‎llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -765,17 +765,35 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
765765
.lowerIf(
766766
all(typeInSet(0, {s8, s16, s32, s64, s128}), typeIs(2, p0)));
767767

768+
LegalityPredicate UseOutlineAtomics = [&ST](const LegalityQuery &Query) {
769+
return ST.outlineAtomics() && !ST.hasLSE();
770+
};
771+
768772
getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
769-
.legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0)))
770-
.customIf([](const LegalityQuery &Query) {
771-
return Query.Types[0].getSizeInBits() == 128;
773+
.legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0),
774+
predNot(UseOutlineAtomics)))
775+
.customIf(all(typeIs(0, s128), predNot(UseOutlineAtomics)))
776+
.customIf([UseOutlineAtomics](const LegalityQuery &Query) {
777+
return Query.Types[0].getSizeInBits() == 128 &&
778+
!UseOutlineAtomics(Query);
772779
})
780+
.libcallIf(all(typeInSet(0, {s8, s16, s32, s64, s128}), typeIs(1, p0),
781+
UseOutlineAtomics))
782+
.clampScalar(0, s32, s64);
783+
784+
getActionDefinitionsBuilder({G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD,
785+
G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR,
786+
G_ATOMICRMW_XOR})
787+
.legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0),
788+
predNot(UseOutlineAtomics)))
789+
.libcallIf(all(typeInSet(0, {s8, s16, s32, s64}), typeIs(1, p0),
790+
UseOutlineAtomics))
773791
.clampScalar(0, s32, s64);
774792

793+
// Do not outline these atomics operations, as per comment in
794+
// AArch64ISelLowering.cpp's shouldExpandAtomicRMWInIR().
775795
getActionDefinitionsBuilder(
776-
{G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND,
777-
G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX,
778-
G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
796+
{G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
779797
.legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0)))
780798
.clampScalar(0, s32, s64);
781799

0 commit comments

Comments
 (0)