Skip to content

Commit 07f1c62

Browse files
asbasb
committed
[RISCV] Add codegen support for RV64A
In order to support codegen RV64A, this patch: * Introduces masked atomics intrinsics for atomicrmw operations and cmpxchg that use the i64 type. These are ultimately lowered to masked operations using lr.w/sc.w, but we need to use these alternate intrinsics for RV64 because i32 is not legal * Modifies RISCVExpandPseudoInsts.cpp to handle PseudoAtomicLoadNand64 and PseudoCmpXchg64 * Modifies the AtomicExpandPass hooks in RISCVTargetLowering to sext/trunc as needed for RV64 and to select the i64 intrinsic IDs when necessary * Adds appropriate patterns to RISCVInstrInfoA.td * Updates test/CodeGen/RISCV/atomic-*.ll to show RV64A support This ends up being a fairly mechanical change, as the logic for RV32A is effectively reused. Differential Revision: https://reviews.llvm.org/D53233 llvm-svn: 351422
1 parent b694030 commit 07f1c62

File tree

7 files changed

+4166
-48
lines changed

7 files changed

+4166
-48
lines changed

‎llvm/include/llvm/IR/IntrinsicsRISCV.td‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,29 @@ def int_riscv_masked_cmpxchg_i32
4141
llvm_i32_ty, llvm_i32_ty],
4242
[IntrArgMemOnly, NoCapture<0>]>;
4343

44+
class MaskedAtomicRMW64Intrinsic
45+
: Intrinsic<[llvm_i64_ty],
46+
[llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, llvm_i64_ty],
47+
[IntrArgMemOnly, NoCapture<0>]>;
48+
49+
class MaskedAtomicRMW64WithSextIntrinsic
50+
: Intrinsic<[llvm_i64_ty],
51+
[llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, llvm_i64_ty,
52+
llvm_i64_ty],
53+
[IntrArgMemOnly, NoCapture<0>]>;
54+
55+
def int_riscv_masked_atomicrmw_xchg_i64 : MaskedAtomicRMW64Intrinsic;
56+
def int_riscv_masked_atomicrmw_add_i64 : MaskedAtomicRMW64Intrinsic;
57+
def int_riscv_masked_atomicrmw_sub_i64 : MaskedAtomicRMW64Intrinsic;
58+
def int_riscv_masked_atomicrmw_nand_i64 : MaskedAtomicRMW64Intrinsic;
59+
def int_riscv_masked_atomicrmw_max_i64 : MaskedAtomicRMW64WithSextIntrinsic;
60+
def int_riscv_masked_atomicrmw_min_i64 : MaskedAtomicRMW64WithSextIntrinsic;
61+
def int_riscv_masked_atomicrmw_umax_i64 : MaskedAtomicRMW64Intrinsic;
62+
def int_riscv_masked_atomicrmw_umin_i64 : MaskedAtomicRMW64Intrinsic;
63+
64+
def int_riscv_masked_cmpxchg_i64
65+
: Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty,
66+
llvm_i64_ty, llvm_i64_ty],
67+
[IntrArgMemOnly, NoCapture<0>]>;
68+
4469
} // TargetPrefix = "riscv"

‎llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp‎

Lines changed: 67 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
8787
case RISCV::PseudoAtomicLoadNand32:
8888
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 32,
8989
NextMBBI);
90+
case RISCV::PseudoAtomicLoadNand64:
91+
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Nand, false, 64,
92+
NextMBBI);
9093
case RISCV::PseudoMaskedAtomicSwap32:
9194
return expandAtomicBinOp(MBB, MBBI, AtomicRMWInst::Xchg, true, 32,
9295
NextMBBI);
@@ -111,6 +114,8 @@ bool RISCVExpandPseudo::expandMI(MachineBasicBlock &MBB,
111114
NextMBBI);
112115
case RISCV::PseudoCmpXchg32:
113116
return expandAtomicCmpXchg(MBB, MBBI, false, 32, NextMBBI);
117+
case RISCV::PseudoCmpXchg64:
118+
return expandAtomicCmpXchg(MBB, MBBI, false, 64, NextMBBI);
114119
case RISCV::PseudoMaskedCmpXchg32:
115120
return expandAtomicCmpXchg(MBB, MBBI, true, 32, NextMBBI);
116121
}
@@ -152,12 +157,61 @@ static unsigned getSCForRMW32(AtomicOrdering Ordering) {
152157
}
153158
}
154159

160+
static unsigned getLRForRMW64(AtomicOrdering Ordering) {
161+
switch (Ordering) {
162+
default:
163+
llvm_unreachable("Unexpected AtomicOrdering");
164+
case AtomicOrdering::Monotonic:
165+
return RISCV::LR_D;
166+
case AtomicOrdering::Acquire:
167+
return RISCV::LR_D_AQ;
168+
case AtomicOrdering::Release:
169+
return RISCV::LR_D;
170+
case AtomicOrdering::AcquireRelease:
171+
return RISCV::LR_D_AQ;
172+
case AtomicOrdering::SequentiallyConsistent:
173+
return RISCV::LR_D_AQ_RL;
174+
}
175+
}
176+
177+
static unsigned getSCForRMW64(AtomicOrdering Ordering) {
178+
switch (Ordering) {
179+
default:
180+
llvm_unreachable("Unexpected AtomicOrdering");
181+
case AtomicOrdering::Monotonic:
182+
return RISCV::SC_D;
183+
case AtomicOrdering::Acquire:
184+
return RISCV::SC_D;
185+
case AtomicOrdering::Release:
186+
return RISCV::SC_D_RL;
187+
case AtomicOrdering::AcquireRelease:
188+
return RISCV::SC_D_RL;
189+
case AtomicOrdering::SequentiallyConsistent:
190+
return RISCV::SC_D_AQ_RL;
191+
}
192+
}
193+
194+
static unsigned getLRForRMW(AtomicOrdering Ordering, int Width) {
195+
if (Width == 32)
196+
return getLRForRMW32(Ordering);
197+
if (Width == 64)
198+
return getLRForRMW64(Ordering);
199+
llvm_unreachable("Unexpected LR width\n");
200+
}
201+
202+
static unsigned getSCForRMW(AtomicOrdering Ordering, int Width) {
203+
if (Width == 32)
204+
return getSCForRMW32(Ordering);
205+
if (Width == 64)
206+
return getSCForRMW64(Ordering);
207+
llvm_unreachable("Unexpected SC width\n");
208+
}
209+
155210
static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
156211
DebugLoc DL, MachineBasicBlock *ThisMBB,
157212
MachineBasicBlock *LoopMBB,
158213
MachineBasicBlock *DoneMBB,
159214
AtomicRMWInst::BinOp BinOp, int Width) {
160-
assert(Width == 32 && "RV64 atomic expansion currently unsupported");
161215
unsigned DestReg = MI.getOperand(0).getReg();
162216
unsigned ScratchReg = MI.getOperand(1).getReg();
163217
unsigned AddrReg = MI.getOperand(2).getReg();
@@ -166,11 +220,11 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
166220
static_cast<AtomicOrdering>(MI.getOperand(4).getImm());
167221

168222
// .loop:
169-
// lr.w dest, (addr)
223+
// lr.[w|d] dest, (addr)
170224
// binop scratch, dest, val
171-
// sc.w scratch, scratch, (addr)
225+
// sc.[w|d] scratch, scratch, (addr)
172226
// bnez scratch, loop
173-
BuildMI(LoopMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
227+
BuildMI(LoopMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
174228
.addReg(AddrReg);
175229
switch (BinOp) {
176230
default:
@@ -184,7 +238,7 @@ static void doAtomicBinOpExpansion(const RISCVInstrInfo *TII, MachineInstr &MI,
184238
.addImm(-1);
185239
break;
186240
}
187-
BuildMI(LoopMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
241+
BuildMI(LoopMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
188242
.addReg(AddrReg)
189243
.addReg(ScratchReg);
190244
BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
@@ -219,7 +273,7 @@ static void doMaskedAtomicBinOpExpansion(
219273
const RISCVInstrInfo *TII, MachineInstr &MI, DebugLoc DL,
220274
MachineBasicBlock *ThisMBB, MachineBasicBlock *LoopMBB,
221275
MachineBasicBlock *DoneMBB, AtomicRMWInst::BinOp BinOp, int Width) {
222-
assert(Width == 32 && "RV64 atomic expansion currently unsupported");
276+
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
223277
unsigned DestReg = MI.getOperand(0).getReg();
224278
unsigned ScratchReg = MI.getOperand(1).getReg();
225279
unsigned AddrReg = MI.getOperand(2).getReg();
@@ -333,7 +387,7 @@ bool RISCVExpandPseudo::expandAtomicMinMaxOp(
333387
MachineBasicBlock::iterator &NextMBBI) {
334388
assert(IsMasked == true &&
335389
"Should only need to expand masked atomic max/min");
336-
assert(Width == 32 && "RV64 atomic expansion currently unsupported");
390+
assert(Width == 32 && "Should never need to expand masked 64-bit operations");
337391

338392
MachineInstr &MI = *MBBI;
339393
DebugLoc DL = MI.getDebugLoc();
@@ -451,7 +505,6 @@ bool RISCVExpandPseudo::expandAtomicMinMaxOp(
451505
bool RISCVExpandPseudo::expandAtomicCmpXchg(
452506
MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, bool IsMasked,
453507
int Width, MachineBasicBlock::iterator &NextMBBI) {
454-
assert(Width == 32 && "RV64 atomic expansion currently unsupported");
455508
MachineInstr &MI = *MBBI;
456509
DebugLoc DL = MI.getDebugLoc();
457510
MachineFunction *MF = MBB.getParent();
@@ -483,18 +536,18 @@ bool RISCVExpandPseudo::expandAtomicCmpXchg(
483536

484537
if (!IsMasked) {
485538
// .loophead:
486-
// lr.w dest, (addr)
539+
// lr.[w|d] dest, (addr)
487540
// bne dest, cmpval, done
488-
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
541+
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
489542
.addReg(AddrReg);
490543
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::BNE))
491544
.addReg(DestReg)
492545
.addReg(CmpValReg)
493546
.addMBB(DoneMBB);
494547
// .looptail:
495-
// sc.w scratch, newval, (addr)
548+
// sc.[w|d] scratch, newval, (addr)
496549
// bnez scratch, loophead
497-
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
550+
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
498551
.addReg(AddrReg)
499552
.addReg(NewValReg);
500553
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))
@@ -507,7 +560,7 @@ bool RISCVExpandPseudo::expandAtomicCmpXchg(
507560
// and scratch, dest, mask
508561
// bne scratch, cmpval, done
509562
unsigned MaskReg = MI.getOperand(5).getReg();
510-
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW32(Ordering)), DestReg)
563+
BuildMI(LoopHeadMBB, DL, TII->get(getLRForRMW(Ordering, Width)), DestReg)
511564
.addReg(AddrReg);
512565
BuildMI(LoopHeadMBB, DL, TII->get(RISCV::AND), ScratchReg)
513566
.addReg(DestReg)
@@ -525,7 +578,7 @@ bool RISCVExpandPseudo::expandAtomicCmpXchg(
525578
// bnez scratch, loophead
526579
insertMaskedMerge(TII, DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
527580
MaskReg, ScratchReg);
528-
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW32(Ordering)), ScratchReg)
581+
BuildMI(LoopTailMBB, DL, TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
529582
.addReg(AddrReg)
530583
.addReg(ScratchReg);
531584
BuildMI(LoopTailMBB, DL, TII->get(RISCV::BNE))

‎llvm/lib/Target/RISCV/RISCVISelLowering.cpp‎

Lines changed: 85 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1728,37 +1728,74 @@ RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
17281728
}
17291729

17301730
static Intrinsic::ID
1731-
getIntrinsicForMaskedAtomicRMWBinOp32(AtomicRMWInst::BinOp BinOp) {
1732-
switch (BinOp) {
1733-
default:
1734-
llvm_unreachable("Unexpected AtomicRMW BinOp");
1735-
case AtomicRMWInst::Xchg:
1736-
return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
1737-
case AtomicRMWInst::Add:
1738-
return Intrinsic::riscv_masked_atomicrmw_add_i32;
1739-
case AtomicRMWInst::Sub:
1740-
return Intrinsic::riscv_masked_atomicrmw_sub_i32;
1741-
case AtomicRMWInst::Nand:
1742-
return Intrinsic::riscv_masked_atomicrmw_nand_i32;
1743-
case AtomicRMWInst::Max:
1744-
return Intrinsic::riscv_masked_atomicrmw_max_i32;
1745-
case AtomicRMWInst::Min:
1746-
return Intrinsic::riscv_masked_atomicrmw_min_i32;
1747-
case AtomicRMWInst::UMax:
1748-
return Intrinsic::riscv_masked_atomicrmw_umax_i32;
1749-
case AtomicRMWInst::UMin:
1750-
return Intrinsic::riscv_masked_atomicrmw_umin_i32;
1731+
getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
1732+
if (XLen == 32) {
1733+
switch (BinOp) {
1734+
default:
1735+
llvm_unreachable("Unexpected AtomicRMW BinOp");
1736+
case AtomicRMWInst::Xchg:
1737+
return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
1738+
case AtomicRMWInst::Add:
1739+
return Intrinsic::riscv_masked_atomicrmw_add_i32;
1740+
case AtomicRMWInst::Sub:
1741+
return Intrinsic::riscv_masked_atomicrmw_sub_i32;
1742+
case AtomicRMWInst::Nand:
1743+
return Intrinsic::riscv_masked_atomicrmw_nand_i32;
1744+
case AtomicRMWInst::Max:
1745+
return Intrinsic::riscv_masked_atomicrmw_max_i32;
1746+
case AtomicRMWInst::Min:
1747+
return Intrinsic::riscv_masked_atomicrmw_min_i32;
1748+
case AtomicRMWInst::UMax:
1749+
return Intrinsic::riscv_masked_atomicrmw_umax_i32;
1750+
case AtomicRMWInst::UMin:
1751+
return Intrinsic::riscv_masked_atomicrmw_umin_i32;
1752+
}
1753+
}
1754+
1755+
if (XLen == 64) {
1756+
switch (BinOp) {
1757+
default:
1758+
llvm_unreachable("Unexpected AtomicRMW BinOp");
1759+
case AtomicRMWInst::Xchg:
1760+
return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
1761+
case AtomicRMWInst::Add:
1762+
return Intrinsic::riscv_masked_atomicrmw_add_i64;
1763+
case AtomicRMWInst::Sub:
1764+
return Intrinsic::riscv_masked_atomicrmw_sub_i64;
1765+
case AtomicRMWInst::Nand:
1766+
return Intrinsic::riscv_masked_atomicrmw_nand_i64;
1767+
case AtomicRMWInst::Max:
1768+
return Intrinsic::riscv_masked_atomicrmw_max_i64;
1769+
case AtomicRMWInst::Min:
1770+
return Intrinsic::riscv_masked_atomicrmw_min_i64;
1771+
case AtomicRMWInst::UMax:
1772+
return Intrinsic::riscv_masked_atomicrmw_umax_i64;
1773+
case AtomicRMWInst::UMin:
1774+
return Intrinsic::riscv_masked_atomicrmw_umin_i64;
1775+
}
17511776
}
1777+
1778+
llvm_unreachable("Unexpected XLen\n");
17521779
}
17531780

17541781
Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
17551782
IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
17561783
Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
1757-
Value *Ordering = Builder.getInt32(static_cast<uint32_t>(AI->getOrdering()));
1784+
unsigned XLen = Subtarget.getXLen();
1785+
Value *Ordering =
1786+
Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
17581787
Type *Tys[] = {AlignedAddr->getType()};
17591788
Function *LrwOpScwLoop = Intrinsic::getDeclaration(
17601789
AI->getModule(),
1761-
getIntrinsicForMaskedAtomicRMWBinOp32(AI->getOperation()), Tys);
1790+
getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
1791+
1792+
if (XLen == 64) {
1793+
Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
1794+
Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
1795+
ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
1796+
}
1797+
1798+
Value *Result;
17621799

17631800
// Must pass the shift amount needed to sign extend the loaded value prior
17641801
// to performing a signed comparison for min/max. ShiftAmt is the number of
@@ -1770,13 +1807,18 @@ Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
17701807
const DataLayout &DL = AI->getModule()->getDataLayout();
17711808
unsigned ValWidth =
17721809
DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
1773-
Value *SextShamt = Builder.CreateSub(
1774-
Builder.getInt32(Subtarget.getXLen() - ValWidth), ShiftAmt);
1775-
return Builder.CreateCall(LrwOpScwLoop,
1776-
{AlignedAddr, Incr, Mask, SextShamt, Ordering});
1810+
Value *SextShamt =
1811+
Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
1812+
Result = Builder.CreateCall(LrwOpScwLoop,
1813+
{AlignedAddr, Incr, Mask, SextShamt, Ordering});
1814+
} else {
1815+
Result =
1816+
Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
17771817
}
17781818

1779-
return Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
1819+
if (XLen == 64)
1820+
Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
1821+
return Result;
17801822
}
17811823

17821824
TargetLowering::AtomicExpansionKind
@@ -1791,10 +1833,21 @@ RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
17911833
Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
17921834
IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
17931835
Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
1794-
Value *Ordering = Builder.getInt32(static_cast<uint32_t>(Ord));
1836+
unsigned XLen = Subtarget.getXLen();
1837+
Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
1838+
Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
1839+
if (XLen == 64) {
1840+
CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
1841+
NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
1842+
Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
1843+
CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
1844+
}
17951845
Type *Tys[] = {AlignedAddr->getType()};
1796-
Function *MaskedCmpXchg = Intrinsic::getDeclaration(
1797-
CI->getModule(), Intrinsic::riscv_masked_cmpxchg_i32, Tys);
1798-
return Builder.CreateCall(MaskedCmpXchg,
1799-
{AlignedAddr, CmpVal, NewVal, Mask, Ordering});
1846+
Function *MaskedCmpXchg =
1847+
Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
1848+
Value *Result = Builder.CreateCall(
1849+
MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
1850+
if (XLen == 64)
1851+
Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
1852+
return Result;
18001853
}

0 commit comments

Comments
 (0)