Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CHERI-RISCV] Inline expansion of capability-based cmpxchg in hybrid mode #490

Open
wants to merge 8 commits into
base: dev
Choose a base branch
from
Open

[CHERI-RISCV] Inline expansion of capability-based cmpxchg in hybrid mode #490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8e402bc
[CHERI-RISC-V] Swap CheriStoreCond_r operand order to match AMO_rr
arichardson Dec 8, 2020
71e651f
[CHERI-RISC-V] Fix capability i8/i16 cmpxchg in hybrid mode
arichardson Nov 4, 2020
140f9fa
[CHERI-RISC-V] Handle capability-based cmpxchg in hybrid mode
arichardson Nov 12, 2020
16ab419
[CHERI-RISC-V] Fix lowering of i8/i16 cmpxchg using a capability pointer
arichardson Sep 30, 2021
3285480
[CHERI-Generic] Add cmpxchg tests with all memory order combinations
arichardson Nov 12, 2020
e3d507c
[CHERI-RISC-V] Insert the required fences for explicit address mode a…
arichardson Sep 30, 2021
04c69db
[CHERI-RISC-V] Support capability-based integer atomic RMW ops
arichardson Oct 1, 2021
ef40de6
[CHERI-RISC-V] Expand capability AtomicRMW XCHG in hybrid mode
arichardson Aug 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion llvm/include/llvm/CodeGen/TargetLowering.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1906,7 +1906,10 @@ class TargetLoweringBase {
/// Note that *unlike* operations above the maximum size, atomic ops
/// are still natively supported below the minimum; they just
/// require a more complex expansion.
unsigned getMinCmpXchgSizeInBits() const { return MinCmpXchgSizeInBits; }
virtual unsigned getMinCmpXchgSizeInBits(const Instruction *I,
const Value *Ptr) const {
return MinCmpXchgSizeInBits;
}

/// Whether the target supports unaligned atomic operations.
bool supportsUnalignedAtomics() const { return SupportsUnalignedAtomics; }
Expand Down
47 changes: 27 additions & 20 deletions llvm/lib/CodeGen/AtomicExpandPass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,8 @@ bool AtomicExpand::runOnFunction(Function &F) {
"invariant broken");
MadeChange = true;
}
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
unsigned MinCASSize =
TLI->getMinCmpXchgSizeInBits(RMWI, RMWI->getPointerOperand()) / 8;
unsigned ValueSize = getAtomicOpSize(RMWI);
if (ValueSize < MinCASSize &&
(Op == AtomicRMWInst::Or || Op == AtomicRMWInst::Xor ||
Expand Down Expand Up @@ -617,7 +618,8 @@ bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {
case TargetLoweringBase::AtomicExpansionKind::None:
return false;
case TargetLoweringBase::AtomicExpansionKind::LLSC: {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
unsigned MinCASSize =
TLI->getMinCmpXchgSizeInBits(AI, AI->getPointerOperand()) / 8;
unsigned ValueSize = getAtomicOpSize(AI);
if (ValueSize < MinCASSize) {
expandPartwordAtomicRMW(AI,
Expand All @@ -633,7 +635,8 @@ bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) {
return true;
}
case TargetLoweringBase::AtomicExpansionKind::CmpXChg: {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
unsigned MinCASSize =
TLI->getMinCmpXchgSizeInBits(AI, AI->getPointerOperand()) / 8;
unsigned ValueSize = getAtomicOpSize(AI);
if (ValueSize < MinCASSize) {
// TODO: Handle atomicrmw fadd/fsub
Expand Down Expand Up @@ -854,9 +857,9 @@ void AtomicExpand::expandPartwordAtomicRMW(

IRBuilder<> Builder(AI);

PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
PartwordMaskValues PMV = createMaskInstrs(
Builder, AI, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
TLI->getMinCmpXchgSizeInBits(AI, AI->getPointerOperand()) / 8);

Value *ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
Expand Down Expand Up @@ -894,9 +897,9 @@ AtomicRMWInst *AtomicExpand::widenPartwordAtomicRMW(AtomicRMWInst *AI) {
Op == AtomicRMWInst::And) &&
"Unable to widen operation");

PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
PartwordMaskValues PMV = createMaskInstrs(
Builder, AI, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
TLI->getMinCmpXchgSizeInBits(AI, AI->getPointerOperand()) / 8);

Value *ValOperand_Shifted =
Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), PMV.WordType),
Expand Down Expand Up @@ -964,6 +967,8 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
Function *F = BB->getParent();
IRBuilder<> Builder(CI);
LLVMContext &Ctx = Builder.getContext();
assert(!F->getParent()->getDataLayout().isFatPointer(Addr->getType()) &&
"Target should have partword capability-based atomics!");

BasicBlock *EndBB =
BB->splitBasicBlock(CI->getIterator(), "partword.cmpxchg.end");
Expand All @@ -976,9 +981,9 @@ bool AtomicExpand::expandPartwordCmpXchg(AtomicCmpXchgInst *CI) {
std::prev(BB->end())->eraseFromParent();
Builder.SetInsertPoint(BB);

PartwordMaskValues PMV =
createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
PartwordMaskValues PMV = createMaskInstrs(
Builder, CI, CI->getCompareOperand()->getType(), Addr, CI->getAlign(),
TLI->getMinCmpXchgSizeInBits(CI, CI->getPointerOperand()) / 8);

// Shift the incoming values over, into the right location in the word.
Value *NewVal_Shifted =
Expand Down Expand Up @@ -1060,9 +1065,9 @@ void AtomicExpand::expandAtomicOpToLLSC(
void AtomicExpand::expandAtomicRMWToMaskedIntrinsic(AtomicRMWInst *AI) {
IRBuilder<> Builder(AI);

PartwordMaskValues PMV =
createMaskInstrs(Builder, AI, AI->getType(), AI->getPointerOperand(),
AI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
PartwordMaskValues PMV = createMaskInstrs(
Builder, AI, AI->getType(), AI->getPointerOperand(), AI->getAlign(),
TLI->getMinCmpXchgSizeInBits(AI, AI->getPointerOperand()) / 8);

// The value operand must be sign-extended for signed min/max so that the
// target's signed comparison instructions can be used. Otherwise, just
Expand All @@ -1088,7 +1093,8 @@ void AtomicExpand::expandAtomicCmpXchgToMaskedIntrinsic(AtomicCmpXchgInst *CI) {

PartwordMaskValues PMV = createMaskInstrs(
Builder, CI, CI->getCompareOperand()->getType(), CI->getPointerOperand(),
CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
CI->getAlign(),
TLI->getMinCmpXchgSizeInBits(CI, CI->getPointerOperand()) / 8);

Value *CmpVal_Shifted = Builder.CreateShl(
Builder.CreateZExt(CI->getCompareOperand(), PMV.WordType), PMV.ShiftAmt,
Expand Down Expand Up @@ -1309,9 +1315,9 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
if (ShouldInsertFencesForAtomic && UseUnconditionalReleaseBarrier)
TLI->emitLeadingFence(Builder, CI, SuccessOrder);

PartwordMaskValues PMV =
createMaskInstrs(Builder, CI, CI->getCompareOperand()->getType(), Addr,
CI->getAlign(), TLI->getMinCmpXchgSizeInBits() / 8);
PartwordMaskValues PMV = createMaskInstrs(
Builder, CI, CI->getCompareOperand()->getType(), Addr, CI->getAlign(),
TLI->getMinCmpXchgSizeInBits(CI, CI->getPointerOperand()) / 8);
Builder.CreateBr(StartBB);

// Start the main loop block now that we've taken care of the preliminaries.
Expand Down Expand Up @@ -1541,7 +1547,8 @@ Value *AtomicExpand::insertRMWCmpXchgLoop(
}

bool AtomicExpand::tryExpandAtomicCmpXchg(AtomicCmpXchgInst *CI) {
unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8;
unsigned MinCASSize =
TLI->getMinCmpXchgSizeInBits(CI, CI->getPointerOperand()) / 8;
unsigned ValueSize = getAtomicOpSize(CI);

switch (TLI->shouldExpandAtomicCmpXchgInIR(CI)) {
Expand Down
Loading