Skip to content

Commit

Permalink
[AIE2P] ISel support for fifo loads
Browse files Browse the repository at this point in the history
  • Loading branch information
@khallouh
khallouh committed Jan 24, 2025
1 parent 299b4c7 commit 98d0684
Show file tree
Hide file tree
Showing 3 changed files with 1,367 additions and 0 deletions.
292 changes: 292 additions & 0 deletions llvm/lib/Target/AIE/aie2p/AIE2PInstructionSelector.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/IR/IntrinsicsAIE2P.h"
#include "llvm/MC/MCContext.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/TypeSize.h"
#include <cassert>
Expand Down Expand Up @@ -87,6 +88,11 @@ class AIE2PInstructionSelector : public AIEBaseInstructionSelector {
Register createDRegSequence(Register ModifierReg, Register IncrReg,
Register SizeReg, Register CountReg,
MachineRegisterInfo &MRI) override;
Register createPLFRRegSequence(Register PtrReg, Register FifoReg,
Register AvailReg, MachineRegisterInfo &MRI);
bool buildAndConstrainFifoLoadCopies(Register Bfp16Vec, Register Mantissa,
Register Exponent,
MachineRegisterInfo &MRI);
Register createDSRegSequence(Register ModifierReg, Register Incr1Reg,
Register Incr2Reg, Register Size1Reg,
Register Count1Reg, Register Size2Reg,
Expand All @@ -103,6 +109,7 @@ class AIE2PInstructionSelector : public AIEBaseInstructionSelector {
bool select1024BitG_AIE_LOAD_STORE(MachineInstr &I, LoadStoreOpcodes &LSO,
AddressingModeInfo &AMI,
MachineRegisterInfo &MRI);
bool selectVLD_FIFO(MachineInstr &I, MachineRegisterInfo &MRI);
bool selectSetI128(MachineInstr &I, MachineOperand &DstReg,
MachineOperand &SrcReg, MachineRegisterInfo &MRI);
bool selectExtractI128(MachineInstr &I, Register DstReg, Register SrcReg,
Expand Down Expand Up @@ -348,6 +355,20 @@ bool AIE2PInstructionSelector::select(MachineInstr &I) {
case Intrinsic::aie2p_v64accfloat_to_v64bfp16ebs16:
case Intrinsic::aie2p_v64bfp16ebs8_to_v64bfp16ebs16:
return selectVCONVbfp16(I, MRI);
case Intrinsic::aie2p_fifo_ld_fill:
case Intrinsic::aie2p_fifo_ld_pop_unaligned:
case Intrinsic::aie2p_fifo_ld_pop_1d_unaligned:
case Intrinsic::aie2p_fifo_ld_pop_544_1d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_576_1d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_544_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_576_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_2d_unaligned:
case Intrinsic::aie2p_fifo_ld_pop_3d_unaligned:
case Intrinsic::aie2p_fifo_ld_pop_544_2d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_576_2d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_544_3d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_576_3d_bfp16:
return selectVLD_FIFO(I, MRI);
default:
return selectImpl(I, *CoverageInfo);
}
Expand Down Expand Up @@ -1534,6 +1555,39 @@ Register AIE2PInstructionSelector::createDRegSequence(
return MI.getReg(0);
}

Register AIE2PInstructionSelector::createPLFRRegSequence(
Register PtrReg, Register FifoReg, Register AvailReg,
MachineRegisterInfo &MRI) {

Register PLFRIn = MRI.createVirtualRegister(&AIE2P::ePSRFLdFRegClass);
MachineInstrBuilder MI =
MIB.buildInstr(TargetOpcode::REG_SEQUENCE, {PLFRIn}, {})
.addReg(PtrReg)
.addImm(AIE2P::sub_ptr)
.addReg(FifoReg)
.addImm(AIE2P::sub_fifo)
.addReg(AvailReg)
.addImm(AIE2P::sub_avail);

return MI.getReg(0);
}
bool AIE2PInstructionSelector::buildAndConstrainFifoLoadCopies(
Register Bfp16Vec, Register Mantissa, Register Exponent,
MachineRegisterInfo &MRI) {

auto CopyMI1 = MIB.buildInstr(TargetOpcode::COPY, {Mantissa}, {})
.addReg(Bfp16Vec, 0, AIE2P::sub_bfp16_x);
auto CopyMI2 = MIB.buildInstr(TargetOpcode::COPY, {Exponent}, {})
.addReg(Bfp16Vec, 0, AIE2P::sub_bfp16_e);

return constrainOperandRegClass(*MF, TRI, MRI, TII, RBI, *CopyMI2,
AIE2P::EXPVEC64RegClass,
CopyMI2->getOperand(0)) &&
constrainOperandRegClass(*MF, TRI, MRI, TII, RBI, *CopyMI1,
AIE2P::VEC512RegClass,
CopyMI1->getOperand(0));
}

Register AIE2PInstructionSelector::createDSRegSequence(
Register ModifierReg, Register Incr1Reg, Register Incr2Reg,
Register Size1Reg, Register Count1Reg, Register Size2Reg,
Expand Down Expand Up @@ -2370,6 +2424,244 @@ bool AIE2PInstructionSelector::selectG_STORE(MachineInstr &I,
return selectImpl(I, *CoverageInfo);
}

unsigned int getLoadFifoOpcode(MachineInstr &I) {
switch (cast<GIntrinsic>(I).getIntrinsicID()) {
case Intrinsic::aie2p_fifo_ld_fill:
return AIE2P::VLDB_FILL_512;
case Intrinsic::aie2p_fifo_ld_pop_unaligned:
return AIE2P::VLDB_POP_512_normal_pop;
case Intrinsic::aie2p_fifo_ld_pop_1d_unaligned:
return AIE2P::VLDB_POP_512_fifo_1d_pop;
case Intrinsic::aie2p_fifo_ld_pop_544_1d_bfp16:
return AIE2P::VLDB_POP_544_fifo_1d_pop;
case Intrinsic::aie2p_fifo_ld_pop_576_1d_bfp16:
return AIE2P::VLDB_POP_576_fifo_1d_pop;
case Intrinsic::aie2p_fifo_ld_pop_544_bfp16:
return AIE2P::VLDB_POP_544_normal_pop;
case Intrinsic::aie2p_fifo_ld_pop_576_bfp16:
return AIE2P::VLDB_POP_576_normal_pop;
case Intrinsic::aie2p_fifo_ld_pop_2d_unaligned:
return AIE2P::VLDB_POP_512_2D;
case Intrinsic::aie2p_fifo_ld_pop_3d_unaligned:
return AIE2P::VLDB_POP_512_3D;
case Intrinsic::aie2p_fifo_ld_pop_544_2d_bfp16:
return AIE2P::VLDB_POP_544_2D;
case Intrinsic::aie2p_fifo_ld_pop_576_2d_bfp16:
return AIE2P::VLDB_POP_576_2D;
case Intrinsic::aie2p_fifo_ld_pop_544_3d_bfp16:
return AIE2P::VLDB_POP_544_3D;
case Intrinsic::aie2p_fifo_ld_pop_576_3d_bfp16:
return AIE2P::VLDB_POP_576_3D;
}
llvm_unreachable("unreachable: Failed to get sparse load opcode");
return AIE2P::INSTRUCTION_LIST_END;
}

bool AIE2PInstructionSelector::selectVLD_FIFO(MachineInstr &I,
MachineRegisterInfo &MRI) {
auto IntrinsicID = cast<GIntrinsic>(I).getIntrinsicID();
MachineInstrBuilder MI;
Register PtrIn;
Register FifoIn;
Register AvailIn;
Register PLFRIn;
switch (IntrinsicID) {
case Intrinsic::aie2p_fifo_ld_fill: {

Register PtrOut = I.getOperand(0).getReg();
Register FifoOut = I.getOperand(1).getReg();
Register AvailOut = I.getOperand(2).getReg();
PtrIn = I.getOperand(4).getReg();
FifoIn = I.getOperand(5).getReg();
AvailIn = I.getOperand(6).getReg();

MI = MIB.buildInstr(getLoadFifoOpcode(I), {PtrOut, FifoOut, AvailOut},
{PtrIn, FifoIn, AvailIn});

I.eraseFromParent();
return constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
}
case Intrinsic::aie2p_fifo_ld_pop_unaligned:
case Intrinsic::aie2p_fifo_ld_pop_1d_unaligned:
case Intrinsic::aie2p_fifo_ld_pop_2d_unaligned:
case Intrinsic::aie2p_fifo_ld_pop_3d_unaligned: {

Register VecOut = I.getOperand(0).getReg();
Register PtrOut = I.getOperand(1).getReg();
Register FifoOut = I.getOperand(2).getReg();
Register AvailOut = I.getOperand(3).getReg();

if (IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_unaligned) {
PtrIn = I.getOperand(5).getReg();
FifoIn = I.getOperand(6).getReg();
AvailIn = I.getOperand(7).getReg();

MI = MIB.buildInstr(getLoadFifoOpcode(I),
{VecOut, PtrOut, FifoOut, AvailOut},
{PtrIn, FifoIn, AvailIn});
} else if (IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_1d_unaligned) {
PtrIn = I.getOperand(5).getReg();
FifoIn = I.getOperand(6).getReg();
AvailIn = I.getOperand(7).getReg();

Register OffsetReg = I.getOperand(8).getReg();
MI = MIB.buildInstr(getLoadFifoOpcode(I),
{VecOut, PtrOut, FifoOut, AvailOut},
{PtrIn, FifoIn, AvailIn, OffsetReg});
} else if (IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_2d_unaligned) {
Register CountOut1Reg = I.getOperand(4).getReg();
PtrIn = I.getOperand(6).getReg();
FifoIn = I.getOperand(7).getReg();
AvailIn = I.getOperand(8).getReg();
Register OffsetReg = I.getOperand(9).getReg();
Register SizeReg = I.getOperand(10).getReg();
Register CountIn1Reg = I.getOperand(11).getReg();
Register IncrReg = I.getOperand(12).getReg();
if (!RBI.constrainGenericRegister(CountOut1Reg, AIE2P::eDCRegClass, MRI))
return false;

Register DReg =
createDRegSequence(OffsetReg, IncrReg, SizeReg, CountIn1Reg, MRI);
MI = MIB.buildInstr(getLoadFifoOpcode(I),
{VecOut, PtrOut, FifoOut, AvailOut, CountOut1Reg},
{PtrIn, FifoIn, AvailIn, DReg});
} else if (IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_3d_unaligned) {
Register CountOut1Reg = I.getOperand(4).getReg();
Register CountOut2Reg = I.getOperand(5).getReg();
PtrIn = I.getOperand(7).getReg();
FifoIn = I.getOperand(8).getReg();
AvailIn = I.getOperand(9).getReg();
Register OffsetReg = I.getOperand(10).getReg();
Register Size1Reg = I.getOperand(11).getReg();
Register CountIn1Reg = I.getOperand(12).getReg();
Register Incr1Reg = I.getOperand(13).getReg();
Register Size2Reg = I.getOperand(14).getReg();
Register CountIn2Reg = I.getOperand(15).getReg();
Register Incr2Reg = I.getOperand(16).getReg();

if (!RBI.constrainGenericRegister(CountOut1Reg,
*TRI.getAddrCountRegClass(), MRI) ||
!RBI.constrainGenericRegister(CountOut2Reg,
*TRI.getAddrCountRegClass(), MRI))
return false;
Register DSReg =
createDSRegSequence(OffsetReg, Incr1Reg, Incr2Reg, Size1Reg,
CountIn1Reg, Size2Reg, CountIn2Reg, MRI);
MI = MIB.buildInstr(
getLoadFifoOpcode(I),
{VecOut, PtrOut, FifoOut, AvailOut, CountOut1Reg, CountOut2Reg},
{PtrIn, FifoIn, AvailIn, DSReg});
} else {
return false;
}

I.eraseFromParent();
return constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
}
case Intrinsic::aie2p_fifo_ld_pop_576_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_544_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_576_1d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_544_1d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_576_2d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_544_2d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_576_3d_bfp16:
case Intrinsic::aie2p_fifo_ld_pop_544_3d_bfp16: {

Register PtrOut = I.getOperand(0).getReg();
Register FifoOut = I.getOperand(1).getReg();
Register AvailOut = I.getOperand(2).getReg();
Register Vec576Out = MRI.createVirtualRegister(&AIE2P::mEXbRegClass);
Register MantVecOut;
Register ExpVecOut;
if (IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_576_bfp16 ||
IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_544_bfp16) {
PtrIn = I.getOperand(6).getReg();
FifoIn = I.getOperand(7).getReg();
AvailIn = I.getOperand(8).getReg();
MantVecOut = I.getOperand(3).getReg();
ExpVecOut = I.getOperand(4).getReg();

MI = MIB.buildInstr(getLoadFifoOpcode(I),
{Vec576Out, PtrOut, FifoOut, AvailOut},
{PtrIn, FifoIn, AvailIn});
} else if (IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_576_1d_bfp16 ||
IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_544_1d_bfp16) {
PtrIn = I.getOperand(6).getReg();
FifoIn = I.getOperand(7).getReg();
AvailIn = I.getOperand(8).getReg();
MantVecOut = I.getOperand(3).getReg();
ExpVecOut = I.getOperand(4).getReg();

Register OffsetReg = I.getOperand(9).getReg();
MI = MIB.buildInstr(getLoadFifoOpcode(I),
{Vec576Out, PtrOut, FifoOut, AvailOut},
{PtrIn, FifoIn, AvailIn, OffsetReg});
} else if (IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_576_2d_bfp16 ||
IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_544_2d_bfp16) {
Register CountOut1Reg = I.getOperand(3).getReg();
PtrIn = I.getOperand(7).getReg();
FifoIn = I.getOperand(8).getReg();
AvailIn = I.getOperand(9).getReg();
Register OffsetReg = I.getOperand(10).getReg();
Register SizeReg = I.getOperand(11).getReg();
Register CountIn1Reg = I.getOperand(12).getReg();
Register IncrReg = I.getOperand(13).getReg();
MantVecOut = I.getOperand(4).getReg();
ExpVecOut = I.getOperand(5).getReg();

if (!RBI.constrainGenericRegister(CountOut1Reg, AIE2P::eDCRegClass, MRI))
return false;
Register DReg =
createDRegSequence(OffsetReg, IncrReg, SizeReg, CountIn1Reg, MRI);
MI = MIB.buildInstr(getLoadFifoOpcode(I),
{Vec576Out, PtrOut, FifoOut, AvailOut, CountOut1Reg},
{PtrIn, FifoIn, AvailIn, DReg});
} else if (IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_576_3d_bfp16 ||
IntrinsicID == Intrinsic::aie2p_fifo_ld_pop_544_3d_bfp16) {
Register CountOut1Reg = I.getOperand(3).getReg();
Register CountOut2Reg = I.getOperand(4).getReg();
PtrIn = I.getOperand(8).getReg();
FifoIn = I.getOperand(9).getReg();
AvailIn = I.getOperand(10).getReg();
Register OffsetReg = I.getOperand(11).getReg();
Register Size1Reg = I.getOperand(12).getReg();
Register CountIn1Reg = I.getOperand(13).getReg();
Register Incr1Reg = I.getOperand(14).getReg();
Register Size2Reg = I.getOperand(15).getReg();
Register CountIn2Reg = I.getOperand(16).getReg();
Register Incr2Reg = I.getOperand(17).getReg();
MantVecOut = I.getOperand(5).getReg();
ExpVecOut = I.getOperand(6).getReg();

if (!RBI.constrainGenericRegister(CountOut1Reg,
*TRI.getAddrCountRegClass(), MRI) ||
!RBI.constrainGenericRegister(CountOut2Reg,
*TRI.getAddrCountRegClass(), MRI))
return false;
Register DSReg =
createDSRegSequence(OffsetReg, Incr1Reg, Incr2Reg, Size1Reg,
CountIn1Reg, Size2Reg, CountIn2Reg, MRI);
MI = MIB.buildInstr(
getLoadFifoOpcode(I),
{Vec576Out, PtrOut, FifoOut, AvailOut, CountOut1Reg, CountOut2Reg},
{PtrIn, FifoIn, AvailIn, DSReg});
} else {
return false;
}

bool CopiesConstrained =
buildAndConstrainFifoLoadCopies(Vec576Out, MantVecOut, ExpVecOut, MRI);

I.eraseFromParent();
return constrainSelectedInstRegOperands(*MI, TII, TRI, RBI) &&
CopiesConstrained;
}
return false;
}

return false;
}

bool AIE2PInstructionSelector::selectG_AIE_LOAD_STORE(
MachineInstr &I, MachineRegisterInfo &MRI) {

Expand Down
Loading

0 comments on commit 98d0684

Please sign in to comment.