mirror of
https://gitlab.com/libeigen/eigen.git
synced 2025-09-12 09:23:12 +08:00
Fixed performance issues for complex VSX and P10 MMA in gebp_kernel (level 3).
This commit is contained in:
Chip Kerchner
David Tellenbach
parent
e7b8643d70
commit
d59ef212e1
@ -31,6 +31,52 @@ struct Packet2cf
|
||||
{
|
||||
EIGEN_STRONG_INLINE explicit Packet2cf() {}
|
||||
EIGEN_STRONG_INLINE explicit Packet2cf(const Packet4f& a) : v(a) {}
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b)
|
||||
{
|
||||
Packet4f v1, v2;
|
||||
|
||||
// Permute and multiply the real parts of a and b
|
||||
v1 = vec_perm(a.v, a.v, p16uc_PSET32_WODD);
|
||||
// Get the imaginary parts of a
|
||||
v2 = vec_perm(a.v, a.v, p16uc_PSET32_WEVEN);
|
||||
// multiply a_re * b
|
||||
v1 = vec_madd(v1, b.v, p4f_ZERO);
|
||||
// multiply a_im * b and get the conjugate result
|
||||
v2 = vec_madd(v2, b.v, p4f_ZERO);
|
||||
v2 = reinterpret_cast<Packet4f>(pxor(v2, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR)));
|
||||
// permute back to a proper order
|
||||
v2 = vec_perm(v2, v2, p16uc_COMPLEX32_REV);
|
||||
|
||||
return Packet2cf(padd<Packet4f>(v1, v2));
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf& operator*=(const Packet2cf& b) {
|
||||
v = pmul(Packet2cf(*this), b).v;
|
||||
return *this;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet2cf operator*(const Packet2cf& b) const {
|
||||
return Packet2cf(*this) *= b;
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf& operator+=(const Packet2cf& b) {
|
||||
v = padd(v, b.v);
|
||||
return *this;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet2cf operator+(const Packet2cf& b) const {
|
||||
return Packet2cf(*this) += b;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet2cf& operator-=(const Packet2cf& b) {
|
||||
v = psub(v, b.v);
|
||||
return *this;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet2cf operator-(const Packet2cf& b) const {
|
||||
return Packet2cf(*this) -= b;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet2cf operator-(void) const {
|
||||
return Packet2cf(vec_neg(v));
|
||||
}
|
||||
|
||||
Packet4f v;
|
||||
};
|
||||
|
||||
@ -81,6 +127,25 @@ template<> EIGEN_STRONG_INLINE Packet2cf ploaddup<Packet2cf>(const std::complex<
|
||||
template<> EIGEN_STRONG_INLINE void pstore <std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { pstore((float*)to, from.v); }
|
||||
template<> EIGEN_STRONG_INLINE void pstoreu<std::complex<float> >(std::complex<float> * to, const Packet2cf& from) { pstoreu((float*)to, from.v); }
|
||||
|
||||
EIGEN_STRONG_INLINE Packet2cf pload2(const std::complex<float>* from0, const std::complex<float>* from1)
|
||||
{
|
||||
Packet4f res0, res1;
|
||||
#ifdef __VSX__
|
||||
__asm__ ("lxsdx %x0,%y1" : "=wa" (res0) : "Z" (*from0));
|
||||
__asm__ ("lxsdx %x0,%y1" : "=wa" (res1) : "Z" (*from1));
|
||||
#ifdef _BIG_ENDIAN
|
||||
__asm__ ("xxpermdi %x0, %x1, %x2, 0" : "=wa" (res0) : "wa" (res0), "wa" (res1));
|
||||
#else
|
||||
__asm__ ("xxpermdi %x0, %x2, %x1, 0" : "=wa" (res0) : "wa" (res0), "wa" (res1));
|
||||
#endif
|
||||
#else
|
||||
*((std::complex<float> *)&res0[0]) = *from0;
|
||||
*((std::complex<float> *)&res1[0]) = *from1;
|
||||
res0 = vec_perm(res0, res1, p16uc_TRANSPOSE64_HI);
|
||||
#endif
|
||||
return Packet2cf(res0);
|
||||
}
|
||||
|
||||
template<> EIGEN_DEVICE_FUNC inline Packet2cf pgather<std::complex<float>, Packet2cf>(const std::complex<float>* from, Index stride)
|
||||
{
|
||||
EIGEN_ALIGN16 std::complex<float> af[2];
|
||||
@ -101,25 +166,6 @@ template<> EIGEN_STRONG_INLINE Packet2cf psub<Packet2cf>(const Packet2cf& a, con
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { return Packet2cf(pnegate(a.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { return Packet2cf(pxor<Packet4f>(a.v, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR))); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pmul<Packet2cf>(const Packet2cf& a, const Packet2cf& b)
|
||||
{
|
||||
Packet4f v1, v2;
|
||||
|
||||
// Permute and multiply the real parts of a and b
|
||||
v1 = vec_perm(a.v, a.v, p16uc_PSET32_WODD);
|
||||
// Get the imaginary parts of a
|
||||
v2 = vec_perm(a.v, a.v, p16uc_PSET32_WEVEN);
|
||||
// multiply a_re * b
|
||||
v1 = vec_madd(v1, b.v, p4f_ZERO);
|
||||
// multiply a_im * b and get the conjugate result
|
||||
v2 = vec_madd(v2, b.v, p4f_ZERO);
|
||||
v2 = reinterpret_cast<Packet4f>(pxor(v2, reinterpret_cast<Packet4f>(p4ui_CONJ_XOR)));
|
||||
// permute back to a proper order
|
||||
v2 = vec_perm(v2, v2, p16uc_COMPLEX32_REV);
|
||||
|
||||
return Packet2cf(padd<Packet4f>(v1, v2));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pand <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pand<Packet4f>(a.v, b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf por <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(por<Packet4f>(a.v, b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet2cf pxor <Packet2cf>(const Packet2cf& a, const Packet2cf& b) { return Packet2cf(pxor<Packet4f>(a.v, b.v)); }
|
||||
@ -239,6 +285,51 @@ struct Packet1cd
|
||||
{
|
||||
EIGEN_STRONG_INLINE Packet1cd() {}
|
||||
EIGEN_STRONG_INLINE explicit Packet1cd(const Packet2d& a) : v(a) {}
|
||||
|
||||
EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b)
|
||||
{
|
||||
Packet2d a_re, a_im, v1, v2;
|
||||
|
||||
// Permute and multiply the real parts of a and b
|
||||
a_re = vec_perm(a.v, a.v, p16uc_PSET64_HI);
|
||||
// Get the imaginary parts of a
|
||||
a_im = vec_perm(a.v, a.v, p16uc_PSET64_LO);
|
||||
// multiply a_re * b
|
||||
v1 = vec_madd(a_re, b.v, p2d_ZERO);
|
||||
// multiply a_im * b and get the conjugate result
|
||||
v2 = vec_madd(a_im, b.v, p2d_ZERO);
|
||||
v2 = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(v2), reinterpret_cast<Packet4ui>(v2), 8));
|
||||
v2 = pxor(v2, reinterpret_cast<Packet2d>(p2ul_CONJ_XOR1));
|
||||
|
||||
return Packet1cd(padd<Packet2d>(v1, v2));
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE Packet1cd& operator*=(const Packet1cd& b) {
|
||||
v = pmul(Packet1cd(*this), b).v;
|
||||
return *this;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet1cd operator*(const Packet1cd& b) const {
|
||||
return Packet1cd(*this) *= b;
|
||||
}
|
||||
|
||||
EIGEN_STRONG_INLINE Packet1cd& operator+=(const Packet1cd& b) {
|
||||
v = padd(v, b.v);
|
||||
return *this;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet1cd operator+(const Packet1cd& b) const {
|
||||
return Packet1cd(*this) += b;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet1cd& operator-=(const Packet1cd& b) {
|
||||
v = psub(v, b.v);
|
||||
return *this;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet1cd operator-(const Packet1cd& b) const {
|
||||
return Packet1cd(*this) -= b;
|
||||
}
|
||||
EIGEN_STRONG_INLINE Packet1cd operator-(void) const {
|
||||
return Packet1cd(vec_neg(v));
|
||||
}
|
||||
|
||||
Packet2d v;
|
||||
};
|
||||
|
||||
@ -290,24 +381,6 @@ template<> EIGEN_STRONG_INLINE Packet1cd psub<Packet1cd>(const Packet1cd& a, con
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { return Packet1cd(pnegate(Packet2d(a.v))); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { return Packet1cd(pxor(a.v, reinterpret_cast<Packet2d>(p2ul_CONJ_XOR2))); }
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pmul<Packet1cd>(const Packet1cd& a, const Packet1cd& b)
|
||||
{
|
||||
Packet2d a_re, a_im, v1, v2;
|
||||
|
||||
// Permute and multiply the real parts of a and b
|
||||
a_re = vec_perm(a.v, a.v, p16uc_PSET64_HI);
|
||||
// Get the imaginary parts of a
|
||||
a_im = vec_perm(a.v, a.v, p16uc_PSET64_LO);
|
||||
// multiply a_re * b
|
||||
v1 = vec_madd(a_re, b.v, p2d_ZERO);
|
||||
// multiply a_im * b and get the conjugate result
|
||||
v2 = vec_madd(a_im, b.v, p2d_ZERO);
|
||||
v2 = reinterpret_cast<Packet2d>(vec_sld(reinterpret_cast<Packet4ui>(v2), reinterpret_cast<Packet4ui>(v2), 8));
|
||||
v2 = pxor(v2, reinterpret_cast<Packet2d>(p2ul_CONJ_XOR1));
|
||||
|
||||
return Packet1cd(padd<Packet2d>(v1, v2));
|
||||
}
|
||||
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pand <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(pand(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd por <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(por(a.v,b.v)); }
|
||||
template<> EIGEN_STRONG_INLINE Packet1cd pxor <Packet1cd>(const Packet1cd& a, const Packet1cd& b) { return Packet1cd(pxor(a.v,b.v)); }
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,3 +1,10 @@
|
||||
//#define EIGEN_POWER_USE_PREFETCH // Use prefetching in gemm routines
|
||||
#ifdef EIGEN_POWER_USE_PREFETCH
|
||||
#define EIGEN_POWER_PREFETCH(p) prefetch(p)
|
||||
#else
|
||||
#define EIGEN_POWER_PREFETCH(p)
|
||||
#endif
|
||||
|
||||
namespace Eigen {
|
||||
|
||||
namespace internal {
|
||||
@ -16,7 +23,7 @@ EIGEN_STRONG_INLINE void gemm_extra_col(
|
||||
Index remaining_cols,
|
||||
const Packet& pAlpha);
|
||||
|
||||
template<typename Scalar, typename Packet, typename DataMapper, typename Index, const Index accRows>
|
||||
template<typename Scalar, typename Packet, typename DataMapper, typename Index, const Index accRows, const Index accCols>
|
||||
EIGEN_STRONG_INLINE void gemm_extra_row(
|
||||
const DataMapper& res,
|
||||
const Scalar* lhs_base,
|
||||
@ -26,6 +33,7 @@ EIGEN_STRONG_INLINE void gemm_extra_row(
|
||||
Index offsetA,
|
||||
Index row,
|
||||
Index col,
|
||||
Index rows,
|
||||
Index cols,
|
||||
Index remaining_rows,
|
||||
const Packet& pAlpha,
|
||||
@ -48,6 +56,71 @@ EIGEN_STRONG_INLINE void gemm_unrolled_col(
|
||||
template<typename Packet>
|
||||
EIGEN_STRONG_INLINE Packet bmask(const int remaining_rows);
|
||||
|
||||
template<typename Scalar, typename Packet, typename Packetc, typename DataMapper, typename Index, const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
|
||||
EIGEN_STRONG_INLINE void gemm_complex_extra_col(
|
||||
const DataMapper& res,
|
||||
const Scalar* lhs_base,
|
||||
const Scalar* rhs_base,
|
||||
Index depth,
|
||||
Index strideA,
|
||||
Index offsetA,
|
||||
Index strideB,
|
||||
Index row,
|
||||
Index col,
|
||||
Index remaining_rows,
|
||||
Index remaining_cols,
|
||||
const Packet& pAlphaReal,
|
||||
const Packet& pAlphaImag);
|
||||
|
||||
template<typename Scalar, typename Packet, typename Packetc, typename DataMapper, typename Index, const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
|
||||
EIGEN_STRONG_INLINE void gemm_complex_extra_row(
|
||||
const DataMapper& res,
|
||||
const Scalar* lhs_base,
|
||||
const Scalar* rhs_base,
|
||||
Index depth,
|
||||
Index strideA,
|
||||
Index offsetA,
|
||||
Index strideB,
|
||||
Index row,
|
||||
Index col,
|
||||
Index rows,
|
||||
Index cols,
|
||||
Index remaining_rows,
|
||||
const Packet& pAlphaReal,
|
||||
const Packet& pAlphaImag,
|
||||
const Packet& pMask);
|
||||
|
||||
template<typename Scalar, typename Packet, typename Packetc, typename DataMapper, typename Index, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
|
||||
EIGEN_STRONG_INLINE void gemm_complex_unrolled_col(
|
||||
const DataMapper& res,
|
||||
const Scalar* lhs_base,
|
||||
const Scalar* rhs_base,
|
||||
Index depth,
|
||||
Index strideA,
|
||||
Index offsetA,
|
||||
Index strideB,
|
||||
Index& row,
|
||||
Index rows,
|
||||
Index col,
|
||||
Index remaining_cols,
|
||||
const Packet& pAlphaReal,
|
||||
const Packet& pAlphaImag);
|
||||
|
||||
template<typename Scalar, typename Packet>
|
||||
EIGEN_STRONG_INLINE Packet ploadLhs(const Scalar* lhs);
|
||||
|
||||
template<typename DataMapper, typename Packet, typename Index, const Index accCols, int N, int StorageOrder>
|
||||
EIGEN_STRONG_INLINE void bload(PacketBlock<Packet,4>& acc, const DataMapper& res, Index row, Index col);
|
||||
|
||||
template<typename DataMapper, typename Packet, typename Index, const Index accCols, int N, int StorageOrder>
|
||||
EIGEN_STRONG_INLINE void bload(PacketBlock<Packet,8>& acc, const DataMapper& res, Index row, Index col);
|
||||
|
||||
template<typename Packet>
|
||||
EIGEN_STRONG_INLINE void bscale(PacketBlock<Packet,4>& acc, PacketBlock<Packet,4>& accZ, const Packet& pAlpha);
|
||||
|
||||
template<typename Packet, int N>
|
||||
EIGEN_STRONG_INLINE void bscalec(PacketBlock<Packet,N>& aReal, PacketBlock<Packet,N>& aImag, const Packet& bReal, const Packet& bImag, PacketBlock<Packet,N>& cReal, PacketBlock<Packet,N>& cImag);
|
||||
|
||||
const static Packet16uc p16uc_SETCOMPLEX32_FIRST = { 0, 1, 2, 3,
|
||||
16, 17, 18, 19,
|
||||
4, 5, 6, 7,
|
||||
@ -68,7 +141,7 @@ const static Packet16uc p16uc_SETCOMPLEX64_SECOND = { 8, 9, 10, 11, 12, 13, 14
|
||||
|
||||
// Grab two decouples real/imaginary PacketBlocks and return two coupled (real/imaginary pairs) PacketBlocks.
|
||||
template<typename Packet, typename Packetc>
|
||||
EIGEN_STRONG_INLINE void bcouple(PacketBlock<Packet,4>& taccReal, PacketBlock<Packet,4>& taccImag, PacketBlock<Packetc,8>& tRes, PacketBlock<Packetc, 4>& acc1, PacketBlock<Packetc, 4>& acc2)
|
||||
EIGEN_STRONG_INLINE void bcouple_common(PacketBlock<Packet,4>& taccReal, PacketBlock<Packet,4>& taccImag, PacketBlock<Packetc, 4>& acc1, PacketBlock<Packetc, 4>& acc2)
|
||||
{
|
||||
acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_FIRST);
|
||||
acc1.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX32_FIRST);
|
||||
@ -79,6 +152,12 @@ EIGEN_STRONG_INLINE void bcouple(PacketBlock<Packet,4>& taccReal, PacketBlock<Pa
|
||||
acc2.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX32_SECOND);
|
||||
acc2.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX32_SECOND);
|
||||
acc2.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX32_SECOND);
|
||||
}
|
||||
|
||||
template<typename Packet, typename Packetc>
|
||||
EIGEN_STRONG_INLINE void bcouple(PacketBlock<Packet,4>& taccReal, PacketBlock<Packet,4>& taccImag, PacketBlock<Packetc,8>& tRes, PacketBlock<Packetc, 4>& acc1, PacketBlock<Packetc, 4>& acc2)
|
||||
{
|
||||
bcouple_common<Packet, Packetc>(taccReal, taccImag, acc1, acc2);
|
||||
|
||||
acc1.packet[0] = padd<Packetc>(tRes.packet[0], acc1.packet[0]);
|
||||
acc1.packet[1] = padd<Packetc>(tRes.packet[1], acc1.packet[1]);
|
||||
@ -91,8 +170,26 @@ EIGEN_STRONG_INLINE void bcouple(PacketBlock<Packet,4>& taccReal, PacketBlock<Pa
|
||||
acc2.packet[3] = padd<Packetc>(tRes.packet[7], acc2.packet[3]);
|
||||
}
|
||||
|
||||
template<typename Packet, typename Packetc>
|
||||
EIGEN_STRONG_INLINE void bcouple_common(PacketBlock<Packet,1>& taccReal, PacketBlock<Packet,1>& taccImag, PacketBlock<Packetc, 1>& acc1, PacketBlock<Packetc, 1>& acc2)
|
||||
{
|
||||
acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_FIRST);
|
||||
|
||||
acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX32_SECOND);
|
||||
}
|
||||
|
||||
template<typename Packet, typename Packetc>
|
||||
EIGEN_STRONG_INLINE void bcouple(PacketBlock<Packet,1>& taccReal, PacketBlock<Packet,1>& taccImag, PacketBlock<Packetc,2>& tRes, PacketBlock<Packetc, 1>& acc1, PacketBlock<Packetc, 1>& acc2)
|
||||
{
|
||||
bcouple_common<Packet, Packetc>(taccReal, taccImag, acc1, acc2);
|
||||
|
||||
acc1.packet[0] = padd<Packetc>(tRes.packet[0], acc1.packet[0]);
|
||||
|
||||
acc2.packet[0] = padd<Packetc>(tRes.packet[1], acc2.packet[0]);
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void bcouple<Packet2d, Packet1cd>(PacketBlock<Packet2d,4>& taccReal, PacketBlock<Packet2d,4>& taccImag, PacketBlock<Packet1cd,8>& tRes, PacketBlock<Packet1cd, 4>& acc1, PacketBlock<Packet1cd, 4>& acc2)
|
||||
EIGEN_STRONG_INLINE void bcouple_common<Packet2d, Packet1cd>(PacketBlock<Packet2d,4>& taccReal, PacketBlock<Packet2d,4>& taccImag, PacketBlock<Packet1cd, 4>& acc1, PacketBlock<Packet1cd, 4>& acc2)
|
||||
{
|
||||
acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_FIRST);
|
||||
acc1.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX64_FIRST);
|
||||
@ -103,16 +200,14 @@ EIGEN_STRONG_INLINE void bcouple<Packet2d, Packet1cd>(PacketBlock<Packet2d,4>& t
|
||||
acc2.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], p16uc_SETCOMPLEX64_SECOND);
|
||||
acc2.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], p16uc_SETCOMPLEX64_SECOND);
|
||||
acc2.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], p16uc_SETCOMPLEX64_SECOND);
|
||||
}
|
||||
|
||||
acc1.packet[0] = padd<Packet1cd>(tRes.packet[0], acc1.packet[0]);
|
||||
acc1.packet[1] = padd<Packet1cd>(tRes.packet[1], acc1.packet[1]);
|
||||
acc1.packet[2] = padd<Packet1cd>(tRes.packet[2], acc1.packet[2]);
|
||||
acc1.packet[3] = padd<Packet1cd>(tRes.packet[3], acc1.packet[3]);
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void bcouple_common<Packet2d, Packet1cd>(PacketBlock<Packet2d,1>& taccReal, PacketBlock<Packet2d,1>& taccImag, PacketBlock<Packet1cd, 1>& acc1, PacketBlock<Packet1cd, 1>& acc2)
|
||||
{
|
||||
acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_FIRST);
|
||||
|
||||
acc2.packet[0] = padd<Packet1cd>(tRes.packet[4], acc2.packet[0]);
|
||||
acc2.packet[1] = padd<Packet1cd>(tRes.packet[5], acc2.packet[1]);
|
||||
acc2.packet[2] = padd<Packet1cd>(tRes.packet[6], acc2.packet[2]);
|
||||
acc2.packet[3] = padd<Packet1cd>(tRes.packet[7], acc2.packet[3]);
|
||||
acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], p16uc_SETCOMPLEX64_SECOND);
|
||||
}
|
||||
|
||||
// This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled.
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
// for linear algebra.
|
||||
//
|
||||
// Copyright (C) 2020 Everton Constantino (everton.constantino@ibm.com)
|
||||
// Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com)
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License v. 2.0. If a copy of the MPL was not distributed
|
||||
@ -22,149 +23,41 @@ namespace Eigen {
|
||||
|
||||
namespace internal {
|
||||
|
||||
const static Packet16uc MMA_p16uc_SETCOMPLEX32_FIRST = { 0, 1, 2, 3,
|
||||
16, 17, 18, 19,
|
||||
4, 5, 6, 7,
|
||||
20, 21, 22, 23};
|
||||
|
||||
const static Packet16uc MMA_p16uc_SETCOMPLEX32_SECOND = { 8, 9, 10, 11,
|
||||
24, 25, 26, 27,
|
||||
12, 13, 14, 15,
|
||||
28, 29, 30, 31};
|
||||
//[a,b],[ai,bi] = [a,ai] - This is equivalent to p16uc_GETREAL64
|
||||
const static Packet16uc MMA_p16uc_SETCOMPLEX64_FIRST = { 0, 1, 2, 3, 4, 5, 6, 7,
|
||||
16, 17, 18, 19, 20, 21, 22, 23};
|
||||
|
||||
//[a,b],[ai,bi] = [b,bi] - This is equivalent to p16uc_GETIMAG64
|
||||
const static Packet16uc MMA_p16uc_SETCOMPLEX64_SECOND = { 8, 9, 10, 11, 12, 13, 14, 15,
|
||||
24, 25, 26, 27, 28, 29, 30, 31};
|
||||
|
||||
|
||||
// Grab two decouples real/imaginary PacketBlocks and return two coupled (real/imaginary pairs) PacketBlocks.
|
||||
template<typename Packet, typename Packetc>
|
||||
EIGEN_STRONG_INLINE void bcoupleMMA(PacketBlock<Packet,4>& taccReal, PacketBlock<Packet,4>& taccImag, PacketBlock<Packetc,8>& tRes, PacketBlock<Packetc, 4>& acc1, PacketBlock<Packetc, 4>& acc2)
|
||||
{
|
||||
acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], MMA_p16uc_SETCOMPLEX32_FIRST);
|
||||
acc1.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], MMA_p16uc_SETCOMPLEX32_FIRST);
|
||||
acc1.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], MMA_p16uc_SETCOMPLEX32_FIRST);
|
||||
acc1.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], MMA_p16uc_SETCOMPLEX32_FIRST);
|
||||
|
||||
acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], MMA_p16uc_SETCOMPLEX32_SECOND);
|
||||
acc2.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], MMA_p16uc_SETCOMPLEX32_SECOND);
|
||||
acc2.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], MMA_p16uc_SETCOMPLEX32_SECOND);
|
||||
acc2.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], MMA_p16uc_SETCOMPLEX32_SECOND);
|
||||
|
||||
acc1.packet[0] = padd<Packetc>(tRes.packet[0], acc1.packet[0]);
|
||||
acc1.packet[1] = padd<Packetc>(tRes.packet[1], acc1.packet[1]);
|
||||
acc1.packet[2] = padd<Packetc>(tRes.packet[2], acc1.packet[2]);
|
||||
acc1.packet[3] = padd<Packetc>(tRes.packet[3], acc1.packet[3]);
|
||||
|
||||
acc2.packet[0] = padd<Packetc>(tRes.packet[4], acc2.packet[0]);
|
||||
acc2.packet[1] = padd<Packetc>(tRes.packet[5], acc2.packet[1]);
|
||||
acc2.packet[2] = padd<Packetc>(tRes.packet[6], acc2.packet[2]);
|
||||
acc2.packet[3] = padd<Packetc>(tRes.packet[7], acc2.packet[3]);
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void bcoupleMMA<Packet2d, Packet1cd>(PacketBlock<Packet2d,4>& taccReal, PacketBlock<Packet2d,4>& taccImag, PacketBlock<Packet1cd,8>& tRes, PacketBlock<Packet1cd, 4>& acc1, PacketBlock<Packet1cd, 4>& acc2)
|
||||
{
|
||||
acc1.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], MMA_p16uc_SETCOMPLEX64_FIRST);
|
||||
acc1.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], MMA_p16uc_SETCOMPLEX64_FIRST);
|
||||
acc1.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], MMA_p16uc_SETCOMPLEX64_FIRST);
|
||||
acc1.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], MMA_p16uc_SETCOMPLEX64_FIRST);
|
||||
|
||||
acc2.packet[0].v = vec_perm(taccReal.packet[0], taccImag.packet[0], MMA_p16uc_SETCOMPLEX64_SECOND);
|
||||
acc2.packet[1].v = vec_perm(taccReal.packet[1], taccImag.packet[1], MMA_p16uc_SETCOMPLEX64_SECOND);
|
||||
acc2.packet[2].v = vec_perm(taccReal.packet[2], taccImag.packet[2], MMA_p16uc_SETCOMPLEX64_SECOND);
|
||||
acc2.packet[3].v = vec_perm(taccReal.packet[3], taccImag.packet[3], MMA_p16uc_SETCOMPLEX64_SECOND);
|
||||
|
||||
acc1.packet[0] = padd<Packet1cd>(tRes.packet[0], acc1.packet[0]);
|
||||
acc1.packet[1] = padd<Packet1cd>(tRes.packet[1], acc1.packet[1]);
|
||||
acc1.packet[2] = padd<Packet1cd>(tRes.packet[2], acc1.packet[2]);
|
||||
acc1.packet[3] = padd<Packet1cd>(tRes.packet[3], acc1.packet[3]);
|
||||
|
||||
acc2.packet[0] = padd<Packet1cd>(tRes.packet[4], acc2.packet[0]);
|
||||
acc2.packet[1] = padd<Packet1cd>(tRes.packet[5], acc2.packet[1]);
|
||||
acc2.packet[2] = padd<Packet1cd>(tRes.packet[6], acc2.packet[2]);
|
||||
acc2.packet[3] = padd<Packet1cd>(tRes.packet[7], acc2.packet[3]);
|
||||
}
|
||||
|
||||
template<typename Scalar, typename Packet>
|
||||
EIGEN_STRONG_INLINE Packet ploadLhsMMA(const Scalar *lhs)
|
||||
{
|
||||
return *((Packet *)lhs);
|
||||
}
|
||||
|
||||
template<typename Packet>
|
||||
EIGEN_STRONG_INLINE PacketBlock<Packet,2> pmul(const PacketBlock<Packet,2>& a, const Packet& b)
|
||||
{
|
||||
PacketBlock<Packet,2> pb;
|
||||
pb.packet[0] = a.packet[0]*b;
|
||||
pb.packet[1] = a.packet[1]*b;
|
||||
return pb;
|
||||
}
|
||||
|
||||
template<typename Scalar, typename Packet>
|
||||
EIGEN_STRONG_INLINE void bsetzeroMMA(__vector_quad* acc)
|
||||
{
|
||||
__builtin_mma_xxsetaccz(acc);
|
||||
}
|
||||
|
||||
template<typename DataMapper, typename Index, typename Packet>
|
||||
template<typename DataMapper, typename Index, typename Packet, const Index accCols>
|
||||
EIGEN_STRONG_INLINE void storeAccumulator(Index i, Index j, const DataMapper& data, const Packet& alpha, __vector_quad* acc)
|
||||
{
|
||||
PacketBlock<Packet, 4> result;
|
||||
__builtin_mma_disassemble_acc(&result.packet, acc);
|
||||
|
||||
result.packet[0] = pmadd<Packet>(alpha, result.packet[0], data.template loadPacket<Packet>(i, j + 0));
|
||||
result.packet[1] = pmadd<Packet>(alpha, result.packet[1], data.template loadPacket<Packet>(i, j + 1));
|
||||
result.packet[2] = pmadd<Packet>(alpha, result.packet[2], data.template loadPacket<Packet>(i, j + 2));
|
||||
result.packet[3] = pmadd<Packet>(alpha, result.packet[3], data.template loadPacket<Packet>(i, j + 3));
|
||||
PacketBlock<Packet, 4> tRes;
|
||||
bload<DataMapper, Packet, Index, accCols, 0, ColMajor>(tRes, data, i, j);
|
||||
|
||||
data.template storePacketBlock<Packet, 4>(i, j, result);
|
||||
bscale<Packet>(tRes, result, alpha);
|
||||
|
||||
data.template storePacketBlock<Packet, 4>(i, j, tRes);
|
||||
}
|
||||
|
||||
template<typename DataMapper, typename Index, typename Packet, typename Packetc, int N>
|
||||
EIGEN_STRONG_INLINE void storeComplexAccumulator(Index i, Index j, const DataMapper& data, const Packet& alphaReal, const Packet& alphaImag, __vector_quad *accReal, __vector_quad *accImag, const int accColsC)
|
||||
template<typename DataMapper, typename Index, typename Packet, typename Packetc, const Index accColsC, int N>
|
||||
EIGEN_STRONG_INLINE void storeComplexAccumulator(Index i, Index j, const DataMapper& data, const Packet& alphaReal, const Packet& alphaImag, __vector_quad* accReal, __vector_quad* accImag)
|
||||
{
|
||||
PacketBlock<Packet, 4> resultReal, resultImag;
|
||||
__builtin_mma_disassemble_acc(&resultReal.packet, accReal);
|
||||
__builtin_mma_disassemble_acc(&resultImag.packet, accImag);
|
||||
|
||||
PacketBlock<Packet,4> taccReal, taccImag;
|
||||
taccReal.packet[0] = pmul<Packet>(resultReal.packet[0], alphaReal);
|
||||
taccReal.packet[1] = pmul<Packet>(resultReal.packet[1], alphaReal);
|
||||
taccReal.packet[2] = pmul<Packet>(resultReal.packet[2], alphaReal);
|
||||
taccReal.packet[3] = pmul<Packet>(resultReal.packet[3], alphaReal);
|
||||
|
||||
taccImag.packet[0] = pmul<Packet>(resultImag.packet[0], alphaReal);
|
||||
taccImag.packet[1] = pmul<Packet>(resultImag.packet[1], alphaReal);
|
||||
taccImag.packet[2] = pmul<Packet>(resultImag.packet[2], alphaReal);
|
||||
taccImag.packet[3] = pmul<Packet>(resultImag.packet[3], alphaReal);
|
||||
|
||||
taccReal.packet[0] = psub<Packet>(taccReal.packet[0], pmul<Packet>(resultImag.packet[0], alphaImag));
|
||||
taccReal.packet[1] = psub<Packet>(taccReal.packet[1], pmul<Packet>(resultImag.packet[1], alphaImag));
|
||||
taccReal.packet[2] = psub<Packet>(taccReal.packet[2], pmul<Packet>(resultImag.packet[2], alphaImag));
|
||||
taccReal.packet[3] = psub<Packet>(taccReal.packet[3], pmul<Packet>(resultImag.packet[3], alphaImag));
|
||||
|
||||
taccImag.packet[0] = pmadd<Packet>(resultReal.packet[0], alphaImag, taccImag.packet[0]);
|
||||
taccImag.packet[1] = pmadd<Packet>(resultReal.packet[1], alphaImag, taccImag.packet[1]);
|
||||
taccImag.packet[2] = pmadd<Packet>(resultReal.packet[2], alphaImag, taccImag.packet[2]);
|
||||
taccImag.packet[3] = pmadd<Packet>(resultReal.packet[3], alphaImag, taccImag.packet[3]);
|
||||
|
||||
PacketBlock<Packetc, 8> tRes;
|
||||
tRes.packet[0] = data.template loadPacket<Packetc>(i + N*accColsC, j + 0);
|
||||
tRes.packet[1] = data.template loadPacket<Packetc>(i + N*accColsC, j + 1);
|
||||
tRes.packet[2] = data.template loadPacket<Packetc>(i + N*accColsC, j + 2);
|
||||
tRes.packet[3] = data.template loadPacket<Packetc>(i + N*accColsC, j + 3);
|
||||
bload<DataMapper, Packetc, Index, accColsC, N, ColMajor>(tRes, data, i, j);
|
||||
|
||||
tRes.packet[4] = data.template loadPacket<Packetc>(i + (N+1)*accColsC, j + 0);
|
||||
tRes.packet[5] = data.template loadPacket<Packetc>(i + (N+1)*accColsC, j + 1);
|
||||
tRes.packet[6] = data.template loadPacket<Packetc>(i + (N+1)*accColsC, j + 2);
|
||||
tRes.packet[7] = data.template loadPacket<Packetc>(i + (N+1)*accColsC, j + 3);
|
||||
PacketBlock<Packet,4> taccReal, taccImag;
|
||||
bscalec<Packet,4>(resultReal, resultImag, alphaReal, alphaImag, taccReal, taccImag);
|
||||
|
||||
PacketBlock<Packetc, 4> acc1, acc2;
|
||||
bcoupleMMA<Packet, Packetc>(taccReal, taccImag, tRes, acc1, acc2);
|
||||
bcouple<Packet, Packetc>(taccReal, taccImag, tRes, acc1, acc2);
|
||||
|
||||
data.template storePacketBlock<Packetc, 4>(i + N*accColsC, j, acc1);
|
||||
data.template storePacketBlock<Packetc, 4>(i + (N+1)*accColsC, j, acc2);
|
||||
@ -182,107 +75,145 @@ EIGEN_STRONG_INLINE void pgerMMA(__vector_quad *acc, const RhsPacket& a, const L
|
||||
}
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void pgerMMA<Packet2d, PacketBlock<Packet2d, 2>, false>(__vector_quad *acc, const PacketBlock<Packet2d,2>& a, const Packet2d& b)
|
||||
template<typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
|
||||
EIGEN_STRONG_INLINE void pgerMMA(__vector_quad* acc, const PacketBlock<Packet2d,2>& a, const Packet2d& b)
|
||||
{
|
||||
__vector_pair* a0 = (__vector_pair *)(&a.packet[0]);
|
||||
if(NegativeAccumulate)
|
||||
{
|
||||
__builtin_mma_xvf64gernp(acc, *a0, (__vector unsigned char)b);
|
||||
} else {
|
||||
__builtin_mma_xvf64gerpp(acc, *a0, (__vector unsigned char)b);
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void pgerMMA<Packet2d, PacketBlock<Packet2d, 2>, true>(__vector_quad *acc, const PacketBlock<Packet2d, 2>& a, const Packet2d& b)
|
||||
{
|
||||
__vector_pair *a0 = (__vector_pair *)(&a.packet[0]);
|
||||
__builtin_mma_xvf64gernp(acc, *a0, (__vector unsigned char)b);
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void pgerMMA<Packet2d, __vector_pair, false>(__vector_quad *acc, const __vector_pair& a, const Packet2d& b)
|
||||
template<typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
|
||||
EIGEN_STRONG_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, const Packet2d& b)
|
||||
{
|
||||
__builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b);
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void pgerMMA<Packet2d, __vector_pair, true>(__vector_quad *acc, const __vector_pair& a, const Packet2d& b)
|
||||
if(NegativeAccumulate)
|
||||
{
|
||||
__builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b);
|
||||
} else {
|
||||
__builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b);
|
||||
}
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void pgerMMA<Packet4f, __vector_pair, false>(__vector_quad *acc, const __vector_pair& a, const Packet4f& b)
|
||||
template<typename LhsPacket, typename RhsPacket, bool NegativeAccumulate>
|
||||
EIGEN_STRONG_INLINE void pgerMMA(__vector_quad* acc, const __vector_pair& a, const Packet4f& b)
|
||||
{
|
||||
// Just for compilation
|
||||
EIGEN_UNUSED_VARIABLE(acc)
|
||||
EIGEN_UNUSED_VARIABLE(a)
|
||||
EIGEN_UNUSED_VARIABLE(b)
|
||||
EIGEN_UNUSED_VARIABLE(acc); // Just for compilation
|
||||
EIGEN_UNUSED_VARIABLE(a);
|
||||
EIGEN_UNUSED_VARIABLE(b);
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void pgerMMA<Packet4f, __vector_pair, true>(__vector_quad *acc, const __vector_pair& a, const Packet4f& b)
|
||||
template<typename Scalar, typename Packet, typename RhsPacket, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
|
||||
EIGEN_STRONG_INLINE void pgercMMA(__vector_quad* accReal, __vector_quad* accImag, const Packet& lhsV, const Packet& lhsVi, const RhsPacket& rhsV, const RhsPacket& rhsVi)
|
||||
{
|
||||
// Just for compilation
|
||||
EIGEN_UNUSED_VARIABLE(acc)
|
||||
EIGEN_UNUSED_VARIABLE(a)
|
||||
EIGEN_UNUSED_VARIABLE(b)
|
||||
pgerMMA<Packet, RhsPacket, false>(accReal, rhsV, lhsV);
|
||||
if(LhsIsReal) {
|
||||
pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
|
||||
} else {
|
||||
if(!RhsIsReal) {
|
||||
pgerMMA<Packet, RhsPacket, ConjugateLhs == ConjugateRhs>(accReal, rhsVi, lhsVi);
|
||||
pgerMMA<Packet, RhsPacket, ConjugateRhs>(accImag, rhsVi, lhsV);
|
||||
} else {
|
||||
EIGEN_UNUSED_VARIABLE(rhsVi);
|
||||
}
|
||||
pgerMMA<Packet, RhsPacket, ConjugateLhs>(accImag, rhsV, lhsVi);
|
||||
}
|
||||
}
|
||||
|
||||
// This is necessary because ploadRhs for double returns a pair of vectors when MMA is enabled.
|
||||
template<typename Scalar, typename Packet>
|
||||
EIGEN_STRONG_INLINE void ploadRhsMMA(const Scalar* rhs, Packet& rhsV)
|
||||
{
|
||||
rhsV = *((Packet *)rhs);
|
||||
rhsV = ploadRhs<Scalar, Packet>((const Scalar*)(rhs));
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void ploadRhsMMA<double, PacketBlock<Packet2d, 2> >(const double* rhs, PacketBlock<Packet2d, 2>& rhsV)
|
||||
{
|
||||
rhsV.packet[0] = *((Packet2d *)rhs );
|
||||
rhsV.packet[1] = *(((Packet2d *)rhs) + 1);
|
||||
rhsV.packet[0] = ploadRhs<double, Packet2d>((const double *)((Packet2d *)rhs ));
|
||||
rhsV.packet[1] = ploadRhs<double, Packet2d>((const double *)(((Packet2d *)rhs) + 1));
|
||||
}
|
||||
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void ploadRhsMMA<double, __vector_pair>(const double* rhs, __vector_pair& rhsV)
|
||||
{
|
||||
__builtin_vsx_assemble_pair(&rhsV, (__vector unsigned char)(*(((Packet2d *)rhs) + 1)), (__vector unsigned char)(*((Packet2d *)rhs)));
|
||||
#if EIGEN_COMP_LLVM
|
||||
__builtin_vsx_assemble_pair(&rhsV,
|
||||
(__vector unsigned char)(ploadRhs<double, Packet2d>((const double *)(((Packet2d *)rhs) + 1))),
|
||||
(__vector unsigned char)(ploadRhs<double, Packet2d>((const double *)((Packet2d *)rhs ))));
|
||||
#else
|
||||
__asm__ ("lxvp %x0,%1" : "=wa" (rhsV) : "Y" (*rhs));
|
||||
#endif
|
||||
}
|
||||
|
||||
#define MICRO_MMA_DST \
|
||||
__vector_quad *accZero0, __vector_quad *accZero1, __vector_quad *accZero2, \
|
||||
__vector_quad *accZero3, __vector_quad *accZero4, __vector_quad *accZero5, \
|
||||
__vector_quad *accZero6, __vector_quad *accZero7
|
||||
|
||||
#define MICRO_MMA_SRC \
|
||||
const Scalar **lhs_ptr0, const Scalar **lhs_ptr1, const Scalar **lhs_ptr2, \
|
||||
const Scalar **lhs_ptr3, const Scalar **lhs_ptr4, const Scalar **lhs_ptr5, \
|
||||
const Scalar **lhs_ptr6, const Scalar **lhs_ptr7
|
||||
|
||||
#define MICRO_MMA_ONE \
|
||||
if (sizeof(Scalar) == sizeof(float)) { \
|
||||
MICRO_MMA<unroll_factor, Scalar, Packet, RhsPacket, accRows, accCols>(\
|
||||
&lhs_ptr0, &lhs_ptr1, &lhs_ptr2, &lhs_ptr3, &lhs_ptr4, &lhs_ptr5, &lhs_ptr6, &lhs_ptr7, \
|
||||
rhs_ptr, \
|
||||
&accZero0, &accZero1, &accZero2, &accZero3, &accZero4, &accZero5, &accZero6, &accZero7); \
|
||||
} else { \
|
||||
MICRO_MMA<unroll_factor, Scalar, Packet, __vector_pair, accRows, accCols>(\
|
||||
&lhs_ptr0, &lhs_ptr1, &lhs_ptr2, &lhs_ptr3, &lhs_ptr4, &lhs_ptr5, &lhs_ptr6, &lhs_ptr7, \
|
||||
rhs_ptr, \
|
||||
&accZero0, &accZero1, &accZero2, &accZero3, &accZero4, &accZero5, &accZero6, &accZero7); \
|
||||
template<>
|
||||
EIGEN_STRONG_INLINE void ploadRhsMMA(const float* rhs, __vector_pair& rhsV)
|
||||
{
|
||||
// Just for compilation
|
||||
EIGEN_UNUSED_VARIABLE(rhs);
|
||||
EIGEN_UNUSED_VARIABLE(rhsV);
|
||||
}
|
||||
|
||||
#define MICRO_MMA_WORK_ONE(iter) \
|
||||
if (N > iter) { \
|
||||
Packet lhsV = ploadLhsMMA<Scalar, Packet>(*lhs_ptr##iter); \
|
||||
pgerMMA<Packet, RhsPacket, false>(accZero##iter, rhsV, lhsV); \
|
||||
*lhs_ptr##iter += accCols; \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(accZero##iter); \
|
||||
EIGEN_UNUSED_VARIABLE(lhs_ptr##iter); \
|
||||
}
|
||||
// PEEL_MMA loop factor.
|
||||
#define PEEL_MMA 7
|
||||
|
||||
#define MICRO_MMA_UNROLL(func) \
|
||||
func(0) func(1) func(2) func(3) func(4) func(5) func(6) func(7)
|
||||
|
||||
#define MICRO_MMA_WORK MICRO_MMA_UNROLL(MICRO_MMA_WORK_ONE)
|
||||
#define MICRO_MMA_LOAD_ONE(iter) \
|
||||
if (unroll_factor > iter) { \
|
||||
lhsV##iter = ploadLhs<Scalar, Packet>(lhs_ptr##iter); \
|
||||
lhs_ptr##iter += accCols; \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(lhsV##iter); \
|
||||
}
|
||||
|
||||
#define MICRO_MMA_WORK_ONE(iter, type, peel) \
|
||||
if (unroll_factor > iter) { \
|
||||
pgerMMA<Packet, type, false>(&accZero##iter, rhsV##peel, lhsV##iter); \
|
||||
}
|
||||
|
||||
#define MICRO_MMA_TYPE_PEEL(func, func2, type, peel) \
|
||||
if (PEEL_MMA > peel) { \
|
||||
Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4, lhsV5, lhsV6, lhsV7; \
|
||||
ploadRhsMMA<Scalar, type>(rhs_ptr + (accRows * peel), rhsV##peel); \
|
||||
MICRO_MMA_UNROLL(func2); \
|
||||
func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) \
|
||||
func(4,type,peel) func(5,type,peel) func(6,type,peel) func(7,type,peel) \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(rhsV##peel); \
|
||||
}
|
||||
|
||||
#define MICRO_MMA_UNROLL_TYPE_PEEL(func, func2, type) \
|
||||
type rhsV0, rhsV1, rhsV2, rhsV3, rhsV4, rhsV5, rhsV6, rhsV7, rhsV8, rhsV9; \
|
||||
MICRO_MMA_TYPE_PEEL(func,func2,type,0); MICRO_MMA_TYPE_PEEL(func,func2,type,1); \
|
||||
MICRO_MMA_TYPE_PEEL(func,func2,type,2); MICRO_MMA_TYPE_PEEL(func,func2,type,3); \
|
||||
MICRO_MMA_TYPE_PEEL(func,func2,type,4); MICRO_MMA_TYPE_PEEL(func,func2,type,5); \
|
||||
MICRO_MMA_TYPE_PEEL(func,func2,type,6); MICRO_MMA_TYPE_PEEL(func,func2,type,7); \
|
||||
MICRO_MMA_TYPE_PEEL(func,func2,type,8); MICRO_MMA_TYPE_PEEL(func,func2,type,9);
|
||||
|
||||
#define MICRO_MMA_UNROLL_TYPE_ONE(func, func2, type) \
|
||||
type rhsV0; \
|
||||
MICRO_MMA_TYPE_PEEL(func,func2,type,0);
|
||||
|
||||
#define MICRO_MMA_ONE_PEEL \
|
||||
if (sizeof(Scalar) == sizeof(float)) { \
|
||||
MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \
|
||||
} else { \
|
||||
MICRO_MMA_UNROLL_TYPE_PEEL(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \
|
||||
} \
|
||||
rhs_ptr += (accRows * PEEL_MMA);
|
||||
|
||||
#define MICRO_MMA_ONE \
|
||||
if (sizeof(Scalar) == sizeof(float)) { \
|
||||
MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, RhsPacket); \
|
||||
} else { \
|
||||
MICRO_MMA_UNROLL_TYPE_ONE(MICRO_MMA_WORK_ONE, MICRO_MMA_LOAD_ONE, __vector_pair); \
|
||||
} \
|
||||
rhs_ptr += accRows;
|
||||
|
||||
#define MICRO_MMA_DST_PTR_ONE(iter) \
|
||||
if (unroll_factor > iter) { \
|
||||
@ -304,33 +235,18 @@ EIGEN_STRONG_INLINE void ploadRhsMMA<double, __vector_pair>(const double *rhs, _
|
||||
|
||||
#define MICRO_MMA_PREFETCH_ONE(iter) \
|
||||
if (unroll_factor > iter) { \
|
||||
prefetch(lhs_ptr##iter); \
|
||||
EIGEN_POWER_PREFETCH(lhs_ptr##iter); \
|
||||
}
|
||||
|
||||
#define MICRO_MMA_PREFETCH MICRO_MMA_UNROLL(MICRO_MMA_PREFETCH_ONE)
|
||||
|
||||
#define MICRO_MMA_STORE_ONE(iter) \
|
||||
if (unroll_factor > iter) { \
|
||||
storeAccumulator<DataMapper, Index, Packet>(row + iter*accCols, col, res, pAlpha, &accZero##iter); \
|
||||
storeAccumulator<DataMapper, Index, Packet, accCols>(row + iter*accCols, col, res, pAlpha, &accZero##iter); \
|
||||
}
|
||||
|
||||
#define MICRO_MMA_STORE MICRO_MMA_UNROLL(MICRO_MMA_STORE_ONE)
|
||||
|
||||
// PEEL_MMA loop factor.
|
||||
#define PEEL_MMA 10
|
||||
|
||||
template<int N, typename Scalar, typename Packet, typename RhsPacket, const Index accRows, const Index accCols>
|
||||
EIGEN_STRONG_INLINE void MICRO_MMA(
|
||||
MICRO_MMA_SRC,
|
||||
const Scalar* &rhs_ptr,
|
||||
MICRO_MMA_DST)
|
||||
{
|
||||
RhsPacket rhsV;
|
||||
ploadRhsMMA<Scalar, RhsPacket>(rhs_ptr, rhsV);
|
||||
MICRO_MMA_WORK
|
||||
rhs_ptr += accRows;
|
||||
}
|
||||
|
||||
template<int unroll_factor, typename Scalar, typename Packet, typename RhsPacket, typename DataMapper, typename Index, const Index accRows, const Index accCols>
|
||||
EIGEN_STRONG_INLINE void gemm_unrolled_MMA_iteration(
|
||||
const DataMapper& res,
|
||||
@ -343,22 +259,20 @@ EIGEN_STRONG_INLINE void gemm_unrolled_MMA_iteration(
|
||||
Index col,
|
||||
const Packet& pAlpha)
|
||||
{
|
||||
asm("#gemm_MMA begin");
|
||||
const Scalar* rhs_ptr = rhs_base;
|
||||
const Scalar* lhs_ptr0, * lhs_ptr1, * lhs_ptr2, * lhs_ptr3, * lhs_ptr4, * lhs_ptr5, * lhs_ptr6, * lhs_ptr7;
|
||||
__vector_quad accZero0, accZero1, accZero2, accZero3, accZero4, accZero5, accZero6, accZero7;
|
||||
|
||||
asm("#unrolled MMA start");
|
||||
MICRO_MMA_SRC_PTR
|
||||
MICRO_MMA_DST_PTR
|
||||
|
||||
Index k = 0;
|
||||
for(; k + PEEL_MMA <= depth; k+= PEEL_MMA)
|
||||
{
|
||||
prefetch(rhs_ptr);
|
||||
EIGEN_POWER_PREFETCH(rhs_ptr);
|
||||
MICRO_MMA_PREFETCH
|
||||
for (int l = 0; l < PEEL_MMA; l++) {
|
||||
MICRO_MMA_ONE
|
||||
}
|
||||
MICRO_MMA_ONE_PEEL
|
||||
}
|
||||
for(; k < depth; k++)
|
||||
{
|
||||
@ -367,7 +281,7 @@ EIGEN_STRONG_INLINE void gemm_unrolled_MMA_iteration(
|
||||
MICRO_MMA_STORE
|
||||
|
||||
row += unroll_factor*accCols;
|
||||
asm("#unrolled MMA end");
|
||||
asm("#gemm_MMA end");
|
||||
}
|
||||
|
||||
template<typename Scalar, typename Index, typename Packet, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols>
|
||||
@ -436,7 +350,7 @@ void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB,
|
||||
|
||||
if(remaining_rows > 0)
|
||||
{
|
||||
gemm_extra_row<Scalar, Packet, DataMapper, Index, accRows>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, cols, remaining_rows, pAlpha, pMask);
|
||||
gemm_extra_row<Scalar, Packet, DataMapper, Index, accRows, accCols>(res, lhs_base, rhs_base, depth, strideA, offsetA, row, col, rows, cols, remaining_rows, pAlpha, pMask);
|
||||
}
|
||||
}
|
||||
|
||||
@ -460,310 +374,264 @@ void gemmMMA(const DataMapper& res, const Scalar* blockA, const Scalar* blockB,
|
||||
}
|
||||
}
|
||||
|
||||
template<typename LhsScalar, typename RhsScalar, typename Scalarc, typename Scalar, typename Index, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper, const int accRows, const int accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
|
||||
void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc,
|
||||
Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
|
||||
{
|
||||
const int remaining_rows = rows % accCols;
|
||||
const int remaining_cols = cols % accRows;
|
||||
const int accColsC = accCols / 2;
|
||||
int advanceCols = 2;
|
||||
int advanceRows = 2;
|
||||
#define accColsC (accCols / 2)
|
||||
#define advanceRows ((LhsIsReal) ? 1 : 2)
|
||||
#define advanceCols ((RhsIsReal) ? 1 : 2)
|
||||
|
||||
if(LhsIsReal) advanceRows = 1;
|
||||
if(RhsIsReal) advanceCols = 1;
|
||||
// PEEL_COMPLEX_MMA loop factor.
|
||||
#define PEEL_COMPLEX_MMA 7
|
||||
|
||||
#define MICRO_COMPLEX_MMA_UNROLL(func) \
|
||||
func(0) func(1) func(2) func(3) func(4)
|
||||
|
||||
#define MICRO_COMPLEX_MMA_LOAD_ONE(iter) \
|
||||
if (unroll_factor > iter) { \
|
||||
lhsV##iter = ploadLhs<Scalar, Packet>(lhs_ptr_real##iter); \
|
||||
lhs_ptr_real##iter += accCols; \
|
||||
if(!LhsIsReal) { \
|
||||
lhsVi##iter = ploadLhs<Scalar, Packet>(lhs_ptr_imag##iter); \
|
||||
lhs_ptr_imag##iter += accCols; \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(lhsVi##iter); \
|
||||
} \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(lhsV##iter); \
|
||||
EIGEN_UNUSED_VARIABLE(lhsVi##iter); \
|
||||
}
|
||||
|
||||
#define MICRO_COMPLEX_MMA_WORK_ONE(iter, type, peel) \
|
||||
if (unroll_factor > iter) { \
|
||||
pgercMMA<Scalar, Packet, type, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(&accReal##iter, &accImag##iter, lhsV##iter, lhsVi##iter, rhsV##peel, rhsVi##peel); \
|
||||
}
|
||||
|
||||
#define MICRO_COMPLEX_MMA_TYPE_PEEL(func, func2, type, peel) \
|
||||
if (PEEL_COMPLEX_MMA > peel) { \
|
||||
Packet lhsV0, lhsV1, lhsV2, lhsV3, lhsV4; \
|
||||
Packet lhsVi0, lhsVi1, lhsVi2, lhsVi3, lhsVi4; \
|
||||
ploadRhsMMA<Scalar, type>(rhs_ptr_real + (accRows * peel), rhsV##peel); \
|
||||
if(!RhsIsReal) { \
|
||||
ploadRhsMMA<Scalar, type>(rhs_ptr_imag + (accRows * peel), rhsVi##peel); \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
|
||||
} \
|
||||
MICRO_COMPLEX_MMA_UNROLL(func2); \
|
||||
func(0,type,peel) func(1,type,peel) func(2,type,peel) func(3,type,peel) func(4,type,peel) \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(rhsV##peel); \
|
||||
EIGEN_UNUSED_VARIABLE(rhsVi##peel); \
|
||||
}
|
||||
|
||||
#define MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(func, func2, type) \
|
||||
type rhsV0, rhsV1, rhsV2, rhsV3, rhsV4, rhsV5, rhsV6, rhsV7, rhsV8, rhsV9; \
|
||||
type rhsVi0, rhsVi1, rhsVi2, rhsVi3, rhsVi4, rhsVi5, rhsVi6, rhsVi7, rhsVi8, rhsVi9; \
|
||||
MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,1); \
|
||||
MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,2); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,3); \
|
||||
MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,4); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,5); \
|
||||
MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,6); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,7); \
|
||||
MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,8); MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,9);
|
||||
|
||||
#define MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(func, func2, type) \
|
||||
type rhsV0, rhsVi0; \
|
||||
MICRO_COMPLEX_MMA_TYPE_PEEL(func,func2,type,0);
|
||||
|
||||
#define MICRO_COMPLEX_MMA_ONE_PEEL \
|
||||
if (sizeof(Scalar) == sizeof(float)) { \
|
||||
MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \
|
||||
} else { \
|
||||
MICRO_COMPLEX_MMA_UNROLL_TYPE_PEEL(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \
|
||||
} \
|
||||
rhs_ptr_real += (accRows * PEEL_COMPLEX_MMA); \
|
||||
if(!RhsIsReal) rhs_ptr_imag += (accRows * PEEL_COMPLEX_MMA);
|
||||
|
||||
#define MICRO_COMPLEX_MMA_ONE \
|
||||
if (sizeof(Scalar) == sizeof(float)) { \
|
||||
MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, RhsPacket); \
|
||||
} else { \
|
||||
MICRO_COMPLEX_MMA_UNROLL_TYPE_ONE(MICRO_COMPLEX_MMA_WORK_ONE, MICRO_COMPLEX_MMA_LOAD_ONE, __vector_pair); \
|
||||
} \
|
||||
rhs_ptr_real += accRows; \
|
||||
if(!RhsIsReal) rhs_ptr_imag += accRows;
|
||||
|
||||
#define MICRO_COMPLEX_MMA_DST_PTR_ONE(iter) \
|
||||
if (unroll_factor > iter) { \
|
||||
bsetzeroMMA<Scalar, Packet>(&accReal##iter); \
|
||||
bsetzeroMMA<Scalar, Packet>(&accImag##iter); \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(accReal##iter); \
|
||||
EIGEN_UNUSED_VARIABLE(accImag##iter); \
|
||||
}
|
||||
|
||||
#define MICRO_COMPLEX_MMA_DST_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_DST_PTR_ONE)
|
||||
|
||||
#define MICRO_COMPLEX_MMA_SRC_PTR_ONE(iter) \
|
||||
if (unroll_factor > iter) { \
|
||||
lhs_ptr_real##iter = lhs_base + ( ((advanceRows*row)/accCols) + iter*advanceRows )*strideA*accCols + accCols*offsetA; \
|
||||
if(!LhsIsReal) { \
|
||||
lhs_ptr_imag##iter = lhs_ptr_real##iter + accCols*strideA; \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \
|
||||
} \
|
||||
} else { \
|
||||
EIGEN_UNUSED_VARIABLE(lhs_ptr_real##iter); \
|
||||
EIGEN_UNUSED_VARIABLE(lhs_ptr_imag##iter); \
|
||||
}
|
||||
|
||||
#define MICRO_COMPLEX_MMA_SRC_PTR MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_SRC_PTR_ONE)
|
||||
|
||||
#define MICRO_COMPLEX_MMA_PREFETCH_ONE(iter) \
|
||||
if (unroll_factor > iter) { \
|
||||
EIGEN_POWER_PREFETCH(lhs_ptr_real##iter); \
|
||||
if(!LhsIsReal) { \
|
||||
EIGEN_POWER_PREFETCH(lhs_ptr_imag##iter); \
|
||||
} \
|
||||
}
|
||||
|
||||
#define MICRO_COMPLEX_MMA_PREFETCH MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_PREFETCH_ONE)
|
||||
|
||||
#define MICRO_COMPLEX_MMA_STORE_ONE(iter) \
|
||||
if (unroll_factor > iter) { \
|
||||
storeComplexAccumulator<DataMapper, Index, Packet, Packetc, accColsC, 0>(row + iter*accCols, col, res, pAlphaReal, pAlphaImag, &accReal##iter, &accImag##iter); \
|
||||
}
|
||||
|
||||
#define MICRO_COMPLEX_MMA_STORE MICRO_COMPLEX_MMA_UNROLL(MICRO_COMPLEX_MMA_STORE_ONE)
|
||||
|
||||
template<int unroll_factor, typename Scalar, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper, typename Index, const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
|
||||
EIGEN_STRONG_INLINE void gemm_complex_unrolled_MMA_iteration(
|
||||
const DataMapper& res,
|
||||
const Scalar* lhs_base,
|
||||
const Scalar* rhs_base,
|
||||
Index depth,
|
||||
Index strideA,
|
||||
Index offsetA,
|
||||
Index strideB,
|
||||
Index& row,
|
||||
Index col,
|
||||
const Packet& pAlphaReal,
|
||||
const Packet& pAlphaImag)
|
||||
{
|
||||
asm("#gemm_complex_MMA begin");
|
||||
const Scalar* rhs_ptr_real = rhs_base;
|
||||
const Scalar* rhs_ptr_imag;
|
||||
if(!RhsIsReal) {
|
||||
rhs_ptr_imag = rhs_base + accRows*strideB;
|
||||
} else {
|
||||
EIGEN_UNUSED_VARIABLE(rhs_ptr_imag);
|
||||
}
|
||||
const Scalar* lhs_ptr_real0, * lhs_ptr_imag0, * lhs_ptr_real1, * lhs_ptr_imag1;
|
||||
const Scalar* lhs_ptr_real2, * lhs_ptr_imag2, * lhs_ptr_real3, * lhs_ptr_imag3;
|
||||
const Scalar* lhs_ptr_real4, * lhs_ptr_imag4;
|
||||
__vector_quad accReal0, accImag0, accReal1, accImag1, accReal2, accImag2, accReal3, accImag3, accReal4, accImag4;
|
||||
|
||||
MICRO_COMPLEX_MMA_SRC_PTR
|
||||
MICRO_COMPLEX_MMA_DST_PTR
|
||||
|
||||
Index k = 0;
|
||||
for(; k + PEEL_COMPLEX_MMA <= depth; k+= PEEL_COMPLEX_MMA)
|
||||
{
|
||||
EIGEN_POWER_PREFETCH(rhs_ptr_real);
|
||||
if(!RhsIsReal) {
|
||||
EIGEN_POWER_PREFETCH(rhs_ptr_imag);
|
||||
}
|
||||
MICRO_COMPLEX_MMA_PREFETCH
|
||||
MICRO_COMPLEX_MMA_ONE_PEEL
|
||||
}
|
||||
for(; k < depth; k++)
|
||||
{
|
||||
MICRO_COMPLEX_MMA_ONE
|
||||
}
|
||||
MICRO_COMPLEX_MMA_STORE
|
||||
|
||||
row += unroll_factor*accCols;
|
||||
asm("#gemm_complex_MMA end");
|
||||
}
|
||||
|
||||
template<typename LhsScalar, typename RhsScalar, typename Scalarc, typename Scalar, typename Index, typename Packet, typename Packetc, typename RhsPacket, typename DataMapper, const Index accRows, const Index accCols, bool ConjugateLhs, bool ConjugateRhs, bool LhsIsReal, bool RhsIsReal>
|
||||
void gemm_complexMMA(const DataMapper& res, const LhsScalar* blockAc, const RhsScalar* blockBc, Index rows, Index depth, Index cols, Scalarc alpha, Index strideA, Index strideB, Index offsetA, Index offsetB)
|
||||
{
|
||||
const Index remaining_rows = rows % accCols;
|
||||
const Index remaining_cols = cols % accRows;
|
||||
|
||||
if( strideA == -1 ) strideA = depth;
|
||||
if( strideB == -1 ) strideB = depth;
|
||||
|
||||
const Packet pAlphaReal = pset1<Packet>(alpha.real());
|
||||
const Packet pAlphaImag = pset1<Packet>(alpha.imag());
|
||||
const Packet pMask = bmask<Packet>((const int)(remaining_rows));
|
||||
|
||||
const Scalar* blockA = (Scalar *) blockAc;
|
||||
const Scalar* blockB = (Scalar *) blockBc;
|
||||
|
||||
Packet conj = pset1<Packet>((Scalar)-1.0f);
|
||||
|
||||
Index col = 0;
|
||||
for(; col + accRows <= cols; col += accRows)
|
||||
{
|
||||
const Scalar *rhs_base = blockB + ( (advanceCols*col)/accRows )*strideB*accRows;
|
||||
const Scalar* rhs_base = blockB + advanceCols*col*strideB + accRows*offsetB;
|
||||
const Scalar* lhs_base = blockA;
|
||||
|
||||
Index row = 0;
|
||||
|
||||
for(; row + accCols <= rows; row += accCols)
|
||||
{
|
||||
const Scalar *rhs_ptr = rhs_base;
|
||||
const Scalar *rhs_ptr_imag = rhs_ptr + accRows*strideB;
|
||||
const Scalar *lhs_ptr = lhs_base + ((advanceRows*row)/accCols)*strideA*accCols;
|
||||
const Scalar *lhs_ptr_imag = lhs_ptr + accCols*strideA;
|
||||
|
||||
__vector_quad accReal, accImag;
|
||||
__builtin_mma_xxsetaccz(&accReal);
|
||||
__builtin_mma_xxsetaccz(&accImag);
|
||||
|
||||
lhs_ptr += accCols*offsetA;
|
||||
if(!LhsIsReal)
|
||||
lhs_ptr_imag += accCols*offsetA;
|
||||
rhs_ptr += accRows*offsetB;
|
||||
if(!RhsIsReal)
|
||||
rhs_ptr_imag += accRows*offsetB;
|
||||
for(Index k = 0; k < depth; k++)
|
||||
{
|
||||
Packet lhsV = ploadLhsMMA<Scalar, Packet>(lhs_ptr);
|
||||
RhsPacket rhsV = ploadRhs<Scalar, RhsPacket>(rhs_ptr);
|
||||
|
||||
Packet lhsVi = ploadLhsMMA<Scalar, Packet>(lhs_ptr_imag);
|
||||
RhsPacket rhsVi = ploadRhs<Scalar, RhsPacket>(rhs_ptr_imag);
|
||||
|
||||
if(ConjugateLhs && !LhsIsReal) lhsVi = pmul<Packet>(lhsVi, conj);
|
||||
if(ConjugateRhs && !RhsIsReal) rhsVi = pmul<Packet>(rhsVi, conj);
|
||||
|
||||
if(LhsIsReal)
|
||||
{
|
||||
pgerMMA<Packet, RhsPacket, false>(&accReal, rhsV, lhsV);
|
||||
pgerMMA<Packet, RhsPacket, false>(&accImag, rhsVi, lhsV);
|
||||
} else if(RhsIsReal) {
|
||||
pgerMMA<Packet, RhsPacket, false>(&accReal, rhsV, lhsV);
|
||||
pgerMMA<Packet, RhsPacket, false>(&accImag, rhsV, lhsVi);
|
||||
} else {
|
||||
pgerMMA<Packet, RhsPacket, false>(&accReal, rhsV, lhsV);
|
||||
pgerMMA<Packet, RhsPacket, true>(&accReal, rhsVi, lhsVi);
|
||||
pgerMMA<Packet, RhsPacket, false>(&accImag, rhsVi, lhsV);
|
||||
pgerMMA<Packet, RhsPacket, false>(&accImag, rhsV, lhsVi);
|
||||
#define MAX_COMPLEX_MMA_UNROLL 4
|
||||
while(row + MAX_COMPLEX_MMA_UNROLL*accCols <= rows) {
|
||||
gemm_complex_unrolled_MMA_iteration<MAX_COMPLEX_MMA_UNROLL, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
|
||||
}
|
||||
|
||||
lhs_ptr += accCols;
|
||||
rhs_ptr += accRows;
|
||||
if(!LhsIsReal)
|
||||
lhs_ptr_imag += accCols;
|
||||
if(!RhsIsReal)
|
||||
rhs_ptr_imag += accRows;
|
||||
}
|
||||
|
||||
storeComplexAccumulator<DataMapper, Index, Packet, Packetc, 0>(row, col, res, pAlphaReal, pAlphaImag, &accReal, &accImag, accColsC);
|
||||
switch( (rows-row)/accCols ) {
|
||||
#if MAX_COMPLEX_MMA_UNROLL > 4
|
||||
case 4:
|
||||
gemm_complex_unrolled_MMA_iteration<4, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
|
||||
break;
|
||||
#endif
|
||||
#if MAX_COMPLEX_MMA_UNROLL > 3
|
||||
case 3:
|
||||
gemm_complex_unrolled_MMA_iteration<3, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
|
||||
break;
|
||||
#endif
|
||||
#if MAX_COMPLEX_MMA_UNROLL > 2
|
||||
case 2:
|
||||
gemm_complex_unrolled_MMA_iteration<2, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
|
||||
break;
|
||||
#endif
|
||||
#if MAX_COMPLEX_MMA_UNROLL > 1
|
||||
case 1:
|
||||
gemm_complex_unrolled_MMA_iteration<1, Scalar, Packet, Packetc, RhsPacket, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, pAlphaReal, pAlphaImag);
|
||||
break;
|
||||
#endif
|
||||
default:
|
||||
break;
|
||||
}
|
||||
#undef MAX_COMPLEX_MMA_UNROLL
|
||||
|
||||
if(remaining_rows > 0)
|
||||
{
|
||||
const Scalar *rhs_ptr = rhs_base;
|
||||
const Scalar *rhs_ptr_imag = rhs_ptr + accRows*strideB;
|
||||
const Scalar *lhs_ptr = lhs_base + ((advanceRows*row)/accCols)*strideA*accCols;
|
||||
const Scalar *lhs_ptr_imag = lhs_ptr + remaining_rows*strideA;
|
||||
|
||||
lhs_ptr += remaining_rows*offsetA;
|
||||
if(!LhsIsReal)
|
||||
lhs_ptr_imag += remaining_rows*offsetA;
|
||||
rhs_ptr += accRows*offsetB;
|
||||
if(!RhsIsReal)
|
||||
rhs_ptr_imag += accRows*offsetB;
|
||||
for(Index k = 0; k < depth; k++)
|
||||
{
|
||||
for(Index arow = 0; arow < remaining_rows; arow++)
|
||||
{
|
||||
Scalar lhs_real = lhs_ptr[arow];
|
||||
Scalar lhs_imag;
|
||||
if(!LhsIsReal) lhs_imag = lhs_ptr_imag[arow];
|
||||
|
||||
Scalarc lhsc;
|
||||
|
||||
lhsc.real(lhs_real);
|
||||
if(!LhsIsReal)
|
||||
{
|
||||
if(ConjugateLhs)
|
||||
lhsc.imag(-lhs_imag);
|
||||
else
|
||||
lhsc.imag(lhs_imag);
|
||||
} else {
|
||||
//Lazy approach for now
|
||||
lhsc.imag((Scalar)0);
|
||||
}
|
||||
|
||||
for(int acol = 0; acol < accRows; acol++ )
|
||||
{
|
||||
Scalar rhs_real = rhs_ptr[acol];
|
||||
Scalar rhs_imag;
|
||||
if(!RhsIsReal) rhs_imag = rhs_ptr_imag[acol];
|
||||
Scalarc rhsc;
|
||||
|
||||
rhsc.real(rhs_real);
|
||||
if(!RhsIsReal)
|
||||
{
|
||||
if(ConjugateRhs)
|
||||
rhsc.imag(-rhs_imag);
|
||||
else
|
||||
rhsc.imag(rhs_imag);
|
||||
} else {
|
||||
//Lazy approach for now
|
||||
rhsc.imag((Scalar)0);
|
||||
}
|
||||
res(row + arow, col + acol) += alpha*lhsc*rhsc;
|
||||
}
|
||||
}
|
||||
rhs_ptr += accRows;
|
||||
lhs_ptr += remaining_rows;
|
||||
if(!LhsIsReal)
|
||||
lhs_ptr_imag += remaining_rows;
|
||||
if(!RhsIsReal)
|
||||
rhs_ptr_imag += accRows;
|
||||
}
|
||||
gemm_complex_extra_row<Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, rows, cols, remaining_rows, pAlphaReal, pAlphaImag, pMask);
|
||||
}
|
||||
}
|
||||
|
||||
if(remaining_cols > 0)
|
||||
{
|
||||
const Scalar *rhs_base = blockB + ( (advanceCols*col)/accRows )*strideB*accRows;
|
||||
const Scalar* rhs_base = blockB + advanceCols*col*strideB + remaining_cols*offsetB;
|
||||
const Scalar* lhs_base = blockA;
|
||||
|
||||
for(; col < cols; col++)
|
||||
{
|
||||
Index row = 0;
|
||||
|
||||
for(; row + accCols <= rows; row += accCols)
|
||||
{
|
||||
const Scalar *rhs_ptr = rhs_base;
|
||||
const Scalar *rhs_ptr_imag = rhs_ptr + remaining_cols*strideB;
|
||||
const Scalar *lhs_ptr = lhs_base + ((advanceRows*row)/accCols)*strideA*accCols;
|
||||
const Scalar *lhs_ptr_imag = lhs_ptr + accCols*strideA;
|
||||
|
||||
lhs_ptr += accCols*offsetA;
|
||||
if(!LhsIsReal)
|
||||
lhs_ptr_imag += accCols*offsetA;
|
||||
rhs_ptr += remaining_cols*offsetB;
|
||||
if(!RhsIsReal)
|
||||
rhs_ptr_imag += remaining_cols*offsetB;
|
||||
Scalarc scalarAcc[4][4];
|
||||
for(Index arow = 0; arow < 4; arow++ )
|
||||
{
|
||||
for(Index acol = 0; acol < 4; acol++ )
|
||||
{
|
||||
scalarAcc[arow][acol].real((Scalar)0.0f);
|
||||
scalarAcc[arow][acol].imag((Scalar)0.0f);
|
||||
}
|
||||
}
|
||||
for(Index k = 0; k < depth; k++)
|
||||
{
|
||||
for(Index arow = 0; arow < accCols; arow++)
|
||||
{
|
||||
Scalar lhs_real = lhs_ptr[arow];
|
||||
Scalar lhs_imag;
|
||||
if(!LhsIsReal)
|
||||
{
|
||||
lhs_imag = lhs_ptr_imag[arow];
|
||||
|
||||
if(ConjugateLhs)
|
||||
lhs_imag *= -1;
|
||||
} else {
|
||||
lhs_imag = (Scalar)0;
|
||||
}
|
||||
|
||||
for(int acol = 0; acol < remaining_cols; acol++ )
|
||||
{
|
||||
Scalar rhs_real = rhs_ptr[acol];
|
||||
Scalar rhs_imag;
|
||||
if(!RhsIsReal)
|
||||
{
|
||||
rhs_imag = rhs_ptr_imag[acol];
|
||||
|
||||
if(ConjugateRhs)
|
||||
rhs_imag *= -1;
|
||||
} else {
|
||||
rhs_imag = (Scalar)0;
|
||||
}
|
||||
|
||||
scalarAcc[arow][acol].real(scalarAcc[arow][acol].real() + lhs_real*rhs_real - lhs_imag*rhs_imag);
|
||||
scalarAcc[arow][acol].imag(scalarAcc[arow][acol].imag() + lhs_imag*rhs_real + lhs_real*rhs_imag);
|
||||
}
|
||||
}
|
||||
rhs_ptr += remaining_cols;
|
||||
lhs_ptr += accCols;
|
||||
if(!RhsIsReal)
|
||||
rhs_ptr_imag += remaining_cols;
|
||||
if(!LhsIsReal)
|
||||
lhs_ptr_imag += accCols;
|
||||
}
|
||||
for(int arow = 0; arow < accCols; arow++ )
|
||||
{
|
||||
for(int acol = 0; acol < remaining_cols; acol++ )
|
||||
{
|
||||
Scalar accR = scalarAcc[arow][acol].real();
|
||||
Scalar accI = scalarAcc[arow][acol].imag();
|
||||
Scalar aR = alpha.real();
|
||||
Scalar aI = alpha.imag();
|
||||
Scalar resR = res(row + arow, col + acol).real();
|
||||
Scalar resI = res(row + arow, col + acol).imag();
|
||||
|
||||
res(row + arow, col + acol).real(resR + accR*aR - accI*aI);
|
||||
res(row + arow, col + acol).imag(resI + accR*aI + accI*aR);
|
||||
}
|
||||
}
|
||||
}
|
||||
gemm_complex_unrolled_col<Scalar, Packet, Packetc, DataMapper, Index, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, rows, col, remaining_cols, pAlphaReal, pAlphaImag);
|
||||
|
||||
if (remaining_rows > 0)
|
||||
{
|
||||
const Scalar *rhs_ptr = rhs_base;
|
||||
const Scalar *rhs_ptr_imag = rhs_ptr + remaining_cols*strideB;
|
||||
const Scalar *lhs_ptr = lhs_base + ((advanceRows*row)/accCols)*strideA*accCols;
|
||||
const Scalar *lhs_ptr_imag = lhs_ptr + remaining_rows*strideA;
|
||||
|
||||
lhs_ptr += remaining_rows*offsetA;
|
||||
if(!LhsIsReal)
|
||||
lhs_ptr_imag += remaining_rows*offsetA;
|
||||
rhs_ptr += remaining_cols*offsetB;
|
||||
if(!RhsIsReal)
|
||||
rhs_ptr_imag += remaining_cols*offsetB;
|
||||
for(Index k = 0; k < depth; k++)
|
||||
{
|
||||
for(Index arow = 0; arow < remaining_rows; arow++)
|
||||
{
|
||||
Scalar lhs_real = lhs_ptr[arow];
|
||||
Scalar lhs_imag;
|
||||
if(!LhsIsReal) lhs_imag = lhs_ptr_imag[arow];
|
||||
Scalarc lhsc;
|
||||
|
||||
lhsc.real(lhs_real);
|
||||
if(!LhsIsReal)
|
||||
{
|
||||
if(ConjugateLhs)
|
||||
lhsc.imag(-lhs_imag);
|
||||
else
|
||||
lhsc.imag(lhs_imag);
|
||||
} else {
|
||||
lhsc.imag((Scalar)0);
|
||||
gemm_complex_extra_col<Scalar, Packet, Packetc, DataMapper, Index, accRows, accCols, ConjugateLhs, ConjugateRhs, LhsIsReal, RhsIsReal>(res, lhs_base, rhs_base, depth, strideA, offsetA, strideB, row, col, remaining_rows, remaining_cols, pAlphaReal, pAlphaImag);
|
||||
}
|
||||
rhs_base++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for(Index acol = 0; acol < remaining_cols; acol++ )
|
||||
{
|
||||
Scalar rhs_real = rhs_ptr[acol];
|
||||
Scalar rhs_imag;
|
||||
if(!RhsIsReal) rhs_imag = rhs_ptr_imag[acol];
|
||||
Scalarc rhsc;
|
||||
|
||||
rhsc.real(rhs_real);
|
||||
if(!RhsIsReal)
|
||||
{
|
||||
if(ConjugateRhs)
|
||||
rhsc.imag(-rhs_imag);
|
||||
else
|
||||
rhsc.imag(rhs_imag);
|
||||
} else {
|
||||
rhsc.imag((Scalar)0);
|
||||
}
|
||||
res(row + arow, col + acol) += alpha*lhsc*rhsc;
|
||||
}
|
||||
}
|
||||
rhs_ptr += remaining_cols;
|
||||
lhs_ptr += remaining_rows;
|
||||
if(!LhsIsReal)
|
||||
lhs_ptr_imag += remaining_rows;
|
||||
if(!RhsIsReal)
|
||||
rhs_ptr_imag += remaining_cols;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#undef accColsC
|
||||
#undef advanceRows
|
||||
#undef advanceCols
|
||||
|
||||
#pragma GCC reset_options
|
||||
} // end namespace internal
|
||||
|
||||
} // end namespace Eigen
|
||||
|
||||
#endif // EIGEN_MATRIX_PRODUCT_MMA_ALTIVEC_H
|
||||
|
||||
|
||||
@ -112,6 +112,11 @@ EIGEN_DECLARE_TEST(product_large)
|
||||
CALL_SUBTEST_1( test_aliasing<float>() );
|
||||
|
||||
CALL_SUBTEST_6( bug_1622<1>() );
|
||||
|
||||
CALL_SUBTEST_7( product(MatrixXcd(internal::random<int>(1,EIGEN_TEST_MAX_SIZE/2), internal::random<int>(1,EIGEN_TEST_MAX_SIZE/2))) );
|
||||
CALL_SUBTEST_8( product(Matrix<double,Dynamic,Dynamic,RowMajor>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) );
|
||||
CALL_SUBTEST_9( product(Matrix<std::complex<float>,Dynamic,Dynamic,RowMajor>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) );
|
||||
CALL_SUBTEST_10( product(Matrix<std::complex<double>,Dynamic,Dynamic,RowMajor>(internal::random<int>(1,EIGEN_TEST_MAX_SIZE), internal::random<int>(1,EIGEN_TEST_MAX_SIZE))) );
|
||||
}
|
||||
|
||||
CALL_SUBTEST_6( product_large_regressions<0>() );
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user