mirror of
https://gitlab.com/libeigen/eigen.git
synced 2025-06-04 18:54:00 +08:00
312c8e77ff
Recent changes have introduced the following build error when compiling with HIPCC --------- unsupported/test/../../Eigen/src/Core/GenericPacketMath.h:254:58: error: 'ldexp': no overloaded function has restriction specifiers that are compatible with the ambient context 'pldexp' --------- The fix for the error is to pick the math function(s) from the global namespace (where they are declared as device functions in the HIP header files) when compiling with HIPCC.
754 lines
28 KiB
C++
754 lines
28 KiB
C++
// This file is part of Eigen, a lightweight C++ template library
|
|
// for linear algebra.
|
|
//
|
|
// Copyright (C) 2008 Gael Guennebaud <gael.guennebaud@inria.fr>
|
|
// Copyright (C) 2006-2008 Benoit Jacob <jacob.benoit.1@gmail.com>
|
|
//
|
|
// This Source Code Form is subject to the terms of the Mozilla
|
|
// Public License v. 2.0. If a copy of the MPL was not distributed
|
|
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
#ifndef EIGEN_GENERIC_PACKET_MATH_H
|
|
#define EIGEN_GENERIC_PACKET_MATH_H
|
|
|
|
namespace Eigen {
|
|
|
|
namespace internal {
|
|
|
|
/** \internal
|
|
* \file GenericPacketMath.h
|
|
*
|
|
* Default implementation for types not supported by the vectorization.
|
|
* In practice these functions are provided to make easier the writing
|
|
* of generic vectorized code.
|
|
*/
|
|
|
|
#ifndef EIGEN_DEBUG_ALIGNED_LOAD
|
|
#define EIGEN_DEBUG_ALIGNED_LOAD
|
|
#endif
|
|
|
|
#ifndef EIGEN_DEBUG_UNALIGNED_LOAD
|
|
#define EIGEN_DEBUG_UNALIGNED_LOAD
|
|
#endif
|
|
|
|
#ifndef EIGEN_DEBUG_ALIGNED_STORE
|
|
#define EIGEN_DEBUG_ALIGNED_STORE
|
|
#endif
|
|
|
|
#ifndef EIGEN_DEBUG_UNALIGNED_STORE
|
|
#define EIGEN_DEBUG_UNALIGNED_STORE
|
|
#endif
|
|
|
|
struct default_packet_traits
|
|
{
|
|
enum {
|
|
HasHalfPacket = 0,
|
|
|
|
HasAdd = 1,
|
|
HasSub = 1,
|
|
HasMul = 1,
|
|
HasNegate = 1,
|
|
HasAbs = 1,
|
|
HasArg = 0,
|
|
HasAbs2 = 1,
|
|
HasMin = 1,
|
|
HasMax = 1,
|
|
HasConj = 1,
|
|
HasSetLinear = 1,
|
|
HasBlend = 0,
|
|
HasReduxp = 1,
|
|
|
|
HasDiv = 0,
|
|
HasSqrt = 0,
|
|
HasRsqrt = 0,
|
|
HasExp = 0,
|
|
HasExpm1 = 0,
|
|
HasLog = 0,
|
|
HasLog1p = 0,
|
|
HasLog10 = 0,
|
|
HasPow = 0,
|
|
|
|
HasSin = 0,
|
|
HasCos = 0,
|
|
HasTan = 0,
|
|
HasASin = 0,
|
|
HasACos = 0,
|
|
HasATan = 0,
|
|
HasSinh = 0,
|
|
HasCosh = 0,
|
|
HasTanh = 0,
|
|
HasLGamma = 0,
|
|
HasDiGamma = 0,
|
|
HasZeta = 0,
|
|
HasPolygamma = 0,
|
|
HasErf = 0,
|
|
HasErfc = 0,
|
|
HasNdtri = 0,
|
|
HasBessel = 0,
|
|
HasIGamma = 0,
|
|
HasIGammaDerA = 0,
|
|
HasGammaSampleDerAlpha = 0,
|
|
HasIGammac = 0,
|
|
HasBetaInc = 0,
|
|
|
|
HasRound = 0,
|
|
HasFloor = 0,
|
|
HasCeil = 0,
|
|
|
|
HasSign = 0
|
|
};
|
|
};
|
|
|
|
template<typename T> struct packet_traits : default_packet_traits
|
|
{
|
|
typedef T type;
|
|
typedef T half;
|
|
enum {
|
|
Vectorizable = 0,
|
|
size = 1,
|
|
AlignedOnScalar = 0,
|
|
HasHalfPacket = 0
|
|
};
|
|
enum {
|
|
HasAdd = 0,
|
|
HasSub = 0,
|
|
HasMul = 0,
|
|
HasNegate = 0,
|
|
HasAbs = 0,
|
|
HasAbs2 = 0,
|
|
HasMin = 0,
|
|
HasMax = 0,
|
|
HasConj = 0,
|
|
HasSetLinear = 0
|
|
};
|
|
};
|
|
|
|
template<typename T> struct packet_traits<const T> : packet_traits<T> { };
|
|
|
|
template <typename Src, typename Tgt> struct type_casting_traits {
|
|
enum {
|
|
VectorizedCast = 0,
|
|
SrcCoeffRatio = 1,
|
|
TgtCoeffRatio = 1
|
|
};
|
|
};
|
|
|
|
|
|
/** \internal \returns static_cast<TgtType>(a) (coeff-wise) */
|
|
template <typename SrcPacket, typename TgtPacket>
|
|
EIGEN_DEVICE_FUNC inline TgtPacket
|
|
pcast(const SrcPacket& a) {
|
|
return static_cast<TgtPacket>(a);
|
|
}
|
|
template <typename SrcPacket, typename TgtPacket>
|
|
EIGEN_DEVICE_FUNC inline TgtPacket
|
|
pcast(const SrcPacket& a, const SrcPacket& /*b*/) {
|
|
return static_cast<TgtPacket>(a);
|
|
}
|
|
|
|
template <typename SrcPacket, typename TgtPacket>
|
|
EIGEN_DEVICE_FUNC inline TgtPacket
|
|
pcast(const SrcPacket& a, const SrcPacket& /*b*/, const SrcPacket& /*c*/, const SrcPacket& /*d*/) {
|
|
return static_cast<TgtPacket>(a);
|
|
}
|
|
|
|
/** \internal \returns reinterpret_cast<Target>(a) */
|
|
template <typename Target, typename Packet>
|
|
EIGEN_DEVICE_FUNC inline Target
|
|
preinterpret(const Packet& a); /* { return reinterpret_cast<const Target&>(a); } */
|
|
|
|
/** \internal \returns a + b (coeff-wise) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
padd(const Packet& a, const Packet& b) { return a+b; }
|
|
|
|
/** \internal \returns a - b (coeff-wise) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
psub(const Packet& a, const Packet& b) { return a-b; }
|
|
|
|
/** \internal \returns -a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pnegate(const Packet& a) { return -a; }
|
|
|
|
/** \internal \returns conj(a) (coeff-wise) */
|
|
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pconj(const Packet& a) { return numext::conj(a); }
|
|
|
|
/** \internal \returns a * b (coeff-wise) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pmul(const Packet& a, const Packet& b) { return a*b; }
|
|
|
|
/** \internal \returns a / b (coeff-wise) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pdiv(const Packet& a, const Packet& b) { return a/b; }
|
|
|
|
/** \internal \returns the min of \a a and \a b (coeff-wise) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pmin(const Packet& a, const Packet& b) { return numext::mini(a, b); }
|
|
|
|
/** \internal \returns the max of \a a and \a b (coeff-wise) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pmax(const Packet& a, const Packet& b) { return numext::maxi(a, b); }
|
|
|
|
/** \internal \returns the absolute value of \a a */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pabs(const Packet& a) { using std::abs; return abs(a); }
|
|
|
|
/** \internal \returns the phase angle of \a a */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
parg(const Packet& a) { using numext::arg; return arg(a); }
|
|
|
|
/** \internal \returns the bitwise and of \a a and \a b */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pand(const Packet& a, const Packet& b) { return a & b; }
|
|
|
|
/** \internal \returns the bitwise or of \a a and \a b */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
por(const Packet& a, const Packet& b) { return a | b; }
|
|
|
|
/** \internal \returns the bitwise xor of \a a and \a b */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pxor(const Packet& a, const Packet& b) { return a ^ b; }
|
|
|
|
/** \internal \returns the bitwise andnot of \a a and \a b */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pandnot(const Packet& a, const Packet& b) { return a & (~b); }
|
|
|
|
/** \internal \returns ones */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
ptrue(const Packet& /*a*/) { Packet b; memset((void*)&b, 0xff, sizeof(b)); return b;}
|
|
|
|
template <typename RealScalar>
|
|
EIGEN_DEVICE_FUNC inline std::complex<RealScalar> ptrue(const std::complex<RealScalar>& /*a*/) {
|
|
RealScalar b;
|
|
b = ptrue(b);
|
|
return std::complex<RealScalar>(b, b);
|
|
}
|
|
|
|
/** \internal \returns the bitwise not of \a a */
|
|
template <typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pnot(const Packet& a) { return pxor(ptrue(a), a);}
|
|
|
|
/** \internal \returns \a a shifted by N bits to the right */
|
|
template<int N> EIGEN_DEVICE_FUNC inline int
|
|
pshiftright(const int& a) { return a >> N; }
|
|
template<int N> EIGEN_DEVICE_FUNC inline long int
|
|
pshiftright(const long int& a) { return a >> N; }
|
|
|
|
/** \internal \returns \a a shifted by N bits to the left */
|
|
template<int N> EIGEN_DEVICE_FUNC inline int
|
|
pshiftleft(const int& a) { return a << N; }
|
|
template<int N> EIGEN_DEVICE_FUNC inline long int
|
|
pshiftleft(const long int& a) { return a << N; }
|
|
|
|
/** \internal \returns the significant and exponent of the underlying floating point numbers
|
|
* See https://en.cppreference.com/w/cpp/numeric/math/frexp
|
|
*/
|
|
template <typename Packet>
|
|
EIGEN_DEVICE_FUNC inline Packet pfrexp(const Packet& a, Packet& exponent) {
|
|
int exp;
|
|
EIGEN_USING_STD_MATH(frexp);
|
|
Packet result = frexp(a, &exp);
|
|
exponent = static_cast<Packet>(exp);
|
|
return result;
|
|
}
|
|
|
|
/** \internal \returns a * 2^exponent
|
|
* See https://en.cppreference.com/w/cpp/numeric/math/ldexp
|
|
*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pldexp(const Packet &a, const Packet &exponent) {
|
|
EIGEN_USING_STD_MATH(ldexp);
|
|
return ldexp(a, static_cast<int>(exponent));
|
|
}
|
|
|
|
/** \internal \returns zeros */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pzero(const Packet& a) { return pxor(a,a); }
|
|
|
|
template<> EIGEN_DEVICE_FUNC inline float pzero<float>(const float& a) {
|
|
EIGEN_UNUSED_VARIABLE(a);
|
|
return 0.f;
|
|
}
|
|
|
|
template<> EIGEN_DEVICE_FUNC inline double pzero<double>(const double& a) {
|
|
EIGEN_UNUSED_VARIABLE(a);
|
|
return 0.;
|
|
}
|
|
|
|
/** \internal \returns bits of \a or \b according to the input bit mask \a mask */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pselect(const Packet& mask, const Packet& a, const Packet& b) {
|
|
return por(pand(a,mask),pandnot(b,mask));
|
|
}
|
|
|
|
template<> EIGEN_DEVICE_FUNC inline float pselect<float>(
|
|
const float& mask, const float& a, const float&b) {
|
|
return numext::equal_strict(mask,0.f) ? b : a;
|
|
}
|
|
|
|
template<> EIGEN_DEVICE_FUNC inline double pselect<double>(
|
|
const double& mask, const double& a, const double& b) {
|
|
return numext::equal_strict(mask,0.) ? b : a;
|
|
}
|
|
|
|
/** \internal \returns a <= b as a bit mask */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pcmp_le(const Packet& a, const Packet& b) { return a<=b ? ptrue(a) : pzero(a); }
|
|
|
|
/** \internal \returns a < b as a bit mask */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pcmp_lt(const Packet& a, const Packet& b) { return a<b ? ptrue(a) : pzero(a); }
|
|
|
|
/** \internal \returns a == b as a bit mask */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pcmp_eq(const Packet& a, const Packet& b) { return a==b ? ptrue(a) : pzero(a); }
|
|
|
|
/** \internal \returns a < b or a==NaN or b==NaN as a bit mask */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pcmp_lt_or_nan(const Packet& a, const Packet& b) { return pnot(pcmp_le(b,a)); }
|
|
|
|
/** \internal \returns a packet version of \a *from, from must be 16 bytes aligned */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pload(const typename unpacket_traits<Packet>::type* from) { return *from; }
|
|
|
|
/** \internal \returns a packet version of \a *from, (un-aligned load) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
ploadu(const typename unpacket_traits<Packet>::type* from) { return *from; }
|
|
|
|
/** \internal \returns a packet version of \a *from, (un-aligned masked load)
|
|
* There is no generic implementation. We only have implementations for specialized
|
|
* cases. Generic case should not be called.
|
|
*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline
|
|
typename enable_if<unpacket_traits<Packet>::masked_load_available, Packet>::type
|
|
ploadu(const typename unpacket_traits<Packet>::type* from, typename unpacket_traits<Packet>::mask_t umask);
|
|
|
|
/** \internal \returns a packet with constant coefficients \a a, e.g.: (a,a,a,a) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pset1(const typename unpacket_traits<Packet>::type& a) { return a; }
|
|
|
|
/** \internal \returns a packet with constant coefficients set from bits */
|
|
template<typename Packet,typename BitsType> EIGEN_DEVICE_FUNC inline Packet
|
|
pset1frombits(BitsType a);
|
|
|
|
/** \internal \returns a packet with constant coefficients \a a[0], e.g.: (a[0],a[0],a[0],a[0]) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pload1(const typename unpacket_traits<Packet>::type *a) { return pset1<Packet>(*a); }
|
|
|
|
/** \internal \returns a packet with elements of \a *from duplicated.
|
|
* For instance, for a packet of 8 elements, 4 scalars will be read from \a *from and
|
|
* duplicated to form: {from[0],from[0],from[1],from[1],from[2],from[2],from[3],from[3]}
|
|
* Currently, this function is only used for scalar * complex products.
|
|
*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet
|
|
ploaddup(const typename unpacket_traits<Packet>::type* from) { return *from; }
|
|
|
|
/** \internal \returns a packet with elements of \a *from quadrupled.
|
|
* For instance, for a packet of 8 elements, 2 scalars will be read from \a *from and
|
|
* replicated to form: {from[0],from[0],from[0],from[0],from[1],from[1],from[1],from[1]}
|
|
* Currently, this function is only used in matrix products.
|
|
* For packet-size smaller or equal to 4, this function is equivalent to pload1
|
|
*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
ploadquad(const typename unpacket_traits<Packet>::type* from)
|
|
{ return pload1<Packet>(from); }
|
|
|
|
/** \internal equivalent to
|
|
* \code
|
|
* a0 = pload1(a+0);
|
|
* a1 = pload1(a+1);
|
|
* a2 = pload1(a+2);
|
|
* a3 = pload1(a+3);
|
|
* \endcode
|
|
* \sa pset1, pload1, ploaddup, pbroadcast2
|
|
*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC
|
|
inline void pbroadcast4(const typename unpacket_traits<Packet>::type *a,
|
|
Packet& a0, Packet& a1, Packet& a2, Packet& a3)
|
|
{
|
|
a0 = pload1<Packet>(a+0);
|
|
a1 = pload1<Packet>(a+1);
|
|
a2 = pload1<Packet>(a+2);
|
|
a3 = pload1<Packet>(a+3);
|
|
}
|
|
|
|
/** \internal equivalent to
|
|
* \code
|
|
* a0 = pload1(a+0);
|
|
* a1 = pload1(a+1);
|
|
* \endcode
|
|
* \sa pset1, pload1, ploaddup, pbroadcast4
|
|
*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC
|
|
inline void pbroadcast2(const typename unpacket_traits<Packet>::type *a,
|
|
Packet& a0, Packet& a1)
|
|
{
|
|
a0 = pload1<Packet>(a+0);
|
|
a1 = pload1<Packet>(a+1);
|
|
}
|
|
|
|
/** \internal \brief Returns a packet with coefficients (a,a+1,...,a+packet_size-1). */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet
|
|
plset(const typename unpacket_traits<Packet>::type& a) { return a; }
|
|
|
|
/** \internal copy the packet \a from to \a *to, \a to must be 16 bytes aligned */
|
|
template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pstore(Scalar* to, const Packet& from)
|
|
{ (*to) = from; }
|
|
|
|
/** \internal copy the packet \a from to \a *to, (un-aligned store) */
|
|
template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pstoreu(Scalar* to, const Packet& from)
|
|
{ (*to) = from; }
|
|
|
|
/** \internal copy the packet \a from to \a *to, (un-aligned store with a mask)
|
|
* There is no generic implementation. We only have implementations for specialized
|
|
* cases. Generic case should not be called.
|
|
*/
|
|
template<typename Scalar, typename Packet>
|
|
EIGEN_DEVICE_FUNC inline
|
|
typename enable_if<unpacket_traits<Packet>::masked_store_available, void>::type
|
|
pstoreu(Scalar* to, const Packet& from, typename unpacket_traits<Packet>::mask_t umask);
|
|
|
|
template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline Packet pgather(const Scalar* from, Index /*stride*/)
|
|
{ return ploadu<Packet>(from); }
|
|
|
|
template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from, Index /*stride*/)
|
|
{ pstore(to, from); }
|
|
|
|
/** \internal tries to do cache prefetching of \a addr */
|
|
template<typename Scalar> EIGEN_DEVICE_FUNC inline void prefetch(const Scalar* addr)
|
|
{
|
|
#if defined(EIGEN_HIP_DEVICE_COMPILE)
|
|
// do nothing
|
|
#elif defined(EIGEN_CUDA_ARCH)
|
|
#if defined(__LP64__)
|
|
// 64-bit pointer operand constraint for inlined asm
|
|
asm(" prefetch.L1 [ %1 ];" : "=l"(addr) : "l"(addr));
|
|
#else
|
|
// 32-bit pointer operand constraint for inlined asm
|
|
asm(" prefetch.L1 [ %1 ];" : "=r"(addr) : "r"(addr));
|
|
#endif
|
|
#elif (!EIGEN_COMP_MSVC) && (EIGEN_COMP_GNUC || EIGEN_COMP_CLANG || EIGEN_COMP_ICC)
|
|
__builtin_prefetch(addr);
|
|
#endif
|
|
}
|
|
|
|
/** \internal \returns the first element of a packet */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type pfirst(const Packet& a)
|
|
{ return a; }
|
|
|
|
/** \internal \returns a packet where the element i contains the sum of the packet of \a vec[i] */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
preduxp(const Packet* vecs) { return vecs[0]; }
|
|
|
|
/** \internal \returns the sum of the elements of \a a*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux(const Packet& a)
|
|
{ return a; }
|
|
|
|
/** \internal \returns the sum of the elements of upper and lower half of \a a if \a a is larger than 4.
|
|
* For a packet {a0, a1, a2, a3, a4, a5, a6, a7}, it returns a half packet {a0+a4, a1+a5, a2+a6, a3+a7}
|
|
* For packet-size smaller or equal to 4, this boils down to a noop.
|
|
*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline
|
|
typename conditional<(unpacket_traits<Packet>::size%8)==0,typename unpacket_traits<Packet>::half,Packet>::type
|
|
predux_half_dowto4(const Packet& a)
|
|
{ return a; }
|
|
|
|
/** \internal \returns the product of the elements of \a a */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_mul(const Packet& a)
|
|
{ return a; }
|
|
|
|
/** \internal \returns the min of the elements of \a a */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_min(const Packet& a)
|
|
{ return a; }
|
|
|
|
/** \internal \returns the max of the elements of \a a */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline typename unpacket_traits<Packet>::type predux_max(const Packet& a)
|
|
{ return a; }
|
|
|
|
/** \internal \returns true if all coeffs of \a a means "true"
|
|
* It is supposed to be called on values returned by pcmp_*.
|
|
*/
|
|
// not needed yet
|
|
// template<typename Packet> EIGEN_DEVICE_FUNC inline bool predux_all(const Packet& a)
|
|
// { return bool(a); }
|
|
|
|
/** \internal \returns true if any coeffs of \a a means "true"
|
|
* It is supposed to be called on values returned by pcmp_*.
|
|
*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline bool predux_any(const Packet& a)
|
|
{
|
|
// Dirty but generic implementation where "true" is assumed to be non 0 and all the sames.
|
|
// It is expected that "true" is either:
|
|
// - Scalar(1)
|
|
// - bits full of ones (NaN for floats),
|
|
// - or first bit equals to 1 (1 for ints, smallest denormal for floats).
|
|
// For all these cases, taking the sum is just fine, and this boils down to a no-op for scalars.
|
|
return bool(predux(a));
|
|
}
|
|
|
|
/** \internal \returns the reversed elements of \a a*/
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet preverse(const Packet& a)
|
|
{ return a; }
|
|
|
|
/** \internal \returns \a a with real and imaginary part flipped (for complex type only) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet pcplxflip(const Packet& a)
|
|
{
|
|
return Packet(numext::imag(a),numext::real(a));
|
|
}
|
|
|
|
/**************************
|
|
* Special math functions
|
|
***************************/
|
|
|
|
/** \internal \returns the sine of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet psin(const Packet& a) { EIGEN_USING_STD_MATH(sin); return sin(a); }
|
|
|
|
/** \internal \returns the cosine of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet pcos(const Packet& a) { EIGEN_USING_STD_MATH(cos); return cos(a); }
|
|
|
|
/** \internal \returns the tan of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet ptan(const Packet& a) { EIGEN_USING_STD_MATH(tan); return tan(a); }
|
|
|
|
/** \internal \returns the arc sine of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet pasin(const Packet& a) { EIGEN_USING_STD_MATH(asin); return asin(a); }
|
|
|
|
/** \internal \returns the arc cosine of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet pacos(const Packet& a) { EIGEN_USING_STD_MATH(acos); return acos(a); }
|
|
|
|
/** \internal \returns the arc tangent of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet patan(const Packet& a) { EIGEN_USING_STD_MATH(atan); return atan(a); }
|
|
|
|
/** \internal \returns the hyperbolic sine of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet psinh(const Packet& a) { EIGEN_USING_STD_MATH(sinh); return sinh(a); }
|
|
|
|
/** \internal \returns the hyperbolic cosine of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet pcosh(const Packet& a) { EIGEN_USING_STD_MATH(cosh); return cosh(a); }
|
|
|
|
/** \internal \returns the hyperbolic tan of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet ptanh(const Packet& a) { EIGEN_USING_STD_MATH(tanh); return tanh(a); }
|
|
|
|
/** \internal \returns the exp of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet pexp(const Packet& a) { EIGEN_USING_STD_MATH(exp); return exp(a); }
|
|
|
|
/** \internal \returns the expm1 of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet pexpm1(const Packet& a) { return numext::expm1(a); }
|
|
|
|
/** \internal \returns the log of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet plog(const Packet& a) { EIGEN_USING_STD_MATH(log); return log(a); }
|
|
|
|
/** \internal \returns the log1p of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet plog1p(const Packet& a) { return numext::log1p(a); }
|
|
|
|
/** \internal \returns the log10 of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet plog10(const Packet& a) { EIGEN_USING_STD_MATH(log10); return log10(a); }
|
|
|
|
/** \internal \returns the square-root of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet psqrt(const Packet& a) { EIGEN_USING_STD_MATH(sqrt); return sqrt(a); }
|
|
|
|
/** \internal \returns the reciprocal square-root of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet prsqrt(const Packet& a) {
|
|
return pdiv(pset1<Packet>(1), psqrt(a));
|
|
}
|
|
|
|
/** \internal \returns the rounded value of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet pround(const Packet& a) { using numext::round; return round(a); }
|
|
|
|
/** \internal \returns the floor of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet pfloor(const Packet& a) { using numext::floor; return floor(a); }
|
|
|
|
/** \internal \returns the ceil of \a a (coeff-wise) */
|
|
template<typename Packet> EIGEN_DECLARE_FUNCTION_ALLOWING_MULTIPLE_DEFINITIONS
|
|
Packet pceil(const Packet& a) { using numext::ceil; return ceil(a); }
|
|
|
|
/***************************************************************************
|
|
* The following functions might not have to be overwritten for vectorized types
|
|
***************************************************************************/
|
|
|
|
/** \internal copy a packet with constant coefficient \a a (e.g., [a,a,a,a]) to \a *to. \a to must be 16 bytes aligned */
|
|
// NOTE: this function must really be templated on the packet type (think about different packet types for the same scalar type)
|
|
template<typename Packet>
|
|
inline void pstore1(typename unpacket_traits<Packet>::type* to, const typename unpacket_traits<Packet>::type& a)
|
|
{
|
|
pstore(to, pset1<Packet>(a));
|
|
}
|
|
|
|
/** \internal \returns a * b + c (coeff-wise) */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pmadd(const Packet& a,
|
|
const Packet& b,
|
|
const Packet& c)
|
|
{ return padd(pmul(a, b),c); }
|
|
|
|
/** \internal \returns a packet version of \a *from.
|
|
* The pointer \a from must be aligned on a \a Alignment bytes boundary. */
|
|
template<typename Packet, int Alignment>
|
|
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt(const typename unpacket_traits<Packet>::type* from)
|
|
{
|
|
if(Alignment >= unpacket_traits<Packet>::alignment)
|
|
return pload<Packet>(from);
|
|
else
|
|
return ploadu<Packet>(from);
|
|
}
|
|
|
|
/** \internal copy the packet \a from to \a *to.
|
|
* The pointer \a from must be aligned on a \a Alignment bytes boundary. */
|
|
template<typename Scalar, typename Packet, int Alignment>
|
|
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE void pstoret(Scalar* to, const Packet& from)
|
|
{
|
|
if(Alignment >= unpacket_traits<Packet>::alignment)
|
|
pstore(to, from);
|
|
else
|
|
pstoreu(to, from);
|
|
}
|
|
|
|
/** \internal \returns a packet version of \a *from.
|
|
* Unlike ploadt, ploadt_ro takes advantage of the read-only memory path on the
|
|
* hardware if available to speedup the loading of data that won't be modified
|
|
* by the current computation.
|
|
*/
|
|
template<typename Packet, int LoadMode>
|
|
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Packet ploadt_ro(const typename unpacket_traits<Packet>::type* from)
|
|
{
|
|
return ploadt<Packet, LoadMode>(from);
|
|
}
|
|
|
|
/** \internal default implementation of palign() allowing partial specialization */
|
|
template<int Offset,typename PacketType>
|
|
struct palign_impl
|
|
{
|
|
// by default data are aligned, so there is nothing to be done :)
|
|
static inline void run(PacketType&, const PacketType&) {}
|
|
};
|
|
|
|
/** \internal update \a first using the concatenation of the packet_size minus \a Offset last elements
|
|
* of \a first and \a Offset first elements of \a second.
|
|
*
|
|
* This function is currently only used to optimize matrix-vector products on unligned matrices.
|
|
* It takes 2 packets that represent a contiguous memory array, and returns a packet starting
|
|
* at the position \a Offset. For instance, for packets of 4 elements, we have:
|
|
* Input:
|
|
* - first = {f0,f1,f2,f3}
|
|
* - second = {s0,s1,s2,s3}
|
|
* Output:
|
|
* - if Offset==0 then {f0,f1,f2,f3}
|
|
* - if Offset==1 then {f1,f2,f3,s0}
|
|
* - if Offset==2 then {f2,f3,s0,s1}
|
|
* - if Offset==3 then {f3,s0,s1,s3}
|
|
*/
|
|
template<int Offset,typename PacketType>
|
|
inline void palign(PacketType& first, const PacketType& second)
|
|
{
|
|
palign_impl<Offset,PacketType>::run(first,second);
|
|
}
|
|
|
|
/***************************************************************************
|
|
* Fast complex products (GCC generates a function call which is very slow)
|
|
***************************************************************************/
|
|
|
|
// Eigen+CUDA does not support complexes.
|
|
#if !defined(EIGEN_GPUCC)
|
|
|
|
template<> inline std::complex<float> pmul(const std::complex<float>& a, const std::complex<float>& b)
|
|
{ return std::complex<float>(a.real()*b.real() - a.imag()*b.imag(), a.imag()*b.real() + a.real()*b.imag()); }
|
|
|
|
template<> inline std::complex<double> pmul(const std::complex<double>& a, const std::complex<double>& b)
|
|
{ return std::complex<double>(a.real()*b.real() - a.imag()*b.imag(), a.imag()*b.real() + a.real()*b.imag()); }
|
|
|
|
#endif
|
|
|
|
|
|
/***************************************************************************
|
|
* PacketBlock, that is a collection of N packets where the number of words
|
|
* in the packet is a multiple of N.
|
|
***************************************************************************/
|
|
template <typename Packet,int N=unpacket_traits<Packet>::size> struct PacketBlock {
|
|
Packet packet[N];
|
|
};
|
|
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline void
|
|
ptranspose(PacketBlock<Packet,1>& /*kernel*/) {
|
|
// Nothing to do in the scalar case, i.e. a 1x1 matrix.
|
|
}
|
|
|
|
/***************************************************************************
|
|
* Selector, i.e. vector of N boolean values used to select (i.e. blend)
|
|
* words from 2 packets.
|
|
***************************************************************************/
|
|
template <size_t N> struct Selector {
|
|
bool select[N];
|
|
};
|
|
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pblend(const Selector<unpacket_traits<Packet>::size>& ifPacket, const Packet& thenPacket, const Packet& elsePacket) {
|
|
return ifPacket.select[0] ? thenPacket : elsePacket;
|
|
}
|
|
|
|
/** \internal \returns \a a with the first coefficient replaced by the scalar b */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pinsertfirst(const Packet& a, typename unpacket_traits<Packet>::type b)
|
|
{
|
|
// Default implementation based on pblend.
|
|
// It must be specialized for higher performance.
|
|
Selector<unpacket_traits<Packet>::size> mask;
|
|
mask.select[0] = true;
|
|
// This for loop should be optimized away by the compiler.
|
|
for(Index i=1; i<unpacket_traits<Packet>::size; ++i)
|
|
mask.select[i] = false;
|
|
return pblend(mask, pset1<Packet>(b), a);
|
|
}
|
|
|
|
/** \internal \returns \a a with the last coefficient replaced by the scalar b */
|
|
template<typename Packet> EIGEN_DEVICE_FUNC inline Packet
|
|
pinsertlast(const Packet& a, typename unpacket_traits<Packet>::type b)
|
|
{
|
|
// Default implementation based on pblend.
|
|
// It must be specialized for higher performance.
|
|
Selector<unpacket_traits<Packet>::size> mask;
|
|
// This for loop should be optimized away by the compiler.
|
|
for(Index i=0; i<unpacket_traits<Packet>::size-1; ++i)
|
|
mask.select[i] = false;
|
|
mask.select[unpacket_traits<Packet>::size-1] = true;
|
|
return pblend(mask, pset1<Packet>(b), a);
|
|
}
|
|
|
|
/***************************************************************************
|
|
* Some generic implementations to be used by implementors
|
|
***************************************************************************/
|
|
|
|
/** Default implementation of pfrexp for float.
|
|
* It is expected to be called by implementers of template<> pfrexp.
|
|
*/
|
|
template<typename Packet> EIGEN_STRONG_INLINE Packet
|
|
pfrexp_float(const Packet& a, Packet& exponent);
|
|
|
|
/** Default implementation of pldexp for float.
|
|
* It is expected to be called by implementers of template<> pldexp.
|
|
*/
|
|
template<typename Packet> EIGEN_STRONG_INLINE Packet
|
|
pldexp_float(Packet a, Packet exponent);
|
|
|
|
} // end namespace internal
|
|
|
|
} // end namespace Eigen
|
|
|
|
#endif // EIGEN_GENERIC_PACKET_MATH_H
|