GitHub-Proxy/eigen
1
0
Fork 0
mirror of https://gitlab.com/libeigen/eigen.git synced 2025-05-01 00:04:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

merging.

Browse Source
This commit is contained in:
Rasmus Munk Larsen 2016-02-01 11:10:30 -08:00
commit 00f9ef6c76
42 changed files with 988 additions and 313 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Eigen/src/CholmodSupport/CholmodSupport.h
View File

@ -274,8 +274,9 @@ class CholmodBase : public SparseSolverBase<Derived>
EIGEN_UNUSED_VARIABLE(size);
eigen_assert(size==b.rows());
// note: cd stands for Cholmod Dense
Rhs& b_ref(b.const_cast_derived());
// Cholmod needs column-major stoarge without inner-stride, which corresponds to the default behavior of Ref.
Ref<const Matrix<typename Rhs::Scalar,Dynamic,Dynamic,ColMajor> > b_ref(b.derived());
cholmod_dense b_cd = viewAsCholmod(b_ref);
cholmod_dense* x_cd = cholmod_solve(CHOLMOD_A, m_cholmodFactor, &b_cd, &m_cholmod);
if(!x_cd)

16
Eigen/src/Core/ArrayBase.h
View File

@ -103,7 +103,7 @@ template<typename Derived> class ArrayBase
/** Special case of the template operator=, in order to prevent the compiler
* from generating a default operator= (issue hit with g++ 4.1)
*/
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator=(const ArrayBase& other)
{
internal::call_assignment(derived(), other.derived());
@ -112,28 +112,28 @@ template<typename Derived> class ArrayBase
/** Set all the entries to \a value.
* \sa DenseBase::setConstant(), DenseBase::fill() */
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator=(const Scalar &value)
{ Base::setConstant(value); return derived(); }
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator+=(const Scalar& scalar);
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator-=(const Scalar& scalar);
template<typename OtherDerived>
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator+=(const ArrayBase<OtherDerived>& other);
template<typename OtherDerived>
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator-=(const ArrayBase<OtherDerived>& other);
template<typename OtherDerived>
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator*=(const ArrayBase<OtherDerived>& other);
template<typename OtherDerived>
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator/=(const ArrayBase<OtherDerived>& other);
public:

34
Eigen/src/Core/AssignEvaluator.h
View File

@ -637,7 +637,7 @@ protected:
***************************************************************************/
template<typename DstXprType, typename SrcXprType, typename Functor>
EIGEN_DEVICE_FUNC void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src, const Functor &func)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src, const Functor &func)
{
eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
@ -654,7 +654,7 @@ EIGEN_DEVICE_FUNC void call_dense_assignment_loop(const DstXprType& dst, const S
}
template<typename DstXprType, typename SrcXprType>
EIGEN_DEVICE_FUNC void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(const DstXprType& dst, const SrcXprType& src)
{
call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar>());
}
@ -688,26 +688,30 @@ struct Assignment;
// does not has to bother about these annoying details.
template<typename Dst, typename Src>
EIGEN_DEVICE_FUNC void call_assignment(Dst& dst, const Src& src)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment(Dst& dst, const Src& src)
{
call_assignment(dst, src, internal::assign_op<typename Dst::Scalar>());
}
template<typename Dst, typename Src>
EIGEN_DEVICE_FUNC void call_assignment(const Dst& dst, const Src& src)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment(const Dst& dst, const Src& src)
{
call_assignment(dst, src, internal::assign_op<typename Dst::Scalar>());
}
// Deal with "assume-aliasing"
template<typename Dst, typename Src, typename Func>
EIGEN_DEVICE_FUNC void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if< evaluator_assume_aliasing<Src>::value, void*>::type = 0)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if< evaluator_assume_aliasing<Src>::value, void*>::type = 0)
{
typename plain_matrix_type<Src>::type tmp(src);
call_assignment_no_alias(dst, tmp, func);
}
template<typename Dst, typename Src, typename Func>
EIGEN_DEVICE_FUNC void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if<!evaluator_assume_aliasing<Src>::value, void*>::type = 0)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if<!evaluator_assume_aliasing<Src>::value, void*>::type = 0)
{
call_assignment_no_alias(dst, src, func);
}
@ -715,14 +719,16 @@ EIGEN_DEVICE_FUNC void call_assignment(Dst& dst, const Src& src, const Func& fun
// by-pass "assume-aliasing"
// When there is no aliasing, we require that 'dst' has been properly resized
template<typename Dst, template <typename> class StorageBase, typename Src, typename Func>
EIGEN_DEVICE_FUNC void call_assignment(NoAlias<Dst,StorageBase>& dst, const Src& src, const Func& func)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment(NoAlias<Dst,StorageBase>& dst, const Src& src, const Func& func)
{
call_assignment_no_alias(dst.expression(), src, func);
}
template<typename Dst, typename Src, typename Func>
EIGEN_DEVICE_FUNC void call_assignment_no_alias(Dst& dst, const Src& src, const Func& func)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment_no_alias(Dst& dst, const Src& src, const Func& func)
{
enum {
NeedToTranspose = ( (int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1)
@ -747,13 +753,15 @@ EIGEN_DEVICE_FUNC void call_assignment_no_alias(Dst& dst, const Src& src, const
Assignment<ActualDstTypeCleaned,Src,Func>::run(actualDst, src, func);
}
template<typename Dst, typename Src>
EIGEN_DEVICE_FUNC void call_assignment_no_alias(Dst& dst, const Src& src)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment_no_alias(Dst& dst, const Src& src)
{
call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar>());
}
template<typename Dst, typename Src, typename Func>
EIGEN_DEVICE_FUNC void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src, const Func& func)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src, const Func& func)
{
Index dstRows = src.rows();
Index dstCols = src.cols();
@ -767,7 +775,8 @@ EIGEN_DEVICE_FUNC void call_assignment_no_alias_no_transpose(Dst& dst, const Src
Assignment<Dst,Src,Func>::run(dst, src, func);
}
template<typename Dst, typename Src>
EIGEN_DEVICE_FUNC void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src)
{
call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar>());
}
@ -779,7 +788,8 @@ template<typename Dst, typename Src> void check_for_aliasing(const Dst &dst, con
template< typename DstXprType, typename SrcXprType, typename Functor, typename Scalar>
struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Scalar>
{
EIGEN_DEVICE_FUNC static void run(DstXprType &dst, const SrcXprType &src, const Functor &func)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
static void run(DstXprType &dst, const SrcXprType &src, const Functor &func)
{
eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());

4
Eigen/src/Core/Block.h
View File

@ -129,8 +129,8 @@ template<typename XprType, int BlockRows, int BlockCols, bool InnerPanel> class
: Impl(xpr, startRow, startCol)
{
EIGEN_STATIC_ASSERT(RowsAtCompileTime!=Dynamic && ColsAtCompileTime!=Dynamic,THIS_METHOD_IS_ONLY_FOR_FIXED_SIZE)
eigen_assert(startRow >= 0 && BlockRows >= 1 && startRow + BlockRows <= xpr.rows()
&& startCol >= 0 && BlockCols >= 1 && startCol + BlockCols <= xpr.cols());
eigen_assert(startRow >= 0 && BlockRows >= 0 && startRow + BlockRows <= xpr.rows()
&& startCol >= 0 && BlockCols >= 0 && startCol + BlockCols <= xpr.cols());
}
/** Dynamic-size constructor

169
Eigen/src/Core/CoreEvaluators.h
View File

@ -148,7 +148,8 @@ struct evaluator<PlainObjectBase<Derived> >
EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
if (IsRowMajor)
return m_data[row * m_outerStride.value() + col];
@ -156,12 +157,14 @@ struct evaluator<PlainObjectBase<Derived> >
return m_data[row + col * m_outerStride.value()];
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_data[index];
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index row, Index col)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index row, Index col)
{
if (IsRowMajor)
return const_cast<Scalar*>(m_data)[row * m_outerStride.value() + col];
@ -169,12 +172,14 @@ struct evaluator<PlainObjectBase<Derived> >
return const_cast<Scalar*>(m_data)[row + col * m_outerStride.value()];
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index index)
{
return const_cast<Scalar*>(m_data)[index];
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
if (IsRowMajor)
@ -184,12 +189,14 @@ struct evaluator<PlainObjectBase<Derived> >
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
return ploadt<PacketType, LoadMode>(m_data + index);
}
template<int StoreMode,typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index row, Index col, const PacketType& x)
{
if (IsRowMajor)
@ -201,6 +208,7 @@ struct evaluator<PlainObjectBase<Derived> >
}
template<int StoreMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index index, const PacketType& x)
{
return pstoret<Scalar, PacketType, StoreMode>(const_cast<Scalar*>(m_data) + index, x);
@ -260,45 +268,53 @@ struct unary_evaluator<Transpose<ArgType>, IndexBased>
typedef typename XprType::Scalar Scalar;
typedef typename XprType::CoeffReturnType CoeffReturnType;
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
return m_argImpl.coeff(col, row);
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_argImpl.coeff(index);
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index row, Index col)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index row, Index col)
{
return m_argImpl.coeffRef(col, row);
}
EIGEN_DEVICE_FUNC typename XprType::Scalar& coeffRef(Index index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename XprType::Scalar& coeffRef(Index index)
{
return m_argImpl.coeffRef(index);
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
return m_argImpl.template packet<LoadMode,PacketType>(col, row);
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
return m_argImpl.template packet<LoadMode,PacketType>(index);
}
template<int StoreMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index row, Index col, const PacketType& x)
{
m_argImpl.template writePacket<StoreMode,PacketType>(col, row, x);
}
template<int StoreMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index index, const PacketType& x)
{
m_argImpl.template writePacket<StoreMode,PacketType>(index, x);
@ -338,23 +354,27 @@ struct evaluator<CwiseNullaryOp<NullaryOp,PlainObjectType> >
typedef typename XprType::CoeffReturnType CoeffReturnType;
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
return m_functor(row, col);
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_functor(index);
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
return m_functor.template packetOp<Index,PacketType>(row, col);
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
return m_functor.template packetOp<Index,PacketType>(index);
@ -380,7 +400,8 @@ struct unary_evaluator<CwiseUnaryOp<UnaryOp, ArgType>, IndexBased >
Alignment = evaluator<ArgType>::Alignment
};
EIGEN_DEVICE_FUNC explicit unary_evaluator(const XprType& op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
explicit unary_evaluator(const XprType& op)
: m_functor(op.functor()),
m_argImpl(op.nestedExpression())
{
@ -390,23 +411,27 @@ struct unary_evaluator<CwiseUnaryOp<UnaryOp, ArgType>, IndexBased >
typedef typename XprType::CoeffReturnType CoeffReturnType;
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
return m_functor(m_argImpl.coeff(row, col));
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_functor(m_argImpl.coeff(index));
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
return m_functor.packetOp(m_argImpl.template packet<LoadMode, PacketType>(row, col));
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
return m_functor.packetOp(m_argImpl.template packet<LoadMode, PacketType>(index));
@ -466,17 +491,20 @@ struct binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>, IndexBased, IndexBase
typedef typename XprType::CoeffReturnType CoeffReturnType;
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
return m_functor(m_lhsImpl.coeff(row, col), m_rhsImpl.coeff(row, col));
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_functor(m_lhsImpl.coeff(index), m_rhsImpl.coeff(index));
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
return m_functor.packetOp(m_lhsImpl.template packet<LoadMode,PacketType>(row, col),
@ -484,6 +512,7 @@ struct binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>, IndexBased, IndexBase
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
return m_functor.packetOp(m_lhsImpl.template packet<LoadMode,PacketType>(index),
@ -523,22 +552,26 @@ struct unary_evaluator<CwiseUnaryView<UnaryOp, ArgType>, IndexBased>
typedef typename XprType::Scalar Scalar;
typedef typename XprType::CoeffReturnType CoeffReturnType;
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
return m_unaryOp(m_argImpl.coeff(row, col));
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_unaryOp(m_argImpl.coeff(index));
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index row, Index col)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index row, Index col)
{
return m_unaryOp(m_argImpl.coeffRef(row, col));
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index index)
{
return m_unaryOp(m_argImpl.coeffRef(index));
}
@ -578,27 +611,32 @@ struct mapbase_evaluator : evaluator_base<Derived>
EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
return m_data[col * m_xpr.colStride() + row * m_xpr.rowStride()];
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_data[index * m_xpr.innerStride()];
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index row, Index col)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index row, Index col)
{
return m_data[col * m_xpr.colStride() + row * m_xpr.rowStride()];
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index index)
{
return m_data[index * m_xpr.innerStride()];
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
PointerType ptr = m_data + row * m_xpr.rowStride() + col * m_xpr.colStride();
@ -606,12 +644,14 @@ struct mapbase_evaluator : evaluator_base<Derived>
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
return internal::ploadt<PacketType, LoadMode>(m_data + index * m_xpr.innerStride());
}
template<int StoreMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index row, Index col, const PacketType& x)
{
PointerType ptr = m_data + row * m_xpr.rowStride() + col * m_xpr.colStride();
@ -619,6 +659,7 @@ struct mapbase_evaluator : evaluator_base<Derived>
}
template<int StoreMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index index, const PacketType& x)
{
internal::pstoret<Scalar, PacketType, StoreMode>(m_data + index * m_xpr.innerStride(), x);
@ -767,33 +808,39 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
RowsAtCompileTime = XprType::RowsAtCompileTime
};
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
return m_argImpl.coeff(m_startRow.value() + row, m_startCol.value() + col);
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return coeff(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0);
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index row, Index col)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index row, Index col)
{
return m_argImpl.coeffRef(m_startRow.value() + row, m_startCol.value() + col);
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index index)
{
return coeffRef(RowsAtCompileTime == 1 ? 0 : index, RowsAtCompileTime == 1 ? index : 0);
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
return m_argImpl.template packet<LoadMode,PacketType>(m_startRow.value() + row, m_startCol.value() + col);
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
return packet<LoadMode,PacketType>(RowsAtCompileTime == 1 ? 0 : index,
@ -801,12 +848,14 @@ struct unary_evaluator<Block<ArgType, BlockRows, BlockCols, InnerPanel>, IndexBa
}
template<int StoreMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index row, Index col, const PacketType& x)
{
return m_argImpl.template writePacket<StoreMode,PacketType>(m_startRow.value() + row, m_startCol.value() + col, x);
}
template<int StoreMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index index, const PacketType& x)
{
return writePacket<StoreMode,PacketType>(RowsAtCompileTime == 1 ? 0 : index,
@ -859,7 +908,7 @@ struct evaluator<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> >
Alignment = EIGEN_PLAIN_ENUM_MIN(evaluator<ThenMatrixType>::Alignment, evaluator<ElseMatrixType>::Alignment)
};
inline EIGEN_DEVICE_FUNC explicit evaluator(const XprType& select)
EIGEN_DEVICE_FUNC explicit evaluator(const XprType& select)
: m_conditionImpl(select.conditionMatrix()),
m_thenImpl(select.thenMatrix()),
m_elseImpl(select.elseMatrix())
@ -869,7 +918,8 @@ struct evaluator<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> >
typedef typename XprType::CoeffReturnType CoeffReturnType;
inline EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
if (m_conditionImpl.coeff(row, col))
return m_thenImpl.coeff(row, col);
@ -877,7 +927,8 @@ struct evaluator<Select<ConditionMatrixType, ThenMatrixType, ElseMatrixType> >
return m_elseImpl.coeff(row, col);
}
inline EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
if (m_conditionImpl.coeff(index))
return m_thenImpl.coeff(index);
@ -921,7 +972,8 @@ struct unary_evaluator<Replicate<ArgType, RowFactor, ColFactor> >
m_cols(replicate.nestedExpression().cols())
{}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
// try to avoid using modulo; this is a pure optimization strategy
const Index actual_row = internal::traits<XprType>::RowsAtCompileTime==1 ? 0
@ -934,7 +986,8 @@ struct unary_evaluator<Replicate<ArgType, RowFactor, ColFactor> >
return m_argImpl.coeff(actual_row, actual_col);
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
// try to avoid using modulo; this is a pure optimization strategy
const Index actual_index = internal::traits<XprType>::RowsAtCompileTime==1
@ -945,6 +998,7 @@ struct unary_evaluator<Replicate<ArgType, RowFactor, ColFactor> >
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
const Index actual_row = internal::traits<XprType>::RowsAtCompileTime==1 ? 0
@ -958,6 +1012,7 @@ struct unary_evaluator<Replicate<ArgType, RowFactor, ColFactor> >
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
const Index actual_index = internal::traits<XprType>::RowsAtCompileTime==1
@ -1008,7 +1063,8 @@ struct evaluator<PartialReduxExpr<ArgType, MemberOp, Direction> >
typedef typename XprType::CoeffReturnType CoeffReturnType;
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(Index i, Index j) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Scalar coeff(Index i, Index j) const
{
if (Direction==Vertical)
return m_functor(m_arg.col(j));
@ -1016,7 +1072,8 @@ struct evaluator<PartialReduxExpr<ArgType, MemberOp, Direction> >
return m_functor(m_arg.row(i));
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Scalar coeff(Index index) const
{
if (Direction==Vertical)
return m_functor(m_arg.col(index));
@ -1051,45 +1108,53 @@ struct evaluator_wrapper_base
typedef typename ArgType::Scalar Scalar;
typedef typename ArgType::CoeffReturnType CoeffReturnType;
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
return m_argImpl.coeff(row, col);
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_argImpl.coeff(index);
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index row, Index col)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index row, Index col)
{
return m_argImpl.coeffRef(row, col);
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index index)
{
return m_argImpl.coeffRef(index);
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
return m_argImpl.template packet<LoadMode,PacketType>(row, col);
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
return m_argImpl.template packet<LoadMode,PacketType>(index);
}
template<int StoreMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index row, Index col, const PacketType& x)
{
m_argImpl.template writePacket<StoreMode>(row, col, x);
}
template<int StoreMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index index, const PacketType& x)
{
m_argImpl.template writePacket<StoreMode>(index, x);
@ -1164,29 +1229,34 @@ struct unary_evaluator<Reverse<ArgType, Direction> >
m_cols(ReverseCol ? reverse.nestedExpression().cols() : 1)
{ }
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index col) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index col) const
{
return m_argImpl.coeff(ReverseRow ? m_rows.value() - row - 1 : row,
ReverseCol ? m_cols.value() - col - 1 : col);
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_argImpl.coeff(m_rows.value() * m_cols.value() - index - 1);
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index row, Index col)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index row, Index col)
{
return m_argImpl.coeffRef(ReverseRow ? m_rows.value() - row - 1 : row,
ReverseCol ? m_cols.value() - col - 1 : col);
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index index)
{
return m_argImpl.coeffRef(m_rows.value() * m_cols.value() - index - 1);
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index row, Index col) const
{
enum {
@ -1201,6 +1271,7 @@ struct unary_evaluator<Reverse<ArgType, Direction> >
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
PacketType packet(Index index) const
{
enum { PacketSize = unpacket_traits<PacketType>::size };
@ -1208,6 +1279,7 @@ struct unary_evaluator<Reverse<ArgType, Direction> >
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index row, Index col, const PacketType& x)
{
// FIXME we could factorize some code with packet(i,j)
@ -1224,6 +1296,7 @@ struct unary_evaluator<Reverse<ArgType, Direction> >
}
template<int LoadMode, typename PacketType>
EIGEN_STRONG_INLINE
void writePacket(Index index, const PacketType& x)
{
enum { PacketSize = unpacket_traits<PacketType>::size };
@ -1267,22 +1340,26 @@ struct evaluator<Diagonal<ArgType, DiagIndex> >
typedef typename internal::conditional<!internal::is_same<typename ArgType::StorageKind,Sparse>::value,
typename XprType::CoeffReturnType,Scalar>::type CoeffReturnType;
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index row, Index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index row, Index) const
{
return m_argImpl.coeff(row + rowOffset(), row + colOffset());
}
EIGEN_DEVICE_FUNC CoeffReturnType coeff(Index index) const
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
CoeffReturnType coeff(Index index) const
{
return m_argImpl.coeff(index + rowOffset(), index + colOffset());
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index row, Index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index row, Index)
{
return m_argImpl.coeffRef(row + rowOffset(), row + colOffset());
}
EIGEN_DEVICE_FUNC Scalar& coeffRef(Index index)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Scalar& coeffRef(Index index)
{
return m_argImpl.coeffRef(index + rowOffset(), index + colOffset());
}

18
Eigen/src/Core/CwiseUnaryOp.h
View File

@ -61,26 +61,26 @@ class CwiseUnaryOp : public CwiseUnaryOpImpl<UnaryOp, XprType, typename internal
typedef typename internal::ref_selector<XprType>::type XprTypeNested;
typedef typename internal::remove_all<XprType>::type NestedExpression;
EIGEN_DEVICE_FUNC
explicit inline CwiseUnaryOp(const XprType& xpr, const UnaryOp& func = UnaryOp())
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
explicit CwiseUnaryOp(const XprType& xpr, const UnaryOp& func = UnaryOp())
: m_xpr(xpr), m_functor(func) {}
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE Index rows() const { return m_xpr.rows(); }
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE Index cols() const { return m_xpr.cols(); }
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Index rows() const { return m_xpr.rows(); }
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Index cols() const { return m_xpr.cols(); }
/** \returns the functor representing the unary operation */
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const UnaryOp& functor() const { return m_functor; }
/** \returns the nested expression */
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const typename internal::remove_all<XprTypeNested>::type&
nestedExpression() const { return m_xpr; }
/** \returns the nested expression */
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
typename internal::remove_all<XprTypeNested>::type&
nestedExpression() { return m_xpr; }

12
Eigen/src/Core/DenseBase.h
View File

@ -275,13 +275,13 @@ template<typename Derived> class DenseBase
/** Copies \a other into *this. \returns a reference to *this. */
template<typename OtherDerived>
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator=(const DenseBase<OtherDerived>& other);
/** Special case of the template operator=, in order to prevent the compiler
* from generating a default operator= (issue hit with g++ 4.1)
*/
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator=(const DenseBase& other);
template<typename OtherDerived>
@ -388,10 +388,10 @@ template<typename Derived> class DenseBase
inline bool hasNaN() const;
inline bool allFinite() const;
EIGEN_DEVICE_FUNC
inline Derived& operator*=(const Scalar& other);
EIGEN_DEVICE_FUNC
inline Derived& operator/=(const Scalar& other);
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator*=(const Scalar& other);
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator/=(const Scalar& other);
typedef typename internal::add_const_on_value_type<typename internal::eval<Derived>::type>::type EvalReturnType;
/** \returns the matrix or vector obtained by evaluating this expression.

2
Eigen/src/Core/GenericPacketMath.h
View File

@ -285,7 +285,7 @@ template<typename Scalar, typename Packet> EIGEN_DEVICE_FUNC inline void pstoreu
{ pstore(to, from); }
/** \internal tries to do cache prefetching of \a addr */
template<typename Scalar> inline void prefetch(const Scalar* addr)
template<typename Scalar> EIGEN_DEVICE_FUNC inline void prefetch(const Scalar* addr)
{
#ifdef __CUDA_ARCH__
#if defined(__LP64__)

8
Eigen/src/Core/MatrixBase.h
View File

@ -135,14 +135,14 @@ template<typename Derived> class MatrixBase
/** Special case of the template operator=, in order to prevent the compiler
* from generating a default operator= (issue hit with g++ 4.1)
*/
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator=(const MatrixBase& other);
// We cannot inherit here via Base::operator= since it is causing
// trouble with MSVC.
template <typename OtherDerived>
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator=(const DenseBase<OtherDerived>& other);
template <typename OtherDerived>
@ -154,10 +154,10 @@ template<typename Derived> class MatrixBase
Derived& operator=(const ReturnByValue<OtherDerived>& other);
template<typename OtherDerived>
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator+=(const MatrixBase<OtherDerived>& other);
template<typename OtherDerived>
EIGEN_DEVICE_FUNC
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
Derived& operator-=(const MatrixBase<OtherDerived>& other);
#ifdef __CUDACC__

61
Eigen/src/Core/functors/NullaryFunctors.h
View File

@ -37,7 +37,7 @@ template<typename Scalar>
struct functor_traits<scalar_identity_op<Scalar> >
{ enum { Cost = NumTraits<Scalar>::AddCost, PacketAccess = false, IsRepeatable = true }; };
template <typename Scalar, typename Packet, bool RandomAccess> struct linspaced_op_impl;
template <typename Scalar, typename Packet, bool RandomAccess, bool IsInteger> struct linspaced_op_impl;
// linear access for packet ops:
// 1) initialization
@ -48,12 +48,12 @@ template <typename Scalar, typename Packet, bool RandomAccess> struct linspaced_
// TODO: Perhaps it's better to initialize lazily (so not in the constructor but in packetOp)
// in order to avoid the padd() in operator() ?
template <typename Scalar, typename Packet>
struct linspaced_op_impl<Scalar,Packet,false>
struct linspaced_op_impl<Scalar,Packet,/*RandomAccess*/false,/*IsInteger*/false>
{
linspaced_op_impl(const Scalar& low, const Scalar& step) :
m_low(low), m_step(step),
m_packetStep(pset1<Packet>(unpacket_traits<Packet>::size*step)),
m_base(padd(pset1<Packet>(low), pmul(pset1<Packet>(step),plset<Packet>(-unpacket_traits<Packet>::size)))) {}
linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) :
m_low(low), m_step(num_steps==1 ? Scalar() : (high-low)/Scalar(num_steps-1)),
m_packetStep(pset1<Packet>(unpacket_traits<Packet>::size*m_step)),
m_base(padd(pset1<Packet>(low), pmul(pset1<Packet>(m_step),plset<Packet>(-unpacket_traits<Packet>::size)))) {}
template<typename Index>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (Index i) const
@ -75,10 +75,10 @@ struct linspaced_op_impl<Scalar,Packet,false>
// 1) each step
// [low, ..., low] + ( [step, ..., step] * ( [i, ..., i] + [0, ..., size] ) )
template <typename Scalar, typename Packet>
struct linspaced_op_impl<Scalar,Packet,true>
struct linspaced_op_impl<Scalar,Packet,/*RandomAccess*/true,/*IsInteger*/false>
{
linspaced_op_impl(const Scalar& low, const Scalar& step) :
m_low(low), m_step(step),
linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) :
m_low(low), m_step(num_steps==1 ? Scalar() : (high-low)/Scalar(num_steps-1)),
m_lowPacket(pset1<Packet>(m_low)), m_stepPacket(pset1<Packet>(m_step)), m_interPacket(plset<Packet>(0)) {}
template<typename Index>
@ -95,6 +95,31 @@ struct linspaced_op_impl<Scalar,Packet,true>
const Packet m_interPacket;
};
template <typename Scalar, typename Packet>
struct linspaced_op_impl<Scalar,Packet,/*RandomAccess*/true,/*IsInteger*/true>
{
linspaced_op_impl(const Scalar& low, const Scalar& high, Index num_steps) :
m_low(low), m_length(high-low), m_divisor(num_steps==1?1:num_steps-1), m_interPacket(plset<Packet>(0))
{}
template<typename Index>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Scalar operator() (Index i) const {
return m_low + (m_length*Scalar(i))/m_divisor;
}
template<typename Index>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
const Packet packetOp(Index i) const {
return internal::padd(pset1<Packet>(m_low), pdiv(pmul(pset1<Packet>(m_length), padd(pset1<Packet>(Scalar(i)),m_interPacket)),
pset1<Packet>(m_divisor))); }
const Scalar m_low;
const Scalar m_length;
const Index m_divisor;
const Packet m_interPacket;
};
// ----- Linspace functor ----------------------------------------------------------------
// Forward declaration (we default to random access which does not really give
@ -102,10 +127,20 @@ struct linspaced_op_impl<Scalar,Packet,true>
// nested expressions).
template <typename Scalar, typename PacketType, bool RandomAccess = true> struct linspaced_op;
template <typename Scalar, typename PacketType, bool RandomAccess> struct functor_traits< linspaced_op<Scalar,PacketType,RandomAccess> >
{ enum { Cost = 1, PacketAccess = packet_traits<Scalar>::HasSetLinear, IsRepeatable = true }; };
{
enum
{
Cost = 1,
PacketAccess = packet_traits<Scalar>::HasSetLinear
&& ((!NumTraits<Scalar>::IsInteger) || packet_traits<Scalar>::HasDiv),
IsRepeatable = true
};
};
template <typename Scalar, typename PacketType, bool RandomAccess> struct linspaced_op
{
linspaced_op(const Scalar& low, const Scalar& high, Index num_steps) : impl((num_steps==1 ? high : low), (num_steps==1 ? Scalar() : (high-low)/Scalar(num_steps-1))) {}
linspaced_op(const Scalar& low, const Scalar& high, Index num_steps)
: impl((num_steps==1 ? high : low),high,num_steps)
{}
template<typename Index>
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (Index i) const { return impl(i); }
@ -134,7 +169,9 @@ template <typename Scalar, typename PacketType, bool RandomAccess> struct linspa
// This proxy object handles the actual required temporaries, the different
// implementations (random vs. sequential access) as well as the
// correct piping to size 2/4 packet operations.
const linspaced_op_impl<Scalar,PacketType,RandomAccess> impl;
// As long as we don't have a Bresenham-like implementation for linear-access and integer types,
// we have to by-pass RandomAccess for integer types. See bug 698.
const linspaced_op_impl<Scalar,PacketType,(NumTraits<Scalar>::IsInteger?true:RandomAccess),NumTraits<Scalar>::IsInteger> impl;
};
// all functors allow linear access, except scalar_identity_op. So we fix here a quick meta

2
Eigen/src/Core/functors/UnaryFunctors.h
View File

@ -666,7 +666,7 @@ struct functor_traits<scalar_floor_op<Scalar> >
template<typename Scalar> struct scalar_ceil_op {
EIGEN_EMPTY_STRUCT_CTOR(scalar_ceil_op)
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar operator() (const Scalar& a) const { return numext::ceil(a); }
typedef typename packet_traits<Scalar>::type Packet;
template <typename Packet>
EIGEN_DEVICE_FUNC inline Packet packetOp(const Packet& a) const { return internal::pceil(a); }
};
template<typename Scalar>

9
Eigen/src/Core/util/XprHelper.h
View File

@ -529,19 +529,18 @@ template <typename A> struct promote_storage_type<const A, A>
* A op A -> A
* A op dense -> dense
* dense op B -> dense
* A * dense -> A
* dense * B -> B
* sparse op dense -> sparse
* dense op sparse -> sparse
* \endcode
*/
template <typename A, typename B, typename Functor> struct cwise_promote_storage_type;
template <typename A, typename Functor> struct cwise_promote_storage_type<A,A,Functor> { typedef A ret; };
template <typename Functor> struct cwise_promote_storage_type<Dense,Dense,Functor> { typedef Dense ret; };
template <typename ScalarA, typename ScalarB> struct cwise_promote_storage_type<Dense,Dense,scalar_product_op<ScalarA,ScalarB> > { typedef Dense ret; };
template <typename A, typename Functor> struct cwise_promote_storage_type<A,Dense,Functor> { typedef Dense ret; };
template <typename B, typename Functor> struct cwise_promote_storage_type<Dense,B,Functor> { typedef Dense ret; };
template <typename A, typename ScalarA, typename ScalarB> struct cwise_promote_storage_type<A,Dense,scalar_product_op<ScalarA,ScalarB> > { typedef A ret; };
template <typename B, typename ScalarA, typename ScalarB> struct cwise_promote_storage_type<Dense,B,scalar_product_op<ScalarA,ScalarB> > { typedef B ret; };
template <typename Functor> struct cwise_promote_storage_type<Sparse,Dense,Functor> { typedef Sparse ret; };
template <typename Functor> struct cwise_promote_storage_type<Dense,Sparse,Functor> { typedef Sparse ret; };
/** \internal Specify the "storage kind" of multiplying an expression of kind A with kind B.
* The template parameter ProductTag permits to specialize the resulting storage kind wrt to

23
Eigen/src/OrderingMethods/Eigen_Colamd.h
View File

@ -98,9 +98,6 @@ namespace internal {
/* === Definitions ========================================================== */
/* ========================================================================== */
#define COLAMD_MAX(a,b) (((a) > (b)) ? (a) : (b))
#define COLAMD_MIN(a,b) (((a) < (b)) ? (a) : (b))
#define ONES_COMPLEMENT(r) (-(r)-1)
/* -------------------------------------------------------------------------- */
@ -735,8 +732,8 @@ static void init_scoring
/* === Extract knobs ==================================================== */
dense_row_count = COLAMD_MAX (0, COLAMD_MIN (knobs [COLAMD_DENSE_ROW] * n_col, n_col)) ;
dense_col_count = COLAMD_MAX (0, COLAMD_MIN (knobs [COLAMD_DENSE_COL] * n_row, n_row)) ;
dense_row_count = numext::maxi(IndexType(0), numext::mini(IndexType(knobs [COLAMD_DENSE_ROW] * n_col), n_col)) ;
dense_col_count = numext::maxi(IndexType(0), numext::mini(IndexType(knobs [COLAMD_DENSE_COL] * n_row), n_row)) ;
COLAMD_DEBUG1 (("colamd: densecount: %d %d\n", dense_row_count, dense_col_count)) ;
max_deg = 0 ;
n_col2 = n_col ;
@ -800,7 +797,7 @@ static void init_scoring
else
{
/* keep track of max degree of remaining rows */
max_deg = COLAMD_MAX (max_deg, deg) ;
max_deg = numext::maxi(max_deg, deg) ;
}
}
COLAMD_DEBUG1 (("colamd: Dense and null rows killed: %d\n", n_row - n_row2)) ;
@ -838,7 +835,7 @@ static void init_scoring
/* add row's external degree */
score += Row [row].shared1.degree - 1 ;
/* guard against integer overflow */
score = COLAMD_MIN (score, n_col) ;
score = numext::mini(score, n_col) ;
}
/* determine pruned column length */
col_length = (IndexType) (new_cp - &A [Col [c].start]) ;
@ -910,7 +907,7 @@ static void init_scoring
head [score] = c ;
/* see if this score is less than current min */
min_score = COLAMD_MIN (min_score, score) ;
min_score = numext::mini(min_score, score) ;
}
@ -1036,7 +1033,7 @@ static IndexType find_ordering /* return the number of garbage collections */
/* === Garbage_collection, if necessary ============================= */
needed_memory = COLAMD_MIN (pivot_col_score, n_col - k) ;
needed_memory = numext::mini(pivot_col_score, n_col - k) ;
if (pfree + needed_memory >= Alen)
{
pfree = Eigen::internal::garbage_collection (n_row, n_col, Row, Col, A, &A [pfree]) ;
@ -1095,7 +1092,7 @@ static IndexType find_ordering /* return the number of garbage collections */
/* clear tag on pivot column */
Col [pivot_col].shared1.thickness = pivot_col_thickness ;
max_deg = COLAMD_MAX (max_deg, pivot_row_degree) ;
max_deg = numext::maxi(max_deg, pivot_row_degree) ;
/* === Kill all rows used to construct pivot row ==================== */
@ -1269,7 +1266,7 @@ static IndexType find_ordering /* return the number of garbage collections */
/* add set difference */
cur_score += row_mark - tag_mark ;
/* integer overflow... */
cur_score = COLAMD_MIN (cur_score, n_col) ;
cur_score = numext::mini(cur_score, n_col) ;
}
/* recompute the column's length */
@ -1382,7 +1379,7 @@ static IndexType find_ordering /* return the number of garbage collections */
cur_score -= Col [col].shared1.thickness ;
/* make sure score is less or equal than the max score */
cur_score = COLAMD_MIN (cur_score, max_score) ;
cur_score = numext::mini(cur_score, max_score) ;
COLAMD_ASSERT (cur_score >= 0) ;
/* store updated score */
@ -1405,7 +1402,7 @@ static IndexType find_ordering /* return the number of garbage collections */
head [cur_score] = col ;
/* see if this score is less than current min */
min_score = COLAMD_MIN (min_score, cur_score) ;
min_score = numext::mini(min_score, cur_score) ;
}

21
Eigen/src/SparseCore/SparseCompressedBase.h
View File

@ -117,6 +117,24 @@ template<typename Derived>
class SparseCompressedBase<Derived>::InnerIterator
{
public:
InnerIterator()
: m_values(0), m_indices(0), m_outer(0), m_id(0), m_end(0)
{}
InnerIterator(const InnerIterator& other)
: m_values(other.m_values), m_indices(other.m_indices), m_outer(other.m_outer), m_id(other.m_id), m_end(other.m_end)
{}
InnerIterator& operator=(const InnerIterator& other)
{
m_values = other.m_values;
m_indices = other.m_indices;
const_cast<OuterType&>(m_outer).setValue(other.m_outer.value());
m_id = other.m_id;
m_end = other.m_end;
return *this;
}
InnerIterator(const SparseCompressedBase& mat, Index outer)
: m_values(mat.valuePtr()), m_indices(mat.innerIndexPtr()), m_outer(outer)
{
@ -162,7 +180,8 @@ class SparseCompressedBase<Derived>::InnerIterator
protected:
const Scalar* m_values;
const StorageIndex* m_indices;
const internal::variable_if_dynamic<Index,Derived::IsVectorAtCompileTime?0:Dynamic> m_outer;
typedef internal::variable_if_dynamic<Index,Derived::IsVectorAtCompileTime?0:Dynamic> OuterType;
const OuterType m_outer;
Index m_id;
Index m_end;
private:

221
Eigen/src/SparseCore/SparseCwiseBinaryOp.h
View File

@ -49,17 +49,10 @@ class CwiseBinaryOpImpl<BinaryOp, Lhs, Rhs, Sparse>
namespace internal {
template<typename BinaryOp, typename Lhs, typename Rhs, typename Derived,
typename _LhsStorageMode = typename traits<Lhs>::StorageKind,
typename _RhsStorageMode = typename traits<Rhs>::StorageKind>
class sparse_cwise_binary_op_inner_iterator_selector;
} // end namespace internal
namespace internal {
// Generic "sparse OP sparse"
template<typename XprType> struct binary_sparse_evaluator;
template<typename BinaryOp, typename Lhs, typename Rhs>
struct binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>, IteratorBased, IteratorBased>
: evaluator_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs> >
@ -153,6 +146,182 @@ protected:
evaluator<Rhs> m_rhsImpl;
};
// dense op sparse
template<typename BinaryOp, typename Lhs, typename Rhs>
struct binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>, IndexBased, IteratorBased>
: evaluator_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs> >
{
protected:
typedef typename evaluator<Rhs>::InnerIterator RhsIterator;
typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType;
typedef typename traits<XprType>::Scalar Scalar;
typedef typename XprType::StorageIndex StorageIndex;
public:
class ReverseInnerIterator;
class InnerIterator
{
enum { IsRowMajor = (int(Rhs::Flags)&RowMajorBit)==RowMajorBit };
public:
EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer)
: m_lhsEval(aEval.m_lhsImpl), m_rhsIter(aEval.m_rhsImpl,outer), m_functor(aEval.m_functor), m_id(-1), m_innerSize(aEval.m_expr.rhs().innerSize())
{
this->operator++();
}
EIGEN_STRONG_INLINE InnerIterator& operator++()
{
++m_id;
if(m_id<m_innerSize)
{
Scalar lhsVal = m_lhsEval.coeff(IsRowMajor?m_rhsIter.outer():m_id,
IsRowMajor?m_id:m_rhsIter.outer());
if(m_rhsIter && m_rhsIter.index()==m_id)
{
m_value = m_functor(lhsVal, m_rhsIter.value());
++m_rhsIter;
}
else
m_value = m_functor(lhsVal, Scalar(0));
}
return *this;
}
EIGEN_STRONG_INLINE Scalar value() const { return m_value; }
EIGEN_STRONG_INLINE StorageIndex index() const { return m_id; }
EIGEN_STRONG_INLINE Index row() const { return IsRowMajor ? m_rhsIter.outer() : m_id; }
EIGEN_STRONG_INLINE Index col() const { return IsRowMajor ? m_id : m_rhsIter.outer(); }
EIGEN_STRONG_INLINE operator bool() const { return m_id<m_innerSize; }
protected:
const evaluator<Lhs> &m_lhsEval;
RhsIterator m_rhsIter;
const BinaryOp& m_functor;
Scalar m_value;
StorageIndex m_id;
StorageIndex m_innerSize;
};
enum {
CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost,
// Expose storage order of the sparse expression
Flags = (XprType::Flags & ~RowMajorBit) | (int(Rhs::Flags)&RowMajorBit)
};
explicit binary_evaluator(const XprType& xpr)
: m_functor(xpr.functor()),
m_lhsImpl(xpr.lhs()),
m_rhsImpl(xpr.rhs()),
m_expr(xpr)
{
EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<BinaryOp>::Cost);
EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
}
inline Index nonZerosEstimate() const {
return m_expr.size();
}
protected:
const BinaryOp m_functor;
evaluator<Lhs> m_lhsImpl;
evaluator<Rhs> m_rhsImpl;
const XprType &m_expr;
};
// sparse op dense
template<typename BinaryOp, typename Lhs, typename Rhs>
struct binary_evaluator<CwiseBinaryOp<BinaryOp, Lhs, Rhs>, IteratorBased, IndexBased>
: evaluator_base<CwiseBinaryOp<BinaryOp, Lhs, Rhs> >
{
protected:
typedef typename evaluator<Lhs>::InnerIterator LhsIterator;
typedef CwiseBinaryOp<BinaryOp, Lhs, Rhs> XprType;
typedef typename traits<XprType>::Scalar Scalar;
typedef typename XprType::StorageIndex StorageIndex;
public:
class ReverseInnerIterator;
class InnerIterator
{
enum { IsRowMajor = (int(Lhs::Flags)&RowMajorBit)==RowMajorBit };
public:
EIGEN_STRONG_INLINE InnerIterator(const binary_evaluator& aEval, Index outer)
: m_lhsIter(aEval.m_lhsImpl,outer), m_rhsEval(aEval.m_rhsImpl), m_functor(aEval.m_functor), m_id(-1), m_innerSize(aEval.m_expr.lhs().innerSize())
{
this->operator++();
}
EIGEN_STRONG_INLINE InnerIterator& operator++()
{
++m_id;
if(m_id<m_innerSize)
{
Scalar rhsVal = m_rhsEval.coeff(IsRowMajor?m_lhsIter.outer():m_id,
IsRowMajor?m_id:m_lhsIter.outer());
if(m_lhsIter && m_lhsIter.index()==m_id)
{
m_value = m_functor(m_lhsIter.value(), rhsVal);
++m_lhsIter;
}
else
m_value = m_functor(Scalar(0),rhsVal);
}
return *this;
}
EIGEN_STRONG_INLINE Scalar value() const { return m_value; }
EIGEN_STRONG_INLINE StorageIndex index() const { return m_id; }
EIGEN_STRONG_INLINE Index row() const { return IsRowMajor ? m_lhsIter.outer() : m_id; }
EIGEN_STRONG_INLINE Index col() const { return IsRowMajor ? m_id : m_lhsIter.outer(); }
EIGEN_STRONG_INLINE operator bool() const { return m_id<m_innerSize; }
protected:
LhsIterator m_lhsIter;
const evaluator<Rhs> &m_rhsEval;
const BinaryOp& m_functor;
Scalar m_value;
StorageIndex m_id;
StorageIndex m_innerSize;
};
enum {
CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost,
// Expose storage order of the sparse expression
Flags = (XprType::Flags & ~RowMajorBit) | (int(Lhs::Flags)&RowMajorBit)
};
explicit binary_evaluator(const XprType& xpr)
: m_functor(xpr.functor()),
m_lhsImpl(xpr.lhs()),
m_rhsImpl(xpr.rhs()),
m_expr(xpr)
{
EIGEN_INTERNAL_CHECK_COST_VALUE(functor_traits<BinaryOp>::Cost);
EIGEN_INTERNAL_CHECK_COST_VALUE(CoeffReadCost);
}
inline Index nonZerosEstimate() const {
return m_expr.size();
}
protected:
const BinaryOp m_functor;
evaluator<Lhs> m_lhsImpl;
evaluator<Rhs> m_rhsImpl;
const XprType &m_expr;
};
// "sparse .* sparse"
template<typename T, typename Lhs, typename Rhs>
struct binary_evaluator<CwiseBinaryOp<scalar_product_op<T>, Lhs, Rhs>, IteratorBased, IteratorBased>
@ -287,7 +456,8 @@ public:
enum {
CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost,
Flags = XprType::Flags
// Expose storage order of the sparse expression
Flags = (XprType::Flags & ~RowMajorBit) | (int(Rhs::Flags)&RowMajorBit)
};
explicit binary_evaluator(const XprType& xpr)
@ -360,7 +530,8 @@ public:
enum {
CoeffReadCost = evaluator<Lhs>::CoeffReadCost + evaluator<Rhs>::CoeffReadCost + functor_traits<BinaryOp>::Cost,
Flags = XprType::Flags
// Expose storage order of the sparse expression
Flags = (XprType::Flags & ~RowMajorBit) | (int(Lhs::Flags)&RowMajorBit)
};
explicit binary_evaluator(const XprType& xpr)
@ -428,6 +599,34 @@ SparseMatrixBase<Derived>::cwiseProduct(const MatrixBase<OtherDerived> &other) c
return typename CwiseProductDenseReturnType<OtherDerived>::Type(derived(), other.derived());
}
template<typename DenseDerived, typename SparseDerived>
EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar>, const DenseDerived, const SparseDerived>
operator+(const MatrixBase<DenseDerived> &a, const SparseMatrixBase<SparseDerived> &b)
{
return CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar>, const DenseDerived, const SparseDerived>(a.derived(), b.derived());
}
template<typename SparseDerived, typename DenseDerived>
EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived>
operator+(const SparseMatrixBase<SparseDerived> &a, const MatrixBase<DenseDerived> &b)
{
return CwiseBinaryOp<internal::scalar_sum_op<typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived>(a.derived(), b.derived());
}
template<typename DenseDerived, typename SparseDerived>
EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar>, const DenseDerived, const SparseDerived>
operator-(const MatrixBase<DenseDerived> &a, const SparseMatrixBase<SparseDerived> &b)
{
return CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar>, const DenseDerived, const SparseDerived>(a.derived(), b.derived());
}
template<typename SparseDerived, typename DenseDerived>
EIGEN_STRONG_INLINE const CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived>
operator-(const SparseMatrixBase<SparseDerived> &a, const MatrixBase<DenseDerived> &b)
{
return CwiseBinaryOp<internal::scalar_difference_op<typename DenseDerived::Scalar>, const SparseDerived, const DenseDerived>(a.derived(), b.derived());
}
} // end namespace Eigen
#endif // EIGEN_SPARSE_CWISE_BINARY_OP_H

11
Eigen/src/SparseQR/SparseQR.h
View File

@ -128,6 +128,17 @@ class SparseQR : public SparseSolverBase<SparseQR<_MatrixType,_OrderingType> >
inline Index cols() const { return m_pmat.cols();}
/** \returns a const reference to the \b sparse upper triangular matrix R of the QR factorization.
* \warning The entries of the returned matrix are not sorted. This means that using it in algorithms
* expecting sorted entries will fail. This include random coefficient accesses (SpaseMatrix::coeff()),
* and coefficient-wise operations. Matrix products and triangular solves are fine though.
*
* To sort the entries, you can assign it to a row-major matrix, and if a column-major matrix
* is required, you can copy it again:
* \code
* SparseMatrix<double> R = qr.matrixR(); // column-major, not sorted!
* SparseMatrix<double,RowMajor> Rr = qr.matrixR(); // row-major, sorted
* SparseMatrix<double> Rc = Rr; // column-major, sorted
* \endcode
*/
const QRMatrixType& matrixR() const { return m_R; }

8
bench/tensors/README Normal file
View File

@ -0,0 +1,8 @@
Each benchmark comes in 2 flavors: one that runs on CPU, and one that runs on GPU.
To compile the CPU benchmarks, simply call:
g++ tensor_benchmarks_cpu.cc benchmark_main.cc -I ../../ -std=c++11 -O3 -DNDEBUG -pthread -mavx -o benchmarks_cpu
To compile the GPU benchmarks, simply call:
nvcc tensor_benchmarks_gpu.cu benchmark_main.cc -I ../../ -std=c++11 -O2 -DNDEBUG -arch compute_35 -o benchmarks_gpu

3
bench/tensors/benchmark.h
View File

@ -41,10 +41,9 @@ class Benchmark {
void RunWithArg(int arg);
};
} // namespace testing
void SetBenchmarkBytesProcessed(int64_t);
void SetBenchmarkFlopsProcessed(int64_t);
void StopBenchmarkTiming();
void StartBenchmarkTiming();
#define BENCHMARK(f) \
static ::testing::Benchmark* _benchmark_##f __attribute__((unused)) = \
(new ::testing::Benchmark(#f, f))

29
bench/tensors/benchmark_main.cc
View File

@ -23,7 +23,7 @@
#include <time.h>
#include <map>
static int64_t g_bytes_processed;
static int64_t g_flops_processed;
static int64_t g_benchmark_total_time_ns;
static int64_t g_benchmark_start_time_ns;
typedef std::map<std::string, ::testing::Benchmark*> BenchmarkMap;
@ -49,12 +49,27 @@ static int Round(int n) {
}
return 10*base;
}
#ifdef __APPLE__
#include <mach/mach_time.h>
static mach_timebase_info_data_t g_time_info;
static void __attribute__((constructor)) init_info() {
mach_timebase_info(&g_time_info);
}
#endif
static int64_t NanoTime() {
#if defined(__APPLE__)
uint64_t t = mach_absolute_time();
return t * g_time_info.numer / g_time_info.denom;
#else
struct timespec t;
t.tv_sec = t.tv_nsec = 0;
clock_gettime(CLOCK_MONOTONIC, &t);
return static_cast<int64_t>(t.tv_sec) * 1000000000LL + t.tv_nsec;
#endif
}
namespace testing {
Benchmark* Benchmark::Arg(int arg) {
args_.push_back(arg);
@ -124,7 +139,7 @@ void Benchmark::Run() {
}
}
void Benchmark::RunRepeatedlyWithArg(int iterations, int arg) {
g_bytes_processed = 0;
g_flops_processed = 0;
g_benchmark_total_time_ns = 0;
g_benchmark_start_time_ns = NanoTime();
if (fn_ != NULL) {
@ -153,10 +168,10 @@ void Benchmark::RunWithArg(int arg) {
}
char throughput[100];
throughput[0] = '\0';
if (g_benchmark_total_time_ns > 0 && g_bytes_processed > 0) {
double mib_processed = static_cast<double>(g_bytes_processed)/1e6;
if (g_benchmark_total_time_ns > 0 && g_flops_processed > 0) {
double mflops_processed = static_cast<double>(g_flops_processed)/1e6;
double seconds = static_cast<double>(g_benchmark_total_time_ns)/1e9;
snprintf(throughput, sizeof(throughput), " %8.2f MiB/s", mib_processed/seconds);
snprintf(throughput, sizeof(throughput), " %8.2f MFlops/s", mflops_processed/seconds);
}
char full_name[100];
if (fn_range_ != NULL) {
@ -175,8 +190,8 @@ void Benchmark::RunWithArg(int arg) {
fflush(stdout);
}
} // namespace testing
void SetBenchmarkBytesProcessed(int64_t x) {
g_bytes_processed = x;
void SetBenchmarkFlopsProcessed(int64_t x) {
g_flops_processed = x;
}
void StopBenchmarkTiming() {
if (g_benchmark_start_time_ns != 0) {

117
bench/tensors/tensor_benchmarks.h
View File

@ -13,8 +13,6 @@ typedef int TensorIndex;
using Eigen::Tensor;
using Eigen::TensorMap;
typedef int64_t int64;
// TODO(bsteiner): also templatize on the input type since we have users
// for int8 as well as floats.
template <typename Device> class BenchmarkSuite {
@ -42,7 +40,21 @@ template <typename Device> class BenchmarkSuite {
device_.memcpy(c_, a_, m_ * m_ * sizeof(float));
}
// Record the number of values copied per second
finalizeBenchmark(m_ * m_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(m_) * m_ * num_iters);
}
void typeCasting(int num_iters) {
eigen_assert(m_ == n_);
const Eigen::array<TensorIndex, 2> sizes = {{m_, k_}};
const TensorMap<Tensor<float, 2, 0, TensorIndex>, Eigen::Aligned> A(a_, sizes);
TensorMap<Tensor<int, 2, 0, TensorIndex>, Eigen::Aligned> B((int*)b_, sizes);
StartBenchmarkTiming();
for (int iter = 0; iter < num_iters; ++iter) {
B.device(device_) = A.cast<int>();
}
// Record the number of values copied per second
finalizeBenchmark(static_cast<int64_t>(m_) * k_ * num_iters);
}
void random(int num_iters) {
@ -55,7 +67,7 @@ template <typename Device> class BenchmarkSuite {
C.device(device_) = C.random();
}
// Record the number of random numbers generated per second
finalizeBenchmark(m_ * m_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(m_) * m_ * num_iters);
}
void slicing(int num_iters) {
@ -84,7 +96,35 @@ template <typename Device> class BenchmarkSuite {
}
// Record the number of values copied from the rhs slice to the lhs slice
// each second
finalizeBenchmark(m_ * m_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(m_) * m_ * num_iters);
}
void rowChip(int num_iters) {
const Eigen::array<TensorIndex, 2> input_size = {{k_, n_}};
const TensorMap<Tensor<float, 2, 0, TensorIndex>, Eigen::Aligned> B(b_, input_size);
const Eigen::array<TensorIndex, 1> output_size = {{n_}};
TensorMap<Tensor<float, 1, 0, TensorIndex>, Eigen::Aligned> C(c_, output_size);
StartBenchmarkTiming();
for (int iter = 0; iter < num_iters; ++iter) {
C.device(device_) = B.chip(iter % k_, 0);
}
// Record the number of values copied from the rhs chip to the lhs.
finalizeBenchmark(static_cast<int64_t>(n_) * num_iters);
}
void colChip(int num_iters) {
const Eigen::array<TensorIndex, 2> input_size= {{k_, n_}};
const TensorMap<Tensor<float, 2, 0, TensorIndex>, Eigen::Aligned> B(b_, input_size);
const Eigen::array<TensorIndex, 1> output_size = {{n_}};
TensorMap<Tensor<float, 1, 0, TensorIndex>, Eigen::Aligned> C(c_, output_size);
StartBenchmarkTiming();
for (int iter = 0; iter < num_iters; ++iter) {
C.device(device_) = B.chip(iter % n_, 1);
}
// Record the number of values copied from the rhs chip to the lhs.
finalizeBenchmark(static_cast<int64_t>(n_) * num_iters);
}
void shuffling(int num_iters) {
@ -101,7 +141,7 @@ template <typename Device> class BenchmarkSuite {
B.device(device_) = A.shuffle(shuffle);
}
// Record the number of values shuffled from A and copied to B each second
finalizeBenchmark(m_ * k_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(m_) * k_ * num_iters);
}
void padding(int num_iters) {
@ -120,7 +160,7 @@ template <typename Device> class BenchmarkSuite {
B.device(device_) = A.pad(paddings);
}
// Record the number of values copied from the padded tensor A each second
finalizeBenchmark(m_ * k_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(m_) * k_ * num_iters);
}
void striding(int num_iters) {
@ -137,7 +177,7 @@ template <typename Device> class BenchmarkSuite {
B.device(device_) = A.stride(strides);
}
// Record the number of values copied from the padded tensor A each second
finalizeBenchmark(m_ * k_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(m_) * k_ * num_iters);
}
void broadcasting(int num_iters) {
@ -147,7 +187,6 @@ template <typename Device> class BenchmarkSuite {
TensorMap<Tensor<float, 2>, Eigen::Aligned> C(c_, size_c);
#ifndef EIGEN_HAS_INDEX_LIST
// nvcc doesn't support cxx11
const Eigen::array<int, 2> broadcast = {{1, n_}};
#else
// Take advantage of cxx11 to give the compiler information it can use to
@ -161,7 +200,7 @@ template <typename Device> class BenchmarkSuite {
C.device(device_) = A.broadcast(broadcast);
}
// Record the number of values broadcasted from A and copied to C each second
finalizeBenchmark(m_ * n_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(m_) * n_ * num_iters);
}
void coeffWiseOp(int num_iters) {
@ -177,7 +216,7 @@ template <typename Device> class BenchmarkSuite {
}
// Record the number of FLOP executed per second (2 multiplications and
// 1 addition per value)
finalizeBenchmark(3 * m_ * m_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(3) * m_ * m_ * num_iters);
}
void algebraicFunc(int num_iters) {
@ -193,7 +232,7 @@ template <typename Device> class BenchmarkSuite {
}
// Record the number of FLOP executed per second (assuming one operation
// per value)
finalizeBenchmark(m_ * m_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(m_) * m_ * num_iters);
}
void transcendentalFunc(int num_iters) {
@ -209,17 +248,23 @@ template <typename Device> class BenchmarkSuite {
}
// Record the number of FLOP executed per second (assuming one operation
// per value)
finalizeBenchmark(m_ * m_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(m_) * m_ * num_iters);
}
// Simple reduction
void reduction(int num_iters) {
// Row reduction
void rowReduction(int num_iters) {
const Eigen::array<TensorIndex, 2> input_size = {{k_, n_}};
const TensorMap<Tensor<float, 2>, Eigen::Aligned> B(b_, input_size);
const TensorMap<Tensor<float, 2, 0, TensorIndex>, Eigen::Aligned> B(b_, input_size);
const Eigen::array<TensorIndex, 1> output_size = {{n_}};
TensorMap<Tensor<float, 1>, Eigen::Aligned> C(c_, output_size);
TensorMap<Tensor<float, 1, 0, TensorIndex>, Eigen::Aligned> C(c_, output_size);
#ifndef EIGEN_HAS_INDEX_LIST
const Eigen::array<TensorIndex, 1> sum_along_dim = {{0}};
#else
// Take advantage of cxx11 to give the compiler information it can use to
// optimize the code.
Eigen::IndexList<Eigen::type2index<0>> sum_along_dim;
#endif
StartBenchmarkTiming();
for (int iter = 0; iter < num_iters; ++iter) {
@ -227,7 +272,33 @@ template <typename Device> class BenchmarkSuite {
}
// Record the number of FLOP executed per second (assuming one operation
// per value)
finalizeBenchmark(m_ * m_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(k_) * n_ * num_iters);
}
// Column reduction
void colReduction(int num_iters) {
const Eigen::array<TensorIndex, 2> input_size = {{k_, n_}};
const TensorMap<Tensor<float, 2, 0, TensorIndex>, Eigen::Aligned> B(
b_, input_size);
const Eigen::array<TensorIndex, 1> output_size = {{k_}};
TensorMap<Tensor<float, 1, 0, TensorIndex>, Eigen::Aligned> C(
c_, output_size);
#ifndef EIGEN_HAS_INDEX_LIST
const Eigen::array<TensorIndex, 1> sum_along_dim = {{1}};
#else
// Take advantage of cxx11 to give the compiler information it can use to
// optimize the code.
Eigen::IndexList<Eigen::type2index<1>> sum_along_dim;
#endif
StartBenchmarkTiming();
for (int iter = 0; iter < num_iters; ++iter) {
C.device(device_) = B.sum(sum_along_dim);
}
// Record the number of FLOP executed per second (assuming one operation
// per value)
finalizeBenchmark(static_cast<int64_t>(k_) * n_ * num_iters);
}
// do a contraction which is equivalent to a matrix multiplication
@ -249,7 +320,7 @@ template <typename Device> class BenchmarkSuite {
}
// Record the number of FLOP executed per second (size_ multiplications and
// additions for each value in the resulting tensor)
finalizeBenchmark(static_cast<int64>(2) * m_ * n_ * k_ * num_iters);
finalizeBenchmark(static_cast<int64_t>(2) * m_ * n_ * k_ * num_iters);
}
void convolution(int num_iters, int kernel_x, int kernel_y) {
@ -268,8 +339,8 @@ template <typename Device> class BenchmarkSuite {
}
// Record the number of FLOP executed per second (kernel_size
// multiplications and additions for each value in the resulting tensor)
finalizeBenchmark(
(m_ - kernel_x + 1) * (n_ - kernel_y + 1) * kernel_x * kernel_y * 2 * num_iters);
finalizeBenchmark(static_cast<int64_t>(2) *
(m_ - kernel_x + 1) * (n_ - kernel_y + 1) * kernel_x * kernel_y * num_iters);
}
private:
@ -287,14 +358,14 @@ template <typename Device> class BenchmarkSuite {
//BenchmarkUseRealTime();
}
inline void finalizeBenchmark(int64 num_items) {
inline void finalizeBenchmark(int64_t num_items) {
#if defined(EIGEN_USE_GPU) && defined(__CUDACC__)
if (Eigen::internal::is_same<Device, Eigen::GpuDevice>::value) {
device_.synchronize();
}
#endif
StopBenchmarkTiming();
SetBenchmarkBytesProcessed(num_items);
SetBenchmarkFlopsProcessed(num_items);
}

28
bench/tensors/tensor_benchmarks_cpu.cc
View File

@ -22,6 +22,10 @@ BM_FuncCPU(memcpy, 4);
BM_FuncCPU(memcpy, 8);
BM_FuncCPU(memcpy, 12);
BM_FuncCPU(typeCasting, 4);
BM_FuncCPU(typeCasting, 8);
BM_FuncCPU(typeCasting, 12);
BM_FuncCPU(random, 4);
BM_FuncCPU(random, 8);
BM_FuncCPU(random, 12);
@ -30,6 +34,14 @@ BM_FuncCPU(slicing, 4);
BM_FuncCPU(slicing, 8);
BM_FuncCPU(slicing, 12);
BM_FuncCPU(rowChip, 4);
BM_FuncCPU(rowChip, 8);
BM_FuncCPU(rowChip, 12);
BM_FuncCPU(colChip, 4);
BM_FuncCPU(colChip, 8);
BM_FuncCPU(colChip, 12);
BM_FuncCPU(shuffling, 4);
BM_FuncCPU(shuffling, 8);
BM_FuncCPU(shuffling, 12);
@ -58,9 +70,13 @@ BM_FuncCPU(transcendentalFunc, 4);
BM_FuncCPU(transcendentalFunc, 8);
BM_FuncCPU(transcendentalFunc, 12);
BM_FuncCPU(reduction, 4);
BM_FuncCPU(reduction, 8);
BM_FuncCPU(reduction, 12);
BM_FuncCPU(rowReduction, 4);
BM_FuncCPU(rowReduction, 8);
BM_FuncCPU(rowReduction, 12);
BM_FuncCPU(colReduction, 4);
BM_FuncCPU(colReduction, 8);
BM_FuncCPU(colReduction, 12);
// Contractions
@ -98,6 +114,12 @@ BM_FuncWithInputDimsCPU(contraction, N, 64, N, 8);
BM_FuncWithInputDimsCPU(contraction, N, 64, N, 12);
BM_FuncWithInputDimsCPU(contraction, N, 64, N, 16);
BM_FuncWithInputDimsCPU(contraction, N, N, 64, 1);
BM_FuncWithInputDimsCPU(contraction, N, N, 64, 4);
BM_FuncWithInputDimsCPU(contraction, N, N, 64, 8);
BM_FuncWithInputDimsCPU(contraction, N, N, 64, 12);
BM_FuncWithInputDimsCPU(contraction, N, N, 64, 16);
BM_FuncWithInputDimsCPU(contraction, 1, N, N, 1);
BM_FuncWithInputDimsCPU(contraction, 1, N, N, 4);
BM_FuncWithInputDimsCPU(contraction, 1, N, N, 8);

9
bench/tensors/tensor_benchmarks_gpu.cu
View File

@ -19,14 +19,20 @@
BENCHMARK_RANGE(BM_##FUNC, 10, 5000);
BM_FuncGPU(memcpy);
BM_FuncGPU(typeCasting);
BM_FuncGPU(random);
BM_FuncGPU(slicing);
BM_FuncGPU(rowChip);
BM_FuncGPU(colChip);
BM_FuncGPU(shuffling);
BM_FuncGPU(padding);
BM_FuncGPU(striding);
BM_FuncGPU(broadcasting);
BM_FuncGPU(coeffWiseOp);
BM_FuncGPU(reduction);
BM_FuncGPU(algebraicFunc);
BM_FuncGPU(transcendentalFunc);
BM_FuncGPU(rowReduction);
BM_FuncGPU(colReduction);
// Contractions
@ -45,6 +51,7 @@ BM_FuncGPU(reduction);
BM_FuncWithInputDimsGPU(contraction, N, N, N);
BM_FuncWithInputDimsGPU(contraction, 64, N, N);
BM_FuncWithInputDimsGPU(contraction, N, 64, N);
BM_FuncWithInputDimsGPU(contraction, N, N, 64);
// Convolutions

7
doc/TutorialSparse.dox
View File

@ -257,7 +257,14 @@ Binary coefficient wise operators can also mix sparse and dense expressions:
\code
sm2 = sm1.cwiseProduct(dm1);
dm2 = sm1 + dm1;
dm2 = dm1 - sm1;
\endcode
Performance-wise, the adding/subtracting sparse and dense matrices is better performed in two steps. For instance, instead of doing <tt>dm2 = sm1 + dm1</tt>, better write:
\code
dm2 = dm1;
dm2 += sm1;
\endcode
This version has the advantage to fully exploit the higher performance of dense storage (no indirection, SIMD, etc.), and to pay the cost of slow sparse evaluation on the few non-zeros of the sparse matrix only.
%Sparse expressions also support transposition:

2
test/adjoint.cpp
View File

@ -45,12 +45,14 @@ template<> struct adjoint_specific<false> {
// check null inputs
VERIFY_IS_APPROX((v1*0).normalized(), (v1*0));
#if (!EIGEN_ARCH_i386) || defined(EIGEN_VECTORIZE)
RealScalar very_small = (std::numeric_limits<RealScalar>::min)();
VERIFY( (v1*very_small).norm() == 0 );
VERIFY_IS_APPROX((v1*very_small).normalized(), (v1*very_small));
v3 = v1*very_small;
v3.normalize();
VERIFY_IS_APPROX(v3, (v1*very_small));
#endif
// check compatibility of dot and adjoint
ref = NumTraits<Scalar>::IsInteger ? 0 : (std::max)((std::max)(v1.norm(),v2.norm()),(std::max)((square * v2).norm(),(square.adjoint() * v1).norm()));

28
test/nullary.cpp
View File

@ -48,30 +48,32 @@ void testVectorType(const VectorType& base)
VectorType m(base);
m.setLinSpaced(size,low,high);
if(!NumTraits<Scalar>::IsInteger)
{
VectorType n(size);
for (int i=0; i<size; ++i)
n(i) = low+i*step;
VERIFY_IS_APPROX(m,n);
}
VectorType n(size);
for (int i=0; i<size; ++i)
n(i) = size==1 ? low : (low + ((high-low)*Scalar(i))/(size-1));
VERIFY_IS_APPROX(m,n);
// random access version
m = VectorType::LinSpaced(size,low,high);
VERIFY_IS_APPROX(m,n);
// Assignment of a RowVectorXd to a MatrixXd (regression test for bug #79).
VERIFY( (MatrixXd(RowVectorXd::LinSpaced(3, 0, 1)) - RowVector3d(0, 0.5, 1)).norm() < std::numeric_limits<Scalar>::epsilon() );
// These guys sometimes fail! This is not good. Any ideas how to fix them!?
//VERIFY( m(m.size()-1) == high );
//VERIFY( m(0) == low );
VERIFY( internal::isApprox(m(m.size()-1),high) );
VERIFY( size==1 || internal::isApprox(m(0),low) );
// sequential access version
m = VectorType::LinSpaced(Sequential,size,low,high);
VERIFY_IS_APPROX(m,n);
// These guys sometimes fail! This is not good. Any ideas how to fix them!?
//VERIFY( m(m.size()-1) == high );
//VERIFY( m(0) == low );
VERIFY( internal::isApprox(m(m.size()-1),high) );
VERIFY( size==1 || internal::isApprox(m(0),low) );
// check whether everything works with row and col major vectors
Matrix<Scalar,Dynamic,1> row_vector(size);
@ -126,5 +128,13 @@ void test_nullary()
CALL_SUBTEST_8( testVectorType(Vector4f()) );
CALL_SUBTEST_8( testVectorType(Matrix<float,8,1>()) );
CALL_SUBTEST_8( testVectorType(Matrix<float,1,1>()) );
CALL_SUBTEST_9( testVectorType(VectorXi(internal::random<int>(1,300))) );
CALL_SUBTEST_9( testVectorType(Matrix<int,1,1>()) );
}
#ifdef EIGEN_TEST_PART_6
// Assignment of a RowVectorXd to a MatrixXd (regression test for bug #79).
VERIFY( (MatrixXd(RowVectorXd::LinSpaced(3, 0, 1)) - RowVector3d(0, 0.5, 1)).norm() < std::numeric_limits<double>::epsilon() );
#endif
}

32
test/sparse_basic.cpp
View File

@ -192,6 +192,11 @@ template<typename SparseMatrixType> void sparse_basic(const SparseMatrixType& re
VERIFY_IS_APPROX(refM4.cwiseProduct(m3), refM4.cwiseProduct(refM3));
// VERIFY_IS_APPROX(m3.cwise()/refM4, refM3.cwise()/refM4);
VERIFY_IS_APPROX(refM4 + m3, refM4 + refM3);
VERIFY_IS_APPROX(m3 + refM4, refM3 + refM4);
VERIFY_IS_APPROX(refM4 - m3, refM4 - refM3);
VERIFY_IS_APPROX(m3 - refM4, refM3 - refM4);
// test aliasing
VERIFY_IS_APPROX((m1 = -m1), (refM1 = -refM1));
VERIFY_IS_APPROX((m1 = m1.transpose()), (refM1 = refM1.transpose().eval()));
@ -455,6 +460,33 @@ template<typename SparseMatrixType> void sparse_basic(const SparseMatrixType& re
refMat1.setIdentity();
VERIFY_IS_APPROX(m1, refMat1);
}
// test array/vector of InnerIterator
{
typedef typename SparseMatrixType::InnerIterator IteratorType;
DenseMatrix refMat2 = DenseMatrix::Zero(rows, cols);
SparseMatrixType m2(rows, cols);
initSparse<Scalar>(density, refMat2, m2);
IteratorType static_array[2];
static_array[0] = IteratorType(m2,0);
static_array[1] = IteratorType(m2,m2.outerSize()-1);
VERIFY( static_array[0] || m2.innerVector(static_array[0].outer()).nonZeros() == 0 );
VERIFY( static_array[1] || m2.innerVector(static_array[1].outer()).nonZeros() == 0 );
if(static_array[0] && static_array[1])
{
++(static_array[1]);
static_array[1] = IteratorType(m2,0);
VERIFY( static_array[1] );
VERIFY( static_array[1].index() == static_array[0].index() );
VERIFY( static_array[1].outer() == static_array[0].outer() );
VERIFY( static_array[1].value() == static_array[0].value() );
}
std::vector<IteratorType> iters(2);
iters[0] = IteratorType(m2,0);
iters[1] = IteratorType(m2,m2.outerSize()-1);
}
}

3
test/stable_norm.cpp
View File

@ -174,7 +174,8 @@ template<typename MatrixType> void stable_norm(const MatrixType& m)
VERIFY_IS_APPROX(vcopy.norm(), RealScalar(1));
VERIFY_IS_APPROX((vbig.stableNormalized()).norm(), RealScalar(1));
VERIFY_IS_APPROX((vsmall.stableNormalized()).norm(), RealScalar(1));
VERIFY_IS_APPROX(vbig, vbig.stableNorm() * vbig.stableNormalized());
RealScalar big_scaling = ((std::numeric_limits<RealScalar>::max)() * RealScalar(1e-4));
VERIFY_IS_APPROX(vbig/big_scaling, (vbig.stableNorm() * vbig.stableNormalized()).eval()/big_scaling);
VERIFY_IS_APPROX(vsmall, vsmall.stableNorm() * vsmall.stableNormalized());
}
}

18
test/zerosized.cpp
View File

@ -25,6 +25,7 @@ template<typename MatrixType> void zeroReduction(const MatrixType& m) {
template<typename MatrixType> void zeroSizedMatrix()
{
MatrixType t1;
typedef typename MatrixType::Scalar Scalar;
if (MatrixType::SizeAtCompileTime == Dynamic || MatrixType::SizeAtCompileTime == 0)
{
@ -45,6 +46,23 @@ template<typename MatrixType> void zeroSizedMatrix()
VERIFY(t1==t2);
}
}
if(MatrixType::MaxColsAtCompileTime!=0 && MatrixType::MaxRowsAtCompileTime!=0)
{
Index rows = MatrixType::RowsAtCompileTime==Dynamic ? internal::random<Index>(1,10) : MatrixType::RowsAtCompileTime;
Index cols = MatrixType::ColsAtCompileTime==Dynamic ? internal::random<Index>(1,10) : MatrixType::ColsAtCompileTime;
MatrixType m(rows,cols);
zeroReduction(m.template block<0,MatrixType::ColsAtCompileTime>(0,0,0,cols));
zeroReduction(m.template block<MatrixType::RowsAtCompileTime,0>(0,0,rows,0));
zeroReduction(m.template block<0,1>(0,0));
zeroReduction(m.template block<1,0>(0,0));
Matrix<Scalar,Dynamic,Dynamic> prod = m.template block<MatrixType::RowsAtCompileTime,0>(0,0,rows,0) * m.template block<0,MatrixType::ColsAtCompileTime>(0,0,0,cols);
VERIFY(prod.rows()==rows && prod.cols()==cols);
VERIFY(prod.isZero());
prod = m.template block<1,0>(0,0) * m.template block<0,1>(0,0);
VERIFY(prod.size()==1);
VERIFY(prod.isZero());
}
}
template<typename VectorType> void zeroSizedVector()

39
unsupported/Eigen/CXX11/src/Core/util/EmulateArray.h
View File

@ -25,6 +25,16 @@ template <typename T, size_t n> class array {
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE const T& operator[] (size_t index) const { return values[index]; }
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE T& front() { return values[0]; }
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE const T& front() const { return values[0]; }
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE T& back() { return values[n-1]; }
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE const T& back() const { return values[n-1]; }
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
static std::size_t size() { return n; }
@ -123,13 +133,33 @@ template <typename T> class array<T, 0> {
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE T& operator[] (size_t) {
eigen_assert(false && "Can't index a zero size array");
return *static_cast<T*>(NULL);
return dummy;
}
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE const T& operator[] (size_t) const {
eigen_assert(false && "Can't index a zero size array");
return *static_cast<const T*>(NULL);
return dummy;
}
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE T& front() {
eigen_assert(false && "Can't index a zero size array");
return dummy;
}
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE const T& front() const {
eigen_assert(false && "Can't index a zero size array");
return dummy;
}
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE T& back() {
eigen_assert(false && "Can't index a zero size array");
return dummy;
}
EIGEN_DEVICE_FUNC
EIGEN_STRONG_INLINE const T& back() const {
eigen_assert(false && "Can't index a zero size array");
return dummy;
}
static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::size_t size() { return 0; }
@ -142,6 +172,9 @@ template <typename T> class array<T, 0> {
eigen_assert(l.size() == 0);
}
#endif
private:
T dummy;
};
namespace internal {

4
unsupported/Eigen/CXX11/src/Tensor/TensorContraction.h
View File

@ -378,7 +378,7 @@ struct TensorContractionEvaluatorBase
}
template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment>
void evalGemv(Scalar* buffer) const {
EIGEN_DEVICE_FUNC void evalGemv(Scalar* buffer) const {
const Index rows = m_i_size;
const Index cols = m_k_size;
@ -516,7 +516,7 @@ struct TensorEvaluator<const TensorContractionOp<Indices, LeftArgType, RightArgT
Base(op, device) { }
template <bool lhs_inner_dim_contiguous, bool rhs_inner_dim_contiguous, bool rhs_inner_dim_reordered, int Alignment>
void evalProduct(Scalar* buffer) const {
EIGEN_DEVICE_FUNC void evalProduct(Scalar* buffer) const {
if (this->m_j_size == 1) {
this->template evalGemv<lhs_inner_dim_contiguous, rhs_inner_dim_contiguous, rhs_inner_dim_reordered, Alignment>(buffer);
return;

8
unsupported/Eigen/CXX11/src/Tensor/TensorContractionBlocking.h
View File

@ -28,7 +28,7 @@ class TensorContractionBlocking {
typedef typename LhsMapper::Scalar LhsScalar;
typedef typename RhsMapper::Scalar RhsScalar;
TensorContractionBlocking(Index k, Index m, Index n, Index num_threads = 1) :
EIGEN_DEVICE_FUNC TensorContractionBlocking(Index k, Index m, Index n, Index num_threads = 1) :
kc_(k), mc_(m), nc_(n)
{
if (ShardingType == ShardByCol) {
@ -41,9 +41,9 @@ class TensorContractionBlocking {
}
}
EIGEN_ALWAYS_INLINE Index kc() const { return kc_; }
EIGEN_ALWAYS_INLINE Index mc() const { return mc_; }
EIGEN_ALWAYS_INLINE Index nc() const { return nc_; }
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index kc() const { return kc_; }
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index mc() const { return mc_; }
EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE Index nc() const { return nc_; }
private:
Index kc_;

8
unsupported/Eigen/CXX11/src/Tensor/TensorConvolution.h
View File

@ -21,7 +21,7 @@ namespace Eigen {
*/
namespace internal {
template <typename Index, typename InputDims, size_t NumKernelDims, int Layout>
template <typename Index, typename InputDims, int NumKernelDims, int Layout>
class IndexMapper {
public:
IndexMapper(const InputDims& input_dims, const array<Index, NumKernelDims>& kernel_dims,
@ -123,7 +123,7 @@ class IndexMapper {
}
inputIndex += p * m_inputStrides[NumKernelDims];
} else {
int limit = 0;
std::ptrdiff_t limit = 0;
if (NumKernelDims < NumDims) {
limit = NumDims - NumKernelDims - 1;
}
@ -147,7 +147,7 @@ class IndexMapper {
}
outputIndex += p * m_outputStrides[NumKernelDims];
} else {
int limit = 0;
std::ptrdiff_t limit = 0;
if (NumKernelDims < NumDims) {
limit = NumDims - NumKernelDims - 1;
}
@ -206,7 +206,7 @@ class IndexMapper {
}
private:
static const size_t NumDims = internal::array_size<InputDims>::value;
static const int NumDims = internal::array_size<InputDims>::value;
array<Index, NumDims> m_inputStrides;
array<Index, NumDims> m_outputStrides;
array<Index, NumDims> m_cudaInputStrides;

8
unsupported/Eigen/CXX11/src/Tensor/TensorReduction.h
View File

@ -345,7 +345,7 @@ template <typename Self, typename Op, typename Device>
struct InnerReducer {
static const bool HasOptimizedImplementation = false;
static void run(const Self&, Op&, const Device&, typename Self::CoeffReturnType*, typename Self::Index, typename Self::Index) {
EIGEN_DEVICE_FUNC static void run(const Self&, Op&, const Device&, typename Self::CoeffReturnType*, typename Self::Index, typename Self::Index) {
eigen_assert(false && "Not implemented");
}
};
@ -355,7 +355,7 @@ template <typename Self, typename Op, typename Device>
struct OuterReducer {
static const bool HasOptimizedImplementation = false;
static void run(const Self&, Op&, const Device&, typename Self::CoeffReturnType*, typename Self::Index, typename Self::Index) {
EIGEN_DEVICE_FUNC static void run(const Self&, Op&, const Device&, typename Self::CoeffReturnType*, typename Self::Index, typename Self::Index) {
eigen_assert(false && "Not implemented");
}
};
@ -463,7 +463,7 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device>
m_outputStrides[i] = m_outputStrides[i - 1] * m_dimensions[i - 1];
}
} else {
m_outputStrides[NumOutputDims - 1] = 1;
m_outputStrides.back() = 1;
for (int i = NumOutputDims - 2; i >= 0; --i) {
m_outputStrides[i] = m_outputStrides[i + 1] * m_dimensions[i + 1];
}
@ -479,7 +479,7 @@ struct TensorEvaluator<const TensorReductionOp<Op, Dims, ArgType>, Device>
input_strides[i] = input_strides[i-1] * input_dims[i-1];
}
} else {
input_strides[NumInputDims - 1] = 1;
input_strides.back() = 1;
for (int i = NumInputDims - 2; i >= 0; --i) {
input_strides[i] = input_strides[i + 1] * input_dims[i + 1];
}

5
unsupported/Eigen/CXX11/src/Tensor/TensorStorage.h
View File

@ -41,7 +41,10 @@ class TensorStorage<T, FixedDimensions, Options_>
private:
static const std::size_t Size = FixedDimensions::total_size;
EIGEN_ALIGN_MAX T m_data[Size];
// Allocate an array of size at least one to prevent compiler warnings.
static const std::size_t MinSize = max_n_1<Size>::size;
EIGEN_ALIGN_MAX T m_data[MinSize];
FixedDimensions m_dimensions;
public:

2
unsupported/test/CMakeLists.txt
View File

@ -158,7 +158,7 @@ if(CUDA_FOUND)
if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
set(CUDA_NVCC_FLAGS "-ccbin /usr/bin/clang" CACHE STRING "nvcc flags" FORCE)
endif()
set(CUDA_NVCC_FLAGS "-std=c++11 -arch compute_30")
set(CUDA_NVCC_FLAGS "-std=c++11 --relaxed-constexpr -arch compute_30")
cuda_include_directories("${CMAKE_CURRENT_BINARY_DIR}" "${CUDA_TOOLKIT_ROOT_DIR}/include")
set(EIGEN_ADD_TEST_FILENAME_EXTENSION "cu")

22
unsupported/test/cxx11_tensor_argmax_cuda.cu
View File

@ -56,6 +56,10 @@ void test_cuda_simple_argmax()
VERIFY_IS_EQUAL(out_max(Eigen::array<DenseIndex, 1>(0)), 72*53*97 - 1);
VERIFY_IS_EQUAL(out_min(Eigen::array<DenseIndex, 1>(0)), 0);
cudaFree(d_in);
cudaFree(d_out_max);
cudaFree(d_out_min);
}
template <int DataLayout>
@ -141,6 +145,9 @@ void test_cuda_argmax_dim()
// Expect max to be in the last index of the reduced dimension
VERIFY_IS_EQUAL(tensor_arg.data()[n], tensor.dimension(dim) - 1);
}
cudaFree(d_in);
cudaFree(d_out);
}
}
@ -227,15 +234,18 @@ void test_cuda_argmin_dim()
// Expect max to be in the last index of the reduced dimension
VERIFY_IS_EQUAL(tensor_arg.data()[n], tensor.dimension(dim) - 1);
}
cudaFree(d_in);
cudaFree(d_out);
}
}
void test_cxx11_tensor_cuda()
{
CALL_SUBTEST(test_cuda_simple_argmax<RowMajor>());
CALL_SUBTEST(test_cuda_simple_argmax<ColMajor>());
CALL_SUBTEST(test_cuda_argmax_dim<RowMajor>());
CALL_SUBTEST(test_cuda_argmax_dim<ColMajor>());
CALL_SUBTEST(test_cuda_argmin_dim<RowMajor>());
CALL_SUBTEST(test_cuda_argmin_dim<ColMajor>());
CALL_SUBTEST_1(test_cuda_simple_argmax<RowMajor>());
CALL_SUBTEST_1(test_cuda_simple_argmax<ColMajor>());
CALL_SUBTEST_2(test_cuda_argmax_dim<RowMajor>());
CALL_SUBTEST_2(test_cuda_argmax_dim<ColMajor>());
CALL_SUBTEST_3(test_cuda_argmin_dim<RowMajor>());
CALL_SUBTEST_3(test_cuda_argmin_dim<ColMajor>());
}

14
unsupported/test/cxx11_tensor_contract_cuda.cu
View File

@ -67,12 +67,16 @@ static void test_cuda_contraction(int m_size, int k_size, int n_size)
t_result = t_left.contract(t_right, dims);
cudaMemcpy(t_result_gpu.data(), d_t_result, t_result_bytes, cudaMemcpyDeviceToHost);
for (size_t i = 0; i < t_result.dimensions().TotalSize(); i++) {
if (fabs(t_result.data()[i] - t_result_gpu.data()[i]) >= 1e-4) {
std::cout << "mismatch detected at index " << i << ": " << t_result.data()[i]
<< " vs " << t_result_gpu.data()[i] << std::endl;
assert(false);
for (size_t i = 0; i < t_result.size(); i++) {
if (fabs(t_result(i) - t_result_gpu(i)) < 1e-4f) {
continue;
}
if (Eigen::internal::isApprox(t_result(i), t_result_gpu(i), 1e-4f)) {
continue;
}
std::cout << "mismatch detected at index " << i << ": " << t_result(i)
<< " vs " << t_result_gpu(i) << std::endl;
assert(false);
}
cudaFree((void*)d_t_left);

131
unsupported/test/cxx11_tensor_cuda.cu
View File

@ -63,6 +63,10 @@ void test_cuda_elementwise_small() {
out(Eigen::array<int, 1>(i)),
in1(Eigen::array<int, 1>(i)) + in2(Eigen::array<int, 1>(i)));
}
cudaFree(d_in1);
cudaFree(d_in2);
cudaFree(d_out);
}
void test_cuda_elementwise()
@ -113,6 +117,11 @@ void test_cuda_elementwise()
}
}
}
cudaFree(d_in1);
cudaFree(d_in2);
cudaFree(d_in3);
cudaFree(d_out);
}
void test_cuda_reduction()
@ -158,10 +167,13 @@ void test_cuda_reduction()
VERIFY_IS_APPROX(out(i,j), expected);
}
}
cudaFree(d_in1);
cudaFree(d_out);
}
template<int DataLayout>
static void test_cuda_contraction()
void test_cuda_contraction()
{
// with these dimensions, the output has 300 * 140 elements, which is
// more than 30 * 1024, which is the number of threads in blocks on
@ -216,10 +228,14 @@ static void test_cuda_contraction()
assert(false);
}
}
cudaFree(d_t_left);
cudaFree(d_t_right);
cudaFree(d_t_result);
}
template<int DataLayout>
static void test_cuda_convolution_1d()
void test_cuda_convolution_1d()
{
Tensor<float, 4, DataLayout> input(74,37,11,137);
Tensor<float, 1, DataLayout> kernel(4);
@ -266,9 +282,13 @@ static void test_cuda_convolution_1d()
}
}
}
cudaFree(d_input);
cudaFree(d_kernel);
cudaFree(d_out);
}
static void test_cuda_convolution_inner_dim_col_major_1d()
void test_cuda_convolution_inner_dim_col_major_1d()
{
Tensor<float, 4, ColMajor> input(74,9,11,7);
Tensor<float, 1, ColMajor> kernel(4);
@ -315,9 +335,13 @@ static void test_cuda_convolution_inner_dim_col_major_1d()
}
}
}
cudaFree(d_input);
cudaFree(d_kernel);
cudaFree(d_out);
}
static void test_cuda_convolution_inner_dim_row_major_1d()
void test_cuda_convolution_inner_dim_row_major_1d()
{
Tensor<float, 4, RowMajor> input(7,9,11,74);
Tensor<float, 1, RowMajor> kernel(4);
@ -364,10 +388,14 @@ static void test_cuda_convolution_inner_dim_row_major_1d()
}
}
}
cudaFree(d_input);
cudaFree(d_kernel);
cudaFree(d_out);
}
template<int DataLayout>
static void test_cuda_convolution_2d()
void test_cuda_convolution_2d()
{
Tensor<float, 4, DataLayout> input(74,37,11,137);
Tensor<float, 2, DataLayout> kernel(3,4);
@ -424,10 +452,14 @@ static void test_cuda_convolution_2d()
}
}
}
cudaFree(d_input);
cudaFree(d_kernel);
cudaFree(d_out);
}
template<int DataLayout>
static void test_cuda_convolution_3d()
void test_cuda_convolution_3d()
{
Tensor<float, 5, DataLayout> input(Eigen::array<int, 5>(74,37,11,137,17));
Tensor<float, 3, DataLayout> kernel(3,4,2);
@ -498,6 +530,10 @@ static void test_cuda_convolution_3d()
}
}
}
cudaFree(d_input);
cudaFree(d_kernel);
cudaFree(d_out);
}
@ -535,6 +571,9 @@ void test_cuda_lgamma(const Scalar stddev)
VERIFY_IS_APPROX(out(i,j), (std::lgamma)(in(i,j)));
}
}
cudaFree(d_in);
cudaFree(d_out);
}
template <typename Scalar>
@ -571,6 +610,9 @@ void test_cuda_erf(const Scalar stddev)
VERIFY_IS_APPROX(out(i,j), (std::erf)(in(i,j)));
}
}
cudaFree(d_in);
cudaFree(d_out);
}
template <typename Scalar>
@ -607,47 +649,50 @@ void test_cuda_erfc(const Scalar stddev)
VERIFY_IS_APPROX(out(i,j), (std::erfc)(in(i,j)));
}
}
cudaFree(d_in);
cudaFree(d_out);
}
void test_cxx11_tensor_cuda()
{
CALL_SUBTEST(test_cuda_elementwise_small());
CALL_SUBTEST(test_cuda_elementwise());
CALL_SUBTEST(test_cuda_reduction());
CALL_SUBTEST(test_cuda_contraction<ColMajor>());
CALL_SUBTEST(test_cuda_contraction<RowMajor>());
CALL_SUBTEST(test_cuda_convolution_1d<ColMajor>());
CALL_SUBTEST(test_cuda_convolution_1d<RowMajor>());
CALL_SUBTEST(test_cuda_convolution_inner_dim_col_major_1d());
CALL_SUBTEST(test_cuda_convolution_inner_dim_row_major_1d());
CALL_SUBTEST(test_cuda_convolution_2d<ColMajor>());
CALL_SUBTEST(test_cuda_convolution_2d<RowMajor>());
CALL_SUBTEST(test_cuda_convolution_3d<ColMajor>());
CALL_SUBTEST(test_cuda_convolution_3d<RowMajor>());
CALL_SUBTEST(test_cuda_lgamma<float>(1.0f));
CALL_SUBTEST(test_cuda_lgamma<float>(100.0f));
CALL_SUBTEST(test_cuda_lgamma<float>(0.01f));
CALL_SUBTEST(test_cuda_lgamma<float>(0.001f));
CALL_SUBTEST(test_cuda_erf<float>(1.0f));
CALL_SUBTEST(test_cuda_erf<float>(100.0f));
CALL_SUBTEST(test_cuda_erf<float>(0.01f));
CALL_SUBTEST(test_cuda_erf<float>(0.001f));
CALL_SUBTEST(test_cuda_erfc<float>(1.0f));
CALL_SUBTEST_1(test_cuda_elementwise_small());
CALL_SUBTEST_1(test_cuda_elementwise());
CALL_SUBTEST_1(test_cuda_reduction());
CALL_SUBTEST_2(test_cuda_contraction<ColMajor>());
CALL_SUBTEST_2(test_cuda_contraction<RowMajor>());
CALL_SUBTEST_3(test_cuda_convolution_1d<ColMajor>());
CALL_SUBTEST_3(test_cuda_convolution_1d<RowMajor>());
CALL_SUBTEST_3(test_cuda_convolution_inner_dim_col_major_1d());
CALL_SUBTEST_3(test_cuda_convolution_inner_dim_row_major_1d());
CALL_SUBTEST_3(test_cuda_convolution_2d<ColMajor>());
CALL_SUBTEST_3(test_cuda_convolution_2d<RowMajor>());
CALL_SUBTEST_3(test_cuda_convolution_3d<ColMajor>());
CALL_SUBTEST_3(test_cuda_convolution_3d<RowMajor>());
CALL_SUBTEST_4(test_cuda_lgamma<float>(1.0f));
CALL_SUBTEST_4(test_cuda_lgamma<float>(100.0f));
CALL_SUBTEST_4(test_cuda_lgamma<float>(0.01f));
CALL_SUBTEST_4(test_cuda_lgamma<float>(0.001f));
CALL_SUBTEST_4(test_cuda_erf<float>(1.0f));
CALL_SUBTEST_4(test_cuda_erf<float>(100.0f));
CALL_SUBTEST_4(test_cuda_erf<float>(0.01f));
CALL_SUBTEST_4(test_cuda_erf<float>(0.001f));
CALL_SUBTEST_4(test_cuda_erfc<float>(1.0f));
// CALL_SUBTEST(test_cuda_erfc<float>(100.0f));
CALL_SUBTEST(test_cuda_erfc<float>(5.0f)); // CUDA erfc lacks precision for large inputs
CALL_SUBTEST(test_cuda_erfc<float>(0.01f));
CALL_SUBTEST(test_cuda_erfc<float>(0.001f));
CALL_SUBTEST(test_cuda_lgamma<double>(1.0));
CALL_SUBTEST(test_cuda_lgamma<double>(100.0));
CALL_SUBTEST(test_cuda_lgamma<double>(0.01));
CALL_SUBTEST(test_cuda_lgamma<double>(0.001));
CALL_SUBTEST(test_cuda_erf<double>(1.0));
CALL_SUBTEST(test_cuda_erf<double>(100.0));
CALL_SUBTEST(test_cuda_erf<double>(0.01));
CALL_SUBTEST(test_cuda_erf<double>(0.001));
CALL_SUBTEST(test_cuda_erfc<double>(1.0));
CALL_SUBTEST_4(test_cuda_erfc<float>(5.0f)); // CUDA erfc lacks precision for large inputs
CALL_SUBTEST_4(test_cuda_erfc<float>(0.01f));
CALL_SUBTEST_4(test_cuda_erfc<float>(0.001f));
CALL_SUBTEST_4(test_cuda_lgamma<double>(1.0));
CALL_SUBTEST_4(test_cuda_lgamma<double>(100.0));
CALL_SUBTEST_4(test_cuda_lgamma<double>(0.01));
CALL_SUBTEST_4(test_cuda_lgamma<double>(0.001));
CALL_SUBTEST_4(test_cuda_erf<double>(1.0));
CALL_SUBTEST_4(test_cuda_erf<double>(100.0));
CALL_SUBTEST_4(test_cuda_erf<double>(0.01));
CALL_SUBTEST_4(test_cuda_erf<double>(0.001));
CALL_SUBTEST_4(test_cuda_erfc<double>(1.0));
// CALL_SUBTEST(test_cuda_erfc<double>(100.0));
CALL_SUBTEST(test_cuda_erfc<double>(5.0)); // CUDA erfc lacks precision for large inputs
CALL_SUBTEST(test_cuda_erfc<double>(0.01));
CALL_SUBTEST(test_cuda_erfc<double>(0.001));
CALL_SUBTEST_4(test_cuda_erfc<double>(5.0)); // CUDA erfc lacks precision for large inputs
CALL_SUBTEST_4(test_cuda_erfc<double>(0.01));
CALL_SUBTEST_4(test_cuda_erfc<double>(0.001));
}

4
unsupported/test/cxx11_tensor_device.cu
View File

@ -383,6 +383,6 @@ static void test_gpu() {
void test_cxx11_tensor_device()
{
CALL_SUBTEST(test_cpu());
CALL_SUBTEST(test_gpu());
CALL_SUBTEST_1(test_cpu());
CALL_SUBTEST_2(test_gpu());
}

12
unsupported/test/cxx11_tensor_empty.cpp
View File

@ -16,16 +16,20 @@ static void test_empty_tensor()
{
Tensor<float, 2> source;
Tensor<float, 2> tgt1 = source;
Tensor<float, 2> tgt2;
tgt2 = source;
Tensor<float, 2> tgt2(source);
Tensor<float, 2> tgt3;
tgt3 = tgt1;
tgt3 = tgt2;
}
static void test_empty_fixed_size_tensor()
{
TensorFixedSize<float, Sizes<0>> source;
TensorFixedSize<float, Sizes<0>> tgt1 = source;
TensorFixedSize<float, Sizes<0>> tgt2;
tgt2 = source;
TensorFixedSize<float, Sizes<0>> tgt2(source);
TensorFixedSize<float, Sizes<0>> tgt3;
tgt3 = tgt1;
tgt3 = tgt2;
}

7
unsupported/test/cxx11_tensor_reduction_cuda.cu
View File

@ -48,9 +48,12 @@ static void test_full_reductions() {
// Check that the CPU and GPU reductions return the same result.
VERIFY_IS_APPROX(full_redux(), full_redux_gpu());
gpu_device.deallocate(gpu_in_ptr);
gpu_device.deallocate(gpu_out_ptr);
}
void test_cxx11_tensor_reduction_cuda() {
CALL_SUBTEST(test_full_reductions<ColMajor>());
CALL_SUBTEST(test_full_reductions<RowMajor>());
CALL_SUBTEST_1(test_full_reductions<ColMajor>());
CALL_SUBTEST_2(test_full_reductions<RowMajor>());
}

85
unsupported/test/cxx11_tensor_thread_pool.cpp
View File

@ -17,7 +17,7 @@
using Eigen::Tensor;
static void test_multithread_elementwise()
void test_multithread_elementwise()
{
Tensor<float, 3> in1(2,3,7);
Tensor<float, 3> in2(2,3,7);
@ -40,7 +40,7 @@ static void test_multithread_elementwise()
}
static void test_multithread_compound_assignment()
void test_multithread_compound_assignment()
{
Tensor<float, 3> in1(2,3,7);
Tensor<float, 3> in2(2,3,7);
@ -64,7 +64,7 @@ static void test_multithread_compound_assignment()
}
template<int DataLayout>
static void test_multithread_contraction()
void test_multithread_contraction()
{
Tensor<float, 4, DataLayout> t_left(30, 50, 37, 31);
Tensor<float, 5, DataLayout> t_right(37, 31, 70, 2, 10);
@ -91,15 +91,20 @@ static void test_multithread_contraction()
for (ptrdiff_t i = 0; i < t_result.size(); i++) {
VERIFY(&t_result.data()[i] != &m_result.data()[i]);
if (fabs(t_result.data()[i] - m_result.data()[i]) >= 1e-4) {
std::cout << "mismatch detected: " << t_result.data()[i] << " vs " << m_result.data()[i] << std::endl;
assert(false);
if (fabs(t_result(i) - m_result(i)) < 1e-4) {
continue;
}
if (Eigen::internal::isApprox(t_result(i), m_result(i), 1e-4f)) {
continue;
}
std::cout << "mismatch detected at index " << i << ": " << t_result(i)
<< " vs " << m_result(i) << std::endl;
assert(false);
}
}
template<int DataLayout>
static void test_contraction_corner_cases()
void test_contraction_corner_cases()
{
Tensor<float, 2, DataLayout> t_left(32, 500);
Tensor<float, 2, DataLayout> t_right(32, 28*28);
@ -186,7 +191,7 @@ static void test_contraction_corner_cases()
}
template<int DataLayout>
static void test_multithread_contraction_agrees_with_singlethread() {
void test_multithread_contraction_agrees_with_singlethread() {
int contract_size = internal::random<int>(1, 5000);
Tensor<float, 3, DataLayout> left(internal::random<int>(1, 80),
@ -229,7 +234,7 @@ static void test_multithread_contraction_agrees_with_singlethread() {
template<int DataLayout>
static void test_multithreaded_reductions() {
void test_multithreaded_reductions() {
const int num_threads = internal::random<int>(3, 11);
ThreadPool thread_pool(num_threads);
Eigen::ThreadPoolDevice thread_pool_device(&thread_pool, num_threads);
@ -239,19 +244,19 @@ static void test_multithreaded_reductions() {
Tensor<float, 2, DataLayout> t1(num_rows, num_cols);
t1.setRandom();
Tensor<float, 1, DataLayout> full_redux(1);
Tensor<float, 0, DataLayout> full_redux;
full_redux = t1.sum();
Tensor<float, 1, DataLayout> full_redux_tp(1);
Tensor<float, 0, DataLayout> full_redux_tp;
full_redux_tp.device(thread_pool_device) = t1.sum();
// Check that the single threaded and the multi threaded reductions return
// the same result.
VERIFY_IS_APPROX(full_redux(0), full_redux_tp(0));
VERIFY_IS_APPROX(full_redux(), full_redux_tp());
}
static void test_memcpy() {
void test_memcpy() {
for (int i = 0; i < 5; ++i) {
const int num_threads = internal::random<int>(3, 11);
@ -270,7 +275,7 @@ static void test_memcpy() {
}
static void test_multithread_random()
void test_multithread_random()
{
Eigen::ThreadPool tp(2);
Eigen::ThreadPoolDevice device(&tp, 2);
@ -278,26 +283,52 @@ static void test_multithread_random()
t.device(device) = t.random<Eigen::internal::NormalRandomGenerator<float>>();
}
template<int DataLayout>
void test_multithread_shuffle()
{
Tensor<float, 4, DataLayout> tensor(17,5,7,11);
tensor.setRandom();
const int num_threads = internal::random<int>(2, 11);
ThreadPool threads(num_threads);
Eigen::ThreadPoolDevice device(&threads, num_threads);
Tensor<float, 4, DataLayout> shuffle(7,5,11,17);
array<ptrdiff_t, 4> shuffles = {{2,1,3,0}};
shuffle.device(device) = tensor.shuffle(shuffles);
for (int i = 0; i < 17; ++i) {
for (int j = 0; j < 5; ++j) {
for (int k = 0; k < 7; ++k) {
for (int l = 0; l < 11; ++l) {
VERIFY_IS_EQUAL(tensor(i,j,k,l), shuffle(k,j,l,i));
}
}
}
}
}
void test_cxx11_tensor_thread_pool()
{
CALL_SUBTEST(test_multithread_elementwise());
CALL_SUBTEST(test_multithread_compound_assignment());
CALL_SUBTEST_1(test_multithread_elementwise());
CALL_SUBTEST_1(test_multithread_compound_assignment());
CALL_SUBTEST(test_multithread_contraction<ColMajor>());
CALL_SUBTEST(test_multithread_contraction<RowMajor>());
CALL_SUBTEST_2(test_multithread_contraction<ColMajor>());
CALL_SUBTEST_2(test_multithread_contraction<RowMajor>());
CALL_SUBTEST(test_multithread_contraction_agrees_with_singlethread<ColMajor>());
CALL_SUBTEST(test_multithread_contraction_agrees_with_singlethread<RowMajor>());
CALL_SUBTEST_3(test_multithread_contraction_agrees_with_singlethread<ColMajor>());
CALL_SUBTEST_3(test_multithread_contraction_agrees_with_singlethread<RowMajor>());
// Exercise various cases that have been problematic in the past.
CALL_SUBTEST(test_contraction_corner_cases<ColMajor>());
CALL_SUBTEST(test_contraction_corner_cases<RowMajor>());
CALL_SUBTEST_4(test_contraction_corner_cases<ColMajor>());
CALL_SUBTEST_4(test_contraction_corner_cases<RowMajor>());
CALL_SUBTEST(test_multithreaded_reductions<ColMajor>());
CALL_SUBTEST(test_multithreaded_reductions<RowMajor>());
CALL_SUBTEST_5(test_multithreaded_reductions<ColMajor>());
CALL_SUBTEST_5(test_multithreaded_reductions<RowMajor>());
CALL_SUBTEST(test_memcpy());
CALL_SUBTEST(test_multithread_random());
CALL_SUBTEST_6(test_memcpy());
CALL_SUBTEST_6(test_multithread_random());
CALL_SUBTEST_6(test_multithread_shuffle<ColMajor>());
CALL_SUBTEST_6(test_multithread_shuffle<RowMajor>());
}