diff --git a/Eigen/src/Core/Redux.h b/Eigen/src/Core/Redux.h index 796e6c4eb..64acc0ffd 100644 --- a/Eigen/src/Core/Redux.h +++ b/Eigen/src/Core/Redux.h @@ -44,9 +44,10 @@ public: }; enum { + MayLinearize = (int(Evaluator::Flags) & LinearAccessBit), MightVectorize = (int(Evaluator::Flags)&ActualPacketAccessBit) && (functor_traits::PacketAccess), - MayLinearVectorize = bool(MightVectorize) && (int(Evaluator::Flags)&LinearAccessBit), + MayLinearVectorize = bool(MightVectorize) && bool(MayLinearize), MaySliceVectorize = bool(MightVectorize) && (int(SliceVectorizedWork)==Dynamic || int(SliceVectorizedWork)>=3) }; @@ -54,6 +55,7 @@ public: enum { Traversal = int(MayLinearVectorize) ? int(LinearVectorizedTraversal) : int(MaySliceVectorize) ? int(SliceVectorizedTraversal) + : int(MayLinearize) ? int(LinearTraversal) : int(DefaultTraversal) }; @@ -142,6 +144,46 @@ struct redux_novec_unroller static EIGEN_STRONG_INLINE Scalar run(const Evaluator&, const Func&) { return Scalar(); } }; +template +struct redux_novec_linear_unroller +{ + enum { + HalfLength = Length/2 + }; + + typedef typename Evaluator::Scalar Scalar; + + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func& func) + { + return func(redux_novec_linear_unroller::run(eval,func), + redux_novec_linear_unroller::run(eval,func)); + } +}; + +template +struct redux_novec_linear_unroller +{ + typedef typename Evaluator::Scalar Scalar; + + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Evaluator &eval, const Func&) + { + return eval.coeff(Start); + } +}; + +// This is actually dead code and will never be called. It is required +// to prevent false warnings regarding failed inlining though +// for 0 length run() will never be called at all. +template +struct redux_novec_linear_unroller +{ + typedef typename Evaluator::Scalar Scalar; + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE Scalar run(const Evaluator&, const Func&) { return Scalar(); } +}; + /*** vectorization ***/ template @@ -180,6 +222,40 @@ struct redux_vec_unroller } }; +template +struct redux_vec_linear_unroller +{ + template + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func& func) + { + enum { + PacketSize = unpacket_traits::size, + HalfLength = Length/2 + }; + + return func.packetOp( + redux_vec_linear_unroller::template run(eval,func), + redux_vec_linear_unroller::template run(eval,func) ); + } +}; + +template +struct redux_vec_linear_unroller +{ + template + EIGEN_DEVICE_FUNC + static EIGEN_STRONG_INLINE PacketType run(const Evaluator &eval, const Func&) + { + enum { + PacketSize = unpacket_traits::size, + index = Start * PacketSize, + alignment = Evaluator::Alignment + }; + return eval.template packet(index); + } +}; + /*************************************************************************** * Part 3 : implementation of all cases ***************************************************************************/ @@ -210,6 +286,23 @@ struct redux_impl } }; +template +struct redux_impl +{ + typedef typename Evaluator::Scalar Scalar; + + template + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE + Scalar run(const Evaluator &eval, const Func& func, const XprType& xpr) + { + eigen_assert(xpr.size()>0 && "you are using an empty matrix"); + Scalar res = eval.coeff(0); + for(Index k = 1; k < xpr.size(); ++k) + res = func(res, eval.coeff(k)); + return res; + } +}; + template struct redux_impl : redux_novec_unroller @@ -224,6 +317,20 @@ struct redux_impl } }; +template +struct redux_impl + : redux_novec_linear_unroller +{ + typedef redux_novec_linear_unroller Base; + typedef typename Evaluator::Scalar Scalar; + template + EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE + Scalar run(const Evaluator &eval, const Func& func, const XprType& /*xpr*/) + { + return Base::run(eval,func); + } +}; + template struct redux_impl { @@ -342,13 +449,13 @@ struct redux_impl EIGEN_ONLY_USED_FOR_DEBUG(xpr) eigen_assert(xpr.rows()>0 && xpr.cols()>0 && "you are using an empty matrix"); if (VectorizedSize > 0) { - Scalar res = func.predux(redux_vec_unroller::template run(eval,func)); + Scalar res = func.predux(redux_vec_linear_unroller::template run(eval,func)); if (VectorizedSize != Size) - res = func(res,redux_novec_unroller::run(eval,func)); + res = func(res,redux_novec_linear_unroller::run(eval,func)); return res; } else { - return redux_novec_unroller::run(eval,func); + return redux_novec_linear_unroller::run(eval,func); } } };